2602 files changed, 610904 insertions, 440488 deletions

diff --git a/.clang-format b/.clang-format
new file mode 100644
index 00000000000..84c2efe3fad
--- /dev/null
+++ b/.clang-format
@@ -0,0 +1,107 @@
+---
+Language:        Cpp
+# BasedOnStyle:  Chromium
+AccessModifierOffset: -1
+AlignAfterOpenBracket: Align
+AlignConsecutiveAssignments: false
+AlignConsecutiveDeclarations: false
+AlignEscapedNewlines: Right
+AlignOperands:   true
+AlignTrailingComments: true
+AllowAllParametersOfDeclarationOnNextLine: false
+AllowShortBlocksOnASingleLine: false
+AllowShortCaseLabelsOnASingleLine: false
+AllowShortFunctionsOnASingleLine: Inline
+AllowShortIfStatementsOnASingleLine: false
+AllowShortLoopsOnASingleLine: false
+AlwaysBreakAfterDefinitionReturnType: All
+AlwaysBreakAfterReturnType: All
+AlwaysBreakBeforeMultilineStrings: true
+AlwaysBreakTemplateDeclarations: true
+BinPackArguments: true
+BinPackParameters: true
+BraceWrapping:
+  AfterClass:      false
+  AfterControlStatement: false
+  AfterEnum:       false
+  AfterFunction:   true
+  AfterNamespace:  false
+  AfterObjCDeclaration: false
+  AfterStruct:     false
+  AfterUnion:      false
+  BeforeCatch:     false
+  BeforeElse:      false
+  IndentBraces:    false
+  SplitEmptyFunction: true
+  SplitEmptyRecord: true
+  SplitEmptyNamespace: true
+BreakBeforeBinaryOperators: None
+BreakBeforeBraces: Linux
+BreakBeforeInheritanceComma: false
+BreakBeforeTernaryOperators: true
+BreakConstructorInitializersBeforeComma: false
+BreakConstructorInitializers: BeforeColon
+BreakAfterJavaFieldAnnotations: false
+BreakStringLiterals: true
+ColumnLimit:     80
+CommentPragmas:  '^ IWYU pragma:'
+CompactNamespaces: false
+ConstructorInitializerAllOnOneLineOrOnePerLine: true
+ConstructorInitializerIndentWidth: 4
+ContinuationIndentWidth: 4
+Cpp11BracedListStyle: true
+DerivePointerAlignment: false
+DisableFormat:   false
+ExperimentalAutoDetectBinPacking: false
+FixNamespaceComments: true
+ForEachMacros:
+  - foreach
+  - Q_FOREACH
+  - BOOST_FOREACH
+IncludeCategories:
+  - Regex:           '^<.*\.h>'
+    Priority:        1
+  - Regex:           '^<.*'
+    Priority:        2
+  - Regex:           '.*'
+    Priority:        3
+IncludeIsMainRegex: '([-_](test|unittest))?$'
+IndentCaseLabels: true
+IndentWidth:     4
+IndentWrappedFunctionNames: false
+JavaScriptQuotes: Leave
+JavaScriptWrapImports: true
+KeepEmptyLinesAtTheStartOfBlocks: false
+MacroBlockBegin: ''
+MacroBlockEnd:   ''
+MaxEmptyLinesToKeep: 1
+NamespaceIndentation: None
+ObjCBlockIndentWidth: 2
+ObjCSpaceAfterProperty: false
+ObjCSpaceBeforeProtocolList: false
+PenaltyBreakAssignment: 200
+PenaltyBreakBeforeFirstCallParameter: 1
+PenaltyBreakComment: 300
+PenaltyBreakFirstLessLess: 120
+PenaltyBreakString: 1000
+PenaltyExcessCharacter: 1000000
+PenaltyReturnTypeOnItsOwnLine: 200
+PointerAlignment: Right
+ReflowComments:  true
+SortIncludes:    false
+SortUsingDeclarations: true
+SpaceAfterCStyleCast: false
+SpaceAfterTemplateKeyword: true
+SpaceBeforeAssignmentOperators: true
+SpaceBeforeParens: ControlStatements
+SpaceInEmptyParentheses: false
+SpacesBeforeTrailingComments: 2
+SpacesInAngles:  false
+SpacesInContainerLiterals: true
+SpacesInCStyleCastParentheses: false
+SpacesInParentheses: false
+SpacesInSquareBrackets: false
+Standard:        Auto
+TabWidth:        8
+UseTab:          Never
+...
diff --git a/.github/ISSUE_TEMPLATE b/.github/ISSUE_TEMPLATE
new file mode 100644
index 00000000000..386ed2d8dd5
--- /dev/null
+++ b/.github/ISSUE_TEMPLATE
@@ -0,0 +1,30 @@
+<!-- Please use this template while reporting an issue, providing as much information as possible. Failure to do so may result in a delayed response. Thank you! -->
+
+**Description of problem:**
+
+
+**The exact command to reproduce the issue**:
+
+
+**The full output of the command that failed**:
+<details>
+
+
+
+</details>
+
+**Expected results:**
+
+
+**Additional info:**
+
+
+**- The output of the `gluster volume info` command**:
+<details>
+
+
+
+</details>
+
+**- The operating system / glusterfs version**:
+
diff --git a/xlators/features/glupy/src/__init__.py.in b/.github/PULL_REQUEST_TEMPLATE
index e69de29bb2d..e69de29bb2d 100644
--- a/xlators/features/glupy/src/__init__.py.in
+++ b/.github/PULL_REQUEST_TEMPLATE
diff --git a/.github/RELEASE_TRACKER_TEMPLATE b/.github/RELEASE_TRACKER_TEMPLATE
new file mode 100644
index 00000000000..502bbd5556c
--- /dev/null
+++ b/.github/RELEASE_TRACKER_TEMPLATE
@@ -0,0 +1,12 @@
+<!-- Please use this template while creating a tracker issue -->
+
+**Description of problem:**
+A tracker issue to track the issues that will be fixed as a part of this release
+
+
+**Major or minor release**:
+
+
+**Release version**:
+
+
diff --git a/.github/stale.yml b/.github/stale.yml
new file mode 100644
index 00000000000..460e327c6ea
--- /dev/null
+++ b/.github/stale.yml
@@ -0,0 +1,25 @@
+# Number of days of inactivity before an issue becomes stale
+daysUntilStale: 210
+# Number of days of inactivity before a stale issue is closed
+daysUntilClose: 15
+# Issues with these labels will never be considered stale
+exemptLabels:
+  - pinned
+  - security
+# Label to use when marking an issue as stale
+staleLabel: wontfix
+
+# Comment to post when marking an issue as stale. Set to `false` to disable
+markComment: >
+  Thank you for your contributions.
+
+  Noticed that this issue is not having any activity in last ~6 months! We
+  are marking this issue as stale because it has not had recent activity.
+
+  It will be closed in 2 weeks if no one responds with a comment here.
+  
+
+# Comment to post when closing a stale issue. Set to `false` to disable
+closeComment: >
+  Closing this issue as there was no update since my last update on issue.
+  If this is an issue which is still valid, feel free to open it.
diff --git a/.gitignore b/.gitignore
index cd52fef1dda..fc5ba586f8e 100644
--- a/.gitignore
+++ b/.gitignore
@@ -3,11 +3,17 @@ autom4te.cache
 build
 config.*
 configure
+cscope.*
+tags
 depcomp
+INSTALL
 install-sh
 ltmain.sh
+Makefile
 Makefile.in
 missing
+stamp-h1
+stamp-h2
 test-driver
 *compile
 *.gcda
@@ -23,53 +29,97 @@ test-driver
 *.patch
 .libs
 .deps
-Makefile
-stamp-h1
+.dirstamp
+
 # Softlinks to test and log
 log
 *.vol
+.clang-format
+
+# cmocka unit tests
+*.log
+*.trs
+*_unittest
 
 # Generated files
-tests/env.rc
-dht_layout_unittest
-mem_pool_unittest
-tests/utils/arequal-checksum
+site.h
+*.py[co]
 api/examples/__init__.py
-api/examples/__init__.py?
 api/examples/setup.py
-xlators/features/glupy/src/__init__.py
-contrib/argp-standalone/libargp.a
-contrib/uuid/uuid_types.h
+api/src/gfapi.map
+cli/src/gluster
+contrib/fuse-util/fusermount-glusterfs
+extras/geo-rep/gsync-sync-gfid
+extras/geo-rep/schedule_georep.py
+extras/snap_scheduler/conf.py
 extras/init.d/glusterd-Debian
+extras/init.d/glusterd-FreeBSD
 extras/init.d/glusterd-Redhat
 extras/init.d/glusterd-SuSE
-extras/init.d/glusterd-FreeBSD
 extras/init.d/glusterd.plist
-extras/systemd/glusterd.service
 extras/ocf/glusterd
 extras/ocf/volume
+extras/run-gluster.tmpfiles
+extras/systemd/glusterd.service
+extras/systemd/gluster-ta-volume.service
+extras/systemd/glusterfssharedstorage.service
 extras/who-wrote-glusterfs/gitdm
-extras/geo-rep/gsync-sync-gfid
+geo-replication/.tox
+geo-replication/gsyncd.conf
+geo-replication/src/gsyncd
+geo-replication/src/peer_gsec_create
+geo-replication/src/peer_mountbroker
+geo-replication/src/peer_mountbroker.py
 geo-replication/src/set_geo_rep_pem_keys.sh
-geo-replication/syncdaemon/configinterface.py
+geo-replication/src/peer_georep-sshkey.py
+geo-replication/syncdaemon.egg-info
+geo-replication/syncdaemon/conf.py
+geo-replication/tests/unit/.coverage
+geo-replication/tests/unit/cover
+geo-replication/tests/unit/coverage.xml
+geo-replication/tests/unit/nosetests.xml
+geo-replication/tests/unit/results.html
 glusterfs-api.pc
 glusterfs.spec
-glusterfsd/src/glusterfsd
 glusterfsd/src/glusterd
 glusterfsd/src/glusterfs
+glusterfsd/src/glusterfsd
+glusterfsd/src/gf_attach
+heal/src/glfsheal
 libgfchangelog.pc
 libglusterfs/src/graph.lex.c
 libglusterfs/src/y.tab.c
 libglusterfs/src/y.tab.h
+libglusterfs/src/defaults.c
+libglusterfs/src/cli1-xdr.h
+libglusterfs/src/protocol-common.h
 libtool
+# copied XDR for cyclic libglusterfs <-> rpc-header dependency
 run-tests.sh
+!tests/basic/fuse/Makefile
+!tests/basic/gfapi/Makefile
+tests/env.rc
+tests/utils/arequal-checksum
+xlators/features/glupy/src/__init__.py
+xlators/features/glupy/src/setup.py
 xlators/mount/fuse/utils/mount.glusterfs
 xlators/mount/fuse/utils/mount_glusterfs
-xlators/features/glupy/src/setup.py
-geo-replication/src/peer_add_secret_pub
-geo-replication/src/peer_gsec_create
-geo-replication/src/gsyncd
-cli/src/gluster
-contrib/fuse-util/fusermount-glusterfs
-geo-replication/src/gsyncd
-api/src/gfapi.map
+extras/peer_add_secret_pub
+tools/gfind_missing_files/gcrawler
+tools/glusterfind/glusterfind
+tools/glusterfind/src/tool.conf
+# Eventing
+events/src/eventsapiconf.py
+extras/systemd/glustereventsd.service
+events/src/eventtypes.py
+libglusterfs/src/eventtypes.h
+extras/init.d/glustereventsd-Debian
+extras/init.d/glustereventsd-FreeBSD
+extras/init.d/glustereventsd-Redhat
+tools/setgfid2path/src/gluster-setgfid2path
+xlators/features/cloudsync/src/cloudsync-autogen-fops.c
+xlators/features/cloudsync/src/cloudsync-autogen-fops.h
+xlators/features/utime/src/utime-autogen-fops.c
+xlators/features/utime/src/utime-autogen-fops.h
+tests/basic/metadisp/ftruncate
+xlators/features/metadisp/src/fops.c
diff --git a/.mailmap b/.mailmap
index d4a13b40065..141a1667ffa 100644
--- a/.mailmap
+++ b/.mailmap
@@ -10,15 +10,20 @@ Amar Tumballi <amarts@redhat.com> <amar@gluster.com> <amar@del.gluster.com>
 Anand Avati <avati@redhat.com> <avati@gluster.com> <avati@dev.gluster.com> <avati@amp.gluster.com> <avati@blackhole.gluster.com>
 Anush Shetty <ashetty@redhat.com> <anush@gluster.com>
 Csaba Henk <csaba@redhat.com> <csaba@gluster.com> <csaba@lowlife.hu> <csaba@zresearch.com>
+Günther Deschner <gd@redhat.com> <gd@samba.org>
 Harshavardhana <fharshav@redhat.com> <harsha@gluster.com> <harsha@zresearch.com> <harsha@dev.gluster.com> <harsha@harshavardhana.net>
+Ji-Hyeon Gim <potatogim@gluesys.com> <potatogim@potatogim.net>
 Justin Clift <justin@gluster.org> <jclift@redhat.com>
-Kaleb S. KEITHLEY <kkeithle@redhat.com> <kkeithle@f16node1.kkeithle.usersys.redhat.com>
+Kaleb S. KEITHLEY <kkeithle@redhat.com> <kkeithle@f16node1.kkeithle.usersys.redhat.com> <kkeithle@linux.keithley.org>
 Kaushal M <kaushal@redhat.com> <kaushal@gluster.com>
 Kaushik BV <kbudiger@redhat.com> <kaushikbv@gluster.com>
 Krishna Srinivas <ksriniva@redhat.com> <krishna@gluster.com> <krishna@zresearch.com> <krishna@guest-laptop>
 Krishnan Parthasarathi <kparthas@redhat.com> <kp@gluster.com>
 Louis Zuckerman <louiszuckerman@gmail.com> <me@louiszuckerman.com>
 M S Vishwanath Bhat <vbhat@redhat.com> <msvbhat@gmail.com> <vishwanath@gluster.com>
+Michael Adam <madam@redhat.com> <obnox@samba.org>
+Oleksandr Natalenko <oleksandr@natalenko.name> <o.natalenko@lanet.ua>
+Patrick Uiterwijk <puiterwijk@fedoraproject.org> <patrick@puiterwijk.org>
 Pavan Sondur <pavan@gluster.com> <pavan@dev.gluster.com>
 Pete Zaitcev <zaitcev@kotori.zaitcev.us> <zaitcev@yahoo.com>
 Pranith Kumar K <pkarampu@redhat.com> <pranithk@gluster.com>
@@ -27,9 +32,12 @@ Raghavendra Bhat <raghavendra@redhat.com> <raghavendrabhat@gluster.com>
 Raghavendra G <rgowdapp@redhat.com> <raghavendra@gluster.com> <raghavendra@zresearch.com>
 Rahul C S <rahulcs@redhat.com> <rahulcssjce@gmail.com>
 Rajesh Amaravathi <rajesh@redhat.com> <rajesh@gluster.com> <rajesh.amaravathi@gmail.com>
+Ravishankar N <ravishankar@redhat.com> <root@ravi2.(none)>
+Sakshi Bansal <sabansal@redhat.com> <sabansal@localhost.localdomain>
 Shehjar Tikoo <shehjart@gluster.com> <shehjart@zresearch.com>
 Venky Shankar <vshankar@redhat.com> <venky@gluster.com>
 Vijay Bellur <vbellur@redhat.com> <vijay@gluster.com> <vijay@dev.gluster.com>
 Vijaykumar Koppad <vkoppad@redhat.com> <vijaykumar.koppad@gmail.com>
+Vijaikumar Mallikarjuna <vmallika@redhat.com>
 Vikas Gorur <vikas@gluster.com> <vikas@zresearch.com>
 shishir gowda <sgowda@redhat.com> <shishirng@gluster.com>
diff --git a/.testignore b/.testignore
new file mode 100644
index 00000000000..fe8f838bf2b
--- /dev/null
+++ b/.testignore
@@ -0,0 +1,64 @@
+.github/ISSUE_TEMPLATE
+.github/PULL_REQUEST_TEMPLATE
+.github/stale.yml
+.gitignore
+.mailmap
+.testignore
+.clang-format
+rfc.sh
+submit-for-review.sh
+AUTHORS
+CONTRIBUTING.md
+COPYING-GPLV2
+COPYING-LGPLV3
+ChangeLog
+INSTALL
+MAINTAINERS
+NEWS
+README.md
+THANKS
+COMMITMENT
+api/examples/README
+api/examples/getvolfile.py
+api/src/README.Symbol_Versions
+build-aux/checkpatch.pl
+contrib/fuse-lib/COPYING.LIB
+contrib/fuse-util/COPYING
+contrib/macfuse/COPYING.txt
+doc/*
+extras/FreeBSD/README.FreeBSD
+extras/Solaris/README.solaris
+extras/Ubuntu/README.Ubuntu
+extras/benchmarking/README
+extras/cliutils/README.md
+extras/command-completion/README
+extras/create_new_xlator/README.md
+extras/glusterfs.vim
+extras/glusterfs-logrotate
+extras/glusterfs-georep-logrotate
+extras/init.d/glusterd-Debian.in
+extras/init.d/glusterd-FreeBSD.in
+extras/init.d/glusterd-Redhat.in
+extras/init.d/glusterd-SuSE.in
+extras/init.d/glusterd.plist.in
+extras/init.d/glustereventsd-Debian.in
+extras/init.d/glustereventsd-FreeBSD.in
+extras/init.d/glustereventsd-Redhat.in
+extras/init.d/rhel5-load-fuse.modules
+extras/logger.conf.example
+extras/snap_scheduler/README.md
+extras/test/ld-preload-test/README
+extras/who-wrote-glusterfs/*
+extras/distributed-testing/*
+geo-replication/syncdaemon/README.md
+geo-replication/test-requirements.txt
+rpc/xdr/src/.gitignore
+tests/README.md
+xlators/experimental/README.md
+xlators/experimental/dht2/README.md
+xlators/experimental/dht2/TODO.md
+xlators/experimental/posix2/README.md
+xlators/experimental/posix2/TODO.md
+xlators/features/glupy/doc/README.md
+xlators/features/glupy/doc/TESTING
+xlators/features/glupy/doc/test.vol
diff --git a/COMMITMENT b/COMMITMENT
new file mode 100644
index 00000000000..16b75efcf29
--- /dev/null
+++ b/COMMITMENT
@@ -0,0 +1,46 @@
+Common Cure Rights Commitment
+Version 1.0
+
+Before filing or continuing to prosecute any legal proceeding or claim
+(other than a Defensive Action) arising from termination of a Covered
+License, we commit to extend to the person or entity ('you') accused
+of violating the Covered License the following provisions regarding
+cure and reinstatement, taken from GPL version 3. As used here, the
+term 'this License' refers to the specific Covered License being
+enforced.
+
+    However, if you cease all violation of this License, then your
+    license from a particular copyright holder is reinstated (a)
+    provisionally, unless and until the copyright holder explicitly
+    and finally terminates your license, and (b) permanently, if the
+    copyright holder fails to notify you of the violation by some
+    reasonable means prior to 60 days after the cessation.
+
+    Moreover, your license from a particular copyright holder is
+    reinstated permanently if the copyright holder notifies you of the
+    violation by some reasonable means, this is the first time you
+    have received notice of violation of this License (for any work)
+    from that copyright holder, and you cure the violation prior to 30
+    days after your receipt of the notice.
+
+We intend this Commitment to be irrevocable, and binding and
+enforceable against us and assignees of or successors to our
+copyrights.
+
+Definitions
+
+'Covered License' means the GNU General Public License, version 2
+(GPLv2), the GNU Lesser General Public License, version 2.1
+(LGPLv2.1), or the GNU Library General Public License, version 2
+(LGPLv2), all as published by the Free Software Foundation.
+
+'Defensive Action' means a legal proceeding or claim that We bring
+against you in response to a prior proceeding or claim initiated by
+you or your affiliate.
+
+'We' means each contributor to this repository as of the date of
+inclusion of this file, including subsidiaries of a corporate
+contributor.
+
+This work is available under a Creative Commons Attribution-ShareAlike
+4.0 International license (https://creativecommons.org/licenses/by-sa/4.0/).
diff --git a/CONTRIBUTING b/CONTRIBUTING
deleted file mode 100644
index 7bccd88d7e5..00000000000
--- a/CONTRIBUTING
+++ /dev/null
@@ -1,25 +0,0 @@
-        Developer's Certificate of Origin 1.1
-
-        By making a contribution to this project, I certify that:
-
-        (a) The contribution was created in whole or in part by me and I
-            have the right to submit it under the open source license
-            indicated in the file; or
-
-        (b) The contribution is based upon previous work that, to the best
-            of my knowledge, is covered under an appropriate open source
-            license and I have the right under that license to submit that
-            work with modifications, whether created in whole or in part
-            by me, under the same open source license (unless I am
-            permitted to submit under a different license), as indicated
-            in the file; or
-
-        (c) The contribution was provided directly to me by some other
-            person who certified (a), (b) or (c) and I have not modified
-            it.
-
-        (d) I understand and agree that this project and the contribution
-            are public and that a record of the contribution (including all
-            personal information I submit with it, including my sign-off) is
-            maintained indefinitely and may be redistributed consistent with
-            this project or the open source license(s) involved.
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
new file mode 100644
index 00000000000..65fc3497104
--- /dev/null
+++ b/CONTRIBUTING.md
@@ -0,0 +1,114 @@
+# GlusterFS project Contribution guidelines
+
+## Development Workflow
+
+We follow most of the details as per the [document here](https://help.github.com/en/github/collaborating-with-issues-and-pull-requests). If you are not aware of the github workflow, it is recommended to go through them before continuing here.
+
+
+#### Get the Repository setup
+
+0. Fork Repository
+   - Fork [GlusterFS repository](https://github.com/gluster/glusterfs/fork).
+
+1. Clone Repository
+   - Clone the glusterfs repo freshly from github using below steps.
+
+```
+   git clone git@github.com:${username}/glusterfs.git
+   cd glusterfs/
+   git remote add upstream git@github.com:gluster/glusterfs.git
+```
+
+About two tasks are one time for the life time. You can continue to use the same repository for all the work in future.
+
+#### Development & Other flows
+
+0. Issue:
+   - Make sure there is an issue filed for the task you are working on.
+   - If it is not filed, open the issue with all the description.
+   - If it is a bug fix, add label "Type:Bug".
+   - If it is an RFC, provide all the documentation, and request for "DocApproved", and "SpecApproved" label.
+
+1. Code:
+   - Start coding
+   - Build and test locally
+   - Make sure clang-format is installed and is run on the patch.
+
+2. Keep up-to-date
+   - GlusterFS is a large project with many developers, so there would be one or the other patch everyday.
+   - It is critical for developer to be up-to-date with `devel` repo to be Conflict-Free when PR is opened.
+   - Git provides many options to keep up-to-date, below is one of them
+```
+   git fetch upstream
+   git rebase upstream/devel
+```
+   - It is recommended you keep pushing to your repo every day, so you don't loose any work.
+     - It can be done by `./rfc.sh` (or `git push origin HEAD:issueNNN`)
+
+2. Commit Message / PR description:
+   - The name of the branch on your personal fork can start with issueNNNN, followed by anything of your choice.
+   - PRs continue to have the title of format "component: \<title\>", like it is practiced now.
+   - When you open a PR, having a reference Issue for the commit is mandatory in GlusterFS.
+   - Commit message can have, either `Fixes: #NNNN` or `Updates: #NNNN` in a separate line in the commit message.
+     - Here, NNNN is the Issue ID in glusterfs repository.
+   - Each commit needs the author to have the "Signed-off-by: Name \<email\>" line.
+     - Can do this by `-s` option for `git commit`.
+   - If the PR is not ready for review, apply the label `work-in-progress`.
+     - Check the availability of "Draft PR" is present for you, if yes, use that instead.
+
+3. Tests:
+   - All the required smoke tests would be auto-triggered.
+     - Developers get a chance to retrigger the smoke tests using **"/recheck smoke"** as comment.
+   - The "regression" tests would be triggered by a comment **"/run regression"** from developers in the [@gluster-maintainers](https://github.com/orgs/gluster/teams/gluster-maintainers) group.
+     - Ask for help as comment in PR if you have any questions about the process!
+
+4. Review Process:
+   - `+2` : is equivalent to "Approve" from the people in the maintainer's group.
+   - `+1` : can be given by a maintainer/reviewer by explicitly stating that in the comment.
+   - `-1` : provide details on required changes and pick "Request Changes" while submitting your review.
+   - `-2` : done by adding the `DO-NOT-MERGE` label.
+
+   - Any further discussions can happen as comments in the PR.
+
+5. Making changes:
+   - There are 2 approaches to submit changes done after addressing review comments.
+     - Commit changes as a new commit on top of the original commits in the branch, and push the changes to same branch (issueNNNN)
+     - Commit changes into the same commit with `--amend` option, and do a push to the same branch with `--force` option.
+
+6. Merging:
+   - GlusterFS project follows 'Squash and Merge' method
+     - This is mainly to preserve the historic Gerrit method of one patch in `git log` for one URL link.
+     - This also makes every merge a complete patch, which has passed all tests.
+   - The merging of the patch is expected to be done by the maintainers.
+     - It can be done when all the tests (smoke and regression) pass.
+     - When the PR has 'Approved' flag from corresponding maintainer.
+   - If you feel there is delay, feel free to add a comment, discuss the same in Slack channel, or send email.
+
+## By contributing to this project, the contributor would need to agree to below.
+
+### Developer's Certificate of Origin 1.1
+
+By making a contribution to this project, I certify that:
+
+(a) The contribution was created in whole or in part by me and I
+    have the right to submit it under the open source license
+    indicated in the file; or
+
+(b) The contribution is based upon previous work that, to the best
+    of my knowledge, is covered under an appropriate open source
+    license and I have the right under that license to submit that
+    work with modifications, whether created in whole or in part
+    by me, under the same open source license (unless I am
+    permitted to submit under a different license), as indicated
+    in the file; or
+
+(c) The contribution was provided directly to me by some other
+    person who certified (a), (b) or (c) and I have not modified
+    it.
+
+(d) I understand and agree that this project and the contribution
+    are public and that a record of the contribution (including all
+    personal information I submit with it, including my sign-off) is
+    maintained indefinitely and may be redistributed consistent with
+    this project or the open source license(s) involved.
+
diff --git a/INSTALL b/INSTALL
index f70ef15b03f..a56390e54fb 100644
--- a/INSTALL
+++ b/INSTALL
@@ -3,18 +3,29 @@ Installation Instructions
 
 0. If you have cloned from git, run ./autogen.sh.
 
-1. Run ./configure after untaring the package.
+1. Run ./configure.
 
  bash# ./configure
  GlusterFS configure summary
  ===========================
- FUSE client        : yes
- Infiniband verbs   : yes
- epoll IO multiplex : yes
- Berkeley-DB        : yes
- libglusterfsclient : yes
- mod_glusterfs      : yes
- argp-standalone    : no
+ GlusterFS configure summary
+  ===========================
+   FUSE client          : yes
+   Infiniband verbs     : yes
+   epoll IO multiplex   : yes
+   argp-standalone      : no
+   fusermount           : yes
+   readline             : yes
+   georeplication       : yes
+   Linux-AIO            : yes
+   Enable Debug         : no
+   Block Device xlator  : yes
+   glupy                : yes
+   Use syslog           : yes
+   XML output           : yes
+   QEMU Block formats   : yes
+   Encryption xlator    : yes
+
  
 The configure summary will tell you what all components will be built with 
 GlusterFS. Other than 'argp-standalone' if something else says 'no', that 
@@ -26,9 +37,11 @@ be used if the system doesn't have a proper argp package installed.
  bash# make 
  bash# make install
 
-Installation complete :-)
+Installation completed :-)
 
  bash# glusterfs --version
 
 Make sure your version is the latest from the release, and the one you 
 just installed :-)
+
+For more information on GlusterFS installation refer# http://docs.gluster.org/en/latest/Developer-guide/Building-GlusterFS/
diff --git a/MAINTAINERS b/MAINTAINERS
index 96aa5746764..953e8755fd9 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -11,7 +11,7 @@ consult gluster-devel@gluster.org and not any specific individual privately.
 
 Descriptions of section entries:
 
-	M: Mail patches to: FullName <address@domain>
+	M: Main contact that knows and takes care of this area
 	L: Mailing list that is relevant to this area
 	W: Web-page with status/info
 	Q: Patchwork web based patch tracking system site
@@ -46,181 +46,409 @@ Descriptions of section entries:
 	      matches patches or files that contain one or more of the words
 	      printk, pr_info or pr_err
 	   One regex pattern per line.  Multiple K: lines acceptable.
+        P: Peer for a component
 
 
 General Project Architects
 --------------------------
 M: Amar Tumballi <amarts@gmail.com>
-M: Anand Avati <avati@redhat.com>
-M: Jeff Darcy <jdarcy@redhat.com>
-M: Kaleb S. Keithley <kkeithle@redhat.com>
-M: Vijay Bellur <vbellur@redhat.com>
-
+M: Xavier Hernandez  <xhernandez@redhat.com>
+P: Pranith Karampuri <pranith.karampuri@phonepe.com>
+P: Atin Mukherjee <amukherj@redhat.com>
 
 xlators:
 --------
+Access Control List (ACL)
+M: Raghavendra Talur <rtalur@redhat.com>
+P: Jiffin Tony Thottan <jthottan@redhat.com>
+S: Maintained
+F: xlators/system/posix-acl/
+
+Arbiter
+M: Ravishankar N <ravishankar@redhat.com>
+P: Pranith Karampuri <pranith.karampuri@phonepe.com>
+S: Maintained
+F: xlators/features/arbiter/
+
 Automatic File Replication (AFR)
-M: Pranith Karampuri <pkarampu@redhat.com>
+M: Pranith Karampuri <pranith.karampuri@phonepe.com>
+M: Ravishankar N <ravishankar@redhat.com>
+P: Karthik US <ksubrahm@redhat.com>
 S: Maintained
 F: xlators/cluster/afr/
 
+Barrier
+M: Raghavendra Bhat <rabhat@redhat.com>
+P: Atin Mukherjee <amukherj@redhat.com>
+S: Maintained
+F: xlators/features/barrier
+
+BitRot
+M: Kotresh HR <khiremat@redhat.com>
+P: Raghavendra Bhat <rabhat@redhat.com>
+S: Maintained
+F: xlators/features/bit-rot/
+
+Changelog
+M: Aravinda V K <avishwan@redhat.com>
+P: Kotresh HR <khiremat@redhat.com>
+S: Maintained
+F: xlators/features/changelog/
+
 Distributed Hashing Table (DHT)
-M: Shishir Gowda <gowda.shishir@gmail.com>
+P: Susant Palai <spalai@redhat.com>
 S: Maintained
 F: xlators/cluster/dht/
 
-Performance
-M: Raghavendra Gowdappa <rgowdapp@redhat.com>
+Erasure Coding
+M: Pranith Karampuri <pranith.karampuri@phonepe.com>
+M: Xavier Hernandez  <xhernandez@redhat.com>
+P: Ashish Pandey <aspandey@redhat.com>
 S: Maintained
-F: xlators/performance/
+F: xlators/cluster/ec/
+
+Error-gen
+M: Raghavendra Talur <rtalur@redhat.com>
+S: Maintained
+F: xlators/debug/error-gen/
+
+FUSE Bridge
+M: Csaba Henk <chenk@redhat.com>
+P: Niels de Vos <ndevos@redhat.com>
+S: Maintained
+F: xlators/mount/
 
 Index
-M: Pranith Karampuri <pkarampu@redhat.com>
+M: Pranith Karampuri <pranith.karampuri@phonepe.com>
+P: Ravishankar N <ravishankar@redhat.com>
 S: Maintained
 F: xlators/features/index/
 
-Quota
-M: Krishnan Parthasarathi <kparthas@redhat.com>
-M: Raghavendra Gowdappa <rgowdapp@redhat.com>
+IO Cache
+P: Mohammed Rafi KC <rafi.kavungal@iternity.com>
 S: Maintained
-F: xlators/features/quota/
+F: xlators/performance/io-cache/
+
+IO Statistics
+M: Krutika Dhananjay <kdhananj@redhat.com>
+M: Shyam Ranganathan <srangana@redhat.com>
+S: Maintained
+F: xlators/debug/io-stats/
+
+IO threads
+M: Pranith Karampuri <pranith.karampuri@phonepe.com>
+P: Ravishankar N <ravishankar@redhat.com>
+S: Maintained
+F: xlators/performance/io-threads/
+
+Leases
+M: Poornima G <pgurusid@redhat.com>
+P: Niels de Vos <ndevos@redhat.com>
+P: Soumya Koduri <skoduri@redhat.com>
+S: Maintained
+F: xlators/features/leases/
+
+Locks
+M: Krutika Dhananjay <kdhananj@redhat.com>
+P: Xavier Hernandez  <xhernandez@redhat.com>
+S: Maintained
+F: xlators/features/locks/
 
 Marker
-M: Raghavendra Gowdappa <rgowdapp@redhat.com>
+M: Kotresh HR <khiremat@redhat.com>
 S: Maintained
 F: xlators/features/marker/
 
-Changelog
-M: Venky Shankar <vshankar@redhat.com>
+Meta
+M: Mohammed Rafi KC <rafi.kavungal@iternity.com>
 S: Maintained
-F: xlators/features/changelog/
+F: xlators/features/meta/
 
-Block Device
-M: M. Mohan Kumar <mohan@in.ibm.com>
+Metadata-cache
+M: Poornima G <pgurusid@redhat.com>
+P: Soumya Koduri <skoduri@redhat.com>
 S: Maintained
-F: xlators/storage/bd/
+F: xlators/performance/md-cache/
 
-FUSE Bridge
-M: Anand Avati <avati@redhat.com>
-M: Brian Foster <bfoster@redhat.com>
-M: Csaba Henk <chenk@redhat.com>
+Negative-lookup Cache
+M: Poornima G <pgurusid@redhat.com>
+P: Pranith Karampuri <pranith.karampuri@phonepe.com>
 S: Maintained
-F: xlators/mount/
+F: xlators/performance/nl-cache/
 
-NFS
-M: Niels de Vos <ndevos@redhat.com>
-S: Maintained
+gNFS
+M: Jiffin Tony Thottan <jthottan@redhat.com>
+P: Xie Changlong <xiechanglong@cmss.chinamobile.com>
+P: Amar Tumballi <amarts@gmail.com>
+S: Odd Fixes
 F: xlators/nfs/server/
 
-Locks
-M: Pranith Karampuri <pkarampu@redhat.com>
+Open-behind
 S: Maintained
-F: xlators/features/locks/
+F: xlators/performance/open-behind/
 
+Posix:
+M: Raghavendra Bhat <raghavendra@redhat.com>
+P: Kotresh HR <khiremat@redhat.com>
+P: Krutika Dhananjay <kdhananj@redhat.com>
+S: Maintained
+F: xlators/storage/posix/
+
+Quick-read
+S: Maintained
+F: xlators/performance/quick-read/
+
+Quota
+M: Shyamsundar Ranganathan <srangana@redhat.com>
+P: Hari Gowtham <hgowtham@redhat.com>
+S: Maintained
+F: xlators/features/quota/
+
+Read-ahead
+P: Csaba Henk <chenk@redhat.com>
+S: Maintained
+F: xlators/performance/read-ahead/
+
+Readdir-ahead
+S: Maintained
+F: xlators/performance/readdir-ahead/
+
+Sharding
+M: Krutika Dhananjay <kdhananj@redhat.com>
+P: Xavier Hernandez  <xhernandez@redhat.com>
+S: Maintained
+F: xlators/features/shard/
+
+Trash
+M: Anoop C S <anoopcs@redhat.com>
+M: Jiffin Tony Thottan <jthottan@redhat.com>
+S: Maintained
+F: xlators/features/trash/
+
+Upcall
+M: Poornima G <pgurusid@redhat.com>
+M: Soumya Koduri <skoduri@redhat.com>
+P: Niels de Vos <ndevos@redhat.com>
+S: Maintained
+F: xlators/features/upcall/
+
+Write-behind
+P: Csaba Henk <chenk@redhat.com>
+S: Maintained
+F: xlators/performance/write-behind/
+
+Write Once Read Many
+P: Karthik US <ksubrahm@redhat.com>
+S: Maintained
+F: xlators/features/read-only/
+
+Cloudsync
+M: Susant Kumar Palai <spalai@redhat.com>
+S: Maintained
+F: xlators/features/cloudsync/
 
 Other bits of code:
 -------------------
+
+Doc
+M: Humble Chirammal <hchiramm@redhat.com>
+M: Raghavendra Talur <rtalur@redhat.com>
+S: Maintained
+F: doc/
+
 Geo Replication
-M: Venky Shankar <vshankar@redhat.com>
+M: Aravinda V K <avishwan@redhat.com>
+M: Kotresh HR <khiremat@redhat.com>
+M: Sunny Kumar <sunkumar@redhat.com>
 S: Maintained
 F: geo-replication/
 
-Glupy
-M: Justin Clift <justin@gluster.org>
+Glusterfind
+M: Aravinda VK <avishwan@redhat.com>
 S: Maintained
-F: xlators/features/glupy/
+F: tools/glusterfind/
 
 libgfapi
-M: Anand Avati <avati@redhat.com>
+M: Niels de Vos <ndevos@redhat.com>
+P: Poornima G <pgurusid@redhat.com>
+P: Shyamsundar Ranganathan <srangana@redhat.com>
+P: Soumya Koduri <skoduri@redhat.com>
 S: Maintained
 F: api/
 
-Management Daemon
-M: Krishnan Parthasarathi <kparthas@redhat.com>
-M: Kaushal Madappa <kmadapp@redhat.com>
+libglusterfs
+M: Amar Tumballi <amarts@gmail.com>
+M: Xavier Hernandez  <xhernandez@redhat.com>
+M: Jeff Darcy <jeff@pl.atyp.us>
+P: Kaleb Keithley <kkeithle@redhat.com>
+P: Niels de Vos <ndevos@redhat.com>
+P: Pranith Karampuri <pranith.karampuri@phonepe.com>
+P: Shyamsundar Ranganathan <srangana@redhat.com>
+S: Maintained
+F: libglusterfs/
+
+xxhash
+M: Aravinda VK <avishwan@redhat.com>
+M: Kotresh HR <khiremat@redhat.com>
+P: Yaniv Kaul <ykaul@redhat.com>
+S: Maintained
+F: contrib/xxhash/
+T: https://github.com/Cyan4973/xxHash.git
+
+Management Daemon - glusterd
+M: Atin Mukherjee <amukherj@redhat.com>
+M: Mohit Agrawal <moagrawa@redhat.com>
+M: Sanju Rakonde <srakonde@redhat.com>
 S: Maintained
 F: cli/
-F: xlators/mgmt/
+F: xlators/mgmt/glusterd/
+
+Protocol
+M: Niels de Vos <ndevos@redhat.com>
+P: Mohammed Rafi KC <rafi.kavungal@iternity.com>
+S: Maintained
+F: xlators/protocol/
 
 Remote Procedure Call subsystem
-M: Raghavendra Gowdappa <rgowdapp@redhat.com>
-M: Amar Tumballi <amarts@gmail.com>
-M: Anand Avati <avati@redhat.com>
+P: Mohit Agrawal <moagrawa@redhat.com>
 S: Maintained
-F: rpc/
+F: rpc/rpc-lib/
+F: rpc/xdr/
 
+Snapshot
+M: Raghavendra Bhat <raghavendra@redhat.com>
+P: Mohammed Rafi KC <rafi.kavungal@iternity.com>
+P: Sunny Kumar <sunkumar@redhat.com>
+S: Maintained
+F: xlators/mgmt/glusterd/src/glusterd-snap*
+F: extras/snap-scheduler.py
+
+Socket subsystem
+P: Krutika Dhananjay <kdhananj@redhat.com>
+P: Milind Changire <mchangir@redhat.com>
+P: Mohammed Rafi KC <rafi.kavungal@iternity.com>
+P: Mohit Agrawal <moagrawa@redhat.com>
+S: Maintained
+F: rpc/rpc-transport/socket/
 
-Distribution Specific:
-----------------------
-Debian Packaging
-M: Patrick Matthäi <pmatthaei@debian.org>
-M: Louis Zuckerman <me@louiszuckerman.com>
+Testing - .t framework
+M: Raghavendra Talur <rtalur@redhat.com>
 S: Maintained
-W: http://packages.qa.debian.org/g/glusterfs.html
+F: tests/
 
-Fedora Packaging
-M: glusterfs-owner@fedoraproject.org
-M: Humble Chirammal <hchiramm@redhat.com>
-M: Kaleb Keithley <kkeithle@redhat.com>
-M: Lalatendu Mohanty <lmohanty@redhat.com>
+Utilities
+M: Aravinda VK <avishwan@redhat.com>
+P: Niels de Vos <ndevos@redhat.com>
+P: Raghavendra Talur <rtalur@redhat.com>
+P: Sachidanda Urs <surs@redhat.com>
+S: Maintained
+F: extras/
+
+Events APIs
+M: Aravinda VK <avishwan@redhat.com>
+S: Maintained
+F: events/
+F: libglusterfs/src/events*
+F: libglusterfs/src/eventtypes*
+F: extras/systemd/glustereventsd*
+
+Distribution Specific:
+----------------------
+Build:
 M: Niels de Vos <ndevos@redhat.com>
+M: Hari Gowtham <hgowtham@redhat.com>
+P: Anoop C S <anoopcs@redhat.com>
+P: Raghavendra Talur <rtalur@redhat.com>
+P: Rinku Kothiya <rkothiya@redhat.com>
 S: Maintained
-W: https://apps.fedoraproject.org/packages/glusterfs
-T: http://pkgs.fedoraproject.org/git/glusterfs.git
 
-FreeBSD port
-M: Harshavardhana <harsha@harshavardhana.net>
+Debian packages on download.gluster.org
+M: packaging@gluster.org
+M: Kaleb Keithley <kkeithle@redhat.com>
+P: Sheetal Pamecha <spamecha@redhat.com>
+P: Shwetha Acharya <sacharya@redhat.com>
 S: Maintained
+W: http://download.gluster.org/pub/gluster/glusterfs/LATEST/Debian/Debian.README
+T: https://github.com/gluster/glusterfs-debian.git
 
-MacOS X port
-M: Dennis Schafroth <dennis@schafroth.com>
-M: Harshavardhana <harsha@harshavardhana.net>
+OpenSuSE
+M: packaging@gluster.org
+M: Kaleb Keithley <kkeithle@redhat.com>
+P: Sheetal Pamecha <spamecha@redhat.com>
+P: Shwetha Acharya <sacharya@redhat.com>
 S: Maintained
+W: https://build.opensuse.org/repositories/home:glusterfs
+W: https://download.gluster.org/pub/gluster/glusterfs/LATEST/SuSE/SuSE.README
+T: https://github.com/gluster/glusterfs-suse.git
 
-NetBSD port
-M: Emmanuel Dreyfus <manu@netbsd.org>
+Packages for the CentOS Storage SIG
+M: centos-devel@centos.org
+M: Niels de Vos <ndevos@redhat.com>
+P: Kaleb Keithley <kkeithle@redhat.com>
 S: Maintained
-W: http://pkgsrc.se/filesystems/glusterfs
+W: https://wiki.centos.org/SpecialInterestGroup/Storage/Gluster
+T: https://github.com/CentOS-Storage-SIG/glusterfs.git
 
-Ubuntu Packaging
-M: Louis Zuckerman <me@louiszuckerman.com>
+Ubuntu PPA
+M: packaging@gluster.org
+M: Kaleb Keithley <kkeithle@redhat.com>
+P: Sheetal Pamecha <spamecha@redhat.com>
+P: Shwetha Acharya <sacharya@redhat.com>
 S: Maintained
+W: https://launchpad.net/~gluster
 W: http://download.gluster.org/pub/gluster/glusterfs/LATEST/Ubuntu/Ubuntu.README
-
+T: https://github.com/gluster/glusterfs-debian.git
 
 Related projects
 ----------------
-Gluster Openstack Swift
-M: Luis Pabon <lpabon@redhat.com>
+Gluster Block
+M: Prasanna Kumar Kalever <prasanna.kalever@redhat.com>
+M: Xiubo Li <xiubli@redhat.com>
 S: Maintained
-T: https://github.com/gluster/gluster-swift.git
+T: https://github.com/gluster/gluster-block.git
 
-GlusterFS Hadoop HCFS plugin
-M: Jay Vyas <jvyas@redhat.com>
+GlusterFS core-utils
+M: Anoop C S <anoopcs@redhat.com>
 S: Maintained
-T: https://github.com/gluster/glusterfs-hadoop.git
+T: https://github.com/gluster/glusterfs-coreutils.git
 
 NFS-Ganesha FSAL plugin
-M: Anand Subramanian <ansubram@redhat.com>
+M: Jiffin Tony Thottan <jthottan@redhat.com>
+M: Kaleb Keithley <kkeithle@redhat.com>
+M: Soumya Koduri <skoduri@redhat.com>
 S: Maintained
 T: git://github.com/nfs-ganesha/nfs-ganesha.git
 F: src/nfs-ganesha~/src/FSAL/FSAL_GLUSTER/
 
 QEMU integration
-M: Bharata B Rao <bharata@linux.vnet.ibm.com>
+M: Niels de Vos <ndevos@redhat.com>
+M: Prasanna Kumar Kalever <prasanna.kalever@redhat.com>
 S: Maintained
 T: git://git.qemu.org/qemu.git
 F: block/gluster.c
 
 Samba VFS plugin
+M: Anoop C S <anoopcs@redhat.com>
 M: Raghavendra Talur <rtalur@redhat.com>
-M: Chris Hertel <chertel@redhat.com>
-M: Jose Rivera <jrivera@redhat.com>
-M: Ira Cooper <icooper@redhat.com>
+M: Michael Adam <madam@redhat.com>
+M: Poornima G <pgurusid@redhat.com>
 S: Maintained
 T: git://git.samba.org/samba.git
 F: source3/modules/vfs_glusterfs.c
 
+Storhaug
+M: Jose A. Rivera <jarrpa@redhat.com>
+P: Kaleb Keithley <kkeithle@redhat.com>
+S: Maintained
+T: https://github.com/linux-ha-storage/storhaug.git
+
+Testing - Glusto-Tests
+M: Jonathan Holloway <jholloway@redhat.com>
+M: Vijay Bhaskar Reddy Avuthu <vavuthu@redhat.com>
+M: Akarsha Rai <akrai@redhat.com>
+S: Maintained
+T: https://github.com/gluster/glusto-tests.git
+
 Wireshark dissectors
 M: Niels de Vos <ndevos@redhat.com>
 S: Maintained
@@ -228,3 +456,55 @@ W: https://forge.gluster.org/wireshark
 T: http://code.wireshark.org/git/wireshark
 F: epan/dissectors/packet-gluster*
 
+Infrastructure
+--------------
+
+Platform
+M: Michael Scherer <misc@redhat.com>
+P: Shyamsundar Ranganathan <srangana@redhat.com>
+P: Amar Tumballi <amarts@gmail.com>
+
+Continuous Integration
+M: Michael Scherer <misc@redhat.com>
+M: Deepshikha Khandelwal <dkhandel@redhat.com>
+P: Niels de Vos <ndevos@redhat.com>
+
+Special Thanks
+--------------
+
+GlusterFS would not be possible without the contributions of:
+
+
+M: Vijay Bellur <vbellur@redhat.com>
+M: Jeff Darcy <jeff@pl.atyp.us>
+M: Shreyas Siravara <sshreyas@fb.com>
+M: Kaushal M <kaushal@redhat.com>
+M: Nigel Babu
+M: Prashanth Pai
+P: Sanoj Unnikrishnan
+P: Milind Changire <mchangir@redhat.com>
+P: Sunil Kumar Acharya <sheggodu@redhat.com>
+M: Samikshan Bairagya <samikshan@gmail.com>
+M: Chris Hertel
+M: M. Mohan Kumar <mohan@in.ibm.com>
+M: Shishir Gowda <gowda.shishir@gmail.com>
+M: Brian Foster <bfoster@redhat.com>
+M: Anand Avati <avati@cs.stanford.edu>
+M: Dennis Schafroth <dennis@schafroth.com>
+M: Harshavardhana <harsha@harshavardhana.net>
+M: Krishnan Parthasarathi
+M: Justin Clift <justin@gluster.org>
+M: Venky Shankar <vshankar@redhat.com>
+M: Shravan Chandrashekar <shravantc99@gmail.com>
+M: Joseph Fernandes
+M: Vijaikumar Mallikarjuna
+M: Anand Subramanian
+M: Bharata B Rao <bharata@linux.vnet.ibm.com>
+M: Rajesh Joseph
+M: Dan Lambright
+M: Jay Vyas
+M: Luis Pabon
+M: Ira Cooper
+M: Shwetha Panduranga
+M: Nithya Balachandran
+M: Raghavendra Gowdappa
diff --git a/Makefile.am b/Makefile.am
index 187bfd5ed16..98ea5c1038d 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -1,45 +1,55 @@
+SOURCES = site.h
+
 EXTRA_DIST = autogen.sh \
-	COPYING-GPLV2 COPYING-LGPLV3 \
-	INSTALL README AUTHORS THANKS NEWS \
+	COPYING-GPLV2 COPYING-LGPLV3 COMMITMENT \
+	INSTALL README.md AUTHORS THANKS NEWS \
 	glusterfs.spec glusterfs-api.pc.in libgfchangelog.pc.in \
 	run-tests.sh \
 	build-aux/pkg-version \
-	build-aux/xdrgen \
-        contrib/argp-standalone \
-        contrib/umountd \
+	contrib/umountd \
 	$(shell find $(top_srcdir)/tests -type f -print)
 
-SUBDIRS = $(ARGP_STANDALONE_DIR) libglusterfs rpc api xlators glusterfsd \
-	$(FUSERMOUNT_SUBDIR) doc extras cli heal @SYNCDAEMON_SUBDIR@ \
-	@UMOUNTD_SUBDIR@
+
+SUBDIRS = $(ARGP_STANDALONE_DIR) libglusterfs rpc libglusterd api \
+	glusterfsd xlators $(FUSERMOUNT_SUBDIR) doc extras cli heal \
+	@SYNCDAEMON_SUBDIR@ @UMOUNTD_SUBDIR@ tools events
 
 pkgconfigdir = @pkgconfigdir@
 pkgconfig_DATA = glusterfs-api.pc libgfchangelog.pc
 
-CLEANFILES =
+CLEANFILES = glusterfs-api.pc libgfchangelog.pc contrib/umountd/Makefile
+
+clean-local:
+	find . -name '*.o' -o -name '*.lo' -o -name '.Po' | xargs rm -f
 
 gitclean: distclean
 	find . -name Makefile.in -exec rm -f {} \;
-	find . -name Makefile -exec rm -f {} \;
 	find . -name mount.glusterfs -exec rm -f {} \;
+	find . -name .deps -o -name .libs | xargs rm -rf
 	rm -fr autom4te.cache
 	rm -f missing aclocal.m4 config.h.in config.guess config.sub ltmain.sh install-sh configure depcomp
-	-rm -fr $(CONTRIBDIR)/argp-standalone/autom4te.cache
-	-rm -f $(CONTRIBDIR)/argp-standalone/aclocal.m4
-	-rm -f $(CONTRIBDIR)/argp-standalone/config.h.in
-	-rm -f $(CONTRIBDIR)/argp-standalone/configure
-	-rm -f $(CONTRIBDIR)/argp-standalone/depcomp
-	-rm -f $(CONTRIBDIR)/argp-standalone/install-sh
-	-rm -f $(CONTRIBDIR)/argp-standalone/missing
 
+# dist-hook gets executed with 'make dist', this is the only target getting
+# executed, a dist-hook in other Makefile.am files seem to get ignored.
 dist-hook: gen-VERSION gen-ChangeLog
-.PHONY: gen-VERSION gen-ChangeLog
+	-rm -fr $(distdir)/contrib/umountd/.deps
+	-rm -f $(distdir)/events/src/eventtypes.py
+	-rm -f $(distdir)/tests/env.rc
+	-cp -f $(top_srcdir)/build-aux/config.sub.dist $(distdir)/config.sub
+	-cp -f $(top_srcdir)/build-aux/config.guess.dist $(distdir)/config.guess
+
+.PHONY: gen-VERSION gen-ChangeLog clang-check
+
+clang-check:
+	@$(top_srcdir)/extras/clang-checker.sh
 
 gen-ChangeLog:
 	(cd $(srcdir) && git diff && echo ===== git log ==== && git log) > $(distdir)/ChangeLog
 
+.PHONY : gen-VERSION
 gen-VERSION:
-	if test -d .git; then                           \
-	  $(top_srcdir)/build-aux/pkg-version --full    \
-	    > $(distdir)/VERSION;                       \
+	if test -d $(top_srcdir)/.git; then             \
+	  cd $(top_srcdir);                             \
+	  ./build-aux/pkg-version --full                \
+	    > $(abs_top_builddir)/$(distdir)/VERSION;   \
 	fi
diff --git a/README b/README
deleted file mode 100644
index ce648743ee9..00000000000
--- a/README
+++ /dev/null
@@ -1 +0,0 @@
-For more info, please visit http://www.gluster.org/.
diff --git a/README.md b/README.md
new file mode 100644
index 00000000000..9d68e033782
--- /dev/null
+++ b/README.md
@@ -0,0 +1,46 @@
+# Gluster
+  Gluster is a software defined distributed storage that can scale to several
+  petabytes. It provides interfaces for object, block and file storage.
+
+## Development
+  The development workflow is documented in [Contributors guide](CONTRIBUTING.md)
+
+## Documentation
+  The Gluster documentation can be found at [Gluster Docs](http://docs.gluster.org).
+
+## Deployment
+  Quick instructions to build and install can be found in [INSTALL](INSTALL) file.
+
+## Testing
+
+  GlusterFS source contains some functional tests under `tests/` directory. All
+  these tests are run against every patch submitted for review. If you want your
+  patch to be tested, please add a `.t` test file as part of your patch submission.
+  You can also submit a patch to only add a `.t` file for the test case you are
+  aware of.
+
+  To run these tests, on your test-machine, just run `./run-tests.sh`. Don't run
+  this on a machine where you have 'production' glusterfs is running, as it would
+  blindly kill all gluster processes in each runs.
+
+  If you are sending a patch, and want to validate one or few specific tests, then
+  run a single test by running the below command.
+
+```
+  bash# /bin/bash ${path_to_gluster}/tests/basic/rpc-coverage.t
+```
+
+  You can also use `prove` tool if available in your machine, as follows.
+
+```
+  bash# prove -vmfe '/bin/bash' ${path_to_gluster}/tests/basic/rpc-coverage.t
+```
+
+
+## Maintainers
+  The list of Gluster maintainers is available in [MAINTAINERS](MAINTAINERS) file.
+
+## License
+  Gluster is dual licensed under [GPLV2](COPYING-GPLV2) and [LGPLV3+](COPYING-LGPLV3).
+
+  Please visit the [Gluster Home Page](http://www.gluster.org/) to find out more about Gluster.
diff --git a/api/examples/getvolfile.py b/api/examples/getvolfile.py
index 184586c632d..3b2c8ab5a15 100755
--- a/api/examples/getvolfile.py
+++ b/api/examples/getvolfile.py
@@ -1,43 +1,45 @@
-#!/usr/bin/python
+#!/usr/bin/python3
 
+from __future__ import print_function
 import ctypes
 import ctypes.util
 
-api = ctypes.CDLL(ctypes.util.find_library("gfapi"))
+api = ctypes.CDLL("libgfapi.so")
 api.glfs_get_volfile.argtypes = [ctypes.c_void_p,
                                  ctypes.c_void_p,
                                  ctypes.c_ulong]
-api.glfs_get_volfile.restype = ctypes.c_long;
+api.glfs_get_volfile.restype = ctypes.c_long
 
-def get_volfile (host, volume):
-	# This is set to a large value to exercise the "buffer not big enough"
-	# path.  More realistically, you'd just start with a huge buffer.
-	BUF_LEN = 0
-	fs = api.glfs_new(volume)
-	#api.glfs_set_logging(fs,"/dev/stderr",7)
-	api.glfs_set_volfile_server(fs,"tcp",host,24007)
-	api.glfs_init(fs)
-	vbuf = ctypes.create_string_buffer(BUF_LEN)
-	vlen = api.glfs_get_volfile(fs,vbuf,BUF_LEN)
-	if vlen < 0:
-		vlen = BUF_LEN - vlen
-		vbuf = ctypes.create_string_buffer(vlen)
-		vlen = api.glfs_get_volfile(fs,vbuf,vlen)
-	api.glfs_fini(fs)
-	if vlen <= 0:
-		return vlen
-	return vbuf.value[:vlen]
+
+def get_volfile(host, volume):
+    # This is set to a large value to exercise the "buffer not big enough"
+    # path.  More realistically, you'd just start with a huge buffer.
+    BUF_LEN = 0
+    fs = api.glfs_new(volume)
+    # api.glfs_set_logging(fs,"/dev/stderr",7)
+    api.glfs_set_volfile_server(fs, "tcp", host, 24007)
+    api.glfs_init(fs)
+    vbuf = ctypes.create_string_buffer(BUF_LEN)
+    vlen = api.glfs_get_volfile(fs, vbuf, BUF_LEN)
+    if vlen < 0:
+        vlen = BUF_LEN - vlen
+        vbuf = ctypes.create_string_buffer(vlen)
+        vlen = api.glfs_get_volfile(fs, vbuf, vlen)
+    api.glfs_fini(fs)
+    if vlen <= 0:
+        return vlen
+    return vbuf.value[:vlen]
 
 if __name__ == "__main__":
-	import sys
+    import sys
 
-	try:
-		res = apply(get_volfile,sys.argv[1:3])
-	except:
-		print "fetching volfile failed (volume not started?)"
+    try:
+        res = get_volfile(*sys.argv[1:3])
+    except:
+        print("fetching volfile failed (volume not started?)")
 
-	try:
-		for line in res.split('\n'):
-			print line
-	except:
-		print "bad return value %s" % res
+    try:
+        for line in res.split('\n'):
+            print(line)
+    except:
+        print("bad return value %s" % res)
diff --git a/api/examples/glfsxmp.c b/api/examples/glfsxmp.c
index 7180a997f13..a55616ef739 100644
--- a/api/examples/glfsxmp.c
+++ b/api/examples/glfsxmp.c
@@ -1,1598 +1,1811 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <errno.h>
-#include "api/glfs.h"
-#include "api/glfs-handles.h"
+#include <glusterfs/api/glfs.h>
+#include <glusterfs/api/glfs-handles.h>
 #include <string.h>
 #include <time.h>
 
+#define TEST_STR_LEN 2048
 
 int
-test_dirops (glfs_t *fs)
+test_dirops(glfs_t *fs)
 {
-        glfs_fd_t *fd = NULL;
-        char buf[512];
-        struct dirent *entry = NULL;
-
-        fd = glfs_opendir (fs, "/");
-        if (!fd) {
-                fprintf (stderr, "/: %s\n", strerror (errno));
-                return -1;
-        }
-
-        fprintf (stderr, "Entries:\n");
-        while (glfs_readdir_r (fd, (struct dirent *)buf, &entry), entry) {
-                fprintf (stderr, "%s: %lu\n", entry->d_name, glfs_telldir (fd));
-        }
-
-        glfs_closedir (fd);
-        return 0;
+    glfs_fd_t *fd = NULL;
+    char buf[512];
+    struct dirent *entry = NULL;
+
+    fd = glfs_opendir(fs, "/");
+    if (!fd) {
+        fprintf(stderr, "/: %s\n", strerror(errno));
+        return -1;
+    }
+
+    fprintf(stderr, "Entries:\n");
+    while (glfs_readdir_r(fd, (struct dirent *)buf, &entry), entry) {
+        fprintf(stderr, "%s: %lu\n", entry->d_name, glfs_telldir(fd));
+    }
+
+    glfs_closedir(fd);
+    return 0;
 }
 
-
 int
-test_xattr (glfs_t *fs)
+test_xattr(glfs_t *fs)
 {
-        char *filename = "/filename2";
-        char buf[512];
-        char *ptr;
-        int ret;
-
-        ret = glfs_setxattr (fs, filename, "user.testkey", "testval", 8, 0);
-        fprintf (stderr, "setxattr(%s): %d (%s)\n", filename, ret,
-                 strerror (errno));
-
-        ret = glfs_setxattr (fs, filename, "user.testkey2", "testval", 8, 0);
-        fprintf (stderr, "setxattr(%s): %d (%s)\n", filename, ret,
-                 strerror (errno));
-
-        ret = glfs_listxattr (fs, filename, buf, 512);
-        fprintf (stderr, "listxattr(%s): %d (%s)\n", filename, ret,
-                 strerror (errno));
-        if (ret < 0)
-                return -1;
-
-        for (ptr = buf; ptr < buf + ret; ptr++) {
-                printf ("key=%s\n", ptr);
-                ptr += strlen (ptr);
-        }
-
-        return 0;
+    char *filename = "/filename2";
+    char *linkfile = "/linkfile";
+    glfs_fd_t *fd = NULL;
+    char buf[512];
+    char *ptr;
+    int ret;
+
+    ret = glfs_setxattr(fs, filename, "user.testkey", "testval", 8, 0);
+    fprintf(stderr, "setxattr(%s): %d (%s)\n", filename, ret, strerror(errno));
+
+    ret = glfs_setxattr(fs, filename, "user.testkey2", "testval", 8, 0);
+    fprintf(stderr, "setxattr(%s): %d (%s)\n", filename, ret, strerror(errno));
+
+    ret = glfs_getxattr(fs, filename, "user.testkey", buf, 512);
+    fprintf(stderr, "getxattr(%s): %d (%s)\n", filename, ret, strerror(errno));
+    if (ret < 0)
+        return -1;
+
+    ret = glfs_listxattr(fs, filename, buf, 512);
+    fprintf(stderr, "listxattr(%s): %d (%s)\n", filename, ret, strerror(errno));
+    if (ret < 0)
+        return -1;
+
+    ret = glfs_symlink(fs, "filename", linkfile);
+    fprintf(stderr, "symlink(%s %s): %s\n", filename, linkfile,
+            strerror(errno));
+    if (ret < 0)
+        return -1;
+
+    ret = glfs_readlink(fs, linkfile, buf, 512);
+    fprintf(stderr, "readlink(%s) : %d (%s)\n", filename, ret, strerror(errno));
+    if (ret < 0)
+        return -1;
+
+    ret = glfs_lsetxattr(fs, filename, "user.testkey3", "testval", 8, 0);
+    fprintf(stderr, "lsetxattr(%s) : %d (%s)\n", linkfile, ret,
+            strerror(errno));
+    if (ret < 0)
+        return -1;
+
+    ret = glfs_llistxattr(fs, linkfile, buf, 512);
+    fprintf(stderr, "llistxattr(%s): %d (%s)\n", filename, ret,
+            strerror(errno));
+    if (ret < 0)
+        return -1;
+
+    ret = glfs_lgetxattr(fs, filename, "user.testkey3", buf, 512);
+    fprintf(stderr, "lgetxattr(%s): %d (%s)\n", linkfile, ret, strerror(errno));
+    if (ret < 0)
+        return -1;
+
+    for (ptr = buf; ptr < buf + ret; ptr++) {
+        printf("key=%s\n", ptr);
+        ptr += strlen(ptr);
+    }
+
+    ret = glfs_removexattr(fs, filename, "user.testkey2");
+    fprintf(stderr, "removexattr(%s): %d (%s)\n", filename, ret,
+            strerror(errno));
+
+    fd = glfs_open(fs, filename, O_RDWR);
+    fprintf(stderr, "open(%s): (%p) %s\n", filename, fd, strerror(errno));
+
+    ret = glfs_fsetxattr(fd, "user.testkey2", "testval", 8, 0);
+    fprintf(stderr, "fsetxattr(%s): %d (%s)\n", filename, ret, strerror(errno));
+
+    ret = glfs_fgetxattr(fd, "user.testkey2", buf, 512);
+    fprintf(stderr, "fgetxattr(%s): %d (%s)\n", filename, ret, strerror(errno));
+
+    ret = glfs_flistxattr(fd, buf, 512);
+    fprintf(stderr, "flistxattr(%s): %d (%s)\n", filename, ret,
+            strerror(errno));
+    if (ret < 0)
+        return -1;
+
+    for (ptr = buf; ptr < buf + ret; ptr++) {
+        printf("key=%s\n", ptr);
+        ptr += strlen(ptr);
+    }
+
+    ret = glfs_fremovexattr(fd, "user.testkey2");
+    fprintf(stderr, "fremovexattr(%s): %d (%s)\n", filename, ret,
+            strerror(errno));
+
+    glfs_close(fd);
+
+    return 0;
 }
 
-
 int
-test_chdir (glfs_t *fs)
+test_chdir(glfs_t *fs)
 {
-        int ret = -1;
-        char *topdir = "/topdir";
-        char *linkdir = "/linkdir";
-        char *subdir = "./subdir";
-        char *respath = NULL;
-        char pathbuf[4096];
-
-        ret = glfs_mkdir (fs, topdir, 0755);
-        if (ret) {
-                fprintf (stderr, "mkdir(%s): %s\n", topdir, strerror (errno));
-                return -1;
-        }
-
-        respath = glfs_getcwd (fs, pathbuf, 4096);
-        fprintf (stdout, "getcwd() = %s\n", respath);
-
-        ret = glfs_symlink (fs, topdir, linkdir);
-        if (ret) {
-                fprintf (stderr, "symlink(%s, %s): %s\n", topdir, linkdir, strerror (errno));
-                return -1;
-        }
-
-        ret = glfs_chdir (fs, linkdir);
-        if (ret) {
-                fprintf (stderr, "chdir(%s): %s\n", linkdir, strerror (errno));
-                return -1;
-        }
-
-        respath = glfs_getcwd (fs, pathbuf, 4096);
-        fprintf (stdout, "getcwd() = %s\n", respath);
-
-        respath = glfs_realpath (fs, subdir, pathbuf);
-        if (respath) {
-                fprintf (stderr, "realpath(%s) worked unexpectedly: %s\n", subdir, respath);
-                return -1;
-        }
-
-        ret = glfs_mkdir (fs, subdir, 0755);
-        if (ret) {
-                fprintf (stderr, "mkdir(%s): %s\n", subdir, strerror (errno));
-                return -1;
-        }
-
-        respath = glfs_realpath (fs, subdir, pathbuf);
-        if (!respath) {
-                fprintf (stderr, "realpath(%s): %s\n", subdir, strerror (errno));
-        } else {
-                fprintf (stdout, "realpath(%s) = %s\n", subdir, respath);
-        }
-
-        ret = glfs_chdir (fs, subdir);
-        if (ret) {
-                fprintf (stderr, "chdir(%s): %s\n", subdir, strerror (errno));
-                return -1;
-        }
-
-        respath = glfs_getcwd (fs, pathbuf, 4096);
-        fprintf (stdout, "getcwd() = %s\n", respath);
-
-        respath = glfs_realpath (fs, "/linkdir/subdir", pathbuf);
-        if (!respath) {
-                fprintf (stderr, "realpath(/linkdir/subdir): %s\n", strerror (errno));
-        } else {
-                fprintf (stdout, "realpath(/linkdir/subdir) = %s\n", respath);
-        }
-
-        return 0;
+    int ret = -1;
+    char *dir = "/dir";
+    char *topdir = "/topdir";
+    char *linkdir = "/linkdir";
+    char *linkdir2 = "/linkdir2";
+    char *subdir = "./subdir";
+    char *respath = NULL;
+    char pathbuf[4096];
+
+    ret = glfs_mkdir(fs, topdir, 0755);
+    fprintf(stderr, "mkdir(%s): %s\n", topdir, strerror(errno));
+    if (ret)
+        return -1;
+
+    ret = glfs_mkdir(fs, dir, 0755);
+    fprintf(stderr, "mkdir(%s): %s\n", dir, strerror(errno));
+    if (ret)
+        return -1;
+
+    respath = glfs_getcwd(fs, pathbuf, 4096);
+    fprintf(stdout, "getcwd() = %s\n", respath);
+
+    ret = glfs_symlink(fs, "topdir", linkdir);
+    if (ret) {
+        fprintf(stderr, "symlink(%s, %s): %s\n", topdir, linkdir,
+                strerror(errno));
+        return -1;
+    }
+
+    ret = glfs_chdir(fs, linkdir);
+    if (ret) {
+        fprintf(stderr, "chdir(%s): %s\n", linkdir, strerror(errno));
+        return -1;
+    }
+
+    respath = glfs_getcwd(fs, pathbuf, 4096);
+    fprintf(stdout, "getcwd() = %s\n", respath);
+
+    respath = glfs_realpath(fs, subdir, pathbuf);
+    if (respath) {
+        fprintf(stderr, "realpath(%s) worked unexpectedly: %s\n", subdir,
+                respath);
+        return -1;
+    }
+
+    ret = glfs_mkdir(fs, subdir, 0755);
+    if (ret) {
+        fprintf(stderr, "mkdir(%s): %s\n", subdir, strerror(errno));
+        return -1;
+    }
+
+    respath = glfs_realpath(fs, subdir, pathbuf);
+    if (!respath) {
+        fprintf(stderr, "realpath(%s): %s\n", subdir, strerror(errno));
+    } else {
+        fprintf(stdout, "realpath(%s) = %s\n", subdir, respath);
+    }
+
+    ret = glfs_chdir(fs, subdir);
+    if (ret) {
+        fprintf(stderr, "chdir(%s): %s\n", subdir, strerror(errno));
+        return -1;
+    }
+
+    respath = glfs_getcwd(fs, pathbuf, 4096);
+    fprintf(stdout, "getcwd() = %s\n", respath);
+
+    respath = glfs_realpath(fs, "/linkdir/subdir", pathbuf);
+    if (!respath) {
+        fprintf(stderr, "realpath(/linkdir/subdir): %s\n", strerror(errno));
+    } else {
+        fprintf(stdout, "realpath(/linkdir/subdir) = %s\n", respath);
+    }
+
+    return 0;
 }
 
 #ifdef DEBUG
 static void
-peek_stat (struct stat *sb)
+peek_stat(struct stat *sb)
 {
-        printf ("Dumping stat information:\n");
-        printf ("File type:                ");
-
-        switch (sb->st_mode & S_IFMT) {
-                case S_IFBLK:  printf ("block device\n");            break;
-                case S_IFCHR:  printf ("character device\n");        break;
-                case S_IFDIR:  printf ("directory\n");               break;
-                case S_IFIFO:  printf ("FIFO/pipe\n");               break;
-                case S_IFLNK:  printf ("symlink\n");                 break;
-                case S_IFREG:  printf ("regular file\n");            break;
-                case S_IFSOCK: printf ("socket\n");                  break;
-                default:       printf ("unknown?\n");                break;
-        }
-
-        printf ("I-node number:            %ld\n", (long) sb->st_ino);
-
-        printf ("Mode:                     %lo (octal)\n",
-                (unsigned long) sb->st_mode);
-
-        printf ("Link count:               %ld\n", (long) sb->st_nlink);
-        printf ("Ownership:                UID=%ld   GID=%ld\n",
-                (long) sb->st_uid, (long) sb->st_gid);
-
-        printf ("Preferred I/O block size: %ld bytes\n",
-                (long) sb->st_blksize);
-        printf ("File size:                %lld bytes\n",
-                (long long) sb->st_size);
-        printf ("Blocks allocated:         %lld\n",
-                (long long) sb->st_blocks);
-
-        printf ("Last status change:       %s", ctime(&sb->st_ctime));
-        printf ("Last file access:         %s", ctime(&sb->st_atime));
-        printf ("Last file modification:   %s", ctime(&sb->st_mtime));
-
-        return;
+    printf("Dumping stat information:\n");
+    printf("File type:                ");
+
+    switch (sb->st_mode & S_IFMT) {
+        case S_IFBLK:
+            printf("block device\n");
+            break;
+        case S_IFCHR:
+            printf("character device\n");
+            break;
+        case S_IFDIR:
+            printf("directory\n");
+            break;
+        case S_IFIFO:
+            printf("FIFO/pipe\n");
+            break;
+        case S_IFLNK:
+            printf("symlink\n");
+            break;
+        case S_IFREG:
+            printf("regular file\n");
+            break;
+        case S_IFSOCK:
+            printf("socket\n");
+            break;
+        default:
+            printf("unknown?\n");
+            break;
+    }
+
+    printf("I-node number:            %ld\n", (long)sb->st_ino);
+
+    printf("Mode:                     %lo (octal)\n",
+           (unsigned long)sb->st_mode);
+
+    printf("Link count:               %ld\n", (long)sb->st_nlink);
+    printf("Ownership:                UID=%ld   GID=%ld\n", (long)sb->st_uid,
+           (long)sb->st_gid);
+
+    printf("Preferred I/O block size: %ld bytes\n", (long)sb->st_blksize);
+    printf("File size:                %lld bytes\n", (long long)sb->st_size);
+    printf("Blocks allocated:         %lld\n", (long long)sb->st_blocks);
+
+    printf("Last status change:       %s", ctime(&sb->st_ctime));
+    printf("Last file access:         %s", ctime(&sb->st_atime));
+    printf("Last file modification:   %s", ctime(&sb->st_mtime));
+
+    return;
 }
 
 static void
-peek_handle (unsigned char *glid)
+peek_handle(unsigned char *glid)
 {
-        int i;
+    int i;
 
-        for (i = 0; i < GFAPI_HANDLE_LENGTH; i++)
-        {
-                printf (":%02x:", glid[i]);
-        }
-        printf ("\n");
+    for (i = 0; i < GFAPI_HANDLE_LENGTH; i++) {
+        printf(":%02x:", glid[i]);
+    }
+    printf("\n");
 }
-#else /* DEBUG */
+#else  /* DEBUG */
 static void
-peek_stat (struct stat *sb)
+peek_stat(struct stat *sb)
 {
-        return;
+    return;
 }
 
 static void
-peek_handle (unsigned char *id)
+peek_handle(unsigned char *id)
 {
-        return;
+    return;
 }
 #endif /* DEBUG */
 
-glfs_t    *fs = NULL;
-char      *full_parent_name = "/testdir", *parent_name = "testdir";
+glfs_t *fs = NULL;
+char *full_parent_name = "/testdir", *parent_name = "testdir";
 
 void
-test_h_unlink (void)
+test_h_unlink(void)
 {
-        char               *my_dir = "unlinkdir";
-        char               *my_file = "file.txt";
-        char               *my_subdir = "dir1";
-        struct glfs_object *parent = NULL, *leaf = NULL, *dir = NULL,
-                           *subdir = NULL, *subleaf = NULL;
-        struct stat         sb;
-        int                 ret;
-
-        printf ("glfs_h_unlink tests: In Progress\n");
-
-        /* Prepare tests */
-        parent = glfs_h_lookupat (fs, NULL, full_parent_name, &sb);
-        if (parent == NULL) {
-                fprintf (stderr, "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
-                         full_parent_name, NULL, strerror (errno));
-                printf ("glfs_h_lookupat tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        dir = glfs_h_mkdir (fs, parent, my_dir, 0644, &sb);
-        if (dir == NULL) {
-                fprintf (stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
-                         my_dir, parent, strerror (errno));
-                printf ("glfs_h_unlink tests: FAILED\n");
-                goto out;
-        }
-
-        leaf = glfs_h_creat (fs, dir, my_file, O_CREAT, 0644, &sb);
-        if (leaf == NULL) {
-                fprintf (stderr, "glfs_h_creat: error creating %s: from (%p),%s\n",
-                         my_file, dir, strerror (errno));
-                printf ("glfs_h_unlink tests: FAILED\n");
-                goto out;
-        }
-
-        subdir = glfs_h_mkdir (fs, dir, my_subdir, 0644, &sb);
-        if (subdir == NULL) {
-                fprintf (stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
-                         my_subdir, dir, strerror (errno));
-                printf ("glfs_h_unlink tests: FAILED\n");
-                goto out;
-        }
-
-        subleaf = glfs_h_creat (fs, subdir, my_file, O_CREAT, 0644, &sb);
-        if (subleaf == NULL) {
-                fprintf (stderr, "glfs_h_creat: error creating %s: from (%p),%s\n",
-                         my_file, subdir, strerror (errno));
-                printf ("glfs_h_unlink tests: FAILED\n");
-                goto out;
-        }
-
-        /* unlink non empty directory */
-        ret = glfs_h_unlink (fs, dir, my_subdir);
-        if ((ret && errno != ENOTEMPTY) || (ret == 0)) {
-                fprintf (stderr, "glfs_h_unlink: error unlinking %s: it is non empty: %s\n",
-                         my_subdir, strerror (errno));
-                printf ("glfs_h_unlink tests: FAILED\n");
-                goto out;
-        }
-
-        /* unlink regular file */
-        ret = glfs_h_unlink (fs, subdir, my_file);
-        if (ret) {
-                fprintf (stderr, "glfs_h_unlink: error unlinking %s: from (%p),%s\n",
-                         my_file, subdir, strerror (errno));
-                printf ("glfs_h_unlink tests: FAILED\n");
-                goto out;
-        }
-
-        /* unlink directory */
-        ret = glfs_h_unlink (fs, dir, my_subdir);
-        if (ret) {
-                fprintf (stderr, "glfs_h_unlink: error unlinking %s: from (%p),%s\n",
-                         my_subdir, dir, strerror (errno));
-                printf ("glfs_h_unlink tests: FAILED\n");
-                goto out;
-        }
-
-        /* unlink regular file */
-        ret = glfs_h_unlink (fs, dir, my_file);
-        if (ret) {
-                fprintf (stderr, "glfs_h_unlink: error unlinking %s: from (%p),%s\n",
-                         my_file, dir, strerror (errno));
-                printf ("glfs_h_unlink tests: FAILED\n");
-                goto out;
-        }
-
-        /* unlink non-existent regular file */
-        ret = glfs_h_unlink (fs, dir, my_file);
-        if ((ret && errno != ENOENT) || (ret == 0)) {
-                fprintf (stderr, "glfs_h_unlink: error unlinking non-existent %s: invalid errno ,%d, %s\n",
-                         my_file, ret, strerror (errno));
-                printf ("glfs_h_unlink tests: FAILED\n");
-                goto out;
-        }
-
-        /* unlink non-existent directory */
-        ret = glfs_h_unlink (fs, dir, my_subdir);
-        if ((ret && errno != ENOENT) || (ret == 0)) {
-                fprintf (stderr, "glfs_h_unlink: error unlinking non-existent %s:  invalid errno ,%d, %s\n",
-                         my_subdir, ret, strerror (errno));
-                printf ("glfs_h_unlink tests: FAILED\n");
-                goto out;
-        }
-
-        /* unlink directory */
-        ret = glfs_h_unlink (fs, parent, my_dir);
-        if (ret) {
-                fprintf (stderr, "glfs_h_unlink: error unlinking %s: from (%p),%s\n",
-                         my_dir, dir, strerror (errno));
-                printf ("glfs_h_unlink tests: FAILED\n");
-                goto out;
-        }
-
-        printf ("glfs_h_unlink tests: PASSED\n");
+    char *my_dir = "unlinkdir";
+    char *my_file = "file.txt";
+    char *my_subdir = "dir1";
+    struct glfs_object *parent = NULL, *leaf = NULL, *dir = NULL,
+                       *subdir = NULL, *subleaf = NULL;
+    struct stat sb;
+    int ret;
+
+    printf("glfs_h_unlink tests: In Progress\n");
+
+    /* Prepare tests */
+    parent = glfs_h_lookupat(fs, NULL, full_parent_name, &sb, 0);
+    if (parent == NULL) {
+        fprintf(stderr,
+                "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                full_parent_name, NULL, strerror(errno));
+        printf("glfs_h_lookupat tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    dir = glfs_h_mkdir(fs, parent, my_dir, 0755, &sb);
+    if (dir == NULL) {
+        fprintf(stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
+                my_dir, parent, strerror(errno));
+        printf("glfs_h_unlink tests: FAILED\n");
+        goto out;
+    }
+
+    leaf = glfs_h_creat(fs, dir, my_file, O_CREAT, 0644, &sb);
+    if (leaf == NULL) {
+        fprintf(stderr, "glfs_h_creat: error creating %s: from (%p),%s\n",
+                my_file, dir, strerror(errno));
+        printf("glfs_h_unlink tests: FAILED\n");
+        goto out;
+    }
+
+    subdir = glfs_h_mkdir(fs, dir, my_subdir, 0755, &sb);
+    if (subdir == NULL) {
+        fprintf(stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
+                my_subdir, dir, strerror(errno));
+        printf("glfs_h_unlink tests: FAILED\n");
+        goto out;
+    }
+
+    subleaf = glfs_h_creat(fs, subdir, my_file, O_CREAT, 0644, &sb);
+    if (subleaf == NULL) {
+        fprintf(stderr, "glfs_h_creat: error creating %s: from (%p),%s\n",
+                my_file, subdir, strerror(errno));
+        printf("glfs_h_unlink tests: FAILED\n");
+        goto out;
+    }
+
+    /* unlink non empty directory */
+    ret = glfs_h_unlink(fs, dir, my_subdir);
+    if ((ret && errno != ENOTEMPTY) || (ret == 0)) {
+        fprintf(stderr,
+                "glfs_h_unlink: error unlinking %s: it is non empty: %s\n",
+                my_subdir, strerror(errno));
+        printf("glfs_h_unlink tests: FAILED\n");
+        goto out;
+    }
+
+    /* unlink regular file */
+    ret = glfs_h_unlink(fs, subdir, my_file);
+    if (ret) {
+        fprintf(stderr, "glfs_h_unlink: error unlinking %s: from (%p),%s\n",
+                my_file, subdir, strerror(errno));
+        printf("glfs_h_unlink tests: FAILED\n");
+        goto out;
+    }
+
+    /* unlink directory */
+    ret = glfs_h_unlink(fs, dir, my_subdir);
+    if (ret) {
+        fprintf(stderr, "glfs_h_unlink: error unlinking %s: from (%p),%s\n",
+                my_subdir, dir, strerror(errno));
+        printf("glfs_h_unlink tests: FAILED\n");
+        goto out;
+    }
+
+    /* unlink regular file */
+    ret = glfs_h_unlink(fs, dir, my_file);
+    if (ret) {
+        fprintf(stderr, "glfs_h_unlink: error unlinking %s: from (%p),%s\n",
+                my_file, dir, strerror(errno));
+        printf("glfs_h_unlink tests: FAILED\n");
+        goto out;
+    }
+
+    /* unlink non-existent regular file */
+    ret = glfs_h_unlink(fs, dir, my_file);
+    if ((ret && errno != ENOENT) || (ret == 0)) {
+        fprintf(stderr,
+                "glfs_h_unlink: error unlinking non-existent %s: invalid errno "
+                ",%d, %s\n",
+                my_file, ret, strerror(errno));
+        printf("glfs_h_unlink tests: FAILED\n");
+        goto out;
+    }
+
+    /* unlink non-existent directory */
+    ret = glfs_h_unlink(fs, dir, my_subdir);
+    if ((ret && errno != ENOENT) || (ret == 0)) {
+        fprintf(stderr,
+                "glfs_h_unlink: error unlinking non-existent %s:  invalid "
+                "errno ,%d, %s\n",
+                my_subdir, ret, strerror(errno));
+        printf("glfs_h_unlink tests: FAILED\n");
+        goto out;
+    }
+
+    /* unlink directory */
+    ret = glfs_h_unlink(fs, parent, my_dir);
+    if (ret) {
+        fprintf(stderr, "glfs_h_unlink: error unlinking %s: from (%p),%s\n",
+                my_dir, dir, strerror(errno));
+        printf("glfs_h_unlink tests: FAILED\n");
+        goto out;
+    }
+
+    printf("glfs_h_unlink tests: PASSED\n");
 
 out:
-        if (dir)
-                glfs_h_close (dir);
-        if (leaf)
-                glfs_h_close (leaf);
-        if (subdir)
-                glfs_h_close (subdir);
-        if (subleaf)
-                glfs_h_close (subleaf);
-        if (parent)
-                glfs_h_close (parent);
-
-        return;
+    if (dir)
+        glfs_h_close(dir);
+    if (leaf)
+        glfs_h_close(leaf);
+    if (subdir)
+        glfs_h_close(subdir);
+    if (subleaf)
+        glfs_h_close(subleaf);
+    if (parent)
+        glfs_h_close(parent);
+
+    return;
 }
 
 void
-test_h_getsetattrs (void)
+test_h_getsetattrs(void)
 {
-        char               *my_dir = "attrdir";
-        char               *my_file = "attrfile.txt";
-        struct glfs_object *parent = NULL, *leaf = NULL, *dir = NULL;
-        struct stat         sb, retsb;
-        int                 ret, valid;
-        struct timespec     timestamp;
-
-        printf("glfs_h_getattrs and setattrs tests: In Progress\n");
-
-        /* Prepare tests */
-        parent = glfs_h_lookupat (fs, NULL, full_parent_name, &sb);
-        if (parent == NULL) {
-                fprintf (stderr, "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
-                         full_parent_name, NULL, strerror (errno));
-                printf ("glfs_h_lookupat tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        dir = glfs_h_mkdir (fs, parent, my_dir, 0644, &sb);
-        if (dir == NULL) {
-                fprintf (stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
-                         my_dir, parent, strerror (errno));
-                printf ("glfs_h_unlink tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        leaf = glfs_h_creat (fs, dir, my_file, O_CREAT, 0644, &sb);
-        if (leaf == NULL) {
-                fprintf (stderr, "glfs_h_creat: error creating %s: from (%p),%s\n",
-                         my_file, dir, strerror (errno));
-                printf ("glfs_h_unlink tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        ret = glfs_h_getattrs (fs, dir, &retsb);
-        if (ret != 0) {
-                fprintf (stderr, "glfs_h_getattrs: error %s: from (%p),%s\n",
-                         my_dir, dir, strerror (errno));
-                printf ("glfs_h_getattrs and setattrs tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&retsb);
-        /* TODO: Compare stat information */
-
-        retsb.st_mode = 00666;
-        retsb.st_uid = 1000;
-        retsb.st_gid = 1001;
-        ret = clock_gettime (CLOCK_REALTIME, &timestamp);
-        if(ret != 0) {
-                fprintf (stderr, "clock_gettime: error %s\n", strerror (errno));
-                printf ("glfs_h_getattrs and setattrs tests: FAILED\n");
-                goto out;
-        }
-        retsb.st_atim = timestamp;
-        retsb.st_mtim = timestamp;
-        valid = GFAPI_SET_ATTR_MODE | GFAPI_SET_ATTR_UID | GFAPI_SET_ATTR_GID |
-        GFAPI_SET_ATTR_ATIME | GFAPI_SET_ATTR_MTIME;
-        peek_stat (&retsb);
-
-        ret = glfs_h_setattrs (fs, dir, &retsb, valid);
-        if (ret != 0) {
-                fprintf (stderr, "glfs_h_setattrs: error %s: from (%p),%s\n",
-                         my_dir, dir, strerror (errno));
-                printf ("glfs_h_getattrs and setattrs tests: FAILED\n");
-                goto out;
-        }
-
-        memset(&retsb, 0, sizeof (struct stat));
-        ret = glfs_h_stat (fs, dir, &retsb);
-        if (ret != 0) {
-                fprintf (stderr, "glfs_h_stat: error %s: from (%p),%s\n",
-                         my_dir, dir, strerror (errno));
-                printf ("glfs_h_getattrs and setattrs tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&retsb);
-
-        printf ("glfs_h_getattrs and setattrs tests: PASSED\n");
+    char *my_dir = "attrdir";
+    char *my_file = "attrfile.txt";
+    struct glfs_object *parent = NULL, *leaf = NULL, *dir = NULL;
+    struct stat sb, retsb;
+    int ret, valid;
+    struct timespec timestamp;
+
+    printf("glfs_h_getattrs and setattrs tests: In Progress\n");
+
+    /* Prepare tests */
+    parent = glfs_h_lookupat(fs, NULL, full_parent_name, &sb, 0);
+    if (parent == NULL) {
+        fprintf(stderr,
+                "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                full_parent_name, NULL, strerror(errno));
+        printf("glfs_h_lookupat tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    dir = glfs_h_mkdir(fs, parent, my_dir, 0755, &sb);
+    if (dir == NULL) {
+        fprintf(stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
+                my_dir, parent, strerror(errno));
+        printf("glfs_h_unlink tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    leaf = glfs_h_creat(fs, dir, my_file, O_CREAT, 0644, &sb);
+    if (leaf == NULL) {
+        fprintf(stderr, "glfs_h_creat: error creating %s: from (%p),%s\n",
+                my_file, dir, strerror(errno));
+        printf("glfs_h_unlink tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    ret = glfs_h_getattrs(fs, dir, &retsb);
+    if (ret != 0) {
+        fprintf(stderr, "glfs_h_getattrs: error %s: from (%p),%s\n", my_dir,
+                dir, strerror(errno));
+        printf("glfs_h_getattrs and setattrs tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&retsb);
+    /* TODO: Compare stat information */
+
+    retsb.st_mode = 00666;
+    retsb.st_uid = 1000;
+    retsb.st_gid = 1001;
+    ret = clock_gettime(CLOCK_REALTIME, &timestamp);
+    if (ret != 0) {
+        fprintf(stderr, "clock_gettime: error %s\n", strerror(errno));
+        printf("glfs_h_getattrs and setattrs tests: FAILED\n");
+        goto out;
+    }
+    retsb.st_atim = timestamp;
+    retsb.st_mtim = timestamp;
+    valid = GFAPI_SET_ATTR_MODE | GFAPI_SET_ATTR_UID | GFAPI_SET_ATTR_GID |
+            GFAPI_SET_ATTR_ATIME | GFAPI_SET_ATTR_MTIME;
+    peek_stat(&retsb);
+
+    ret = glfs_h_setattrs(fs, dir, &retsb, valid);
+    if (ret != 0) {
+        fprintf(stderr, "glfs_h_setattrs: error %s: from (%p),%s\n", my_dir,
+                dir, strerror(errno));
+        printf("glfs_h_getattrs and setattrs tests: FAILED\n");
+        goto out;
+    }
+
+    memset(&retsb, 0, sizeof(struct stat));
+    ret = glfs_h_stat(fs, dir, &retsb);
+    if (ret != 0) {
+        fprintf(stderr, "glfs_h_stat: error %s: from (%p),%s\n", my_dir, dir,
+                strerror(errno));
+        printf("glfs_h_getattrs and setattrs tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&retsb);
+
+    printf("glfs_h_getattrs and setattrs tests: PASSED\n");
 out:
-        if (parent)
-                glfs_h_close (parent);
-        if (leaf)
-                glfs_h_close (leaf);
-        if (dir)
-                glfs_h_close (dir);
-
-        return;
+    if (parent)
+        glfs_h_close(parent);
+    if (leaf)
+        glfs_h_close(leaf);
+    if (dir)
+        glfs_h_close(dir);
+
+    return;
 }
 
 void
-test_h_truncate (void)
+test_h_truncate(void)
 {
-        char               *my_dir = "truncatedir";
-        char               *my_file = "file.txt";
-        struct glfs_object *root = NULL, *parent = NULL, *leaf = NULL;
-        struct stat         sb;
-        glfs_fd_t          *fd = NULL;
-        char                buf[32];
-        off_t               offset = 0;
-        int                 ret = 0;
-
-        printf("glfs_h_truncate tests: In Progress\n");
-
-        /* Prepare tests */
-        root = glfs_h_lookupat (fs, NULL, full_parent_name, &sb);
-        if (root == NULL) {
-                fprintf (stderr, "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
-                         full_parent_name, NULL, strerror (errno));
-                printf ("glfs_h_truncate tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        parent = glfs_h_mkdir (fs, root, my_dir, 0644, &sb);
-        if (parent == NULL) {
-                fprintf (stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
-                         my_dir, root, strerror (errno));
-                printf ("glfs_h_truncate tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        leaf = glfs_h_creat (fs, parent, my_file, O_CREAT, 0644, &sb);
-        if (leaf == NULL) {
-                fprintf (stderr, "glfs_h_creat: error creating %s: from (%p),%s\n",
-                         my_file, parent, strerror (errno));
-                printf ("glfs_h_truncate tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        fd = glfs_h_open (fs, leaf, O_RDWR);
-        if (fd == NULL) {
-                fprintf (stderr, "glfs_h_open: error on open of %s: %s\n",
-                         my_file, strerror (errno));
-                printf ("glfs_h_truncate tests: FAILED\n");
-                goto out;
-        }
-
-        memcpy (buf, "abcdefghijklmnopqrstuvwxyz012345", 32);
-        ret = glfs_write (fd, buf, 32, 0);
-
-        /* run tests */
-        /* truncate lower */
-        offset = 30;
-        ret = glfs_h_truncate (fs, leaf, offset);
-        if (ret != 0) {
-                fprintf (stderr, "glfs_h_truncate: error creating %s: from (%p),%s\n",
-                         my_file, parent, strerror (errno));
-                printf ("glfs_h_truncate tests: FAILED\n");
-                goto out;
-        }
-        ret = glfs_h_getattrs (fs, leaf, &sb);
-        if (ret != 0) {
-                fprintf (stderr, "glfs_h_getattrs: error for %s (%p),%s\n",
-                         my_file, leaf, strerror (errno));
-                printf ("glfs_h_truncate tests: FAILED\n");
-                goto out;
-        }
-        if (sb.st_size != offset) {
-                fprintf (stderr, "glfs_h_truncate: post size mismatch\n");
-                printf ("glfs_h_truncate tests: FAILED\n");
-                goto out;
-        }
-
-        /* truncate higher */
-        offset = 32;
-        ret = glfs_h_truncate (fs, leaf, offset);
-        if (ret != 0) {
-                fprintf (stderr, "glfs_h_truncate: error creating %s: from (%p),%s\n",
-                         my_file, parent, strerror (errno));
-                printf ("glfs_h_truncate tests: FAILED\n");
-                goto out;
-        }
-        ret = glfs_h_getattrs (fs, leaf, &sb);
-        if (ret != 0) {
-                fprintf (stderr, "glfs_h_getattrs: error for %s (%p),%s\n",
-                         my_file, leaf, strerror (errno));
-                printf ("glfs_h_truncate tests: FAILED\n");
-                goto out;
-        }
-        if (sb.st_size != offset) {
-                fprintf (stderr, "glfs_h_truncate: post size mismatch\n");
-                printf ("glfs_h_truncate tests: FAILED\n");
-                goto out;
-        }
-
-        /* truncate equal */
-        offset = 30;
-        ret = glfs_h_truncate (fs, leaf, offset);
-        if (ret != 0) {
-                fprintf (stderr, "glfs_h_truncate: error creating %s: from (%p),%s\n",
-                         my_file, parent, strerror (errno));
-                printf ("glfs_h_truncate tests: FAILED\n");
-                goto out;
-        }
-        ret = glfs_h_getattrs (fs, leaf, &sb);
-        if (ret != 0) {
-                fprintf (stderr, "glfs_h_getattrs: error for %s (%p),%s\n",
-                         my_file, leaf, strerror (errno));
-                printf ("glfs_h_truncate tests: FAILED\n");
-                goto out;
-        }
-        if (sb.st_size != offset) {
-                fprintf (stderr, "glfs_h_truncate: post size mismatch\n");
-                printf ("glfs_h_truncate tests: FAILED\n");
-                goto out;
-        }
-
-        printf ("glfs_h_truncate tests: PASSED\n");
+    char *my_dir = "truncatedir";
+    char *my_file = "file.txt";
+    struct glfs_object *root = NULL, *parent = NULL, *leaf = NULL;
+    struct stat sb;
+    glfs_fd_t *fd = NULL;
+    char buf[32];
+    off_t offset = 0;
+    int ret = 0;
+
+    printf("glfs_h_truncate tests: In Progress\n");
+
+    /* Prepare tests */
+    root = glfs_h_lookupat(fs, NULL, full_parent_name, &sb, 0);
+    if (root == NULL) {
+        fprintf(stderr,
+                "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                full_parent_name, NULL, strerror(errno));
+        printf("glfs_h_truncate tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    parent = glfs_h_mkdir(fs, root, my_dir, 0755, &sb);
+    if (parent == NULL) {
+        fprintf(stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
+                my_dir, root, strerror(errno));
+        printf("glfs_h_truncate tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    leaf = glfs_h_creat(fs, parent, my_file, O_CREAT, 0644, &sb);
+    if (leaf == NULL) {
+        fprintf(stderr, "glfs_h_creat: error creating %s: from (%p),%s\n",
+                my_file, parent, strerror(errno));
+        printf("glfs_h_truncate tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    fd = glfs_h_open(fs, leaf, O_RDWR);
+    if (fd == NULL) {
+        fprintf(stderr, "glfs_h_open: error on open of %s: %s\n", my_file,
+                strerror(errno));
+        printf("glfs_h_truncate tests: FAILED\n");
+        goto out;
+    }
+
+    memcpy(buf, "abcdefghijklmnopqrstuvwxyz012345", 32);
+    ret = glfs_write(fd, buf, 32, 0);
+
+    /* run tests */
+    /* truncate lower */
+    offset = 30;
+    ret = glfs_h_truncate(fs, leaf, offset);
+    if (ret != 0) {
+        fprintf(stderr, "glfs_h_truncate: error creating %s: from (%p),%s\n",
+                my_file, parent, strerror(errno));
+        printf("glfs_h_truncate tests: FAILED\n");
+        goto out;
+    }
+    ret = glfs_h_getattrs(fs, leaf, &sb);
+    if (ret != 0) {
+        fprintf(stderr, "glfs_h_getattrs: error for %s (%p),%s\n", my_file,
+                leaf, strerror(errno));
+        printf("glfs_h_truncate tests: FAILED\n");
+        goto out;
+    }
+    if (sb.st_size != offset) {
+        fprintf(stderr, "glfs_h_truncate: post size mismatch\n");
+        printf("glfs_h_truncate tests: FAILED\n");
+        goto out;
+    }
+
+    /* truncate higher */
+    offset = 32;
+    ret = glfs_h_truncate(fs, leaf, offset);
+    if (ret != 0) {
+        fprintf(stderr, "glfs_h_truncate: error creating %s: from (%p),%s\n",
+                my_file, parent, strerror(errno));
+        printf("glfs_h_truncate tests: FAILED\n");
+        goto out;
+    }
+    ret = glfs_h_getattrs(fs, leaf, &sb);
+    if (ret != 0) {
+        fprintf(stderr, "glfs_h_getattrs: error for %s (%p),%s\n", my_file,
+                leaf, strerror(errno));
+        printf("glfs_h_truncate tests: FAILED\n");
+        goto out;
+    }
+    if (sb.st_size != offset) {
+        fprintf(stderr, "glfs_h_truncate: post size mismatch\n");
+        printf("glfs_h_truncate tests: FAILED\n");
+        goto out;
+    }
+
+    /* truncate equal */
+    offset = 30;
+    ret = glfs_h_truncate(fs, leaf, offset);
+    if (ret != 0) {
+        fprintf(stderr, "glfs_h_truncate: error creating %s: from (%p),%s\n",
+                my_file, parent, strerror(errno));
+        printf("glfs_h_truncate tests: FAILED\n");
+        goto out;
+    }
+    ret = glfs_h_getattrs(fs, leaf, &sb);
+    if (ret != 0) {
+        fprintf(stderr, "glfs_h_getattrs: error for %s (%p),%s\n", my_file,
+                leaf, strerror(errno));
+        printf("glfs_h_truncate tests: FAILED\n");
+        goto out;
+    }
+    if (sb.st_size != offset) {
+        fprintf(stderr, "glfs_h_truncate: post size mismatch\n");
+        printf("glfs_h_truncate tests: FAILED\n");
+        goto out;
+    }
+
+    printf("glfs_h_truncate tests: PASSED\n");
 out:
-        if (fd)
-                glfs_close (fd);
-        if (root)
-                glfs_h_close (root);
-        if (parent)
-                glfs_h_close (parent);
-        if (leaf)
-                glfs_h_close (leaf);
-
-        return;
+    if (fd)
+        glfs_close(fd);
+    if (root)
+        glfs_h_close(root);
+    if (parent)
+        glfs_h_close(parent);
+    if (leaf)
+        glfs_h_close(leaf);
+
+    return;
 }
 
 void
-test_h_links (void)
+test_h_links(void)
 {
-        char               *my_dir = "linkdir";
-        char               *my_file = "file.txt";
-        char               *my_symlnk = "slnk.txt";
-        char               *my_lnk = "lnk.txt";
-        char               *linksrc_dir = "dir1";
-        char               *linktgt_dir = "dir2";
-        struct glfs_object *root = NULL, *parent = NULL, *leaf = NULL,
-                           *dirsrc = NULL, *dirtgt = NULL, *dleaf = NULL;
-        struct glfs_object *ln1 = NULL;
-        struct stat         sb;
-        int                 ret;
-        char               *buf = NULL;
-
-        printf("glfs_h_link(s) tests: In Progress\n");
-
-        /* Prepare tests */
-        root = glfs_h_lookupat (fs, NULL, full_parent_name, &sb);
-        if (root == NULL) {
-                fprintf (stderr, "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
-                         full_parent_name, NULL, strerror (errno));
-                printf ("glfs_h_link(s) tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        parent = glfs_h_mkdir (fs, root, my_dir, 0644, &sb);
-        if (parent == NULL) {
-                fprintf (stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
-                         my_dir, root, strerror (errno));
-                printf ("glfs_h_link(s) tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        leaf = glfs_h_creat (fs, parent, my_file, O_CREAT, 0644, &sb);
-        if (leaf == NULL) {
-                fprintf (stderr, "glfs_h_creat: error creating %s: from (%p),%s\n",
-                         my_file, parent, strerror (errno));
-                printf ("glfs_h_link(s) tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        dirsrc = glfs_h_mkdir (fs, parent, linksrc_dir, 0644, &sb);
-        if (dirsrc == NULL) {
-                fprintf (stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
-                         linksrc_dir, parent, strerror (errno));
-                printf ("glfs_h_link(s) tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        dirtgt = glfs_h_mkdir (fs, parent, linktgt_dir, 0644, &sb);
-        if (dirtgt == NULL) {
-                fprintf (stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
-                         linktgt_dir, parent, strerror (errno));
-                printf ("glfs_h_link(s) tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        dleaf = glfs_h_creat (fs, dirsrc, my_file, O_CREAT, 0644, &sb);
-        if (dleaf == NULL) {
-                fprintf (stderr, "glfs_h_creat: error creating %s: from (%p),%s\n",
-                         my_file, dirsrc, strerror (errno));
-                printf ("glfs_h_link(s) tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        /* run tests */
-        /* sym link: /testdir/linkdir/file.txt to ./slnk.txt */
-        ln1 = glfs_h_symlink (fs, parent, my_symlnk, "./file.txt", &sb);
-        if (ln1 == NULL) {
-                fprintf (stderr, "glfs_h_symlink: error creating %s: from (%p),%s\n",
-                         my_symlnk, parent, strerror (errno));
-                printf ("glfs_h_link(s) tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        buf = calloc (1024, sizeof(char));
-        if (buf == NULL) {
-                fprintf (stderr, "Error allocating memory\n");
-                printf ("glfs_h_link(s) tests: FAILED\n");
-                goto out;
-        }
-
-        ret = glfs_h_readlink (fs, ln1, buf, 1024);
-        if (ret <= 0) {
-                fprintf (stderr, "glfs_h_readlink: error reading %s: from (%p),%s\n",
-                         my_symlnk, ln1, strerror (errno));
-                printf ("glfs_h_link(s) tests: FAILED\n");
-                goto out;
-        }
-        if (!(strncmp (buf, my_symlnk, strlen (my_symlnk)))) {
-                fprintf (stderr, "glfs_h_readlink: error mismatch in link name: actual %s: retrieved %s\n",
-                         my_symlnk, buf);
-                printf ("glfs_h_link(s) tests: FAILED\n");
-                goto out;
-        }
-
-        /* link: /testdir/linkdir/file.txt to ./lnk.txt */
-        ret = glfs_h_link (fs, leaf, parent, my_lnk);
-        if (ret != 0) {
-                fprintf (stderr, "glfs_h_link: error creating %s: from (%p),%s\n",
-                         my_lnk, parent, strerror (errno));
-                printf ("glfs_h_link(s) tests: FAILED\n");
-                goto out;
-        }
-        /* TODO: Should write content to a file and read from the link */
-
-        /* link: /testdir/linkdir/dir1/file.txt to ../dir2/slnk.txt */
-        ret = glfs_h_link (fs, dleaf, dirtgt, my_lnk);
-        if (ret != 0) {
-                fprintf (stderr, "glfs_h_link: error creating %s: from (%p),%s\n",
-                         my_lnk, dirtgt, strerror (errno));
-                printf ("glfs_h_link(s) tests: FAILED\n");
-                goto out;
-        }
-        /* TODO: Should write content to a file and read from the link */
-
-        printf ("glfs_h_link(s) tests: PASSED\n");
+    char *my_dir = "linkdir";
+    char *my_file = "file.txt";
+    char *my_symlnk = "slnk.txt";
+    char *my_lnk = "lnk.txt";
+    char *linksrc_dir = "dir1";
+    char *linktgt_dir = "dir2";
+    struct glfs_object *root = NULL, *parent = NULL, *leaf = NULL,
+                       *dirsrc = NULL, *dirtgt = NULL, *dleaf = NULL;
+    struct glfs_object *ln1 = NULL;
+    struct stat sb;
+    int ret;
+    char *buf = NULL;
+
+    printf("glfs_h_link(s) tests: In Progress\n");
+
+    /* Prepare tests */
+    root = glfs_h_lookupat(fs, NULL, full_parent_name, &sb, 0);
+    if (root == NULL) {
+        fprintf(stderr,
+                "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                full_parent_name, NULL, strerror(errno));
+        printf("glfs_h_link(s) tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    parent = glfs_h_mkdir(fs, root, my_dir, 0755, &sb);
+    if (parent == NULL) {
+        fprintf(stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
+                my_dir, root, strerror(errno));
+        printf("glfs_h_link(s) tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    leaf = glfs_h_creat(fs, parent, my_file, O_CREAT, 0644, &sb);
+    if (leaf == NULL) {
+        fprintf(stderr, "glfs_h_creat: error creating %s: from (%p),%s\n",
+                my_file, parent, strerror(errno));
+        printf("glfs_h_link(s) tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    dirsrc = glfs_h_mkdir(fs, parent, linksrc_dir, 0755, &sb);
+    if (dirsrc == NULL) {
+        fprintf(stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
+                linksrc_dir, parent, strerror(errno));
+        printf("glfs_h_link(s) tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    dirtgt = glfs_h_mkdir(fs, parent, linktgt_dir, 0755, &sb);
+    if (dirtgt == NULL) {
+        fprintf(stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
+                linktgt_dir, parent, strerror(errno));
+        printf("glfs_h_link(s) tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    dleaf = glfs_h_creat(fs, dirsrc, my_file, O_CREAT, 0644, &sb);
+    if (dleaf == NULL) {
+        fprintf(stderr, "glfs_h_creat: error creating %s: from (%p),%s\n",
+                my_file, dirsrc, strerror(errno));
+        printf("glfs_h_link(s) tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    /* run tests */
+    /* sym link: /testdir/linkdir/file.txt to ./slnk.txt */
+    ln1 = glfs_h_symlink(fs, parent, my_symlnk, "./file.txt", &sb);
+    if (ln1 == NULL) {
+        fprintf(stderr, "glfs_h_symlink: error creating %s: from (%p),%s\n",
+                my_symlnk, parent, strerror(errno));
+        printf("glfs_h_link(s) tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    buf = calloc(1024, sizeof(char));
+    if (buf == NULL) {
+        fprintf(stderr, "Error allocating memory\n");
+        printf("glfs_h_link(s) tests: FAILED\n");
+        goto out;
+    }
+
+    ret = glfs_h_readlink(fs, ln1, buf, 1024);
+    if (ret <= 0) {
+        fprintf(stderr, "glfs_h_readlink: error reading %s: from (%p),%s\n",
+                my_symlnk, ln1, strerror(errno));
+        printf("glfs_h_link(s) tests: FAILED\n");
+        goto out;
+    }
+    if (!(strncmp(buf, my_symlnk, strlen(my_symlnk)))) {
+        fprintf(stderr,
+                "glfs_h_readlink: error mismatch in link name: actual %s: "
+                "retrieved %s\n",
+                my_symlnk, buf);
+        printf("glfs_h_link(s) tests: FAILED\n");
+        goto out;
+    }
+
+    /* link: /testdir/linkdir/file.txt to ./lnk.txt */
+    ret = glfs_h_link(fs, leaf, parent, my_lnk);
+    if (ret != 0) {
+        fprintf(stderr, "glfs_h_link: error creating %s: from (%p),%s\n",
+                my_lnk, parent, strerror(errno));
+        printf("glfs_h_link(s) tests: FAILED\n");
+        goto out;
+    }
+    /* TODO: Should write content to a file and read from the link */
+
+    /* link: /testdir/linkdir/dir1/file.txt to ../dir2/slnk.txt */
+    ret = glfs_h_link(fs, dleaf, dirtgt, my_lnk);
+    if (ret != 0) {
+        fprintf(stderr, "glfs_h_link: error creating %s: from (%p),%s\n",
+                my_lnk, dirtgt, strerror(errno));
+        printf("glfs_h_link(s) tests: FAILED\n");
+        goto out;
+    }
+    /* TODO: Should write content to a file and read from the link */
+
+    printf("glfs_h_link(s) tests: PASSED\n");
 
 out:
-        if (root)
-                glfs_h_close (root);
-        if (parent)
-                glfs_h_close (parent);
-        if (leaf)
-                glfs_h_close (leaf);
-        if (dirsrc)
-                glfs_h_close (dirsrc);
-        if (dirtgt)
-                glfs_h_close (dirtgt);
-        if (dleaf)
-                glfs_h_close (dleaf);
-        if (ln1)
-                glfs_h_close (ln1);
-        if (buf)
-                free (buf);
-
-        return;
+    if (root)
+        glfs_h_close(root);
+    if (parent)
+        glfs_h_close(parent);
+    if (leaf)
+        glfs_h_close(leaf);
+    if (dirsrc)
+        glfs_h_close(dirsrc);
+    if (dirtgt)
+        glfs_h_close(dirtgt);
+    if (dleaf)
+        glfs_h_close(dleaf);
+    if (ln1)
+        glfs_h_close(ln1);
+    if (buf)
+        free(buf);
+
+    return;
 }
 
 void
-test_h_rename (void)
+test_h_rename(void)
 {
-        char               *my_dir = "renamedir";
-        char               *my_file = "file.txt";
-        char               *src_dir = "dir1";
-        char               *tgt_dir = "dir2";
-        struct glfs_object *root = NULL, *parent = NULL, *leaf = NULL,
-                           *dirsrc = NULL, *dirtgt = NULL, *dleaf = NULL;
-        struct stat         sb;
-        int                 ret;
-
-        printf("glfs_h_rename tests: In Progress\n");
-
-        /* Prepare tests */
-        root = glfs_h_lookupat (fs, NULL, full_parent_name, &sb);
-        if (root == NULL) {
-                fprintf (stderr, "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
-                         full_parent_name, NULL, strerror (errno));
-                printf ("glfs_h_rename tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        parent = glfs_h_mkdir (fs, root, my_dir, 0644, &sb);
-        if (parent == NULL) {
-                fprintf (stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
-                         my_dir, root, strerror (errno));
-                printf ("glfs_h_rename tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        leaf = glfs_h_creat (fs, parent, my_file, O_CREAT, 0644, &sb);
-        if (leaf == NULL) {
-                fprintf (stderr, "glfs_h_creat: error creating %s: from (%p),%s\n",
-                         my_file, parent, strerror (errno));
-                printf ("glfs_h_rename tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        dirsrc = glfs_h_mkdir (fs, parent, src_dir, 0644, &sb);
-        if (dirsrc == NULL) {
-                fprintf (stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
-                         src_dir, parent, strerror (errno));
-                printf ("glfs_h_rename tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        dirtgt = glfs_h_mkdir (fs, parent, tgt_dir, 0644, &sb);
-        if (dirtgt == NULL) {
-                fprintf (stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
-                         tgt_dir, parent, strerror (errno));
-                printf ("glfs_h_rename tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        dleaf = glfs_h_creat (fs, dirsrc, my_file, O_CREAT, 0644, &sb);
-        if (dleaf == NULL) {
-                fprintf (stderr, "glfs_h_creat: error creating %s: from (%p),%s\n",
-                         my_file, dirsrc, strerror (errno));
-                printf ("glfs_h_rename tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        /* run tests */
-        /* Rename file.txt -> file1.txt */
-        ret = glfs_h_rename (fs, parent, "file.txt", parent, "file1.txt");
-        if (ret != 0) {
-                fprintf (stderr, "glfs_h_rename: error renaming %s to %s (%s)\n",
-                         "file.txt", "file1.txt", strerror (errno));
-                printf ("glfs_h_rename tests: FAILED\n");
-                goto out;
-        }
-
-        /* rename dir1/file.txt -> file.txt */
-        ret = glfs_h_rename (fs, dirsrc, "file.txt", parent, "file.txt");
-        if (ret != 0) {
-                fprintf (stderr, "glfs_h_rename: error renaming %s/%s to %s (%s)\n",
-                         src_dir, "file.txt", "file.txt", strerror (errno));
-                printf ("glfs_h_rename tests: FAILED\n");
-                goto out;
-        }
-
-        /* rename file1.txt -> file.txt (exists) */
-        ret = glfs_h_rename (fs, parent, "file1.txt", parent, "file.txt");
-        if (ret != 0) {
-                fprintf (stderr, "glfs_h_rename: error renaming %s to %s (%s)\n",
-                         "file.txt", "file.txt", strerror (errno));
-                printf ("glfs_h_rename tests: FAILED\n");
-                goto out;
-        }
-
-        /* rename dir1 -> dir3 */
-        ret = glfs_h_rename (fs, parent, "dir1", parent, "dir3");
-        if (ret != 0) {
-                fprintf (stderr, "glfs_h_rename: error renaming %s to %s (%s)\n",
-                         "dir1", "dir3", strerror (errno));
-                printf ("glfs_h_rename tests: FAILED\n");
-                goto out;
-        }
-
-        /* rename dir2 ->dir3 (exists) */
-        ret = glfs_h_rename (fs, parent, "dir2", parent, "dir3");
-        if (ret != 0) {
-                fprintf (stderr, "glfs_h_rename: error renaming %s to %s (%s)\n",
-                         "dir2", "dir3", strerror (errno));
-                printf ("glfs_h_rename tests: FAILED\n");
-                goto out;
-        }
-
-        /* rename file.txt -> dir3 (fail) */
-        ret = glfs_h_rename (fs, parent, "file.txt", parent, "dir3");
-        if (ret == 0) {
-                fprintf (stderr, "glfs_h_rename: NO error renaming %s to %s (%s)\n",
-                         "file.txt", "dir3", strerror (errno));
-                printf ("glfs_h_rename tests: FAILED\n");
-                goto out;
-        }
-
-        /* rename dir3 -> file.txt (fail) */
-        ret = glfs_h_rename (fs, parent, "dir3", parent, "file.txt");
-        if (ret == 0) {
-                fprintf (stderr, "glfs_h_rename: NO error renaming %s to %s (%s)\n",
-                         "dir3", "file.txt", strerror (errno));
-                printf ("glfs_h_rename tests: FAILED\n");
-                goto out;
-        }
-
-        printf ("glfs_h_rename tests: PASSED\n");
+    char *my_dir = "renamedir";
+    char *my_file = "file.txt";
+    char *src_dir = "dir1";
+    char *tgt_dir = "dir2";
+    struct glfs_object *root = NULL, *parent = NULL, *leaf = NULL,
+                       *dirsrc = NULL, *dirtgt = NULL, *dleaf = NULL;
+    struct stat sb;
+    int ret;
+
+    printf("glfs_h_rename tests: In Progress\n");
+
+    /* Prepare tests */
+    root = glfs_h_lookupat(fs, NULL, full_parent_name, &sb, 0);
+    if (root == NULL) {
+        fprintf(stderr,
+                "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                full_parent_name, NULL, strerror(errno));
+        printf("glfs_h_rename tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    parent = glfs_h_mkdir(fs, root, my_dir, 0755, &sb);
+    if (parent == NULL) {
+        fprintf(stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
+                my_dir, root, strerror(errno));
+        printf("glfs_h_rename tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    leaf = glfs_h_creat(fs, parent, my_file, O_CREAT, 0644, &sb);
+    if (leaf == NULL) {
+        fprintf(stderr, "glfs_h_creat: error creating %s: from (%p),%s\n",
+                my_file, parent, strerror(errno));
+        printf("glfs_h_rename tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    dirsrc = glfs_h_mkdir(fs, parent, src_dir, 0755, &sb);
+    if (dirsrc == NULL) {
+        fprintf(stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
+                src_dir, parent, strerror(errno));
+        printf("glfs_h_rename tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    dirtgt = glfs_h_mkdir(fs, parent, tgt_dir, 0755, &sb);
+    if (dirtgt == NULL) {
+        fprintf(stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
+                tgt_dir, parent, strerror(errno));
+        printf("glfs_h_rename tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    dleaf = glfs_h_creat(fs, dirsrc, my_file, O_CREAT, 0644, &sb);
+    if (dleaf == NULL) {
+        fprintf(stderr, "glfs_h_creat: error creating %s: from (%p),%s\n",
+                my_file, dirsrc, strerror(errno));
+        printf("glfs_h_rename tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    /* run tests */
+    /* Rename file.txt -> file1.txt */
+    ret = glfs_h_rename(fs, parent, "file.txt", parent, "file1.txt");
+    if (ret != 0) {
+        fprintf(stderr, "glfs_h_rename: error renaming %s to %s (%s)\n",
+                "file.txt", "file1.txt", strerror(errno));
+        printf("glfs_h_rename tests: FAILED\n");
+        goto out;
+    }
+
+    /* rename dir1/file.txt -> file.txt */
+    ret = glfs_h_rename(fs, dirsrc, "file.txt", parent, "file.txt");
+    if (ret != 0) {
+        fprintf(stderr, "glfs_h_rename: error renaming %s/%s to %s (%s)\n",
+                src_dir, "file.txt", "file.txt", strerror(errno));
+        printf("glfs_h_rename tests: FAILED\n");
+        goto out;
+    }
+
+    /* rename file1.txt -> file.txt (exists) */
+    ret = glfs_h_rename(fs, parent, "file1.txt", parent, "file.txt");
+    if (ret != 0) {
+        fprintf(stderr, "glfs_h_rename: error renaming %s to %s (%s)\n",
+                "file.txt", "file.txt", strerror(errno));
+        printf("glfs_h_rename tests: FAILED\n");
+        goto out;
+    }
+
+    /* rename dir1 -> dir3 */
+    ret = glfs_h_rename(fs, parent, "dir1", parent, "dir3");
+    if (ret != 0) {
+        fprintf(stderr, "glfs_h_rename: error renaming %s to %s (%s)\n", "dir1",
+                "dir3", strerror(errno));
+        printf("glfs_h_rename tests: FAILED\n");
+        goto out;
+    }
+
+    /* rename dir2 ->dir3 (exists) */
+    ret = glfs_h_rename(fs, parent, "dir2", parent, "dir3");
+    if (ret != 0) {
+        fprintf(stderr, "glfs_h_rename: error renaming %s to %s (%s)\n", "dir2",
+                "dir3", strerror(errno));
+        printf("glfs_h_rename tests: FAILED\n");
+        goto out;
+    }
+
+    /* rename file.txt -> dir3 (fail) */
+    ret = glfs_h_rename(fs, parent, "file.txt", parent, "dir3");
+    if (ret == 0) {
+        fprintf(stderr, "glfs_h_rename: NO error renaming %s to %s (%s)\n",
+                "file.txt", "dir3", strerror(errno));
+        printf("glfs_h_rename tests: FAILED\n");
+        goto out;
+    }
+
+    /* rename dir3 -> file.txt (fail) */
+    ret = glfs_h_rename(fs, parent, "dir3", parent, "file.txt");
+    if (ret == 0) {
+        fprintf(stderr, "glfs_h_rename: NO error renaming %s to %s (%s)\n",
+                "dir3", "file.txt", strerror(errno));
+        printf("glfs_h_rename tests: FAILED\n");
+        goto out;
+    }
+
+    printf("glfs_h_rename tests: PASSED\n");
 
 out:
-        if (root)
-                glfs_h_close (root);
-        if (parent)
-                glfs_h_close (parent);
-        if (leaf)
-                glfs_h_close (leaf);
-        if (dirsrc)
-                glfs_h_close (dirsrc);
-        if (dirtgt)
-                glfs_h_close (dirtgt);
-        if (dleaf)
-                glfs_h_close (dleaf);
-
-        return;
+    if (root)
+        glfs_h_close(root);
+    if (parent)
+        glfs_h_close(parent);
+    if (leaf)
+        glfs_h_close(leaf);
+    if (dirsrc)
+        glfs_h_close(dirsrc);
+    if (dirtgt)
+        glfs_h_close(dirtgt);
+    if (dleaf)
+        glfs_h_close(dleaf);
+
+    return;
 }
 
 void
-assimilatetime (struct timespec *ts, struct timespec ts_st,
-                struct timespec ts_ed)
+assimilatetime(struct timespec *ts, struct timespec ts_st,
+               struct timespec ts_ed)
 {
-        if ((ts_ed.tv_nsec - ts_st.tv_nsec) < 0) {
-                ts->tv_sec += ts_ed.tv_sec - ts_st.tv_sec - 1;
-                ts->tv_nsec += 1000000000 + ts_ed.tv_nsec - ts_st.tv_nsec;
-        } else {
-                ts->tv_sec += ts_ed.tv_sec - ts_st.tv_sec;
-                ts->tv_nsec += ts_ed.tv_nsec - ts_st.tv_nsec;
-        }
-
-        if (ts->tv_nsec > 1000000000) {
-                ts->tv_nsec = ts->tv_nsec - 1000000000;
-                ts->tv_sec += 1;
-        }
-
-        return;
+    if ((ts_ed.tv_nsec - ts_st.tv_nsec) < 0) {
+        ts->tv_sec += ts_ed.tv_sec - ts_st.tv_sec - 1;
+        ts->tv_nsec += 1000000000 + ts_ed.tv_nsec - ts_st.tv_nsec;
+    } else {
+        ts->tv_sec += ts_ed.tv_sec - ts_st.tv_sec;
+        ts->tv_nsec += ts_ed.tv_nsec - ts_st.tv_nsec;
+    }
+
+    if (ts->tv_nsec > 1000000000) {
+        ts->tv_nsec = ts->tv_nsec - 1000000000;
+        ts->tv_sec += 1;
+    }
+
+    return;
 }
 
 #define MAX_FILES_CREATE 10
-#define MAXPATHNAME      512
+#define MAXPATHNAME 512
 void
-test_h_performance (void)
+test_h_performance(void)
 {
-        char               *my_dir = "perftest",
-                           *full_dir_path="/testdir/perftest";
-        char               *my_file = "file_", my_file_name[MAXPATHNAME];
-        struct glfs_object *parent = NULL, *leaf = NULL, *dir = NULL;
-        struct stat         sb;
-        int                 ret, i;
-        struct glfs_fd     *fd;
-        struct timespec     c_ts = {0, 0}, c_ts_st, c_ts_ed;
-        struct timespec     o_ts = {0, 0}, o_ts_st, o_ts_ed;
-
-        printf("glfs_h_performance tests: In Progress\n");
-
-        /* Prepare tests */
-        parent = glfs_h_lookupat (fs, NULL, full_parent_name, &sb);
-        if (parent == NULL) {
-                fprintf (stderr, "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
-                         full_parent_name, NULL, strerror (errno));
-                printf ("glfs_h_performance tests: FAILED\n");
-                goto out;
-        }
-
-        dir = glfs_h_mkdir (fs, parent, my_dir, 0644, &sb);
-        if (dir == NULL) {
-                fprintf (stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
-                         my_dir, parent, strerror (errno));
-                printf ("glfs_h_performance tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        /* create performance */
-        ret = clock_gettime (CLOCK_REALTIME, &o_ts_st);
-        if(ret != 0) {
-                fprintf (stderr, "clock_gettime: error %s\n", strerror (errno));
-                printf ("glfs_h_getattrs and setattrs tests: FAILED\n");
-                goto out;
+    char *my_dir = "perftest", *full_dir_path = "/testdir/perftest";
+    char *my_file = "file_", my_file_name[MAXPATHNAME];
+    struct glfs_object *parent = NULL, *leaf = NULL, *dir = NULL;
+    struct stat sb;
+    int ret, i;
+    struct glfs_fd *fd;
+    struct timespec c_ts = {0, 0}, c_ts_st, c_ts_ed;
+    struct timespec o_ts = {0, 0}, o_ts_st, o_ts_ed;
+
+    printf("glfs_h_performance tests: In Progress\n");
+
+    /* Prepare tests */
+    parent = glfs_h_lookupat(fs, NULL, full_parent_name, &sb, 0);
+    if (parent == NULL) {
+        fprintf(stderr,
+                "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                full_parent_name, NULL, strerror(errno));
+        printf("glfs_h_performance tests: FAILED\n");
+        goto out;
+    }
+
+    dir = glfs_h_mkdir(fs, parent, my_dir, 0755, &sb);
+    if (dir == NULL) {
+        fprintf(stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
+                my_dir, parent, strerror(errno));
+        printf("glfs_h_performance tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    /* create performance */
+    ret = clock_gettime(CLOCK_REALTIME, &o_ts_st);
+    if (ret != 0) {
+        fprintf(stderr, "clock_gettime: error %s\n", strerror(errno));
+        printf("glfs_h_getattrs and setattrs tests: FAILED\n");
+        goto out;
+    }
+
+    for (i = 0; i < MAX_FILES_CREATE; i++) {
+        sprintf(my_file_name, "%s%d", my_file, i);
+
+        ret = clock_gettime(CLOCK_REALTIME, &c_ts_st);
+        if (ret != 0) {
+            fprintf(stderr, "clock_gettime: error %s\n", strerror(errno));
+            printf("glfs_h_getattrs and setattrs tests: FAILED\n");
+            goto out;
         }
 
-        for (i = 0; i < MAX_FILES_CREATE; i++) {
-                sprintf (my_file_name, "%s%d", my_file, i);
-
-                ret = clock_gettime (CLOCK_REALTIME, &c_ts_st);
-                if(ret != 0) {
-                        fprintf (stderr, "clock_gettime: error %s\n",
-                                 strerror (errno));
-                        printf ("glfs_h_getattrs and setattrs tests: FAILED\n");
-                        goto out;
-                }
-
-                leaf = glfs_h_lookupat (fs, dir, my_file_name, &sb);
-                if (leaf != NULL) {
-                        fprintf (stderr, "glfs_h_lookup: exists %s\n",
-                                 my_file_name);
-                        printf ("glfs_h_performance tests: FAILED\n");
-                        goto out;
-                }
-
-                leaf = glfs_h_creat (fs, dir, my_file_name, O_CREAT, 0644, &sb);
-                if (leaf == NULL) {
-                        fprintf (stderr, "glfs_h_creat: error creating %s: from (%p),%s\n",
-                                 my_file, dir, strerror (errno));
-                        printf ("glfs_h_performance tests: FAILED\n");
-                        goto out;
-                }
-
-                ret = clock_gettime (CLOCK_REALTIME, &c_ts_ed);
-                if(ret != 0) {
-                        fprintf (stderr, "clock_gettime: error %s\n",
-                                 strerror (errno));
-                        printf ("glfs_h_getattrs and setattrs tests: FAILED\n");
-                        goto out;
-                }
-
-                assimilatetime (&c_ts, c_ts_st, c_ts_ed);
-                glfs_h_close (leaf); leaf = NULL;
+        leaf = glfs_h_lookupat(fs, dir, my_file_name, &sb, 0);
+        if (leaf != NULL) {
+            fprintf(stderr, "glfs_h_lookup: exists %s\n", my_file_name);
+            printf("glfs_h_performance tests: FAILED\n");
+            goto out;
         }
 
-        ret = clock_gettime (CLOCK_REALTIME, &o_ts_ed);
-        if(ret != 0) {
-                fprintf (stderr, "clock_gettime: error %s\n", strerror (errno));
-                printf ("glfs_h_getattrs and setattrs tests: FAILED\n");
-                goto out;
+        leaf = glfs_h_creat(fs, dir, my_file_name, O_CREAT, 0644, &sb);
+        if (leaf == NULL) {
+            fprintf(stderr, "glfs_h_creat: error creating %s: from (%p),%s\n",
+                    my_file, dir, strerror(errno));
+            printf("glfs_h_performance tests: FAILED\n");
+            goto out;
         }
 
-        assimilatetime (&o_ts, o_ts_st, o_ts_ed);
-
-        printf ("Creation performance (handle based):\n\t# empty files:%d\n",
-                MAX_FILES_CREATE);
-        printf ("\tOverall time:\n\t\tSecs:%ld\n\t\tnSecs:%ld\n",
-                o_ts.tv_sec, o_ts.tv_nsec);
-        printf ("\tcreate call time time:\n\t\tSecs:%ld\n\t\tnSecs:%ld\n",
-                c_ts.tv_sec, c_ts.tv_nsec);
-
-        /* create using path */
-        c_ts.tv_sec = o_ts.tv_sec = 0;
-        c_ts.tv_nsec = o_ts.tv_nsec = 0;
-
-        sprintf (my_file_name, "%s1", full_dir_path);
-        ret = glfs_mkdir (fs, my_file_name, 0644);
+        ret = clock_gettime(CLOCK_REALTIME, &c_ts_ed);
         if (ret != 0) {
-                fprintf (stderr, "glfs_mkdir: error creating %s: from (%p),%s\n",
-                         my_dir, parent, strerror (errno));
-                printf ("glfs_h_performance tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        ret = clock_gettime (CLOCK_REALTIME, &o_ts_st);
-        if(ret != 0) {
-                fprintf (stderr, "clock_gettime: error %s\n", strerror (errno));
-                printf ("glfs_h_getattrs and setattrs tests: FAILED\n");
-                goto out;
+            fprintf(stderr, "clock_gettime: error %s\n", strerror(errno));
+            printf("glfs_h_getattrs and setattrs tests: FAILED\n");
+            goto out;
+        }
+
+        assimilatetime(&c_ts, c_ts_st, c_ts_ed);
+        glfs_h_close(leaf);
+        leaf = NULL;
+    }
+
+    ret = clock_gettime(CLOCK_REALTIME, &o_ts_ed);
+    if (ret != 0) {
+        fprintf(stderr, "clock_gettime: error %s\n", strerror(errno));
+        printf("glfs_h_getattrs and setattrs tests: FAILED\n");
+        goto out;
+    }
+
+    assimilatetime(&o_ts, o_ts_st, o_ts_ed);
+
+    printf("Creation performance (handle based):\n\t# empty files:%d\n",
+           MAX_FILES_CREATE);
+    printf("\tOverall time:\n\t\tSecs:%ld\n\t\tnSecs:%ld\n", o_ts.tv_sec,
+           o_ts.tv_nsec);
+    printf("\tcreate call time time:\n\t\tSecs:%ld\n\t\tnSecs:%ld\n",
+           c_ts.tv_sec, c_ts.tv_nsec);
+
+    /* create using path */
+    c_ts.tv_sec = o_ts.tv_sec = 0;
+    c_ts.tv_nsec = o_ts.tv_nsec = 0;
+
+    sprintf(my_file_name, "%s1", full_dir_path);
+    ret = glfs_mkdir(fs, my_file_name, 0755);
+    if (ret != 0) {
+        fprintf(stderr, "glfs_mkdir: error creating %s: from (%p),%s\n", my_dir,
+                parent, strerror(errno));
+        printf("glfs_h_performance tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    ret = clock_gettime(CLOCK_REALTIME, &o_ts_st);
+    if (ret != 0) {
+        fprintf(stderr, "clock_gettime: error %s\n", strerror(errno));
+        printf("glfs_h_getattrs and setattrs tests: FAILED\n");
+        goto out;
+    }
+
+    for (i = 0; i < MAX_FILES_CREATE; i++) {
+        sprintf(my_file_name, "%s1/%sn%d", full_dir_path, my_file, i);
+
+        ret = clock_gettime(CLOCK_REALTIME, &c_ts_st);
+        if (ret != 0) {
+            fprintf(stderr, "clock_gettime: error %s\n", strerror(errno));
+            printf("glfs_h_getattrs and setattrs tests: FAILED\n");
+            goto out;
         }
 
-        for (i = 0; i < MAX_FILES_CREATE; i++) {
-                sprintf (my_file_name, "%s1/%sn%d", full_dir_path, my_file, i);
-
-                ret = clock_gettime (CLOCK_REALTIME, &c_ts_st);
-                if(ret != 0) {
-                        fprintf (stderr, "clock_gettime: error %s\n",
-                                 strerror (errno));
-                        printf ("glfs_h_getattrs and setattrs tests: FAILED\n");
-                        goto out;
-                }
-
-                ret = glfs_stat (fs, my_file_name, &sb);
-                if (ret == 0) {
-                        fprintf (stderr, "glfs_stat: exists %s\n",
-                                 my_file_name);
-                        printf ("glfs_h_performance tests: FAILED\n");
-                        goto out;
-                }
-
-                fd = glfs_creat (fs, my_file_name, O_CREAT, 0644);
-                if (fd == NULL) {
-                        fprintf (stderr, "glfs_creat: error creating %s: from (%p),%s\n",
-                                 my_file, dir, strerror (errno));
-                        printf ("glfs_h_performance tests: FAILED\n");
-                        goto out;
-                }
-
-                ret = clock_gettime (CLOCK_REALTIME, &c_ts_ed);
-                if(ret != 0) {
-                        fprintf (stderr, "clock_gettime: error %s\n",
-                                 strerror (errno));
-                        printf ("glfs_h_getattrs and setattrs tests: FAILED\n");
-                        goto out;
-                }
-
-                assimilatetime (&c_ts, c_ts_st, c_ts_ed);
-                glfs_close (fd);
+        ret = glfs_stat(fs, my_file_name, &sb);
+        if (ret == 0) {
+            fprintf(stderr, "glfs_stat: exists %s\n", my_file_name);
+            printf("glfs_h_performance tests: FAILED\n");
+            goto out;
         }
 
-        ret = clock_gettime (CLOCK_REALTIME, &o_ts_ed);
-        if(ret != 0) {
-                fprintf (stderr, "clock_gettime: error %s\n", strerror (errno));
-                printf ("glfs_h_getattrs and setattrs tests: FAILED\n");
-                goto out;
+        fd = glfs_creat(fs, my_file_name, O_CREAT, 0644);
+        if (fd == NULL) {
+            fprintf(stderr, "glfs_creat: error creating %s: from (%p),%s\n",
+                    my_file, dir, strerror(errno));
+            printf("glfs_h_performance tests: FAILED\n");
+            goto out;
         }
 
-        assimilatetime (&o_ts, o_ts_st, o_ts_ed);
-
-        printf ("Creation performance (path based):\n\t# empty files:%d\n",
-                MAX_FILES_CREATE);
-        printf ("\tOverall time:\n\t\tSecs:%ld\n\t\tnSecs:%ld\n",
-                o_ts.tv_sec, o_ts.tv_nsec);
-        printf ("\tcreate call time time:\n\t\tSecs:%ld\n\t\tnSecs:%ld\n",
-                c_ts.tv_sec, c_ts.tv_nsec);
+        ret = clock_gettime(CLOCK_REALTIME, &c_ts_ed);
+        if (ret != 0) {
+            fprintf(stderr, "clock_gettime: error %s\n", strerror(errno));
+            printf("glfs_h_getattrs and setattrs tests: FAILED\n");
+            goto out;
+        }
+
+        assimilatetime(&c_ts, c_ts_st, c_ts_ed);
+        glfs_close(fd);
+    }
+
+    ret = clock_gettime(CLOCK_REALTIME, &o_ts_ed);
+    if (ret != 0) {
+        fprintf(stderr, "clock_gettime: error %s\n", strerror(errno));
+        printf("glfs_h_getattrs and setattrs tests: FAILED\n");
+        goto out;
+    }
+
+    assimilatetime(&o_ts, o_ts_st, o_ts_ed);
+
+    printf("Creation performance (path based):\n\t# empty files:%d\n",
+           MAX_FILES_CREATE);
+    printf("\tOverall time:\n\t\tSecs:%ld\n\t\tnSecs:%ld\n", o_ts.tv_sec,
+           o_ts.tv_nsec);
+    printf("\tcreate call time time:\n\t\tSecs:%ld\n\t\tnSecs:%ld\n",
+           c_ts.tv_sec, c_ts.tv_nsec);
 out:
-        return;
+    return;
 }
 
 int
-test_handleops (int argc, char *argv[])
+test_handleops(int argc, char *argv[])
 {
-        int                 ret = 0;
-        glfs_fd_t          *fd = NULL;
-        struct stat         sb = {0, };
-        struct glfs_object *root = NULL, *parent = NULL, *leaf = NULL,
-                           *tmp = NULL;
-        char                readbuf[32], writebuf[32];
-        unsigned char       leaf_handle[GFAPI_HANDLE_LENGTH];
-
-        char *full_leaf_name = "/testdir/testfile.txt",
-             *leaf_name = "testfile.txt",
-             *relative_leaf_name = "testdir/testfile.txt";
-        char *leaf_name1 = "testfile1.txt";
-        char *full_newparent_name = "/testdir/dir1",
-             *newparent_name = "dir1";
-        char *full_newnod_name = "/testdir/nod1",
-             *newnod_name = "nod1";
-
-        /* Initialize test area */
-        ret = glfs_mkdir (fs, full_parent_name, 0644);
-        if (ret != 0 && errno != EEXIST) {
-                fprintf (stderr, "%s: (%p) %s\n", full_parent_name, fd,
-                        strerror (errno));
-                printf ("Test initialization failed on volume %s\n", argv[1]);
-                goto out;
-        }
-        else if (ret != 0) {
-                printf ("Found test directory %s to be existing\n",
-                        full_parent_name);
-                printf ("Cleanup test directory and restart tests\n");
-                goto out;
-        }
-
-        fd = glfs_creat (fs, full_leaf_name, O_CREAT, 0644);
-        if (fd == NULL) {
-                fprintf (stderr, "%s: (%p) %s\n", full_leaf_name, fd,
-                        strerror (errno));
-                printf ("Test initialization failed on volume %s\n", argv[1]);
-                goto out;
-        }
-        glfs_close (fd);
-
-        printf ("Initialized the test area, within volume %s\n", argv[1]);
-
-        /* Handle based APIs test area */
-
-        /* glfs_lookupat test */
-        printf ("glfs_h_lookupat tests: In Progress\n");
-        /* start at root of the volume */
-        root = glfs_h_lookupat (fs, NULL, "/", &sb);
-        if (root == NULL) {
-                fprintf (stderr, "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
-                         "/", NULL, strerror (errno));
-                printf ("glfs_h_lookupat tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        /* lookup a parent within root */
-        parent = glfs_h_lookupat (fs, root, parent_name, &sb);
-        if (parent == NULL) {
-                fprintf (stderr, "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
-                         parent_name, root, strerror (errno));
-                printf ("glfs_h_lookupat tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        /* lookup a leaf/child within the parent */
-        leaf = glfs_h_lookupat (fs, parent, leaf_name, &sb);
-        if (leaf == NULL) {
-                fprintf (stderr, "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
-                         leaf_name, parent, strerror (errno));
-                printf ("glfs_h_lookupat tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        /* reset */
-        glfs_h_close (root); root = NULL;
-        glfs_h_close (leaf); leaf = NULL;
-        glfs_h_close (parent); parent = NULL;
-
-        /* check absolute paths */
-        root = glfs_h_lookupat (fs, NULL, "/", &sb);
-        if (root == NULL) {
-                fprintf (stderr, "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
-                         "/", NULL, strerror (errno));
-                printf ("glfs_h_lookupat tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        parent = glfs_h_lookupat (fs, NULL, full_parent_name, &sb);
-        if (parent == NULL) {
-                fprintf (stderr, "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
-                         full_parent_name, root, strerror (errno));
-                printf ("glfs_h_lookupat tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        leaf = glfs_h_lookupat (fs, NULL, full_leaf_name, &sb);
-        if (leaf == NULL) {
-                fprintf (stderr, "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
-                         full_leaf_name, parent, strerror (errno));
-                printf ("glfs_h_lookupat tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        /* reset */
-        glfs_h_close (leaf); leaf = NULL;
-
-        /* check multiple component paths */
-        leaf = glfs_h_lookupat (fs, root, relative_leaf_name, &sb);
-        if (leaf == NULL) {
-                fprintf (stderr, "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
-                         relative_leaf_name, parent, strerror (errno));
-                goto out;
-        }
-        peek_stat (&sb);
-
-        /* reset */
-        glfs_h_close (root); root = NULL;
-        glfs_h_close (parent); parent = NULL;
-
-        /* check symlinks in path */
-
-        /* TODO: -ve test cases */
-        /* parent invalid
-        * path invalid
-        * path does not exist after some components
-        * no parent, but relative path
-        * parent and full path? -ve?
-        */
-
-        printf ("glfs_h_lookupat tests: PASSED\n");
-
-        /* glfs_openat test */
-        printf ("glfs_h_open tests: In Progress\n");
-        fd = glfs_h_open (fs, leaf, O_RDWR);
-        if (fd == NULL) {
-                fprintf (stderr, "glfs_h_open: error on open of %s: %s\n",
-                         full_leaf_name, strerror (errno));
-                printf ("glfs_h_open tests: FAILED\n");
-                goto out;
-        }
-
-        /* test read/write based on fd */
-        memcpy (writebuf, "abcdefghijklmnopqrstuvwxyz012345", 32);
-        ret = glfs_write (fd, writebuf, 32, 0);
-
-        glfs_lseek (fd, 0, SEEK_SET);
-
-        ret = glfs_read (fd, readbuf, 32, 0);
-        if (memcmp (readbuf, writebuf, 32)) {
-                printf ("Failed to read what I wrote: %s %s\n", readbuf,
-                        writebuf);
-                glfs_close (fd);
-                printf ("glfs_h_open tests: FAILED\n");
-                goto out;
-        }
-
-        glfs_h_close (leaf); leaf = NULL;
-        glfs_close (fd);
-
-        printf ("glfs_h_open tests: PASSED\n");
-
-        /* Create tests */
-        printf ("glfs_h_creat tests: In Progress\n");
-        parent = glfs_h_lookupat (fs, NULL, full_parent_name, &sb);
-        if (parent == NULL) {
-                fprintf (stderr, "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
-                         full_parent_name, root, strerror (errno));
-                printf ("glfs_h_creat tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        leaf = glfs_h_creat (fs, parent, leaf_name1, O_CREAT, 0644, &sb);
-        if (leaf == NULL) {
-                fprintf (stderr, "glfs_h_creat: error on create of %s: from (%p),%s\n",
-                         leaf_name1, parent, strerror (errno));
-                printf ("glfs_h_creat tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        glfs_h_close (leaf); leaf = NULL;
-
-        leaf = glfs_h_creat (fs, parent, leaf_name1, O_CREAT | O_EXCL, 0644,
-                            &sb);
-        if (leaf != NULL || errno != EEXIST) {
-                fprintf (stderr, "glfs_h_creat: existing file, leaf = (%p), errno = %s\n",
-                        leaf, strerror (errno));
-                printf ("glfs_h_creat tests: FAILED\n");
-                if (leaf != NULL) {
-                        glfs_h_close (leaf); leaf = NULL;
-                }
-        }
-
-        tmp = glfs_h_creat (fs, root, parent_name, O_CREAT, 0644, &sb);
-        if (tmp != NULL || !(errno == EISDIR || errno == EINVAL)) {
-                fprintf (stderr, "glfs_h_creat: dir create, tmp = (%p), errno = %s\n",
-                        leaf, strerror (errno));
-                printf ("glfs_h_creat tests: FAILED\n");
-                if (tmp != NULL) {
-                        glfs_h_close (tmp); tmp = NULL;
-                }
-        }
-
-        /* TODO: Other combinations and -ve cases as applicable */
-        printf ("glfs_h_creat tests: PASSED\n");
-
-        /* extract handle and create from handle test */
-        printf ("glfs_h_extract_handle and glfs_h_create_from_handle tests: In Progress\n");
-        /* TODO: Change the lookup to creat below for a GIFD recovery falure,
-         * that needs to be fixed */
-        leaf = glfs_h_lookupat (fs, parent, leaf_name1, &sb);
-        if (leaf == NULL) {
-                fprintf (stderr, "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
-                         leaf_name1, parent, strerror (errno));
-                printf ("glfs_h_extract_handle tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        ret = glfs_h_extract_handle (leaf, leaf_handle,
-                                             GFAPI_HANDLE_LENGTH);
-        if (ret < 0) {
-                fprintf (stderr, "glfs_h_extract_handle: error extracting handle of %s: %s\n",
-                         full_leaf_name, strerror (errno));
-                printf ("glfs_h_extract_handle tests: FAILED\n");
-                goto out;
-        }
-        peek_handle (leaf_handle);
+    int ret = 0;
+    glfs_fd_t *fd = NULL;
+    struct stat sb = {
+        0,
+    };
+    struct glfs_object *root = NULL, *parent = NULL, *leaf = NULL, *tmp = NULL;
+    char readbuf[32], writebuf[32];
+    unsigned char leaf_handle[GFAPI_HANDLE_LENGTH];
+
+    char *full_leaf_name = "/testdir/testfile.txt", *leaf_name = "testfile.txt",
+         *relative_leaf_name = "testdir/testfile.txt";
+    char *leaf_name1 = "testfile1.txt";
+    char *full_newparent_name = "/testdir/dir1", *newparent_name = "dir1";
+    char *full_newnod_name = "/testdir/nod1", *newnod_name = "nod1";
+
+    /* Initialize test area */
+    ret = glfs_mkdir(fs, full_parent_name, 0755);
+    if (ret != 0 && errno != EEXIST) {
+        fprintf(stderr, "%s: (%p) %s\n", full_parent_name, fd, strerror(errno));
+        printf("Test initialization failed on volume %s\n", argv[1]);
+        goto out;
+    } else if (ret != 0) {
+        printf("Found test directory %s to be existing\n", full_parent_name);
+        printf("Cleanup test directory and restart tests\n");
+        goto out;
+    }
+
+    fd = glfs_creat(fs, full_leaf_name, O_CREAT, 0644);
+    if (fd == NULL) {
+        fprintf(stderr, "%s: (%p) %s\n", full_leaf_name, fd, strerror(errno));
+        printf("Test initialization failed on volume %s\n", argv[1]);
+        goto out;
+    }
+    glfs_close(fd);
+
+    printf("Initialized the test area, within volume %s\n", argv[1]);
+
+    /* Handle based APIs test area */
+
+    /* glfs_lookupat test */
+    printf("glfs_h_lookupat tests: In Progress\n");
+    /* start at root of the volume */
+    root = glfs_h_lookupat(fs, NULL, "/", &sb, 0);
+    if (root == NULL) {
+        fprintf(stderr,
+                "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n", "/",
+                NULL, strerror(errno));
+        printf("glfs_h_lookupat tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    /* lookup a parent within root */
+    parent = glfs_h_lookupat(fs, root, parent_name, &sb, 0);
+    if (parent == NULL) {
+        fprintf(stderr,
+                "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                parent_name, root, strerror(errno));
+        printf("glfs_h_lookupat tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    /* lookup a leaf/child within the parent */
+    leaf = glfs_h_lookupat(fs, parent, leaf_name, &sb, 0);
+    if (leaf == NULL) {
+        fprintf(stderr,
+                "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                leaf_name, parent, strerror(errno));
+        printf("glfs_h_lookupat tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    /* reset */
+    glfs_h_close(root);
+    root = NULL;
+    glfs_h_close(leaf);
+    leaf = NULL;
+    glfs_h_close(parent);
+    parent = NULL;
+
+    /* check absolute paths */
+    root = glfs_h_lookupat(fs, NULL, "/", &sb, 0);
+    if (root == NULL) {
+        fprintf(stderr,
+                "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n", "/",
+                NULL, strerror(errno));
+        printf("glfs_h_lookupat tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    parent = glfs_h_lookupat(fs, NULL, full_parent_name, &sb, 0);
+    if (parent == NULL) {
+        fprintf(stderr,
+                "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                full_parent_name, root, strerror(errno));
+        printf("glfs_h_lookupat tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    leaf = glfs_h_lookupat(fs, NULL, full_leaf_name, &sb, 0);
+    if (leaf == NULL) {
+        fprintf(stderr,
+                "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                full_leaf_name, parent, strerror(errno));
+        printf("glfs_h_lookupat tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    /* reset */
+    glfs_h_close(leaf);
+    leaf = NULL;
+
+    /* check multiple component paths */
+    leaf = glfs_h_lookupat(fs, root, relative_leaf_name, &sb, 0);
+    if (leaf == NULL) {
+        fprintf(stderr,
+                "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                relative_leaf_name, parent, strerror(errno));
+        goto out;
+    }
+    peek_stat(&sb);
+
+    /* reset */
+    glfs_h_close(root);
+    root = NULL;
+    glfs_h_close(parent);
+    parent = NULL;
+
+    /* check symlinks in path */
+
+    /* TODO: -ve test cases */
+    /* parent invalid
+     * path invalid
+     * path does not exist after some components
+     * no parent, but relative path
+     * parent and full path? -ve?
+     */
+
+    printf("glfs_h_lookupat tests: PASSED\n");
+
+    /* glfs_openat test */
+    printf("glfs_h_open tests: In Progress\n");
+    fd = glfs_h_open(fs, leaf, O_RDWR);
+    if (fd == NULL) {
+        fprintf(stderr, "glfs_h_open: error on open of %s: %s\n",
+                full_leaf_name, strerror(errno));
+        printf("glfs_h_open tests: FAILED\n");
+        goto out;
+    }
+
+    /* test read/write based on fd */
+    memcpy(writebuf, "abcdefghijklmnopqrstuvwxyz012345", 32);
+    ret = glfs_write(fd, writebuf, 32, 0);
+
+    glfs_lseek(fd, 0, SEEK_SET);
+
+    ret = glfs_read(fd, readbuf, 32, 0);
+    if (memcmp(readbuf, writebuf, 32)) {
+        printf("Failed to read what I wrote: %s %s\n", readbuf, writebuf);
+        glfs_close(fd);
+        printf("glfs_h_open tests: FAILED\n");
+        goto out;
+    }
+
+    glfs_h_close(leaf);
+    leaf = NULL;
+    glfs_close(fd);
+
+    printf("glfs_h_open tests: PASSED\n");
+
+    /* Create tests */
+    printf("glfs_h_creat tests: In Progress\n");
+    parent = glfs_h_lookupat(fs, NULL, full_parent_name, &sb, 0);
+    if (parent == NULL) {
+        fprintf(stderr,
+                "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                full_parent_name, root, strerror(errno));
+        printf("glfs_h_creat tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    leaf = glfs_h_creat(fs, parent, leaf_name1, O_CREAT, 0644, &sb);
+    if (leaf == NULL) {
+        fprintf(stderr, "glfs_h_creat: error on create of %s: from (%p),%s\n",
+                leaf_name1, parent, strerror(errno));
+        printf("glfs_h_creat tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    glfs_h_close(leaf);
+    leaf = NULL;
+
+    leaf = glfs_h_creat(fs, parent, leaf_name1, O_CREAT | O_EXCL, 0644, &sb);
+    if (leaf != NULL || errno != EEXIST) {
+        fprintf(stderr,
+                "glfs_h_creat: existing file, leaf = (%p), errno = %s\n", leaf,
+                strerror(errno));
+        printf("glfs_h_creat tests: FAILED\n");
+        if (leaf != NULL) {
+            glfs_h_close(leaf);
+            leaf = NULL;
+        }
+    }
+
+    tmp = glfs_h_creat(fs, root, parent_name, O_CREAT, 0644, &sb);
+    if (tmp != NULL || !(errno == EISDIR || errno == EINVAL)) {
+        fprintf(stderr, "glfs_h_creat: dir create, tmp = (%p), errno = %s\n",
+                leaf, strerror(errno));
+        printf("glfs_h_creat tests: FAILED\n");
+        if (tmp != NULL) {
+            glfs_h_close(tmp);
+            tmp = NULL;
+        }
+    }
+
+    /* TODO: Other combinations and -ve cases as applicable */
+    printf("glfs_h_creat tests: PASSED\n");
+
+    /* extract handle and create from handle test */
+    printf(
+        "glfs_h_extract_handle and glfs_h_create_from_handle tests: In "
+        "Progress\n");
+    /* TODO: Change the lookup to create below for a GIFD recovery failure,
+     * that needs to be fixed */
+    leaf = glfs_h_lookupat(fs, parent, leaf_name1, &sb, 0);
+    if (leaf == NULL) {
+        fprintf(stderr,
+                "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                leaf_name1, parent, strerror(errno));
+        printf("glfs_h_extract_handle tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    ret = glfs_h_extract_handle(leaf, leaf_handle, GFAPI_HANDLE_LENGTH);
+    if (ret < 0) {
+        fprintf(stderr,
+                "glfs_h_extract_handle: error extracting handle of %s: %s\n",
+                full_leaf_name, strerror(errno));
+        printf("glfs_h_extract_handle tests: FAILED\n");
+        goto out;
+    }
+    peek_handle(leaf_handle);
+
+    glfs_h_close(leaf);
+    leaf = NULL;
+
+    leaf = glfs_h_create_from_handle(fs, leaf_handle, GFAPI_HANDLE_LENGTH, &sb);
+    if (leaf == NULL) {
+        fprintf(
+            stderr,
+            "glfs_h_create_from_handle: error on create of %s: from (%p),%s\n",
+            leaf_name1, leaf_handle, strerror(errno));
+        printf("glfs_h_create_from_handle tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    fd = glfs_h_open(fs, leaf, O_RDWR);
+    if (fd == NULL) {
+        fprintf(stderr, "glfs_h_open: error on open of %s: %s\n",
+                full_leaf_name, strerror(errno));
+        printf("glfs_h_create_from_handle tests: FAILED\n");
+        goto out;
+    }
+
+    /* test read/write based on fd */
+    memcpy(writebuf, "abcdefghijklmnopqrstuvwxyz012345", 32);
+    ret = glfs_write(fd, writebuf, 32, 0);
+
+    glfs_lseek(fd, 0, SEEK_SET);
+
+    ret = glfs_read(fd, readbuf, 32, 0);
+    if (memcmp(readbuf, writebuf, 32)) {
+        printf("Failed to read what I wrote: %s %s\n", writebuf, writebuf);
+        printf("glfs_h_create_from_handle tests: FAILED\n");
+        glfs_close(fd);
+        goto out;
+    }
+
+    glfs_close(fd);
+    glfs_h_close(leaf);
+    leaf = NULL;
+    glfs_h_close(parent);
+    parent = NULL;
+
+    printf(
+        "glfs_h_extract_handle and glfs_h_create_from_handle tests: PASSED\n");
+
+    /* Mkdir tests */
+    printf("glfs_h_mkdir tests: In Progress\n");
+
+    ret = glfs_rmdir(fs, full_newparent_name);
+    if (ret && errno != ENOENT) {
+        fprintf(stderr, "glfs_rmdir: Failed for %s: %s\n", full_newparent_name,
+                strerror(errno));
+        printf("glfs_h_mkdir tests: FAILED\n");
+        goto out;
+    }
+
+    parent = glfs_h_lookupat(fs, NULL, full_parent_name, &sb, 0);
+    if (parent == NULL) {
+        fprintf(stderr,
+                "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                full_parent_name, root, strerror(errno));
+        printf("glfs_h_mkdir tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    leaf = glfs_h_mkdir(fs, parent, newparent_name, 0755, &sb);
+    if (leaf == NULL) {
+        fprintf(stderr, "glfs_h_mkdir: error on mkdir of %s: from (%p),%s\n",
+                newparent_name, parent, strerror(errno));
+        printf("glfs_h_mkdir tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    glfs_h_close(leaf);
+    leaf = NULL;
+
+    leaf = glfs_h_mkdir(fs, parent, newparent_name, 0755, &sb);
+    if (leaf != NULL || errno != EEXIST) {
+        fprintf(stderr,
+                "glfs_h_mkdir: existing directory, leaf = (%p), errno = %s\n",
+                leaf, strerror(errno));
+        printf("glfs_h_mkdir tests: FAILED\n");
+        if (leaf != NULL) {
+            glfs_h_close(leaf);
+            leaf = NULL;
+        }
+    }
+
+    glfs_h_close(parent);
+    parent = NULL;
+
+    printf("glfs_h_mkdir tests: PASSED\n");
+
+    /* Mknod tests */
+    printf("glfs_h_mknod tests: In Progress\n");
+    ret = glfs_unlink(fs, full_newnod_name);
+    if (ret && errno != ENOENT) {
+        fprintf(stderr, "glfs_unlink: Failed for %s: %s\n", full_newnod_name,
+                strerror(errno));
+        printf("glfs_h_mknod tests: FAILED\n");
+        goto out;
+    }
+
+    parent = glfs_h_lookupat(fs, NULL, full_parent_name, &sb, 0);
+    if (parent == NULL) {
+        fprintf(stderr,
+                "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                full_parent_name, root, strerror(errno));
+        printf("glfs_h_mknod tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    leaf = glfs_h_mknod(fs, parent, newnod_name, S_IFIFO, 0, &sb);
+    if (leaf == NULL) {
+        fprintf(stderr, "glfs_h_mkdir: error on mkdir of %s: from (%p),%s\n",
+                newnod_name, parent, strerror(errno));
+        printf("glfs_h_mknod tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    /* TODO: create op on a FIFO node hangs, need to check and fix
+    tmp = glfs_h_creat (fs, parent, newnod_name, O_CREAT, 0644, &sb);
+    if (tmp != NULL || errno != EINVAL) {
+            fprintf (stderr, "glfs_h_creat: node create, tmp = (%p), errno =
+    %s\n", tmp, strerror (errno)); printf ("glfs_h_creat/mknod tests:
+    FAILED\n"); if (tmp != NULL) { glfs_h_close(tmp); tmp = NULL;
+            }
+    } */
+
+    glfs_h_close(leaf);
+    leaf = NULL;
+
+    leaf = glfs_h_mknod(fs, parent, newnod_name, 0644, 0, &sb);
+    if (leaf != NULL || errno != EEXIST) {
+        fprintf(stderr,
+                "glfs_h_mknod: existing node, leaf = (%p), errno = %s\n", leaf,
+                strerror(errno));
+        printf("glfs_h_mknod tests: FAILED\n");
+        if (leaf != NULL) {
+            glfs_h_close(leaf);
+            leaf = NULL;
+        }
+    }
+
+    glfs_h_close(parent);
+    parent = NULL;
+
+    printf("glfs_h_mknod tests: PASSED\n");
+
+    /* unlink tests */
+    test_h_unlink();
+
+    /* TODO: opendir tests */
+
+    /* getattr tests */
+    test_h_getsetattrs();
+
+    /* TODO: setattr tests */
+
+    /* truncate tests */
+    test_h_truncate();
+
+    /* link tests */
+    test_h_links();
+
+    /* rename tests */
+    test_h_rename();
+
+    /* performance tests */
+    test_h_performance();
+
+    /* END: New APIs test area */
 
-        glfs_h_close (leaf); leaf = NULL;
-
-        leaf = glfs_h_create_from_handle (fs, leaf_handle, GFAPI_HANDLE_LENGTH,
-                                          &sb);
-        if (leaf == NULL) {
-                fprintf (stderr, "glfs_h_create_from_handle: error on create of %s: from (%p),%s\n",
-                         leaf_name1, leaf_handle, strerror (errno));
-                printf ("glfs_h_create_from_handle tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        fd = glfs_h_open (fs, leaf, O_RDWR);
-        if (fd == NULL) {
-                fprintf (stderr, "glfs_h_open: error on open of %s: %s\n",
-                         full_leaf_name, strerror (errno));
-                printf ("glfs_h_create_from_handle tests: FAILED\n");
-                goto out;
-        }
-
-        /* test read/write based on fd */
-        memcpy (writebuf, "abcdefghijklmnopqrstuvwxyz012345", 32);
-        ret = glfs_write (fd, writebuf, 32, 0);
-
-        glfs_lseek (fd, 0, SEEK_SET);
-
-        ret = glfs_read (fd, readbuf, 32, 0);
-        if (memcmp (readbuf, writebuf, 32)) {
-                printf ("Failed to read what I wrote: %s %s\n", writebuf,
-                        writebuf);
-                printf ("glfs_h_create_from_handle tests: FAILED\n");
-                glfs_close (fd);
-                goto out;
-        }
-
-        glfs_close (fd);
-        glfs_h_close (leaf); leaf = NULL;
-        glfs_h_close (parent); parent = NULL;
-
-        printf ("glfs_h_extract_handle and glfs_h_create_from_handle tests: PASSED\n");
-
-        /* Mkdir tests */
-        printf ("glfs_h_mkdir tests: In Progress\n");
+out:
+    /* Cleanup glfs handles */
+    if (root)
+        glfs_h_close(root);
+    if (parent)
+        glfs_h_close(parent);
+    if (leaf)
+        glfs_h_close(leaf);
+
+    return ret;
+}
 
-        ret = glfs_rmdir (fs, full_newparent_name);
-        if (ret && errno != ENOENT) {
-                fprintf (stderr, "glfs_rmdir: Failed for %s: %s\n",
-                         full_newparent_name, strerror (errno));
-                printf ("glfs_h_mkdir tests: FAILED\n");
-                goto out;
-        }
+int
+test_write_apis(glfs_t *fs)
+{
+    /* Add more content here */
+    /* Some apis we can get are */
+    /*
+      0. glfs_set_xlator_option()
+
+      Read/Write combinations:
+      . glfs_{p,}readv/{p,}writev
+      . glfs_pread/pwrite
+
+      tests/basic/gfapi/gfapi-async-calls-test.c
+      . glfs_read_async/write_async
+      . glfs_pread_async/pwrite_async
+      . glfs_readv_async/writev_async
+      . glfs_preadv_async/pwritev_async
+
+      . ftruncate/ftruncate_async
+      . fsync/fsync_async
+      . fdatasync/fdatasync_async
+
+    */
+
+    glfs_fd_t *fd = NULL;
+    char *filename = "/filename2";
+    int flags = O_RDWR;
+    char *buf = "some bytes!";
+    char writestr[TEST_STR_LEN];
+    struct iovec iov = {&writestr, TEST_STR_LEN};
+    int ret, i;
+
+    for (i = 0; i < TEST_STR_LEN; i++)
+        writestr[i] = 0x11;
+
+    fd = glfs_open(fs, filename, flags);
+    if (!fd)
+        fprintf(stderr, "open(%s): (%p) %s\n", filename, fd, strerror(errno));
+
+    ret = glfs_writev(fd, &iov, 1, flags);
+    if (ret < 0) {
+        fprintf(stderr, "writev(%s): %d (%s)\n", filename, ret,
+                strerror(errno));
+    }
+
+    ret = glfs_pwrite(fd, buf, 10, 4, flags, NULL, NULL);
+    if (ret < 0) {
+        fprintf(stderr, "pwrite(%s): %d (%s)\n", filename, ret,
+                strerror(errno));
+    }
+
+    ret = glfs_pwritev(fd, &iov, 1, 4, flags);
+    if (ret < 0) {
+        fprintf(stderr, "pwritev(%s): %d (%s)\n", filename, ret,
+                strerror(errno));
+    }
+
+    return 0;
+}
 
-        parent = glfs_h_lookupat (fs, NULL, full_parent_name, &sb);
-        if (parent == NULL) {
-                fprintf (stderr, "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
-                         full_parent_name, root, strerror (errno));
-                printf ("glfs_h_mkdir tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
+int
+test_metadata_ops(glfs_t *fs, glfs_t *fs2)
+{
+    glfs_fd_t *fd = NULL;
+    glfs_fd_t *fd2 = NULL;
+    struct stat sb = {
+        0,
+    };
+    struct glfs_stat gsb = {
+        0,
+    };
+    struct statvfs sfs;
+    char readbuf[32];
+    char writebuf[32];
 
-        leaf = glfs_h_mkdir (fs, parent, newparent_name, 0644, &sb);
-        if (leaf == NULL) {
-                fprintf (stderr, "glfs_h_mkdir: error on mkdir of %s: from (%p),%s\n",
-                         newparent_name, parent, strerror (errno));
-                printf ("glfs_h_mkdir tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        glfs_h_close (leaf); leaf = NULL;
-
-        leaf = glfs_h_mkdir (fs, parent, newparent_name, 0644, &sb);
-        if (leaf != NULL || errno != EEXIST) {
-                fprintf (stderr, "glfs_h_mkdir: existing directory, leaf = (%p), errno = %s\n",
-                         leaf, strerror (errno));
-                printf ("glfs_h_mkdir tests: FAILED\n");
-                if (leaf != NULL) {
-                        glfs_h_close (leaf); leaf = NULL;
-                }
-        }
+    char *filename = "/filename2";
+    int ret;
 
-        glfs_h_close (parent); parent = NULL;
+    ret = glfs_lstat(fs, filename, &sb);
+    fprintf(stderr, "lstat(%s): (%d) %s\n", filename, ret, strerror(errno));
 
-        printf ("glfs_h_mkdir tests: PASSED\n");
+    fd = glfs_creat(fs, filename, O_RDWR, 0644);
+    fprintf(stderr, "creat(%s): (%p) %s\n", filename, fd, strerror(errno));
 
-        /* Mknod tests */
-        printf ("glfs_h_mknod tests: In Progress\n");
-        ret = glfs_unlink (fs, full_newnod_name);
-        if (ret && errno != ENOENT) {
-                fprintf (stderr, "glfs_unlink: Failed for %s: %s\n",
-                         full_newnod_name, strerror (errno));
-                printf ("glfs_h_mknod tests: FAILED\n");
-                goto out;
-        }
+    fd2 = glfs_open(fs2, filename, O_RDWR);
+    fprintf(stderr, "open(%s): (%p) %s\n", filename, fd, strerror(errno));
 
-        parent = glfs_h_lookupat (fs, NULL, full_parent_name, &sb);
-        if (parent == NULL) {
-                fprintf (stderr, "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
-                         full_parent_name, root, strerror (errno));
-                printf ("glfs_h_mknod tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
+    glfs_lseek(fd2, 0, SEEK_SET);
 
-        leaf = glfs_h_mknod (fs, parent, newnod_name, S_IFIFO, 0, &sb);
-        if (leaf == NULL) {
-                fprintf (stderr, "glfs_h_mkdir: error on mkdir of %s: from (%p),%s\n",
-                         newnod_name, parent, strerror (errno));
-                printf ("glfs_h_mknod tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        /* TODO: creat op on a FIFO node hangs, need to check and fix
-        tmp = glfs_h_creat (fs, parent, newnod_name, O_CREAT, 0644, &sb);
-        if (tmp != NULL || errno != EINVAL) {
-                fprintf (stderr, "glfs_h_creat: node create, tmp = (%p), errno = %s\n",
-                        tmp, strerror (errno));
-                printf ("glfs_h_creat/mknod tests: FAILED\n");
-                if (tmp != NULL) {
-                        glfs_h_close(tmp); tmp = NULL;
-                }
-        } */
-
-        glfs_h_close (leaf); leaf = NULL;
-
-        leaf = glfs_h_mknod (fs, parent, newnod_name, 0644, 0, &sb);
-        if (leaf != NULL || errno != EEXIST) {
-                fprintf (stderr, "glfs_h_mknod: existing node, leaf = (%p), errno = %s\n",
-                         leaf, strerror (errno));
-                printf ("glfs_h_mknod tests: FAILED\n");
-                if (leaf != NULL) {
-                        glfs_h_close (leaf); leaf = NULL;
-                }
-        }
+    ret = glfs_read(fd2, readbuf, 32, 0);
 
-        glfs_h_close (parent); parent = NULL;
+    printf("read %d, %s", ret, readbuf);
 
-        printf ("glfs_h_mknod tests: PASSED\n");
+    /* get stat */
+    ret = glfs_fstat(fd2, &sb);
 
-        /* unlink tests */
-        test_h_unlink ();
+    ret = glfs_access(fs, filename, R_OK);
 
-        /* TODO: opendir tests */
+    /* set stat */
+    /* TODO: got some errors, need to fix */
+    /* ret = glfs_fsetattr(fd2, &gsb); */
 
-        /* getattr tests */
-        test_h_getsetattrs ();
+    glfs_close(fd);
+    glfs_close(fd2);
 
-        /* TODO: setattr tests */
+    filename = "/filename3";
+    ret = glfs_mknod(fs, filename, S_IFIFO, 0);
+    fprintf(stderr, "%s: (%d) %s\n", filename, ret, strerror(errno));
 
-        /* truncate tests */
-        test_h_truncate();
+    ret = glfs_lstat(fs, filename, &sb);
+    fprintf(stderr, "%s: (%d) %s\n", filename, ret, strerror(errno));
 
-        /* link tests */
-        test_h_links ();
+    ret = glfs_rename(fs, filename, "/filename4");
+    fprintf(stderr, "rename(%s): (%d) %s\n", filename, ret, strerror(errno));
 
-        /* rename tests */
-        test_h_rename ();
+    ret = glfs_unlink(fs, "/filename4");
+    fprintf(stderr, "unlink(%s): (%d) %s\n", "/filename4", ret,
+            strerror(errno));
 
-        /* performance tests */
-        test_h_performance ();
+    filename = "/dirname2";
+    ret = glfs_mkdir(fs, filename, 0);
+    fprintf(stderr, "%s: (%d) %s\n", filename, ret, strerror(errno));
 
-        /* END: New APIs test area */
+    ret = glfs_lstat(fs, filename, &sb);
+    fprintf(stderr, "lstat(%s): (%d) %s\n", filename, ret, strerror(errno));
 
-out:
-        /* Cleanup glfs handles */
-        if (root)
-                glfs_h_close (root);
-        if (parent)
-                glfs_h_close (parent);
-        if (leaf)
-                glfs_h_close (leaf);
-
-        return ret;
+    ret = glfs_rmdir(fs, filename);
+    fprintf(stderr, "rmdir(%s): (%d) %s\n", filename, ret, strerror(errno));
 }
-
 int
-main (int argc, char *argv[])
+main(int argc, char *argv[])
 {
-        glfs_t    *fs2 = NULL;
-        int        ret = 0;
-        glfs_fd_t *fd = NULL;
-        glfs_fd_t *fd2 = NULL;
-        struct stat sb = {0, };
-        char       readbuf[32];
-        char       writebuf[32];
-
-        char      *filename = "/filename2";
-
-        if (argc != 3) {
-                printf ("Expect following args\n\t%s <volname> <hostname>\n", argv[0]);
-                return -1;
-        }
-
-        fs = glfs_new (argv[1]);
-        if (!fs) {
-                fprintf (stderr, "glfs_new: returned NULL\n");
-                return 1;
-        }
-
-//      ret = glfs_set_volfile (fs, "/tmp/posix.vol");
-
-        ret = glfs_set_volfile_server (fs, "tcp", argv[2], 24007);
-
-//      ret = glfs_set_volfile_server (fs, "unix", "/tmp/gluster.sock", 0);
-
-        ret = glfs_set_logging (fs, "/dev/stderr", 7);
+    glfs_t *fs2 = NULL;
+    int ret = 0;
+    glfs_fd_t *fd = NULL;
+    glfs_fd_t *fd2 = NULL;
+    struct stat sb = {
+        0,
+    };
+    struct glfs_stat gsb = {
+        0,
+    };
+    struct statvfs sfs;
+    char readbuf[32];
+    char writebuf[32];
 
-        ret = glfs_init (fs);
+    char *filename = "/filename2";
 
-        fprintf (stderr, "glfs_init: returned %d\n", ret);
+    if (argc != 3) {
+        printf("Expect following args\n\t%s <volname> <hostname>\n", argv[0]);
+        return -1;
+    }
 
-        sleep (2);
+    fs = glfs_new(argv[1]);
+    if (!fs) {
+        fprintf(stderr, "glfs_new: returned NULL\n");
+        return 1;
+    }
 
-        fs2 = glfs_new (argv[1]);
-        if (!fs2) {
-                fprintf (stderr, "glfs_new: returned NULL\n");
-                return 1;
-        }
-
-
-//      ret = glfs_set_volfile (fs2, "/tmp/posix.vol");
-
-        ret = glfs_set_volfile_server (fs2, "tcp", argv[2], 24007);
+    //      ret = glfs_set_volfile (fs, "/tmp/posix.vol");
 
-        ret = glfs_set_logging (fs2, "/dev/stderr", 7);
+    ret = glfs_set_volfile_server(fs, "tcp", argv[2], 24007);
 
-        ret = glfs_init (fs2);
+    //      ret = glfs_set_volfile_server (fs, "unix", "/tmp/gluster.sock", 0);
 
-        fprintf (stderr, "glfs_init: returned %d\n", ret);
+    ret = glfs_set_logging(fs, "/dev/stderr", 7);
 
-        ret = glfs_lstat (fs, filename, &sb);
-        fprintf (stderr, "%s: (%d) %s\n", filename, ret, strerror (errno));
+    ret = glfs_init(fs);
 
-        fd = glfs_creat (fs, filename, O_RDWR, 0644);
-        fprintf (stderr, "%s: (%p) %s\n", filename, fd, strerror (errno));
+    fprintf(stderr, "glfs_init: returned %d\n", ret);
 
-        fd2 = glfs_open (fs2, filename, O_RDWR);
-        fprintf (stderr, "%s: (%p) %s\n", filename, fd, strerror (errno));
+    if (ret)
+        goto out;
 
-        sprintf (writebuf, "hi there\n");
-        ret = glfs_write (fd, writebuf, 32, 0);
+    sleep(2);
 
-        glfs_lseek (fd2, 0, SEEK_SET);
+    fs2 = glfs_new(argv[1]);
+    if (!fs2) {
+        fprintf(stderr, "glfs_new: returned NULL\n");
+        return 1;
+    }
 
-        ret = glfs_read (fd2, readbuf, 32, 0);
+    //      ret = glfs_set_volfile (fs2, "/tmp/posix.vol");
 
-        printf ("read %d, %s", ret, readbuf);
+    ret = glfs_set_volfile_server(fs2, "tcp", argv[2], 24007);
 
-        glfs_close (fd);
-        glfs_close (fd2);
+    ret = glfs_set_logging(fs2, "/dev/stderr", 7);
 
-        filename = "/filename3";
-        ret = glfs_mknod (fs, filename, S_IFIFO, 0);
-        fprintf (stderr, "%s: (%d) %s\n", filename, ret, strerror (errno));
+    ret = glfs_init(fs2);
 
-        ret = glfs_lstat (fs, filename, &sb);
-        fprintf (stderr, "%s: (%d) %s\n", filename, ret, strerror (errno));
+    fprintf(stderr, "glfs_init: returned %d\n", ret);
 
+    test_metadata_ops(fs, fs2);
 
-        ret = glfs_rename (fs, filename, "/filename4");
-        fprintf (stderr, "rename(%s): (%d) %s\n", filename, ret,
-                 strerror (errno));
+    test_dirops(fs);
 
-        ret = glfs_unlink (fs, "/filename4");
-        fprintf (stderr, "unlink(%s): (%d) %s\n", "/filename4", ret,
-                 strerror (errno));
+    test_xattr(fs);
 
-        filename = "/dirname2";
-        ret = glfs_mkdir (fs, filename, 0);
-        fprintf (stderr, "%s: (%d) %s\n", filename, ret, strerror (errno));
+    test_chdir(fs);
 
-        ret = glfs_lstat (fs, filename, &sb);
-        fprintf (stderr, "lstat(%s): (%d) %s\n", filename, ret, strerror (errno));
+    test_handleops(argc, argv);
+    // done
 
-        ret = glfs_rmdir (fs, filename);
-        fprintf (stderr, "rmdir(%s): (%d) %s\n", filename, ret, strerror (errno));
+    /* Test some extra apis */
+    test_write_apis(fs);
 
-        test_dirops (fs);
+    glfs_statvfs(fs, "/", &sfs);
 
-        test_xattr (fs);
+    glfs_fini(fs);
+    glfs_fini(fs2);
 
-        test_chdir (fs);
-
-        test_handleops (argc, argv);
-        // done
-
-        glfs_fini (fs);
-        glfs_fini (fs2);
-
-        return ret;
+    ret = 0;
+out:
+    return ret;
 }
diff --git a/api/src/Makefile.am b/api/src/Makefile.am
index 0a2101fe549..7f9a7d17b35 100644
--- a/api/src/Makefile.am
+++ b/api/src/Makefile.am
@@ -1,22 +1,27 @@
 lib_LTLIBRARIES = libgfapi.la
-noinst_HEADERS = glfs-mem-types.h glfs-internal.h
+noinst_HEADERS = glfs-mem-types.h glfs-internal.h gfapi-messages.h
 libgfapi_HEADERS = glfs.h glfs-handles.h
 libgfapidir = $(includedir)/glusterfs/api
 
+EXTRA_DIST = gfapi.map gfapi.aliases
+
 libgfapi_la_SOURCES = glfs.c glfs-mgmt.c glfs-fops.c glfs-resolve.c \
 	glfs-handleops.c
 libgfapi_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la \
 	$(top_builddir)/rpc/rpc-lib/src/libgfrpc.la \
-	$(top_builddir)/rpc/xdr/src/libgfxdr.la \
-	$(GF_LDADD)
+	$(top_builddir)/rpc/xdr/src/libgfxdr.la
 
-libgfapi_la_CPPFLAGS = $(GF_CPPFLAGS) -D__USE_FILE_OFFSET64 \
-	-I$(top_srcdir)/libglusterfs/src \
-	-I$(top_srcdir)/rpc/rpc-lib/src \
-	-I$(top_srcdir)/rpc/xdr/src
+libgfapi_la_LDFLAGS = -version-info $(GFAPI_LT_VERSION) $(GF_LDFLAGS) \
+	$(GFAPI_EXTRA_LDFLAGS) $(ACL_LIBS)
 
-libgfapi_la_LDFLAGS = -version-info $(GFAPI_LT_VERSION) \
-	-Wl,--version-script=gfapi.map
+libgfapi_la_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
+        -I$(top_srcdir)/rpc/rpc-lib/src \
+        -I$(top_srcdir)/rpc/xdr/src \
+        -I$(top_builddir)/rpc/xdr/src \
+        -DDATADIR=\"$(localstatedir)\" \
+        -D__USE_FILE_OFFSET64 -D__USE_LARGEFILE64
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
 
 xlator_LTLIBRARIES = api.la
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/mount
@@ -27,12 +32,12 @@ $(install_xlatorLTLIBRARIES): install-libLTLIBRARIES
 
 api_la_SOURCES = glfs-master.c
 api_la_DEPENDENCIES = libgfapi.la
-api_la_LDFLAGS = -module -avoid-version
+api_la_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
+	-I$(top_srcdir)/rpc/xdr/src \
+	-I$(top_builddir)/rpc/xdr/src
+api_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
+#api_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS) $(GF_LDFLAGS)
 api_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la \
 	$(top_builddir)/rpc/rpc-lib/src/libgfrpc.la \
 	$(top_builddir)/rpc/xdr/src/libgfxdr.la \
 	$(top_builddir)/api/src/libgfapi.la
-
-AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
-
-AM_CFLAGS = -Wall $(GF_CFLAGS)
diff --git a/api/src/README.Symbol_Versions b/api/src/README.Symbol_Versions
new file mode 100644
index 00000000000..b6ec95f9311
--- /dev/null
+++ b/api/src/README.Symbol_Versions
@@ -0,0 +1,3 @@
+
+See ../../doc/developer-guide/gfapi-symbol-versions.md
+
diff --git a/api/src/gfapi-messages.h b/api/src/gfapi-messages.h
new file mode 100644
index 00000000000..b9223940416
--- /dev/null
+++ b/api/src/gfapi-messages.h
@@ -0,0 +1,147 @@
+/*
+ *   Copyright (c) 2015-2018 Red Hat, Inc. <http://www.redhat.com>
+ *   This file is part of GlusterFS.
+ *
+ *   This file is licensed to you under your choice of the GNU Lesser
+ *   General Public License, version 3 or any later version (LGPLv3 or
+ *   later), or the GNU General Public License, version 2 (GPLv2), in all
+ *   cases as published by the Free Software Foundation.
+ *   */
+
+#ifndef _GFAPI_MESSAGES_H__
+#define _GFAPI_MESSAGES_H__
+
+#include <glusterfs/glfs-message-id.h>
+
+/* To add new message IDs, append new identifiers at the end of the list.
+ *
+ * Never remove a message ID. If it's not used anymore, you can rename it or
+ * leave it as it is, but not delete it. This is to prevent reutilization of
+ * IDs by other messages.
+ *
+ * The component name must match one of the entries defined in
+ * glfs-message-id.h.
+ */
+
+GLFS_MSGID(API, API_MSG_MEM_ACCT_INIT_FAILED, API_MSG_MASTER_XLATOR_INIT_FAILED,
+           API_MSG_GFAPI_XLATOR_INIT_FAILED, API_MSG_VOLFILE_OPEN_FAILED,
+           API_MSG_VOL_SPEC_FILE_ERROR, API_MSG_GLFS_FSOBJ_NULL,
+           API_MSG_INVALID_ENTRY, API_MSG_FSMUTEX_LOCK_FAILED,
+           API_MSG_COND_WAIT_FAILED, API_MSG_FSMUTEX_UNLOCK_FAILED,
+           API_MSG_INODE_REFRESH_FAILED, API_MSG_GRAPH_CONSTRUCT_FAILED,
+           API_MSG_API_XLATOR_ERROR, API_MSG_XDR_PAYLOAD_FAILED,
+           API_MSG_GET_VOLINFO_CBK_FAILED, API_MSG_FETCH_VOLUUID_FAILED,
+           API_MSG_INSUFF_SIZE, API_MSG_FRAME_CREAT_FAILED,
+           API_MSG_DICT_SET_FAILED, API_MSG_XDR_DECODE_FAILED,
+           API_MSG_GET_VOLFILE_FAILED, API_MSG_WRONG_OPVERSION,
+           API_MSG_DICT_SERIALIZE_FAILED, API_MSG_REMOTE_HOST_CONN_FAILED,
+           API_MSG_VOLFILE_SERVER_EXHAUST, API_MSG_CREATE_RPC_CLIENT_FAILED,
+           API_MSG_REG_NOTIFY_FUNC_FAILED, API_MSG_REG_CBK_FUNC_FAILED,
+           API_MSG_GET_CWD_FAILED, API_MSG_FGETXATTR_FAILED,
+           API_MSG_LOCKINFO_KEY_MISSING, API_MSG_FSETXATTR_FAILED,
+           API_MSG_FSYNC_FAILED, API_MSG_FDCREATE_FAILED,
+           API_MSG_INODE_PATH_FAILED, API_MSG_SYNCOP_OPEN_FAILED,
+           API_MSG_LOCK_MIGRATE_FAILED, API_MSG_OPENFD_SKIPPED,
+           API_MSG_FIRST_LOOKUP_GRAPH_FAILED, API_MSG_CWD_GRAPH_REF_FAILED,
+           API_MSG_SWITCHED_GRAPH, API_MSG_XDR_RESPONSE_DECODE_FAILED,
+           API_MSG_VOLFILE_INFO, API_MSG_VOLFILE_CONNECTING, API_MSG_NEW_GRAPH,
+           API_MSG_ALLOC_FAILED, API_MSG_CREATE_HANDLE_FAILED,
+           API_MSG_INODE_LINK_FAILED, API_MSG_STATEDUMP_FAILED,
+           API_MSG_XREADDIRP_R_FAILED, API_MSG_LOCK_INSERT_MERGE_FAILED,
+           API_MSG_SETTING_LOCK_TYPE_FAILED, API_MSG_INODE_FIND_FAILED,
+           API_MSG_FDCTX_SET_FAILED, API_MSG_UPCALL_SYNCOP_FAILED,
+           API_MSG_INVALID_ARG, API_MSG_UPCALL_EVENT_NULL_RECEIVED,
+           API_MSG_FLAGS_HANDLE, API_MSG_FDCREATE_FAILED_ON_GRAPH,
+           API_MSG_TRANS_RDMA_DEP, API_MSG_TRANS_NOT_SUPPORTED,
+           API_MSG_FS_NOT_INIT, API_MSG_INVALID_SYSRQ,
+           API_MSG_DECODE_XDR_FAILED, API_MSG_NULL, API_MSG_CALL_NOT_SUCCESSFUL,
+           API_MSG_CALL_NOT_VALID, API_MSG_UNABLE_TO_DEL,
+           API_MSG_REMOTE_HOST_DISCONN, API_MSG_HANDLE_NOT_SET);
+
+#define API_MSG_ALLOC_FAILED_STR "Upcall allocation failed"
+#define API_MSG_LOCK_INSERT_MERGE_FAILED_STR                                   \
+    "Lock insertion and splitting/merging failed"
+#define API_MSG_SETTING_LOCK_TYPE_FAILED_STR "Setting lock type failed"
+
+#define API_MSG_INVALID_ARG_STR "Invalid"
+#define API_MSG_INVALID_ENTRY_STR "Upcall entry validation failed"
+#define API_MSG_INODE_FIND_FAILED_STR "Unable to find inode entry"
+#define API_MSG_CREATE_HANDLE_FAILED_STR "handle creation failed"
+#define API_MSG_UPCALL_EVENT_NULL_RECEIVED_STR                                 \
+    "Upcall_EVENT_NULL received. Skipping it"
+#define API_MSG_UPCALL_SYNCOP_FAILED_STR "Synctask for upcall failed"
+#define API_MSG_FDCREATE_FAILED_STR "Allocating anonymous fd failed"
+#define API_MSG_XREADDIRP_R_FAILED_STR "glfs_x_readdirp_r failed"
+#define API_MSG_FDCTX_SET_FAILED_STR "Setting fd ctx failed"
+#define API_MSG_FLAGS_HANDLE_STR "arg not set. Flags handled are"
+#define API_MSG_INODE_REFRESH_FAILED_STR "inode refresh failed"
+#define API_MSG_INODE_LINK_FAILED_STR "inode linking failed"
+#define API_MSG_GET_CWD_FAILED_STR "Failed to get cwd"
+#define API_MSG_FGETXATTR_FAILED_STR "fgetxattr failed"
+#define API_MSG_LOCKINFO_KEY_MISSING_STR "missing lockinfo key"
+#define API_MSG_FSYNC_FAILED_STR "fsync() failed"
+#define API_MSG_FDCREATE_FAILED_ON_GRAPH_STR "fd_create failed on graph"
+#define API_MSG_INODE_PATH_FAILED_STR "inode_path failed"
+#define API_MSG_SYNCOP_OPEN_FAILED_STR "syncop_open failed"
+#define API_MSG_LOCK_MIGRATE_FAILED_STR "lock migration failed on graph"
+#define API_MSG_OPENFD_SKIPPED_STR "skipping openfd in graph"
+#define API_MSG_FIRST_LOOKUP_GRAPH_FAILED_STR "first lookup on graph failed"
+#define API_MSG_CWD_GRAPH_REF_FAILED_STR "cwd refresh of graph failed"
+#define API_MSG_SWITCHED_GRAPH_STR "switched to graph"
+#define API_MSG_FSETXATTR_FAILED_STR "fsetxattr failed"
+#define API_MSG_MEM_ACCT_INIT_FAILED_STR "Memory accounting init failed"
+#define API_MSG_MASTER_XLATOR_INIT_FAILED_STR                                  \
+    "master xlator for initialization failed"
+#define API_MSG_GFAPI_XLATOR_INIT_FAILED_STR                                   \
+    "failed to initialize gfapi translator"
+#define API_MSG_VOLFILE_OPEN_FAILED_STR "volume file open failed"
+#define API_MSG_VOL_SPEC_FILE_ERROR_STR "Cannot reach volume specification file"
+#define API_MSG_TRANS_RDMA_DEP_STR                                             \
+    "transport RDMA is deprecated, falling back to tcp"
+#define API_MSG_TRANS_NOT_SUPPORTED_STR                                        \
+    "transport is not supported, possible values tcp|unix"
+#define API_MSG_GLFS_FSOBJ_NULL_STR "fs is NULL"
+#define API_MSG_FS_NOT_INIT_STR "fs is not properly initialized"
+#define API_MSG_FSMUTEX_LOCK_FAILED_STR                                        \
+    "pthread lock on glfs mutex, returned error"
+#define API_MSG_FSMUTEX_UNLOCK_FAILED_STR                                      \
+    "pthread unlock on glfs mutex, returned error"
+#define API_MSG_COND_WAIT_FAILED_STR "cond wait failed"
+#define API_MSG_INVALID_SYSRQ_STR "not a valid sysrq"
+#define API_MSG_GRAPH_CONSTRUCT_FAILED_STR "failed to construct the graph"
+#define API_MSG_API_XLATOR_ERROR_STR                                           \
+    "api master xlator cannot be specified in volume file"
+#define API_MSG_STATEDUMP_FAILED_STR "statedump failed"
+#define API_MSG_DECODE_XDR_FAILED_STR                                          \
+    "Failed to decode xdr response for GF_CBK_STATEDUMP"
+#define API_MSG_NULL_STR "NULL"
+#define API_MSG_XDR_PAYLOAD_FAILED_STR "failed to create XDR payload"
+#define API_MSG_CALL_NOT_SUCCESSFUL_STR                                        \
+    "GET_VOLUME_INFO RPC call is not successful"
+#define API_MSG_XDR_RESPONSE_DECODE_FAILED_STR                                 \
+    "Failed to decode xdr response for GET_VOLUME_INFO"
+#define API_MSG_CALL_NOT_VALID_STR                                             \
+    "Response received for GET_VOLUME_INFO RPC is not valid"
+#define API_MSG_GET_VOLINFO_CBK_FAILED_STR                                     \
+    "In GET_VOLUME_INFO cbk, received error"
+#define API_MSG_FETCH_VOLUUID_FAILED_STR "Unable to fetch volume UUID"
+#define API_MSG_INSUFF_SIZE_STR "Insufficient size passed"
+#define API_MSG_FRAME_CREAT_FAILED_STR "failed to create the frame"
+#define API_MSG_DICT_SET_FAILED_STR "failed to set"
+#define API_MSG_XDR_DECODE_FAILED_STR "XDR decoding error"
+#define API_MSG_GET_VOLFILE_FAILED_STR "failed to get the volume file"
+#define API_MSG_VOLFILE_INFO_STR "No change in volfile, continuing"
+#define API_MSG_UNABLE_TO_DEL_STR "unable to delete file"
+#define API_MSG_WRONG_OPVERSION_STR                                            \
+    "Server is operating at an op-version which is not supported"
+#define API_MSG_DICT_SERIALIZE_FAILED_STR "Failed to serialize dictionary"
+#define API_MSG_REMOTE_HOST_CONN_FAILED_STR "Failed to connect to remote-host"
+#define API_MSG_REMOTE_HOST_DISCONN_STR "disconnected from remote-host"
+#define API_MSG_VOLFILE_SERVER_EXHAUST_STR "Exhausted all volfile servers"
+#define API_MSG_VOLFILE_CONNECTING_STR "connecting to next volfile server"
+#define API_MSG_CREATE_RPC_CLIENT_FAILED_STR "failed to create rpc clnt"
+#define API_MSG_REG_NOTIFY_FUNC_FAILED_STR "failed to register notify function"
+#define API_MSG_REG_CBK_FUNC_FAILED_STR "failed to register callback function"
+#define API_MSG_NEW_GRAPH_STR "New graph coming up"
+#define API_MSG_HANDLE_NOT_SET_STR "handle not set. Flags handled for xstat are"
+#endif /* !_GFAPI_MESSAGES_H__ */
diff --git a/api/src/gfapi.aliases b/api/src/gfapi.aliases
new file mode 100644
index 00000000000..bc639e6b99f
--- /dev/null
+++ b/api/src/gfapi.aliases
@@ -0,0 +1,201 @@
+
+_priv_glfs_loc_touchup _glfs_loc_touchup$GFAPI_PRIVATE_3.4.0
+_priv_glfs_active_subvol _glfs_active_subvol$GFAPI_PRIVATE_3.4.0
+_priv_glfs_subvol_done _glfs_subvol_done$GFAPI_PRIVATE_3.4.0
+_priv_glfs_init_done _glfs_init_done$GFAPI_PRIVATE_3.4.0
+_priv_glfs_resolve_at _glfs_resolve_at$GFAPI_PRIVATE_3.4.0
+
+_pub_glfs_new _glfs_new$GFAPI_3.4.0
+_pub_glfs_set_volfile _glfs_set_volfile$GFAPI_3.4.0
+_pub_glfs_set_volfile_server _glfs_set_volfile_server$GFAPI_3.4.0
+_pub_glfs_set_logging _glfs_set_logging$GFAPI_3.4.0
+_pub_glfs_init _glfs_init$GFAPI_3.4.0
+_pub_glfs_fini _glfs_fini$GFAPI_3.4.0
+_pub_glfs_open _glfs_open$GFAPI_3.4.0
+_pub_glfs_creat _glfs_creat$GFAPI_3.4.0
+_pub_glfs_close _glfs_close$GFAPI_3.4.0
+_pub_glfs_from_glfd _glfs_from_glfd$GFAPI_3.4.0
+_pub_glfs_set_xlator_option _glfs_set_xlator_option$GFAPI_3.4.0
+_pub_glfs_read _glfs_read$GFAPI_3.4.0
+_pub_glfs_write _glfs_write$GFAPI_3.4.0
+_pub_glfs_readv _glfs_readv$GFAPI_3.4.0
+_pub_glfs_writev _glfs_writev$GFAPI_3.4.0
+_pub_glfs_pread34 _glfs_pread$GFAPI_3.4.0
+_pub_glfs_pwrite34 _glfs_pwrite$GFAPI_3.4.0
+_pub_glfs_pread_async34 _glfs_pread_async$GFAPI_3.4.0
+_pub_glfs_pwrite_async34 _glfs_pwrite_async$GFAPI_3.4.0
+_pub_glfs_preadv _glfs_preadv$GFAPI_3.4.0
+_pub_glfs_pwritev _glfs_pwritev$GFAPI_3.4.0
+_pub_glfs_lseek _glfs_lseek$GFAPI_3.4.0
+_pub_glfs_ftruncate34 _glfs_ftruncate$GFAPI_3.4.0
+_pub_glfs_ftruncate_async34 _glfs_ftruncate_async$GFAPI_3.4.0
+_pub_glfs_lstat _glfs_lstat$GFAPI_3.4.0
+_pub_glfs_stat _glfs_stat$GFAPI_3.4.0
+_pub_glfs_fstat _glfs_fstat$GFAPI_3.4.0
+_pub_glfs_fsync34 _glfs_fsync$GFAPI_3.4.0
+_pub_glfs_fsync_async34 _glfs_fsync_async$GFAPI_3.4.0
+_pub_glfs_fdatasync34 _glfs_fdatasync$GFAPI_3.4.0
+_pub_glfs_fdatasync_async34 _glfs_fdatasync_async$GFAPI_3.4.0
+_pub_glfs_access _glfs_access$GFAPI_3.4.0
+_pub_glfs_symlink _glfs_symlink$GFAPI_3.4.0
+_pub_glfs_readlink _glfs_readlink$GFAPI_3.4.0
+_pub_glfs_mknod _glfs_mknod$GFAPI_3.4.0
+_pub_glfs_mkdir _glfs_mkdir$GFAPI_3.4.0
+_pub_glfs_unlink _glfs_unlink$GFAPI_3.4.0
+_pub_glfs_rmdir _glfs_rmdir$GFAPI_3.4.0
+_pub_glfs_rename _glfs_rename$GFAPI_3.4.0
+_pub_glfs_link _glfs_link$GFAPI_3.4.0
+_pub_glfs_opendir _glfs_opendir$GFAPI_3.4.0
+_pub_glfs_readdir_r _glfs_readdir_r$GFAPI_3.4.0
+_pub_glfs_readdirplus_r _glfs_readdirplus_r$GFAPI_3.4.0
+_pub_glfs_telldir _glfs_telldir$GFAPI_3.4.0
+_pub_glfs_seekdir _glfs_seekdir$GFAPI_3.4.0
+_pub_glfs_closedir _glfs_closedir$GFAPI_3.4.0
+_pub_glfs_statvfs _glfs_statvfs$GFAPI_3.4.0
+_pub_glfs_chmod _glfs_chmod$GFAPI_3.4.0
+_pub_glfs_fchmod _glfs_fchmod$GFAPI_3.4.0
+_pub_glfs_chown _glfs_chown$GFAPI_3.4.0
+_pub_glfs_lchown _glfs_lchown$GFAPI_3.4.0
+_pub_glfs_fchown _glfs_fchown$GFAPI_3.4.0
+_pub_glfs_utimens _glfs_utimens$GFAPI_3.4.0
+_pub_glfs_lutimens _glfs_lutimens$GFAPI_3.4.0
+_pub_glfs_futimens _glfs_futimens$GFAPI_3.4.0
+_pub_glfs_getxattr _glfs_getxattr$GFAPI_3.4.0
+_pub_glfs_lgetxattr _glfs_lgetxattr$GFAPI_3.4.0
+_pub_glfs_fgetxattr _glfs_fgetxattr$GFAPI_3.4.0
+_pub_glfs_listxattr _glfs_listxattr$GFAPI_3.4.0
+_pub_glfs_llistxattr _glfs_llistxattr$GFAPI_3.4.0
+_pub_glfs_flistxattr _glfs_flistxattr$GFAPI_3.4.0
+_pub_glfs_setxattr _glfs_setxattr$GFAPI_3.4.0
+_pub_glfs_lsetxattr _glfs_lsetxattr$GFAPI_3.4.0
+_pub_glfs_fsetxattr _glfs_fsetxattr$GFAPI_3.4.0
+_pub_glfs_removexattr _glfs_removexattr$GFAPI_3.4.0
+_pub_glfs_lremovexattr _glfs_lremovexattr$GFAPI_3.4.0
+_pub_glfs_fremovexattr _glfs_fremovexattr$GFAPI_3.4.0
+_pub_glfs_getcwd _glfs_getcwd$GFAPI_3.4.0
+_pub_glfs_chdir _glfs_chdir$GFAPI_3.4.0
+_pub_glfs_fchdir _glfs_fchdir$GFAPI_3.4.0
+_pub_glfs_realpath34 _glfs_realpath$GFAPI_3.4.0
+_pub_glfs_posix_lock _glfs_posix_lock$GFAPI_3.4.0
+_pub_glfs_dup _glfs_dup$GFAPI_3.4.0
+
+_pub_glfs_setfsuid _glfs_setfsuid$GFAPI_3.4.2
+_pub_glfs_setfsgid _glfs_setfsgid$GFAPI_3.4.2
+_pub_glfs_setfsgroups _glfs_setfsgroups$GFAPI_3.4.2
+_pub_glfs_h_lookupat34 _glfs_h_lookupat$GFAPI_3.4.2
+_pub_glfs_h_creat _glfs_h_creat$GFAPI_3.4.2
+_pub_glfs_h_mkdir _glfs_h_mkdir$GFAPI_3.4.2
+_pub_glfs_h_mknod _glfs_h_mknod$GFAPI_3.4.2
+_pub_glfs_h_symlink _glfs_h_symlink$GFAPI_3.4.2
+_pub_glfs_h_unlink _glfs_h_unlink$GFAPI_3.4.2
+_pub_glfs_h_close _glfs_h_close$GFAPI_3.4.2
+_pub_glfs_h_truncate _glfs_h_truncate$GFAPI_3.4.2
+_pub_glfs_h_stat _glfs_h_stat$GFAPI_3.4.2
+_pub_glfs_h_getattrs _glfs_h_getattrs$GFAPI_3.4.2
+_pub_glfs_h_setattrs _glfs_h_setattrs$GFAPI_3.4.2
+_pub_glfs_h_readlink _glfs_h_readlink$GFAPI_3.4.2
+_pub_glfs_h_link _glfs_h_link$GFAPI_3.4.2
+_pub_glfs_h_rename _glfs_h_rename$GFAPI_3.4.2
+_pub_glfs_h_extract_handle _glfs_h_extract_handle$GFAPI_3.4.2
+_pub_glfs_h_create_from_handle _glfs_h_create_from_handle$GFAPI_3.4.2
+_pub_glfs_h_opendir _glfs_h_opendir$GFAPI_3.4.2
+_pub_glfs_h_open _glfs_h_open$GFAPI_3.4.2
+
+_pub_glfs_get_volumeid _glfs_get_volumeid$GFAPI_3.5.0
+_pub_glfs_readdir _glfs_readdir$GFAPI_3.5.0
+_pub_glfs_readdirplus _glfs_readdirplus$GFAPI_3.5.0
+_pub_glfs_fallocate _glfs_fallocate$GFAPI_3.5.0
+_pub_glfs_discard _glfs_discard$GFAPI_3.5.0
+_pub_glfs_zerofill _glfs_zerofill$GFAPI_3.5.0
+_pub_glfs_caller_specific_init _glfs_caller_specific_init$GFAPI_3.5.0
+_pub_glfs_h_setxattrs _glfs_h_setxattrs$GFAPI_3.5.0
+
+_pub_glfs_unset_volfile_server _glfs_unset_volfile_server$GFAPI_3.5.1
+_pub_glfs_h_getxattrs _glfs_h_getxattrs$GFAPI_3.5.1
+_pub_glfs_h_removexattrs _glfs_h_removexattrs$GFAPI_3.5.1
+
+_pub_glfs_get_volfile _glfs_get_volfile$GFAPI_3.6.0
+_pub_glfs_h_access _glfs_h_access$GFAPI_3.6.0
+
+_pub_glfs_h_poll_upcall370 _glfs_h_poll_upcall$GFAPI_3.7.0
+_pub_glfs_h_acl_set _glfs_h_acl_set$GFAPI_3.7.0
+_pub_glfs_h_acl_get _glfs_h_acl_get$GFAPI_3.7.0
+_pub_glfs_h_statfs _glfs_h_statfs$GFAPI_3.7.0
+_pub_glfs_h_anonymous_read _glfs_h_anonymous_read$GFAPI_3.7.0
+_pub_glfs_h_anonymous_write _glfs_h_anonymous_write$GFAPI_3.7.0
+
+_priv_glfs_free_from_ctx _glfs_free_from_ctx$GFAPI_PRIVATE_3.7.0
+_priv_glfs_new_from_ctx _glfs_new_from_ctx$GFAPI_PRIVATE_3.7.0
+_priv_glfs_resolve _glfs_resolve$GFAPI_PRIVATE_3.7.0
+_priv_glfs_process_upcall_event _glfs_process_upcall_event$GFAPI_PRIVATE_3.7.0
+
+_pub_glfs_h_lookupat _glfs_h_lookupat$GFAPI_3.7.4
+
+_pub_glfs_truncate _glfs_truncate$GFAPI_3.7.15
+
+_pub_glfs_free _glfs_free$GFAPI_3.7.16
+_pub_glfs_h_poll_upcall _glfs_h_poll_upcall$GFAPI_3.7.16
+_pub_glfs_upcall_get_fs _glfs_upcall_get_fs$GFAPI_3.7.16
+_pub_glfs_upcall_get_reason _glfs_upcall_get_reason$GFAPI_3.7.16
+_pub_glfs_upcall_get_event _glfs_upcall_get_event$GFAPI_3.7.16
+_pub_glfs_upcall_inode_get_object _glfs_upcall_inode_get_object$GFAPI_3.7.16
+_pub_glfs_upcall_inode_get_flags _glfs_upcall_inode_get_flags$GFAPI_3.7.16
+_pub_glfs_upcall_inode_get_stat _glfs_upcall_inode_get_stat$GFAPI_3.7.16
+_pub_glfs_upcall_inode_get_expire _glfs_upcall_inode_get_expire$GFAPI_3.7.16
+_pub_glfs_upcall_inode_get_pobject _glfs_upcall_inode_get_pobject$GFAPI_3.7.16
+_pub_glfs_upcall_inode_get_pstat _glfs_upcall_inode_get_pstat$GFAPI_3.7.16
+_pub_glfs_upcall_inode_get_oldpobject _glfs_upcall_inode_get_oldpobject$GFAPI_3.7.16
+_pub_glfs_upcall_inode_get_oldpstat _glfs_upcall_inode_get_oldpstat$GFAPI_3.7.16
+
+_pub_glfs_realpath _glfs_realpath$GFAPI_3.7.17
+
+_pub_glfs_sysrq _glfs_sysrq$GFAPI_3.10.0
+
+_pub_glfs_fd_set_lkowner _glfs_fd_set_lkowner$GFAPI_3.10.7
+
+_pub_glfs_xreaddirplus_r _glfs_xreaddirplus_r$GFAPI_3.11.0
+_pub_glfs_xreaddirplus_r_get_stat _glfs_xreaddirplus_r_get_stat$GFAPI_3.11.0
+_pub_glfs_xreaddirplus_r_get_object _glfs_xreaddirplus_r_get_object$GFAPI_3.11.0
+_pub_glfs_object_copy _glfs_object_copy$GFAPI_3.11.0
+
+_priv_glfs_ipc _glfs_ipc$GFAPI_3.12.0
+
+_pub_glfs_upcall_register _glfs_upcall_register$GFAPI_3.13.0
+_pub_glfs_upcall_unregister _glfs_upcall_unregister$GFAPI_3.13.0
+
+_pub_glfs_setfsleaseid _glfs_setfsleaseid$GFAPI_4.0.0
+_pub_glfs_file_lock _glfs_file_lock$GFAPI_4.0.0
+_pub_glfs_lease _glfs_lease$GFAPI_4.0.0
+_pub_glfs_h_lease _glfs_h_lease$GFAPI_4.0.0
+_pub_glfs_upcall_lease_get_object _glfs_upcall_lease_get_object$GFAPI_4.1.6
+_pub_glfs_upcall_lease_get_lease_type _glfs_upcall_lease_get_lease_type$GFAPI_4.1.6
+
+_priv_glfs_statx _glfs_statx$GFAPI_6.0
+_priv_glfs_iatt_from_statx _glfs_iatt_from_statx$GFAPI_6.0
+_priv_glfs_setfspid _glfs_setfspid$GFAPI_6.1
+
+_pub_glfs_read_async _glfs_read_async$GFAPI_6.0
+_pub_glfs_write_async _glfs_write_async$GFAPI_6.0
+_pub_glfs_readv_async _glfs_readv_async$GFAPI_6.0
+_pub_glfs_writev_async _glfs_writev_async$GFAPI_6.0
+_pub_glfs_pread _glfs_pread$GFAPI_6.0
+_pub_glfs_pwrite _glfs_pwrite$GFAPI_6.0
+_pub_glfs_pread_async _glfs_pread_async$GFAPI_6.0
+_pub_glfs_pwrite_async _glfs_pwrite_async$GFAPI_6.0
+_pub_glfs_preadv_async _glfs_preadv_async$GFAPI_6.0
+_pub_glfs_pwritev_async _glfs_pwritev_async$GFAPI_6.0
+_pub_glfs_fsync _glfs_fsync$GFAPI_6.0
+_pub_glfs_fsync_async _glfs_fsync_async$GFAPI_6.0
+_pub_glfs_fdatasync _glfs_fdatasync$GFAPI_6.0
+_pub_glfs_fdatasync_async _glfs_fdatasync_async$GFAPI_6.0
+_pub_glfs_ftruncate _glfs_ftruncate$GFAPI_6.0
+_pub_glfs_ftruncate_async _glfs_ftruncate_async$GFAPI_6.0
+_pub_glfs_discard_async _glfs_discard_async$GFAPI_6.0
+_pub_glfs_zerofill_async _glfs_zerofill_async$GFAPI_6.0
+_pub_glfs_copy_file_range _glfs_copy_file_range$GFAPI_6.0
+_pub_glfs_fsetattr _glfs_fsetattr$GFAPI_6.0
+_pub_glfs_setattr _glfs_setattr$GFAPI_6.0
+
+_pub_glfs_set_statedump_path _glfs_set_statedump_path@GFAPI_7.0
+
+_pub_glfs_h_creat_open _glfs_h_creat_open@GFAPI_6.6
diff --git a/api/src/gfapi.map.in b/api/src/gfapi.map
index 0819633fef6..228ac47c084 100644
--- a/api/src/gfapi.map.in
+++ b/api/src/gfapi.map
@@ -39,7 +39,6 @@ GFAPI_3.4.0 {
 		glfs_preadv_async;
 		glfs_pwritev_async;
 		glfs_lseek;
-		glfs_truncate;
 		glfs_ftruncate;
 		glfs_ftruncate_async;
 		glfs_lstat;
@@ -98,7 +97,6 @@ GFAPI_3.4.2 {
 		glfs_setfsuid;
 		glfs_setfsgid;
 		glfs_setfsgroups;
-		glfs_h_lookupat;
 		glfs_h_creat;
 		glfs_h_mkdir;
 		glfs_h_mknod;
@@ -116,6 +114,7 @@ GFAPI_3.4.2 {
 		glfs_h_create_from_handle;
 		glfs_h_opendir;
 		glfs_h_open;
+		glfs_h_lookupat;
 } GFAPI_3.4.0;
 
 GFAPI_3.5.0 {
@@ -145,3 +144,140 @@ GFAPI_3.6.0 {
 		glfs_h_access;
 } GFAPI_3.5.1;
 
+GFAPI_3.7.0 {
+	global:
+		glfs_h_poll_upcall;
+		glfs_h_acl_set;
+		glfs_h_acl_get;
+		glfs_h_statfs;
+		glfs_h_anonymous_read;
+		glfs_h_anonymous_write;
+} GFAPI_3.6.0;
+
+GFAPI_PRIVATE_3.7.0 {
+	global:
+		glfs_free_from_ctx;
+		glfs_new_from_ctx;
+		glfs_resolve;
+		glfs_process_upcall_event;
+} GFAPI_3.7.0;
+
+GFAPI_3.7.4 {
+	global:
+		glfs_h_lookupat;
+} GFAPI_PRIVATE_3.7.0;
+
+GFAPI_3.7.15 {
+	global:
+		glfs_truncate;
+} GFAPI_3.7.4;
+
+GFAPI_3.7.16 {
+	global:
+		glfs_free;
+		glfs_upcall_get_fs;
+		glfs_upcall_get_reason;
+		glfs_upcall_get_event;
+		glfs_upcall_inode_get_object;
+		glfs_upcall_inode_get_flags;
+		glfs_upcall_inode_get_stat;
+		glfs_upcall_inode_get_expire;
+		glfs_upcall_inode_get_pobject;
+		glfs_upcall_inode_get_pstat;
+		glfs_upcall_inode_get_oldpobject;
+		glfs_upcall_inode_get_oldpstat;
+		glfs_h_poll_upcall;
+} GFAPI_3.7.15;
+
+GFAPI_3.7.17 {
+	global:
+		glfs_realpath;
+} GFAPI_3.7.16;
+
+GFAPI_3.10.0 {
+	global:
+		glfs_sysrq;
+} GFAPI_3.7.17;
+
+GFAPI_3.10.7 {
+	global:
+		glfs_fd_set_lkowner;
+} GFAPI_3.10.0;
+
+GFAPI_3.11.0 {
+		glfs_xreaddirplus_r;
+		glfs_xreaddirplus_r_get_stat;
+		glfs_xreaddirplus_r_get_object;
+		glfs_object_copy;
+} GFAPI_3.10.7;
+
+GFAPI_PRIVATE_3.12.0 {
+	global:
+		glfs_ipc;
+} GFAPI_3.11.0;
+
+GFAPI_3.13.0 {
+	global:
+		glfs_upcall_register;
+		glfs_upcall_unregister;
+} GFAPI_PRIVATE_3.12.0;
+
+GFAPI_4.0.0 {
+	global:
+		glfs_setfsleaseid;
+		glfs_file_lock;
+		glfs_lease;
+		glfs_h_lease;
+} GFAPI_3.13.0;
+
+GFAPI_4.1.6 {
+	global:
+		glfs_upcall_lease_get_object;
+		glfs_upcall_lease_get_lease_type;
+} GFAPI_4.0.0;
+
+GFAPI_PRIVATE_6.0 {
+	global:
+		glfs_statx;
+		glfs_iatt_from_statx;
+} GFAPI_4.1.6;
+
+GFAPI_6.0 {
+	global:
+		glfs_read_async;
+		glfs_write_async;
+		glfs_readv_async;
+		glfs_writev_async;
+		glfs_pread;
+		glfs_pwrite;
+		glfs_pread_async;
+		glfs_pwrite_async;
+		glfs_preadv_async;
+		glfs_pwritev_async;
+		glfs_fsync;
+		glfs_fsync_async;
+		glfs_fdatasync;
+		glfs_fdatasync_async;
+		glfs_ftruncate;
+		glfs_ftruncate_async;
+		glfs_discard_async;
+		glfs_zerofill_async;
+		glfs_copy_file_range;
+		glfs_setattr;
+		glfs_fsetattr;
+} GFAPI_PRIVATE_6.0;
+
+GFAPI_PRIVATE_6.1 {
+	global:
+		glfs_setfspid;
+} GFAPI_6.0;
+
+GFAPI_6.6 {
+	global:
+		glfs_h_creat_open;
+} GFAPI_PRIVATE_6.1;
+
+GFAPI_7.0 {
+	global:
+		glfs_set_statedump_path;
+} GFAPI_6.6;
diff --git a/api/src/glfs-fops.c b/api/src/glfs-fops.c
index ddfd89a28a0..6aa3c5602d1 100644
--- a/api/src/glfs-fops.c
+++ b/api/src/glfs-fops.c
@@ -1,6 +1,6 @@
 
 /*
-  Copyright (c) 2012 Red Hat, Inc. <http://www.redhat.com>
+  Copyright (c) 2012-2018 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
   This file is licensed to you under your choice of the GNU Lesser
@@ -9,13 +9,22 @@
   cases as published by the Free Software Foundation.
 */
 
+/* for SEEK_HOLE and SEEK_DATA */
+#ifndef _GNU_SOURCE
+#define _GNU_SOURCE
+#endif
+
+#include <unistd.h>
 
 #include "glfs-internal.h"
 #include "glfs-mem-types.h"
-#include "syncop.h"
+#include <glusterfs/syncop.h>
 #include "glfs.h"
-#include "compat-errno.h"
+#include "gfapi-messages.h"
+#include <glusterfs/compat-errno.h>
 #include <limits.h>
+#include "glusterfs3.h"
+#include <glusterfs/iatt.h>
 
 #ifdef NAME_MAX
 #define GF_NAME_MAX NAME_MAX
@@ -23,3475 +32,6414 @@
 #define GF_NAME_MAX 255
 #endif
 
+struct upcall_syncop_args {
+    struct glfs *fs;
+    struct gf_upcall upcall_data;
+};
+
 #define READDIRBUF_SIZE (sizeof(struct dirent) + GF_NAME_MAX + 1)
 
+typedef void (*glfs_io_cbk34)(glfs_fd_t *fd, ssize_t ret, void *data);
+
+/*
+ * This function will mark glfd for deletion and decrement its refcount.
+ */
 int
-glfs_loc_link (loc_t *loc, struct iatt *iatt)
+glfs_mark_glfd_for_deletion(struct glfs_fd *glfd)
 {
-	int ret = -1;
-	inode_t *linked_inode = NULL;
+    LOCK(&glfd->lock);
+    {
+        glfd->state = GLFD_CLOSE;
+    }
+    UNLOCK(&glfd->lock);
 
-	if (!loc->inode) {
-		errno = EINVAL;
-		return -1;
-	}
+    GF_REF_PUT(glfd);
 
-	linked_inode = inode_link (loc->inode, loc->parent, loc->name, iatt);
-	if (linked_inode) {
-		inode_lookup (linked_inode);
-		inode_unref (linked_inode);
-		ret = 0;
-	} else {
-		ret = -1;
-		errno = ENOMEM;
-	}
-
-	return ret;
+    return 0;
 }
 
+/* This function is useful for all async fops. There is chance that glfd is
+ * closed before async fop is completed. When glfd is closed we change the
+ * state to GLFD_CLOSE.
+ *
+ * This function will return _gf_true if the glfd is still valid else return
+ * _gf_false.
+ */
+gf_boolean_t
+glfs_is_glfd_still_valid(struct glfs_fd *glfd)
+{
+    gf_boolean_t ret = _gf_false;
+
+    LOCK(&glfd->lock);
+    {
+        if (glfd->state != GLFD_CLOSE)
+            ret = _gf_true;
+    }
+    UNLOCK(&glfd->lock);
+
+    return ret;
+}
 
 void
-glfs_iatt_to_stat (struct glfs *fs, struct iatt *iatt, struct stat *stat)
+glfd_set_state_bind(struct glfs_fd *glfd)
 {
-	iatt_to_stat (iatt, stat);
-	stat->st_dev = fs->dev_id;
-}
+    LOCK(&glfd->lock);
+    {
+        glfd->state = GLFD_OPEN;
+    }
+    UNLOCK(&glfd->lock);
 
+    fd_bind(glfd->fd);
+    glfs_fd_bind(glfd);
 
+    return;
+}
+
+/*
+ * This routine is called when an upcall event of type
+ * 'GF_UPCALL_CACHE_INVALIDATION' is received.
+ * It makes a copy of the contents of the upcall cache-invalidation
+ * data received into an entry which is stored in the upcall list
+ * maintained by gfapi.
+ */
 int
-glfs_loc_unlink (loc_t *loc)
+glfs_get_upcall_cache_invalidation(struct gf_upcall *to_up_data,
+                                   struct gf_upcall *from_up_data)
 {
-	inode_unlink (loc->inode, loc->parent, loc->name);
+    struct gf_upcall_cache_invalidation *ca_data = NULL;
+    struct gf_upcall_cache_invalidation *f_ca_data = NULL;
+    int ret = -1;
 
-	return 0;
-}
+    GF_VALIDATE_OR_GOTO(THIS->name, to_up_data, out);
+    GF_VALIDATE_OR_GOTO(THIS->name, from_up_data, out);
 
+    f_ca_data = from_up_data->data;
+    GF_VALIDATE_OR_GOTO(THIS->name, f_ca_data, out);
 
-struct glfs_fd *
-pub_glfs_open (struct glfs *fs, const char *path, int flags)
-{
-	int              ret = -1;
-	struct glfs_fd  *glfd = NULL;
-	xlator_t        *subvol = NULL;
-	loc_t            loc = {0, };
-	struct iatt      iatt = {0, };
-	int              reval = 0;
+    ca_data = GF_CALLOC(1, sizeof(*ca_data), glfs_mt_upcall_entry_t);
 
-	__glfs_entry_fs (fs);
+    if (!ca_data) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, errno, API_MSG_ALLOC_FAILED, "entry",
+                NULL);
+        goto out;
+    }
 
-	subvol = priv_glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+    to_up_data->data = ca_data;
 
-	glfd = glfs_fd_new (fs);
-	if (!glfd)
-		goto out;
+    ca_data->flags = f_ca_data->flags;
+    ca_data->expire_time_attr = f_ca_data->expire_time_attr;
+    ca_data->stat = f_ca_data->stat;
+    ca_data->p_stat = f_ca_data->p_stat;
+    ca_data->oldp_stat = f_ca_data->oldp_stat;
 
-retry:
-	ret = glfs_resolve (fs, subvol, path, &loc, &iatt, reval);
+    ret = 0;
+out:
+    return ret;
+}
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+int
+glfs_get_upcall_lease(struct gf_upcall *to_up_data,
+                      struct gf_upcall *from_up_data)
+{
+    struct gf_upcall_recall_lease *ca_data = NULL;
+    struct gf_upcall_recall_lease *f_ca_data = NULL;
+    int ret = -1;
 
-	if (ret)
-		goto out;
+    GF_VALIDATE_OR_GOTO(THIS->name, to_up_data, out);
+    GF_VALIDATE_OR_GOTO(THIS->name, from_up_data, out);
 
-	if (IA_ISDIR (iatt.ia_type)) {
-		ret = -1;
-		errno = EISDIR;
-		goto out;
-	}
+    f_ca_data = from_up_data->data;
+    GF_VALIDATE_OR_GOTO(THIS->name, f_ca_data, out);
 
-	if (!IA_ISREG (iatt.ia_type)) {
-		ret = -1;
-		errno = EINVAL;
-		goto out;
-	}
+    ca_data = GF_CALLOC(1, sizeof(*ca_data), glfs_mt_upcall_entry_t);
 
-	if (glfd->fd) {
-		/* Retry. Safe to touch glfd->fd as we
-		   still have not glfs_fd_bind() yet.
-		*/
-		fd_unref (glfd->fd);
-		glfd->fd = NULL;
-	}
+    if (!ca_data) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, errno, API_MSG_ALLOC_FAILED, "entry",
+                NULL);
+        goto out;
+    }
 
-	glfd->fd = fd_create (loc.inode, getpid());
-	if (!glfd->fd) {
-		ret = -1;
-		errno = ENOMEM;
-		goto out;
-	}
+    to_up_data->data = ca_data;
 
-	ret = syncop_open (subvol, &loc, flags, glfd->fd);
-        DECODE_SYNCOP_ERR (ret);
+    ca_data->lease_type = f_ca_data->lease_type;
+    gf_uuid_copy(ca_data->tid, f_ca_data->tid);
+    ca_data->dict = f_ca_data->dict;
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    ret = 0;
 out:
-	loc_wipe (&loc);
-
-	if (ret && glfd) {
-		glfs_fd_destroy (glfd);
-		glfd = NULL;
-	} else if (glfd) {
-                glfd->fd->flags = flags;
-		fd_bind (glfd->fd);
-		glfs_fd_bind (glfd);
-	}
+    return ret;
+}
+int
+glfs_loc_link(loc_t *loc, struct iatt *iatt)
+{
+    int ret = -1;
+    inode_t *old_inode = NULL;
+    uint64_t ctx_value = LOOKUP_NOT_NEEDED;
+
+    if (!loc->inode) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    old_inode = loc->inode;
+
+    /* If the inode already exists in the cache, the inode
+     * returned here points to the existing one. We need
+     * to update loc.inode accordingly.
+     */
+    loc->inode = inode_link(loc->inode, loc->parent, loc->name, iatt);
+    if (loc->inode) {
+        inode_ctx_set(loc->inode, THIS, &ctx_value);
+        inode_lookup(loc->inode);
+        inode_unref(old_inode);
+        ret = 0;
+    } else {
+        ret = -1;
+    }
+
+    return ret;
+}
 
-	priv_glfs_subvol_done (fs, subvol);
+void
+glfs_iatt_to_stat(struct glfs *fs, struct iatt *iatt, struct stat *stat)
+{
+    iatt_to_stat(iatt, stat);
+    stat->st_dev = fs->dev_id;
+}
 
-	return glfd;
+void
+glfs_iatt_to_statx(struct glfs *fs, const struct iatt *iatt,
+                   struct glfs_stat *statx)
+{
+    statx->glfs_st_mask = 0;
+
+    statx->glfs_st_mode = 0;
+    if (IATT_TYPE_VALID(iatt->ia_flags)) {
+        statx->glfs_st_mode |= st_mode_type_from_ia(iatt->ia_type);
+        statx->glfs_st_mask |= GLFS_STAT_TYPE;
+    }
+
+    if (IATT_MODE_VALID(iatt->ia_flags)) {
+        statx->glfs_st_mode |= st_mode_prot_from_ia(iatt->ia_prot);
+        statx->glfs_st_mask |= GLFS_STAT_MODE;
+    }
+
+    if (IATT_NLINK_VALID(iatt->ia_flags)) {
+        statx->glfs_st_nlink = iatt->ia_nlink;
+        statx->glfs_st_mask |= GLFS_STAT_NLINK;
+    }
+
+    if (IATT_UID_VALID(iatt->ia_flags)) {
+        statx->glfs_st_uid = iatt->ia_uid;
+        statx->glfs_st_mask |= GLFS_STAT_UID;
+    }
+
+    if (IATT_GID_VALID(iatt->ia_flags)) {
+        statx->glfs_st_gid = iatt->ia_gid;
+        statx->glfs_st_mask |= GLFS_STAT_GID;
+    }
+
+    if (IATT_ATIME_VALID(iatt->ia_flags)) {
+        statx->glfs_st_atime.tv_sec = iatt->ia_atime;
+        statx->glfs_st_atime.tv_nsec = iatt->ia_atime_nsec;
+        statx->glfs_st_mask |= GLFS_STAT_ATIME;
+    }
+
+    if (IATT_MTIME_VALID(iatt->ia_flags)) {
+        statx->glfs_st_mtime.tv_sec = iatt->ia_mtime;
+        statx->glfs_st_mtime.tv_nsec = iatt->ia_mtime_nsec;
+        statx->glfs_st_mask |= GLFS_STAT_MTIME;
+    }
+
+    if (IATT_CTIME_VALID(iatt->ia_flags)) {
+        statx->glfs_st_ctime.tv_sec = iatt->ia_ctime;
+        statx->glfs_st_ctime.tv_nsec = iatt->ia_ctime_nsec;
+        statx->glfs_st_mask |= GLFS_STAT_CTIME;
+    }
+
+    if (IATT_BTIME_VALID(iatt->ia_flags)) {
+        statx->glfs_st_btime.tv_sec = iatt->ia_btime;
+        statx->glfs_st_btime.tv_nsec = iatt->ia_btime_nsec;
+        statx->glfs_st_mask |= GLFS_STAT_BTIME;
+    }
+
+    if (IATT_INO_VALID(iatt->ia_flags)) {
+        statx->glfs_st_ino = iatt->ia_ino;
+        statx->glfs_st_mask |= GLFS_STAT_INO;
+    }
+
+    if (IATT_SIZE_VALID(iatt->ia_flags)) {
+        statx->glfs_st_size = iatt->ia_size;
+        statx->glfs_st_mask |= GLFS_STAT_SIZE;
+    }
+
+    if (IATT_BLOCKS_VALID(iatt->ia_flags)) {
+        statx->glfs_st_blocks = iatt->ia_blocks;
+        statx->glfs_st_mask |= GLFS_STAT_BLOCKS;
+    }
+
+    /* unconditionally present, encode as is */
+    statx->glfs_st_blksize = iatt->ia_blksize;
+    statx->glfs_st_rdev_major = ia_major(iatt->ia_rdev);
+    statx->glfs_st_rdev_minor = ia_minor(iatt->ia_rdev);
+    statx->glfs_st_dev_major = ia_major(fs->dev_id);
+    statx->glfs_st_dev_minor = ia_minor(fs->dev_id);
+
+    /* At present we do not read any localFS attributes and pass them along,
+     * so setting this to 0. As we start supporting file attributes we can
+     * populate the same here as well */
+    statx->glfs_st_attributes = 0;
+    statx->glfs_st_attributes_mask = 0;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_open, 3.4.0);
+GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_iatt_from_statx, 6.0)
+void
+priv_glfs_iatt_from_statx(struct iatt *iatt, const struct glfs_stat *statx)
+{
+    /* Most code in xlators are not checking validity flags before accessing
+    the items. Hence zero everything before setting valid items */
+    memset(iatt, 0, sizeof(struct iatt));
+
+    if (GLFS_STAT_TYPE_VALID(statx->glfs_st_mask)) {
+        iatt->ia_type = ia_type_from_st_mode(statx->glfs_st_mode);
+        iatt->ia_flags |= IATT_TYPE;
+    }
+
+    if (GLFS_STAT_MODE_VALID(statx->glfs_st_mask)) {
+        iatt->ia_prot = ia_prot_from_st_mode(statx->glfs_st_mode);
+        iatt->ia_flags |= IATT_MODE;
+    }
+
+    if (GLFS_STAT_NLINK_VALID(statx->glfs_st_mask)) {
+        iatt->ia_nlink = statx->glfs_st_nlink;
+        iatt->ia_flags |= IATT_NLINK;
+    }
+
+    if (GLFS_STAT_UID_VALID(statx->glfs_st_mask)) {
+        iatt->ia_uid = statx->glfs_st_uid;
+        iatt->ia_flags |= IATT_UID;
+    }
+
+    if (GLFS_STAT_GID_VALID(statx->glfs_st_mask)) {
+        iatt->ia_gid = statx->glfs_st_gid;
+        iatt->ia_flags |= IATT_GID;
+    }
+
+    if (GLFS_STAT_ATIME_VALID(statx->glfs_st_mask)) {
+        iatt->ia_atime = statx->glfs_st_atime.tv_sec;
+        iatt->ia_atime_nsec = statx->glfs_st_atime.tv_nsec;
+        iatt->ia_flags |= IATT_ATIME;
+    }
+
+    if (GLFS_STAT_MTIME_VALID(statx->glfs_st_mask)) {
+        iatt->ia_mtime = statx->glfs_st_mtime.tv_sec;
+        iatt->ia_mtime_nsec = statx->glfs_st_mtime.tv_nsec;
+        iatt->ia_flags |= IATT_MTIME;
+    }
+
+    if (GLFS_STAT_CTIME_VALID(statx->glfs_st_mask)) {
+        iatt->ia_ctime = statx->glfs_st_ctime.tv_sec;
+        iatt->ia_ctime_nsec = statx->glfs_st_ctime.tv_nsec;
+        iatt->ia_flags |= IATT_CTIME;
+    }
+
+    if (GLFS_STAT_BTIME_VALID(statx->glfs_st_mask)) {
+        iatt->ia_btime = statx->glfs_st_btime.tv_sec;
+        iatt->ia_btime_nsec = statx->glfs_st_btime.tv_nsec;
+        iatt->ia_flags |= IATT_BTIME;
+    }
+
+    if (GLFS_STAT_INO_VALID(statx->glfs_st_mask)) {
+        iatt->ia_ino = statx->glfs_st_ino;
+        iatt->ia_flags |= IATT_INO;
+    }
+
+    if (GLFS_STAT_SIZE_VALID(statx->glfs_st_mask)) {
+        iatt->ia_size = statx->glfs_st_size;
+        iatt->ia_flags |= IATT_SIZE;
+    }
+
+    if (GLFS_STAT_BLOCKS_VALID(statx->glfs_st_mask)) {
+        iatt->ia_blocks = statx->glfs_st_blocks;
+        iatt->ia_flags |= IATT_BLOCKS;
+    }
+
+    /* unconditionally present, encode as is */
+    iatt->ia_blksize = statx->glfs_st_blksize;
+    iatt->ia_rdev = makedev(statx->glfs_st_rdev_major,
+                            statx->glfs_st_rdev_minor);
+    iatt->ia_dev = makedev(statx->glfs_st_dev_major, statx->glfs_st_dev_minor);
+    iatt->ia_attributes = statx->glfs_st_attributes;
+    iatt->ia_attributes_mask = statx->glfs_st_attributes_mask;
+}
 
+void
+glfsflags_from_gfapiflags(struct glfs_stat *stat, int *glvalid)
+{
+    *glvalid = 0;
+    if (stat->glfs_st_mask & GLFS_STAT_MODE) {
+        *glvalid |= GF_SET_ATTR_MODE;
+    }
+
+    if (stat->glfs_st_mask & GLFS_STAT_SIZE) {
+        *glvalid |= GF_SET_ATTR_SIZE;
+    }
+
+    if (stat->glfs_st_mask & GLFS_STAT_UID) {
+        *glvalid |= GF_SET_ATTR_UID;
+    }
+
+    if (stat->glfs_st_mask & GLFS_STAT_GID) {
+        *glvalid |= GF_SET_ATTR_GID;
+    }
+
+    if (stat->glfs_st_mask & GLFS_STAT_ATIME) {
+        *glvalid |= GF_SET_ATTR_ATIME;
+    }
+
+    if (stat->glfs_st_mask & GLFS_STAT_MTIME) {
+        *glvalid |= GF_SET_ATTR_MTIME;
+    }
+}
 
 int
-pub_glfs_close (struct glfs_fd *glfd)
+glfs_loc_unlink(loc_t *loc)
 {
-	xlator_t  *subvol = NULL;
-	int        ret = -1;
-	fd_t      *fd = NULL;
-	struct glfs *fs = NULL;
+    inode_unlink(loc->inode, loc->parent, loc->name);
 
-	__glfs_entry_fd (glfd);
+    /* since glfs_h_* objects hold a reference to inode
+     * it is safe to keep lookup count to '0' */
+    if (!inode_has_dentry(loc->inode))
+        inode_forget(loc->inode, 0);
 
-	subvol = priv_glfs_active_subvol (glfd->fs);
-        if (!subvol) {
-                ret = -1;
-                errno = EIO;
-                goto out;
-        }
+    return 0;
+}
 
-	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
-	if (!fd) {
-		ret = -1;
-		errno = EBADFD;
-		goto out;
-	}
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_open, 3.4.0)
+struct glfs_fd *
+pub_glfs_open(struct glfs *fs, const char *path, int flags)
+{
+    int ret = -1;
+    struct glfs_fd *glfd = NULL;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    int reval = 0;
+    dict_t *fop_attr = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    glfd = glfs_fd_new(fs);
+    if (!glfd)
+        goto out;
 
-	ret = syncop_flush (subvol, fd);
-        DECODE_SYNCOP_ERR (ret);
-out:
-	fs = glfd->fs;
-	glfs_fd_destroy (glfd);
+retry:
+    ret = glfs_resolve(fs, subvol, path, &loc, &iatt, reval);
 
-	if (fd)
-		fd_unref (fd);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	priv_glfs_subvol_done (fs, subvol);
+    if (ret)
+        goto out;
 
-	return ret;
-}
+    if (IA_ISDIR(iatt.ia_type)) {
+        ret = -1;
+        errno = EISDIR;
+        goto out;
+    }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_close, 3.4.0);
+    if (!IA_ISREG(iatt.ia_type)) {
+        ret = -1;
+        errno = EINVAL;
+        goto out;
+    }
 
+    if (glfd->fd) {
+        /* Retry. Safe to touch glfd->fd as we
+           still have not glfs_fd_bind() yet.
+        */
+        fd_unref(glfd->fd);
+        glfd->fd = NULL;
+    }
+
+    glfd->fd = fd_create(loc.inode, getpid());
+    if (!glfd->fd) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+    glfd->fd->flags = flags;
+
+    ret = get_fop_attr_thrd_key(&fop_attr);
+    if (ret)
+        gf_msg_debug("gfapi", 0, "Getting leaseid from thread failed");
+
+    ret = syncop_open(subvol, &loc, flags, glfd->fd, fop_attr, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
+out:
+    loc_wipe(&loc);
 
-int
-pub_glfs_lstat (struct glfs *fs, const char *path, struct stat *stat)
-{
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	loc_t            loc = {0, };
-	struct iatt      iatt = {0, };
-	int              reval = 0;
+    if (fop_attr)
+        dict_unref(fop_attr);
 
-	__glfs_entry_fs (fs);
+    if (ret && glfd) {
+        GF_REF_PUT(glfd);
+        glfd = NULL;
+    } else if (glfd) {
+        glfd_set_state_bind(glfd);
+    }
 
-	subvol = priv_glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
-retry:
-	ret = glfs_lresolve (fs, subvol, path, &loc, &iatt, reval);
+    glfs_subvol_done(fs, subvol);
+
+    __GLFS_EXIT_FS;
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+invalid_fs:
+    return glfd;
+}
 
-	if (ret == 0 && stat)
-		glfs_iatt_to_stat (fs, &iatt, stat);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_close, 3.4.0)
+int
+pub_glfs_close(struct glfs_fd *glfd)
+{
+    xlator_t *subvol = NULL;
+    int ret = -1;
+    fd_t *fd = NULL;
+    struct glfs *fs = NULL;
+    dict_t *fop_attr = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    if (glfd->lk_owner.len != 0) {
+        ret = syncopctx_setfslkowner(&glfd->lk_owner);
+        if (ret)
+            goto out;
+    }
+    ret = get_fop_attr_thrd_key(&fop_attr);
+    if (ret)
+        gf_msg_debug("gfapi", 0, "Getting leaseid from thread failed");
+
+    ret = syncop_flush(subvol, fd, fop_attr, NULL);
+    DECODE_SYNCOP_ERR(ret);
 out:
-	loc_wipe (&loc);
+    fs = glfd->fs;
 
-	priv_glfs_subvol_done (fs, subvol);
+    if (fd)
+        fd_unref(fd);
+    if (fop_attr)
+        dict_unref(fop_attr);
 
-	return ret;
-}
+    glfs_mark_glfd_for_deletion(glfd);
+    glfs_subvol_done(fs, subvol);
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_lstat, 3.4.0);
+    __GLFS_EXIT_FS;
 
+invalid_fs:
+    return ret;
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_lstat, 3.4.0)
 int
-pub_glfs_stat (struct glfs *fs, const char *path, struct stat *stat)
+pub_glfs_lstat(struct glfs *fs, const char *path, struct stat *stat)
 {
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	loc_t            loc = {0, };
-	struct iatt      iatt = {0, };
-	int              reval = 0;
-
-	__glfs_entry_fs (fs);
-
-	subvol = priv_glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
 retry:
-	ret = glfs_resolve (fs, subvol, path, &loc, &iatt, reval);
+    ret = glfs_lresolve(fs, subvol, path, &loc, &iatt, reval);
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	if (ret == 0 && stat)
-		glfs_iatt_to_stat (fs, &iatt, stat);
+    if (ret == 0 && stat)
+        glfs_iatt_to_stat(fs, &iatt, stat);
 out:
-	loc_wipe (&loc);
-
-	priv_glfs_subvol_done (fs, subvol);
+    loc_wipe(&loc);
 
-	return ret;
-}
+    glfs_subvol_done(fs, subvol);
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_stat, 3.4.0);
+    __GLFS_EXIT_FS;
 
+invalid_fs:
+    return ret;
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_stat, 3.4.0)
 int
-pub_glfs_fstat (struct glfs_fd *glfd, struct stat *stat)
+pub_glfs_stat(struct glfs *fs, const char *path, struct stat *stat)
 {
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	struct iatt      iatt = {0, };
-	fd_t            *fd = NULL;
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+retry:
+    ret = glfs_resolve(fs, subvol, path, &loc, &iatt, reval);
 
-	__glfs_entry_fd (glfd);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	subvol = priv_glfs_active_subvol (glfd->fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+    if (ret == 0 && stat)
+        glfs_iatt_to_stat(fs, &iatt, stat);
+out:
+    loc_wipe(&loc);
 
-	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
-	if (!fd) {
-		ret = -1;
-		errno = EBADFD;
-		goto out;
-	}
+    glfs_subvol_done(fs, subvol);
 
-	ret = syncop_fstat (subvol, fd, &iatt);
-        DECODE_SYNCOP_ERR (ret);
+    __GLFS_EXIT_FS;
 
-	if (ret == 0 && stat)
-		glfs_iatt_to_stat (glfd->fs, &iatt, stat);
-out:
-	if (fd)
-		fd_unref (fd);
+invalid_fs:
+    return ret;
+}
 
-	priv_glfs_subvol_done (glfd->fs, subvol);
+GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_statx, 6.0)
+int
+priv_glfs_statx(struct glfs *fs, const char *path, const unsigned int mask,
+                struct glfs_stat *statxbuf)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    if (path == NULL) {
+        ret = -1;
+        errno = EINVAL;
+        goto out;
+    }
+
+    if (mask & ~GLFS_STAT_ALL) {
+        ret = -1;
+        errno = EINVAL;
+        goto out;
+    }
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
 
-	return ret;
-}
+retry:
+    ret = glfs_resolve(fs, subvol, path, &loc, &iatt, reval);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fstat, 3.4.0);
+    if (ret == 0 && statxbuf)
+        glfs_iatt_to_statx(fs, &iatt, statxbuf);
+out:
+    loc_wipe(&loc);
 
+    glfs_subvol_done(fs, subvol);
 
-struct glfs_fd *
-pub_glfs_creat (struct glfs *fs, const char *path, int flags, mode_t mode)
-{
-	int              ret = -1;
-	struct glfs_fd  *glfd = NULL;
-	xlator_t        *subvol = NULL;
-	loc_t            loc = {0, };
-	struct iatt      iatt = {0, };
-	uuid_t           gfid;
-	dict_t          *xattr_req = NULL;
-	int              reval = 0;
-
-	__glfs_entry_fs (fs);
-
-	subvol = priv_glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
-
-	xattr_req = dict_new ();
-	if (!xattr_req) {
-		ret = -1;
-		errno = ENOMEM;
-		goto out;
-	}
-
-	uuid_generate (gfid);
-	ret = dict_set_static_bin (xattr_req, "gfid-req", gfid, 16);
-	if (ret) {
-		ret = -1;
-		errno = ENOMEM;
-		goto out;
-	}
-
-	glfd = glfs_fd_new (fs);
-	if (!glfd)
-		goto out;
-
-	/* This must be glfs_resolve() and NOT glfs_lresolve().
-	   That is because open("name", O_CREAT) where "name"
-	   is a danging symlink must create the dangling
-	   destinataion.
-	*/
-retry:
-	ret = glfs_resolve (fs, subvol, path, &loc, &iatt, reval);
-
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
-
-	if (ret == -1 && errno != ENOENT)
-		/* Any other type of error is fatal */
-		goto out;
-
-	if (ret == -1 && errno == ENOENT && !loc.parent)
-		/* The parent directory or an ancestor even
-		   higher does not exist
-		*/
-		goto out;
-
-	if (loc.inode) {
-		if (flags & O_EXCL) {
-			ret = -1;
-			errno = EEXIST;
-			goto out;
-		}
-
-		if (IA_ISDIR (iatt.ia_type)) {
-			ret = -1;
-			errno = EISDIR;
-			goto out;
-		}
-
-		if (!IA_ISREG (iatt.ia_type)) {
-			ret = -1;
-			errno = EINVAL;
-			goto out;
-		}
-	}
-
-	if (ret == -1 && errno == ENOENT) {
-		loc.inode = inode_new (loc.parent->table);
-		if (!loc.inode) {
-			ret = -1;
-			errno = ENOMEM;
-			goto out;
-		}
-	}
-
-	if (glfd->fd) {
-		/* Retry. Safe to touch glfd->fd as we
-		   still have not glfs_fd_bind() yet.
-		*/
-		fd_unref (glfd->fd);
-		glfd->fd = NULL;
-	}
-
-	glfd->fd = fd_create (loc.inode, getpid());
-	if (!glfd->fd) {
-		ret = -1;
-		errno = ENOMEM;
-		goto out;
-	}
-
-	if (ret == 0) {
-		ret = syncop_open (subvol, &loc, flags, glfd->fd);
-                DECODE_SYNCOP_ERR (ret);
-	} else {
-		ret = syncop_create (subvol, &loc, flags, mode, glfd->fd,
-				     xattr_req, &iatt);
-                DECODE_SYNCOP_ERR (ret);
-	}
-
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
-
-	if (ret == 0)
-		ret = glfs_loc_link (&loc, &iatt);
-out:
-	loc_wipe (&loc);
-
-	if (xattr_req)
-		dict_unref (xattr_req);
-
-	if (ret && glfd) {
-		glfs_fd_destroy (glfd);
-		glfd = NULL;
-	} else if (glfd) {
-                glfd->fd->flags = flags;
-		fd_bind (glfd->fd);
-		glfs_fd_bind (glfd);
-	}
-
-	priv_glfs_subvol_done (fs, subvol);
-
-	return glfd;
-}
-
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_creat, 3.4.0);
+    __GLFS_EXIT_FS;
 
+invalid_fs:
+    return ret;
+}
 
-off_t
-pub_glfs_lseek (struct glfs_fd *glfd, off_t offset, int whence)
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fstat, 3.4.0)
+int
+pub_glfs_fstat(struct glfs_fd *glfd, struct stat *stat)
 {
-	struct stat sb = {0, };
-	int         ret = -1;
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    struct iatt iatt = {
+        0,
+    };
+    fd_t *fd = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    GF_REF_GET(glfd);
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    ret = syncop_fstat(subvol, fd, &iatt, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    if (ret == 0 && stat)
+        glfs_iatt_to_stat(glfd->fs, &iatt, stat);
+out:
+    if (fd)
+        fd_unref(fd);
+    if (glfd)
+        GF_REF_PUT(glfd);
 
-	__glfs_entry_fd (glfd);
+    glfs_subvol_done(glfd->fs, subvol);
 
-	switch (whence) {
-	case SEEK_SET:
-		glfd->offset = offset;
-		break;
-	case SEEK_CUR:
-		glfd->offset += offset;
-		break;
-	case SEEK_END:
-		ret = pub_glfs_fstat (glfd, &sb);
-		if (ret) {
-			/* seek cannot fail :O */
-			break;
-		}
-		glfd->offset = sb.st_size + offset;
-		break;
-	}
+    __GLFS_EXIT_FS;
 
-	return glfd->offset;
+invalid_fs:
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_lseek, 3.4.0);
-
-
-ssize_t
-pub_glfs_preadv (struct glfs_fd *glfd, const struct iovec *iovec, int iovcnt,
-                 off_t offset, int flags)
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_creat, 3.4.0)
+struct glfs_fd *
+pub_glfs_creat(struct glfs *fs, const char *path, int flags, mode_t mode)
 {
-	xlator_t       *subvol = NULL;
-	ssize_t         ret = -1;
-	ssize_t         size = -1;
-	struct iovec   *iov = NULL;
-	int             cnt = 0;
-	struct iobref  *iobref = NULL;
-	fd_t           *fd = NULL;
-
-	__glfs_entry_fd (glfd);
+    int ret = -1;
+    struct glfs_fd *glfd = NULL;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    uuid_t gfid;
+    dict_t *xattr_req = NULL;
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    xattr_req = dict_new();
+    if (!xattr_req) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gf_uuid_generate(gfid);
+    ret = dict_set_gfuuid(xattr_req, "gfid-req", gfid, true);
+    if (ret) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    glfd = glfs_fd_new(fs);
+    if (!glfd)
+        goto out;
+
+    /* This must be glfs_resolve() and NOT glfs_lresolve().
+       That is because open("name", O_CREAT) where "name"
+       is a danging symlink must create the dangling
+       destination.
+    */
+retry:
+    ret = glfs_resolve(fs, subvol, path, &loc, &iatt, reval);
 
-	subvol = priv_glfs_active_subvol (glfd->fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
-	if (!fd) {
-		ret = -1;
-		errno = EBADFD;
-		goto out;
-	}
+    if (ret == -1 && errno != ENOENT)
+        /* Any other type of error is fatal */
+        goto out;
 
-	size = iov_length (iovec, iovcnt);
+    if (ret == -1 && errno == ENOENT && !loc.parent)
+        /* The parent directory or an ancestor even
+           higher does not exist
+        */
+        goto out;
 
-	ret = syncop_readv (subvol, fd, size, offset, 0, &iov, &cnt, &iobref);
-        DECODE_SYNCOP_ERR (ret);
-	if (ret <= 0)
-		goto out;
+    if (loc.inode) {
+        if (flags & O_EXCL) {
+            ret = -1;
+            errno = EEXIST;
+            goto out;
+        }
 
-	size = iov_copy (iovec, iovcnt, iov, cnt); /* FIXME!!! */
+        if (IA_ISDIR(iatt.ia_type)) {
+            ret = -1;
+            errno = EISDIR;
+            goto out;
+        }
 
-	glfd->offset = (offset + size);
+        if (!IA_ISREG(iatt.ia_type)) {
+            ret = -1;
+            errno = EINVAL;
+            goto out;
+        }
+    }
+
+    if (ret == -1 && errno == ENOENT) {
+        loc.inode = inode_new(loc.parent->table);
+        if (!loc.inode) {
+            ret = -1;
+            errno = ENOMEM;
+            goto out;
+        }
+    }
 
-	ret = size;
+    if (glfd->fd) {
+        /* Retry. Safe to touch glfd->fd as we
+           still have not glfs_fd_bind() yet.
+        */
+        fd_unref(glfd->fd);
+        glfd->fd = NULL;
+    }
+
+    glfd->fd = fd_create(loc.inode, getpid());
+    if (!glfd->fd) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+    glfd->fd->flags = flags;
+
+    if (get_fop_attr_thrd_key(&xattr_req))
+        gf_msg_debug("gfapi", 0, "Getting leaseid from thread failed");
+    if (ret == 0) {
+        ret = syncop_open(subvol, &loc, flags, glfd->fd, xattr_req, NULL);
+        DECODE_SYNCOP_ERR(ret);
+    } else {
+        ret = syncop_create(subvol, &loc, flags, mode, glfd->fd, &iatt,
+                            xattr_req, NULL);
+        DECODE_SYNCOP_ERR(ret);
+    }
+
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
+
+    if (ret == 0)
+        ret = glfs_loc_link(&loc, &iatt);
 out:
-        if (iov)
-                GF_FREE (iov);
-        if (iobref)
-                iobref_unref (iobref);
+    loc_wipe(&loc);
+
+    if (xattr_req)
+        dict_unref(xattr_req);
+
+    if (ret && glfd) {
+        GF_REF_PUT(glfd);
+        glfd = NULL;
+    } else if (glfd) {
+        glfd_set_state_bind(glfd);
+    }
 
-	if (fd)
-		fd_unref (fd);
+    glfs_subvol_done(fs, subvol);
 
-	priv_glfs_subvol_done (glfd->fs, subvol);
+    __GLFS_EXIT_FS;
 
-	return ret;
+invalid_fs:
+    return glfd;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_preadv, 3.4.0);
+#ifdef HAVE_SEEK_HOLE
+static int
+glfs_seek(struct glfs_fd *glfd, off_t offset, int whence)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    fd_t *fd = NULL;
+    gf_seek_what_t what = 0;
+    off_t off = -1;
+
+    switch (whence) {
+        case SEEK_DATA:
+            what = GF_SEEK_DATA;
+            break;
+        case SEEK_HOLE:
+            what = GF_SEEK_HOLE;
+            break;
+        default:
+            /* other SEEK_* do not make sense, all operations get an offset
+             * and the position in the fd is not tracked */
+            errno = EINVAL;
+            goto out;
+    }
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        errno = EBADFD;
+        goto done;
+    }
+
+    ret = syncop_seek(subvol, fd, offset, what, NULL, &off);
+    DECODE_SYNCOP_ERR(ret);
+
+    if (ret != -1)
+        glfd->offset = off;
+
+done:
+    if (fd)
+        fd_unref(fd);
+
+    glfs_subvol_done(glfd->fs, subvol);
 
+out:
+    return ret;
+}
+#endif
 
-ssize_t
-pub_glfs_read (struct glfs_fd *glfd, void *buf, size_t count, int flags)
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_lseek, 3.4.0)
+off_t
+pub_glfs_lseek(struct glfs_fd *glfd, off_t offset, int whence)
 {
-	struct iovec iov = {0, };
-	ssize_t      ret = 0;
+    struct stat sb = {
+        0,
+    };
+    int ret = -1;
+    off_t off = -1;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    GF_REF_GET(glfd);
+
+    switch (whence) {
+        case SEEK_SET:
+            glfd->offset = offset;
+            ret = 0;
+            break;
+        case SEEK_CUR:
+            glfd->offset += offset;
+            ret = 0;
+            break;
+        case SEEK_END:
+            ret = pub_glfs_fstat(glfd, &sb);
+            if (ret) {
+                /* seek cannot fail :O */
+                break;
+            }
+            glfd->offset = sb.st_size + offset;
+            break;
+#ifdef HAVE_SEEK_HOLE
+        case SEEK_DATA:
+        case SEEK_HOLE:
+            ret = glfs_seek(glfd, offset, whence);
+            break;
+#endif
+        default:
+            errno = EINVAL;
+    }
 
-	iov.iov_base = buf;
-	iov.iov_len = count;
+    if (glfd)
+        GF_REF_PUT(glfd);
 
-	ret = pub_glfs_preadv (glfd, &iov, 1, glfd->offset, flags);
+    __GLFS_EXIT_FS;
 
-	return ret;
-}
+    if (ret != -1)
+        off = glfd->offset;
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_read, 3.4.0);
+    return off;
 
+invalid_fs:
+    return -1;
+}
 
-ssize_t
-pub_glfs_pread (struct glfs_fd *glfd, void *buf, size_t count, off_t offset,
-                int flags)
+static ssize_t
+glfs_preadv_common(struct glfs_fd *glfd, const struct iovec *iovec, int iovcnt,
+                   off_t offset, int flags, struct glfs_stat *poststat)
 {
-	struct iovec iov = {0, };
-	ssize_t      ret = 0;
-
-	iov.iov_base = buf;
-	iov.iov_len = count;
+    xlator_t *subvol = NULL;
+    ssize_t ret = -1;
+    ssize_t size = -1;
+    struct iovec *iov = NULL;
+    int cnt = 0;
+    struct iobref *iobref = NULL;
+    fd_t *fd = NULL;
+    struct iatt iatt = {
+        0,
+    };
+    dict_t *fop_attr = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    GF_REF_GET(glfd);
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    size = iov_length(iovec, iovcnt);
+
+    ret = get_fop_attr_thrd_key(&fop_attr);
+    if (ret)
+        gf_msg_debug("gfapi", 0, "Getting leaseid from thread failed");
+
+    ret = syncop_readv(subvol, fd, size, offset, 0, &iov, &cnt, &iobref, &iatt,
+                       fop_attr, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    if (ret >= 0 && poststat)
+        glfs_iatt_to_statx(glfd->fs, &iatt, poststat);
+
+    if (ret <= 0)
+        goto out;
+
+    size = iov_copy(iovec, iovcnt, iov, cnt); /* FIXME!!! */
+
+    glfd->offset = (offset + size);
+
+    ret = size;
+out:
+    if (iov)
+        GF_FREE(iov);
+    if (iobref)
+        iobref_unref(iobref);
 
-	ret = pub_glfs_preadv (glfd, &iov, 1, offset, flags);
+    if (fd)
+        fd_unref(fd);
+    if (glfd)
+        GF_REF_PUT(glfd);
+    if (fop_attr)
+        dict_unref(fop_attr);
 
-	return ret;
-}
+    glfs_subvol_done(glfd->fs, subvol);
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_pread, 3.4.0);
+    __GLFS_EXIT_FS;
 
+invalid_fs:
+    return ret;
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_preadv, 3.4.0)
 ssize_t
-pub_glfs_readv (struct glfs_fd *glfd, const struct iovec *iov, int count,
-                int flags)
+pub_glfs_preadv(struct glfs_fd *glfd, const struct iovec *iovec, int iovcnt,
+                off_t offset, int flags)
 {
-	ssize_t      ret = 0;
-
-	ret = pub_glfs_preadv (glfd, iov, count, glfd->offset, flags);
-
-	return ret;
+    return glfs_preadv_common(glfd, iovec, iovcnt, offset, flags, NULL);
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_readv, 3.4.0);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_read, 3.4.0)
+ssize_t
+pub_glfs_read(struct glfs_fd *glfd, void *buf, size_t count, int flags)
+{
+    struct iovec iov = {
+        0,
+    };
+    ssize_t ret = 0;
 
+    if (glfd == NULL) {
+        errno = EBADF;
+        return -1;
+    }
 
-struct glfs_io {
-	struct glfs_fd      *glfd;
-	int                  op;
-	off_t                offset;
-	struct iovec        *iov;
-	int                  count;
-	int                  flags;
-	glfs_io_cbk          fn;
-	void                *data;
-};
+    iov.iov_base = buf;
+    iov.iov_len = count;
 
+    ret = pub_glfs_preadv(glfd, &iov, 1, glfd->offset, flags);
 
-static int
-glfs_io_async_cbk (int ret, call_frame_t *frame, void *data)
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC(glfs_pread34, glfs_pread, 3.4.0)
+ssize_t
+pub_glfs_pread34(struct glfs_fd *glfd, void *buf, size_t count, off_t offset,
+                 int flags)
 {
-	struct glfs_io  *gio = data;
+    struct iovec iov = {
+        0,
+    };
+    ssize_t ret = 0;
 
-	gio->fn (gio->glfd, ret, gio->data);
+    iov.iov_base = buf;
+    iov.iov_len = count;
 
-	GF_FREE (gio->iov);
-	GF_FREE (gio);
+    ret = pub_glfs_preadv(glfd, &iov, 1, offset, flags);
 
-	return 0;
+    return ret;
 }
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_pread, 6.0)
 ssize_t
-pub_glfs_pwritev (struct glfs_fd *, const struct iovec *, int, off_t, int);
+pub_glfs_pread(struct glfs_fd *glfd, void *buf, size_t count, off_t offset,
+               int flags, struct glfs_stat *poststat)
+{
+    struct iovec iov = {
+        0,
+    };
+    ssize_t ret = 0;
 
-int
-pub_glfs_ftruncate (struct glfs_fd *, off_t);
+    iov.iov_base = buf;
+    iov.iov_len = count;
 
-int
-pub_glfs_fdatasync (struct glfs_fd *);
+    ret = glfs_preadv_common(glfd, &iov, 1, offset, flags, poststat);
 
-int
-pub_glfs_fsync (struct glfs_fd *glfd);
+    return ret;
+}
 
-int
-pub_glfs_discard (struct glfs_fd *, off_t, size_t);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_readv, 3.4.0)
+ssize_t
+pub_glfs_readv(struct glfs_fd *glfd, const struct iovec *iov, int count,
+               int flags)
+{
+    ssize_t ret = 0;
 
-int
-pub_glfs_zerofill (struct glfs_fd *, off_t, off_t);
+    if (glfd == NULL) {
+        errno = EBADF;
+        return -1;
+    }
 
-static int
-glfs_io_async_task (void *data)
-{
-	struct glfs_io *gio = data;
-	ssize_t         ret = 0;
-
-	switch (gio->op) {
-	case GF_FOP_WRITE:
-		ret = pub_glfs_pwritev (gio->glfd, gio->iov, gio->count,
-				    gio->offset, gio->flags);
-		break;
-	case GF_FOP_FTRUNCATE:
-		ret = pub_glfs_ftruncate (gio->glfd, gio->offset);
-		break;
-	case GF_FOP_FSYNC:
-		if (gio->flags)
-			ret = pub_glfs_fdatasync (gio->glfd);
-		else
-			ret = pub_glfs_fsync (gio->glfd);
-		break;
-	case GF_FOP_DISCARD:
-		ret = pub_glfs_discard (gio->glfd, gio->offset, gio->count);
-		break;
-        case GF_FOP_ZEROFILL:
-                ret = pub_glfs_zerofill(gio->glfd, gio->offset, gio->count);
-                break;
-	}
+    ret = pub_glfs_preadv(glfd, iov, count, glfd->offset, flags);
 
-	return (int) ret;
+    return ret;
 }
 
+struct glfs_io {
+    struct glfs_fd *glfd;
+    int op;
+    off_t offset;
+    struct iovec *iov;
+    int count;
+    int flags;
+    gf_boolean_t oldcb;
+    union {
+        glfs_io_cbk34 fn34;
+        glfs_io_cbk fn;
+    };
+    void *data;
+};
 
-int
-glfs_preadv_async_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		       int op_ret, int op_errno, struct iovec *iovec,
-		       int count, struct iatt *stbuf, struct iobref *iobref,
-		       dict_t *xdata)
+static int
+glfs_io_async_cbk(int op_ret, int op_errno, call_frame_t *frame, void *cookie,
+                  struct iovec *iovec, int count, struct iatt *prebuf,
+                  struct iatt *postbuf)
 {
-	struct glfs_io *gio = NULL;
-	xlator_t       *subvol = NULL;
-	struct glfs    *fs = NULL;
-	struct glfs_fd *glfd = NULL;
-
-
-	gio = frame->local;
-	frame->local = NULL;
-	subvol = cookie;
-	glfd = gio->glfd;
-	fs = glfd->fs;
-
-	if (op_ret <= 0)
-		goto out;
+    struct glfs_io *gio = NULL;
+    xlator_t *subvol = NULL;
+    struct glfs *fs = NULL;
+    struct glfs_fd *glfd = NULL;
+    int ret = -1;
+    struct glfs_stat prestat = {}, *prestatp = NULL;
+    struct glfs_stat poststat = {}, *poststatp = NULL;
+
+    GF_VALIDATE_OR_GOTO("gfapi", frame, inval);
+    GF_VALIDATE_OR_GOTO("gfapi", cookie, inval);
+
+    gio = frame->local;
+    frame->local = NULL;
+    subvol = cookie;
+    glfd = gio->glfd;
+    fs = glfd->fs;
+
+    if (!glfs_is_glfd_still_valid(glfd))
+        goto err;
+
+    if (op_ret <= 0) {
+        goto out;
+    } else if (gio->op == GF_FOP_READ) {
+        if (!iovec) {
+            op_ret = -1;
+            op_errno = EINVAL;
+            goto out;
+        }
 
-	op_ret = iov_copy (gio->iov, gio->count, iovec, count);
+        op_ret = iov_copy(gio->iov, gio->count, iovec, count);
+        glfd->offset = gio->offset + op_ret;
+    } else if (gio->op == GF_FOP_WRITE) {
+        glfd->offset = gio->offset + gio->iov->iov_len;
+    }
 
-	glfd->offset = gio->offset + op_ret;
 out:
-	errno = op_errno;
-	gio->fn (gio->glfd, op_ret, gio->data);
+    errno = op_errno;
+    if (gio->oldcb) {
+        gio->fn34(gio->glfd, op_ret, gio->data);
+    } else {
+        if (prebuf) {
+            prestatp = &prestat;
+            glfs_iatt_to_statx(fs, prebuf, prestatp);
+        }
 
-	GF_FREE (gio->iov);
-	GF_FREE (gio);
-	STACK_DESTROY (frame->root);
-	priv_glfs_subvol_done (fs, subvol);
+        if (postbuf) {
+            poststatp = &poststat;
+            glfs_iatt_to_statx(fs, postbuf, poststatp);
+        }
 
-	return 0;
+        gio->fn(gio->glfd, op_ret, prestatp, poststatp, gio->data);
+    }
+err:
+    fd_unref(glfd->fd);
+    /* Since the async operation is complete
+     * release the ref taken during the start
+     * of async operation
+     */
+    GF_REF_PUT(glfd);
+
+    GF_FREE(gio->iov);
+    GF_FREE(gio);
+    STACK_DESTROY(frame->root);
+    glfs_subvol_done(fs, subvol);
+
+    ret = 0;
+inval:
+    return ret;
 }
 
+static int
+glfs_preadv_async_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int op_ret, int op_errno, struct iovec *iovec, int count,
+                      struct iatt *stbuf, struct iobref *iobref, dict_t *xdata)
+{
+    glfs_io_async_cbk(op_ret, op_errno, frame, cookie, iovec, count, NULL,
+                      stbuf);
 
-int
-pub_glfs_preadv_async (struct glfs_fd *glfd, const struct iovec *iovec,
-                       int count, off_t offset, int flags, glfs_io_cbk fn,
-                       void *data)
+    return 0;
+}
+
+static int
+glfs_preadv_async_common(struct glfs_fd *glfd, const struct iovec *iovec,
+                         int count, off_t offset, int flags, gf_boolean_t oldcb,
+                         glfs_io_cbk fn, void *data)
 {
-	struct glfs_io *gio = NULL;
-	int             ret = 0;
-	call_frame_t   *frame = NULL;
-	xlator_t       *subvol = NULL;
-	glfs_t         *fs = NULL;
-	fd_t           *fd = NULL;
-
-	__glfs_entry_fd (glfd);
-
-	subvol = priv_glfs_active_subvol (glfd->fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
-
-	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
-	if (!fd) {
-		ret = -1;
-		errno = EBADFD;
-		goto out;
-	}
-
-	fs = glfd->fs;
-
-	frame = syncop_create_frame (THIS);
-	if (!frame) {
-		ret = -1;
-		errno = ENOMEM;
-		goto out;
-	}
-
-	gio = GF_CALLOC (1, sizeof (*gio), glfs_mt_glfs_io_t);
-	if (!gio) {
-		ret = -1;
-		errno = ENOMEM;
-		goto out;
-	}
-
-	gio->iov = iov_dup (iovec, count);
-	if (!gio->iov) {
-		ret = -1;
-		errno = ENOMEM;
-		goto out;
-	}
-
-	gio->op     = GF_FOP_READ;
-	gio->glfd   = glfd;
-	gio->count  = count;
-	gio->offset = offset;
-	gio->flags  = flags;
-	gio->fn     = fn;
-	gio->data   = data;
-
-	frame->local = gio;
-
-	STACK_WIND_COOKIE (frame, glfs_preadv_async_cbk, subvol, subvol,
-			   subvol->fops->readv, fd, iov_length (iovec, count),
-			   offset, flags, NULL);
+    struct glfs_io *gio = NULL;
+    int ret = 0;
+    call_frame_t *frame = NULL;
+    xlator_t *subvol = NULL;
+    struct glfs *fs = NULL;
+    fd_t *fd = NULL;
+    dict_t *fop_attr = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    GF_REF_GET(glfd);
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    fs = glfd->fs;
+
+    frame = syncop_create_frame(THIS);
+    if (!frame) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gio = GF_CALLOC(1, sizeof(*gio), glfs_mt_glfs_io_t);
+    if (!gio) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gio->iov = iov_dup(iovec, count);
+    if (!gio->iov) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gio->op = GF_FOP_READ;
+    gio->glfd = glfd;
+    gio->count = count;
+    gio->offset = offset;
+    gio->flags = flags;
+    gio->oldcb = oldcb;
+    gio->fn = fn;
+    gio->data = data;
+
+    frame->local = gio;
+
+    ret = get_fop_attr_thrd_key(&fop_attr);
+    if (ret)
+        gf_msg_debug("gfapi", 0, "Getting leaseid from thread failed");
+
+    STACK_WIND_COOKIE(frame, glfs_preadv_async_cbk, subvol, subvol,
+                      subvol->fops->readv, fd, iov_length(iovec, count), offset,
+                      flags, fop_attr);
 
 out:
-        if (ret) {
-                if (gio) {
-                        GF_FREE (gio->iov);
-                        GF_FREE (gio);
-                }
-                if (frame) {
-                        STACK_DESTROY (frame->root);
-                }
-		priv_glfs_subvol_done (fs, subvol);
-	}
+    if (ret) {
+        if (fd)
+            fd_unref(fd);
+        if (glfd)
+            GF_REF_PUT(glfd);
+        if (gio) {
+            GF_FREE(gio->iov);
+            GF_FREE(gio);
+        }
+        if (frame) {
+            STACK_DESTROY(frame->root);
+        }
+        glfs_subvol_done(fs, subvol);
+    }
+    if (fop_attr)
+        dict_unref(fop_attr);
 
-	if (fd)
-		fd_unref (fd);
+    __GLFS_EXIT_FS;
 
-	return ret;
-}
+    return ret;
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_preadv_async, 3.4.0);
+invalid_fs:
+    return -1;
+}
 
+GFAPI_SYMVER_PUBLIC(glfs_preadv_async34, glfs_preadv_async, 3.4.0)
+int
+pub_glfs_preadv_async34(struct glfs_fd *glfd, const struct iovec *iovec,
+                        int count, off_t offset, int flags, glfs_io_cbk34 fn,
+                        void *data)
+{
+    return glfs_preadv_async_common(glfd, iovec, count, offset, flags, _gf_true,
+                                    (void *)fn, data);
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_preadv_async, 6.0)
 int
-pub_glfs_read_async (struct glfs_fd *glfd, void *buf, size_t count, int flags,
-                     glfs_io_cbk fn, void *data)
+pub_glfs_preadv_async(struct glfs_fd *glfd, const struct iovec *iovec,
+                      int count, off_t offset, int flags, glfs_io_cbk fn,
+                      void *data)
 {
-	struct iovec iov = {0, };
-	ssize_t      ret = 0;
+    return glfs_preadv_async_common(glfd, iovec, count, offset, flags,
+                                    _gf_false, fn, data);
+}
 
-	iov.iov_base = buf;
-	iov.iov_len = count;
+GFAPI_SYMVER_PUBLIC(glfs_read_async34, glfs_read_async, 3.4.0)
+int
+pub_glfs_read_async34(struct glfs_fd *glfd, void *buf, size_t count, int flags,
+                      glfs_io_cbk34 fn, void *data)
+{
+    struct iovec iov = {
+        0,
+    };
+    ssize_t ret = 0;
 
-	ret = pub_glfs_preadv_async (glfd, &iov, 1, glfd->offset, flags, fn, data);
+    if (glfd == NULL) {
+        errno = EBADF;
+        return -1;
+    }
 
-	return ret;
-}
+    iov.iov_base = buf;
+    iov.iov_len = count;
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_read_async, 3.4.0);
+    ret = glfs_preadv_async_common(glfd, &iov, 1, glfd->offset, flags, _gf_true,
+                                   (void *)fn, data);
 
+    return ret;
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_read_async, 6.0)
 int
-pub_glfs_pread_async (struct glfs_fd *glfd, void *buf, size_t count,
-                      off_t offset, int flags, glfs_io_cbk fn, void *data)
+pub_glfs_read_async(struct glfs_fd *glfd, void *buf, size_t count, int flags,
+                    glfs_io_cbk fn, void *data)
 {
-	struct iovec iov = {0, };
-	ssize_t      ret = 0;
+    struct iovec iov = {
+        0,
+    };
+    ssize_t ret = 0;
 
-	iov.iov_base = buf;
-	iov.iov_len = count;
+    if (glfd == NULL) {
+        errno = EBADF;
+        return -1;
+    }
 
-	ret = pub_glfs_preadv_async (glfd, &iov, 1, offset, flags, fn, data);
+    iov.iov_base = buf;
+    iov.iov_len = count;
 
-	return ret;
-}
-
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_pread_async, 3.4.0);
+    ret = glfs_preadv_async_common(glfd, &iov, 1, glfd->offset, flags,
+                                   _gf_false, fn, data);
 
+    return ret;
+}
 
+GFAPI_SYMVER_PUBLIC(glfs_pread_async34, glfs_pread_async, 3.4.0)
 int
-pub_glfs_readv_async (struct glfs_fd *glfd, const struct iovec *iov, int count,
-                      int flags, glfs_io_cbk fn, void *data)
+pub_glfs_pread_async34(struct glfs_fd *glfd, void *buf, size_t count,
+                       off_t offset, int flags, glfs_io_cbk34 fn, void *data)
 {
-	ssize_t      ret = 0;
+    struct iovec iov = {
+        0,
+    };
+    ssize_t ret = 0;
 
-	ret = pub_glfs_preadv_async (glfd, iov, count, glfd->offset, flags,
-				  fn, data);
-	return ret;
-}
+    iov.iov_base = buf;
+    iov.iov_len = count;
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_readv_async, 3.4.0);
+    ret = glfs_preadv_async_common(glfd, &iov, 1, offset, flags, _gf_true,
+                                   (void *)fn, data);
 
+    return ret;
+}
 
-ssize_t
-pub_glfs_pwritev (struct glfs_fd *glfd, const struct iovec *iovec, int iovcnt,
-                  off_t offset, int flags)
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_pread_async, 6.0)
+int
+pub_glfs_pread_async(struct glfs_fd *glfd, void *buf, size_t count,
+                     off_t offset, int flags, glfs_io_cbk fn, void *data)
 {
-	xlator_t       *subvol = NULL;
-	int             ret = -1;
-	size_t          size = -1;
-	struct iobref  *iobref = NULL;
-	struct iobuf   *iobuf = NULL;
-	struct iovec    iov = {0, };
-	fd_t           *fd = NULL;
+    struct iovec iov = {
+        0,
+    };
+    ssize_t ret = 0;
 
-	__glfs_entry_fd (glfd);
+    iov.iov_base = buf;
+    iov.iov_len = count;
 
-	subvol = priv_glfs_active_subvol (glfd->fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+    ret = glfs_preadv_async_common(glfd, &iov, 1, offset, flags, _gf_false, fn,
+                                   data);
 
-	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
-	if (!fd) {
-		ret = -1;
-		errno = EBADFD;
-		goto out;
-	}
+    return ret;
+}
 
-	size = iov_length (iovec, iovcnt);
+GFAPI_SYMVER_PUBLIC(glfs_readv_async34, glfs_readv_async, 3.4.0)
+int
+pub_glfs_readv_async34(struct glfs_fd *glfd, const struct iovec *iov, int count,
+                       int flags, glfs_io_cbk34 fn, void *data)
+{
+    ssize_t ret = 0;
 
-	iobuf = iobuf_get2 (subvol->ctx->iobuf_pool, size);
-	if (!iobuf) {
-		ret = -1;
-		errno = ENOMEM;
-		goto out;
-	}
+    if (glfd == NULL) {
+        errno = EBADF;
+        return -1;
+    }
 
-	iobref = iobref_new ();
-	if (!iobref) {
-		iobuf_unref (iobuf);
-		errno = ENOMEM;
-		ret = -1;
-		goto out;
-	}
+    ret = glfs_preadv_async_common(glfd, iov, count, glfd->offset, flags,
+                                   _gf_true, (void *)fn, data);
+    return ret;
+}
 
-	ret = iobref_add (iobref, iobuf);
-	if (ret) {
-		iobuf_unref (iobuf);
-		iobref_unref (iobref);
-		errno = ENOMEM;
-		ret = -1;
-		goto out;
-	}
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_readv_async, 6.0)
+int
+pub_glfs_readv_async(struct glfs_fd *glfd, const struct iovec *iov, int count,
+                     int flags, glfs_io_cbk fn, void *data)
+{
+    ssize_t ret = 0;
 
-	iov_unload (iobuf_ptr (iobuf), iovec, iovcnt);  /* FIXME!!! */
+    if (glfd == NULL) {
+        errno = EBADF;
+        return -1;
+    }
 
-	iov.iov_base = iobuf_ptr (iobuf);
-	iov.iov_len = size;
+    ret = glfs_preadv_async_common(glfd, iov, count, glfd->offset, flags,
+                                   _gf_false, fn, data);
+    return ret;
+}
 
-	ret = syncop_writev (subvol, fd, &iov, 1, offset, iobref, flags);
-        DECODE_SYNCOP_ERR (ret);
+static ssize_t
+glfs_pwritev_common(struct glfs_fd *glfd, const struct iovec *iovec, int iovcnt,
+                    off_t offset, int flags, struct glfs_stat *prestat,
+                    struct glfs_stat *poststat)
+{
+    xlator_t *subvol = NULL;
+    int ret = -1;
+    struct iobref *iobref = NULL;
+    struct iobuf *iobuf = NULL;
+    struct iovec iov = {
+        0,
+    };
+    fd_t *fd = NULL;
+    struct iatt preiatt =
+                    {
+                        0,
+                    },
+                postiatt = {
+                    0,
+                };
+    dict_t *fop_attr = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    GF_REF_GET(glfd);
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    ret = iobuf_copy(subvol->ctx->iobuf_pool, iovec, iovcnt, &iobref, &iobuf,
+                     &iov);
+    if (ret)
+        goto out;
+
+    ret = get_fop_attr_thrd_key(&fop_attr);
+    if (ret)
+        gf_msg_debug("gfapi", 0, "Getting leaseid from thread failed");
+
+    ret = syncop_writev(subvol, fd, &iov, 1, offset, iobref, flags, &preiatt,
+                        &postiatt, fop_attr, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    if (ret >= 0) {
+        if (prestat)
+            glfs_iatt_to_statx(glfd->fs, &preiatt, prestat);
+        if (poststat)
+            glfs_iatt_to_statx(glfd->fs, &postiatt, poststat);
+    }
+
+    if (ret <= 0)
+        goto out;
+
+    glfd->offset = (offset + iov.iov_len);
+out:
+    if (iobuf)
+        iobuf_unref(iobuf);
+    if (iobref)
+        iobref_unref(iobref);
+    if (fd)
+        fd_unref(fd);
+    if (glfd)
+        GF_REF_PUT(glfd);
+    if (fop_attr)
+        dict_unref(fop_attr);
+
+    glfs_subvol_done(glfd->fs, subvol);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
 
-	iobuf_unref (iobuf);
-	iobref_unref (iobref);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_copy_file_range, 6.0)
+ssize_t
+pub_glfs_copy_file_range(struct glfs_fd *glfd_in, off64_t *off_in,
+                         struct glfs_fd *glfd_out, off64_t *off_out, size_t len,
+                         unsigned int flags, struct glfs_stat *statbuf,
+                         struct glfs_stat *prestat, struct glfs_stat *poststat)
+{
+    xlator_t *subvol = NULL;
+    int ret = -1;
+    fd_t *fd_in = NULL;
+    fd_t *fd_out = NULL;
+    struct iatt preiatt =
+                    {
+                        0,
+                    },
+                iattbuf =
+                    {
+                        0,
+                    },
+                postiatt = {
+                    0,
+                };
+    dict_t *fop_attr = NULL;
+    off64_t pos_in;
+    off64_t pos_out;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd_in, invalid_fs);
+    __GLFS_ENTRY_VALIDATE_FD(glfd_out, invalid_fs);
+
+    GF_REF_GET(glfd_in);
+    GF_REF_GET(glfd_out);
+
+    if (glfd_in->fs != glfd_out->fs) {
+        ret = -1;
+        errno = EXDEV;
+        goto out;
+    }
+
+    subvol = glfs_active_subvol(glfd_in->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd_in = glfs_resolve_fd(glfd_in->fs, subvol, glfd_in);
+    if (!fd_in) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    fd_out = glfs_resolve_fd(glfd_out->fs, subvol, glfd_out);
+    if (!fd_out) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    /*
+     * This is based on how the vfs layer in the kernel handles
+     * copy_file_range call. Upon receiving it follows the
+     * below method to consider the offset.
+     * if (off_in != NULL)
+     *    use the value off_in to perform the op
+     * else if off_in == NULL
+     *    use the current file offset position to perform the op
+     *
+     * For gfapi, glfd->offset is used. For a freshly opened
+     * fd, the offset is set to 0.
+     */
+    if (off_in)
+        pos_in = *off_in;
+    else
+        pos_in = glfd_in->offset;
+
+    if (off_out)
+        pos_out = *off_out;
+    else
+        pos_out = glfd_out->offset;
+
+    ret = get_fop_attr_thrd_key(&fop_attr);
+    if (ret)
+        gf_msg_debug("gfapi", 0, "Getting leaseid from thread failed");
+
+    ret = syncop_copy_file_range(subvol, fd_in, pos_in, fd_out, pos_out, len,
+                                 flags, &iattbuf, &preiatt, &postiatt, fop_attr,
+                                 NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    if (ret >= 0) {
+        pos_in += ret;
+        pos_out += ret;
+
+        if (off_in)
+            *off_in = pos_in;
+        if (off_out)
+            *off_out = pos_out;
+
+        if (statbuf)
+            glfs_iatt_to_statx(glfd_in->fs, &iattbuf, statbuf);
+        if (prestat)
+            glfs_iatt_to_statx(glfd_in->fs, &preiatt, prestat);
+        if (poststat)
+            glfs_iatt_to_statx(glfd_in->fs, &postiatt, poststat);
+    }
+
+    if (ret <= 0)
+        goto out;
+
+    /*
+     * If *off_in is NULL, then there is no offset info that can
+     * obtained from the input argument. Hence follow below method.
+     *  If *off_in is NULL, then
+     *     glfd->offset = offset + ret;
+     * else
+     *     do nothing.
+     *
+     * According to the man page of copy_file_range, if off_in is
+     * NULL, then the offset of the source file is advanced by
+     * the return value of the fop. The same applies to off_out as
+     * well. Otherwise, if *off_in is not NULL, then the offset
+     * is not advanced by the filesystem. The entity which sends
+     * the copy_file_range call is supposed to advance the offset
+     * value in its buffer (pointed to by *off_in or *off_out)
+     * by the return value of copy_file_range.
+     */
+    if (!off_in)
+        glfd_in->offset += ret;
+
+    if (!off_out)
+        glfd_out->offset += ret;
 
-	if (ret <= 0)
-		goto out;
+out:
+    if (fd_in)
+        fd_unref(fd_in);
+    if (fd_out)
+        fd_unref(fd_out);
+    if (glfd_in)
+        GF_REF_PUT(glfd_in);
+    if (glfd_out)
+        GF_REF_PUT(glfd_out);
+    if (fop_attr)
+        dict_unref(fop_attr);
+
+    glfs_subvol_done(glfd_in->fs, subvol);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
 
-	glfd->offset = (offset + size);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_pwritev, 3.4.0)
+ssize_t
+pub_glfs_pwritev(struct glfs_fd *glfd, const struct iovec *iovec, int iovcnt,
+                 off_t offset, int flags)
+{
+    return glfs_pwritev_common(glfd, iovec, iovcnt, offset, flags, NULL, NULL);
+}
 
-out:
-	if (fd)
-		fd_unref (fd);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_write, 3.4.0)
+ssize_t
+pub_glfs_write(struct glfs_fd *glfd, const void *buf, size_t count, int flags)
+{
+    struct iovec iov = {
+        0,
+    };
+    ssize_t ret = 0;
 
-	priv_glfs_subvol_done (glfd->fs, subvol);
+    if (glfd == NULL) {
+        errno = EBADF;
+        return -1;
+    }
 
-	return ret;
-}
+    iov.iov_base = (void *)buf;
+    iov.iov_len = count;
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_pwritev, 3.4.0);
+    ret = pub_glfs_pwritev(glfd, &iov, 1, glfd->offset, flags);
 
+    return ret;
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_writev, 3.4.0)
 ssize_t
-pub_glfs_write (struct glfs_fd *glfd, const void *buf, size_t count, int flags)
+pub_glfs_writev(struct glfs_fd *glfd, const struct iovec *iov, int count,
+                int flags)
 {
-	struct iovec iov = {0, };
-	ssize_t      ret = 0;
+    ssize_t ret = 0;
 
-	iov.iov_base = (void *) buf;
-	iov.iov_len = count;
+    if (glfd == NULL) {
+        errno = EBADF;
+        return -1;
+    }
 
-	ret = pub_glfs_pwritev (glfd, &iov, 1, glfd->offset, flags);
+    ret = pub_glfs_pwritev(glfd, iov, count, glfd->offset, flags);
 
-	return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_write, 3.4.0);
-
-
+GFAPI_SYMVER_PUBLIC(glfs_pwrite34, glfs_pwrite, 3.4.0)
 ssize_t
-pub_glfs_writev (struct glfs_fd *glfd, const struct iovec *iov, int count,
-                 int flags)
+pub_glfs_pwrite34(struct glfs_fd *glfd, const void *buf, size_t count,
+                  off_t offset, int flags)
 {
-	ssize_t      ret = 0;
-
-	ret = pub_glfs_pwritev (glfd, iov, count, glfd->offset, flags);
+    struct iovec iov = {
+        0,
+    };
+    ssize_t ret = 0;
 
-	return ret;
-}
+    iov.iov_base = (void *)buf;
+    iov.iov_len = count;
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_writev, 3.4.0);
+    ret = pub_glfs_pwritev(glfd, &iov, 1, offset, flags);
 
+    return ret;
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_pwrite, 6.0)
 ssize_t
-pub_glfs_pwrite (struct glfs_fd *glfd, const void *buf, size_t count,
-                 off_t offset, int flags)
+pub_glfs_pwrite(struct glfs_fd *glfd, const void *buf, size_t count,
+                off_t offset, int flags, struct glfs_stat *prestat,
+                struct glfs_stat *poststat)
 {
-	struct iovec iov = {0, };
-	ssize_t      ret = 0;
+    struct iovec iov = {
+        0,
+    };
+    ssize_t ret = 0;
 
-	iov.iov_base = (void *) buf;
-	iov.iov_len = count;
+    iov.iov_base = (void *)buf;
+    iov.iov_len = count;
 
-	ret = pub_glfs_pwritev (glfd, &iov, 1, offset, flags);
+    ret = glfs_pwritev_common(glfd, &iov, 1, offset, flags, prestat, poststat);
 
-	return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_pwrite, 3.4.0);
+extern glfs_t *
+pub_glfs_from_glfd(glfs_fd_t *);
 
+static int
+glfs_pwritev_async_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int op_ret, int op_errno, struct iatt *prebuf,
+                       struct iatt *postbuf, dict_t *xdata)
+{
+    glfs_io_async_cbk(op_ret, op_errno, frame, cookie, NULL, 0, prebuf,
+                      postbuf);
+
+    return 0;
+}
 
-extern glfs_t *pub_glfs_from_glfd (glfs_fd_t *);
+static int
+glfs_pwritev_async_common(struct glfs_fd *glfd, const struct iovec *iovec,
+                          int count, off_t offset, int flags,
+                          gf_boolean_t oldcb, glfs_io_cbk fn, void *data)
+{
+    struct glfs_io *gio = NULL;
+    int ret = -1;
+    call_frame_t *frame = NULL;
+    xlator_t *subvol = NULL;
+    fd_t *fd = NULL;
+    struct iobref *iobref = NULL;
+    struct iobuf *iobuf = NULL;
+    dict_t *fop_attr = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    /* Need to take explicit ref so that the fd
+     * is not destroyed before the fop is complete
+     */
+    GF_REF_GET(glfd);
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        errno = EBADFD;
+        goto out;
+    }
+
+    gio = GF_CALLOC(1, sizeof(*gio), glfs_mt_glfs_io_t);
+    if (!gio) {
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gio->op = GF_FOP_WRITE;
+    gio->glfd = glfd;
+    gio->offset = offset;
+    gio->flags = flags;
+    gio->oldcb = oldcb;
+    gio->fn = fn;
+    gio->data = data;
+    gio->count = 1;
+    gio->iov = GF_CALLOC(gio->count, sizeof(*(gio->iov)), gf_common_mt_iovec);
+    if (!gio->iov) {
+        errno = ENOMEM;
+        goto out;
+    }
+
+    ret = iobuf_copy(subvol->ctx->iobuf_pool, iovec, count, &iobref, &iobuf,
+                     gio->iov);
+    if (ret)
+        goto out;
+
+    frame = syncop_create_frame(THIS);
+    if (!frame) {
+        errno = ENOMEM;
+        ret = -1;
+        goto out;
+    }
+
+    frame->local = gio;
+
+    ret = get_fop_attr_thrd_key(&fop_attr);
+    if (ret)
+        gf_msg_debug("gfapi", 0, "Getting leaseid from thread failed");
+
+    STACK_WIND_COOKIE(frame, glfs_pwritev_async_cbk, subvol, subvol,
+                      subvol->fops->writev, fd, gio->iov, gio->count, offset,
+                      flags, iobref, fop_attr);
+
+    ret = 0;
+out:
+    if (ret) {
+        if (fd)
+            fd_unref(fd);
+        if (glfd)
+            GF_REF_PUT(glfd);
+        GF_FREE(gio);
+        /*
+         * If there is any error condition check after the frame
+         * creation, we have to destroy the frame root.
+         */
+        glfs_subvol_done(glfd->fs, subvol);
+    }
+    if (fop_attr)
+        dict_unref(fop_attr);
+
+    if (iobuf)
+        iobuf_unref(iobuf);
+    if (iobref)
+        iobref_unref(iobref);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
 
+GFAPI_SYMVER_PUBLIC(glfs_pwritev_async34, glfs_pwritev_async, 3.4.0)
 int
-pub_glfs_pwritev_async (struct glfs_fd *glfd, const struct iovec *iovec,
-                        int count, off_t offset, int flags, glfs_io_cbk fn,
-                        void *data)
+pub_glfs_pwritev_async34(struct glfs_fd *glfd, const struct iovec *iovec,
+                         int count, off_t offset, int flags, glfs_io_cbk34 fn,
+                         void *data)
 {
-	struct glfs_io *gio = NULL;
-	int             ret = 0;
+    return glfs_pwritev_async_common(glfd, iovec, count, offset, flags,
+                                     _gf_true, (void *)fn, data);
+}
 
-	gio = GF_CALLOC (1, sizeof (*gio), glfs_mt_glfs_io_t);
-	if (!gio) {
-		errno = ENOMEM;
-		return -1;
-	}
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_pwritev_async, 6.0)
+int
+pub_glfs_pwritev_async(struct glfs_fd *glfd, const struct iovec *iovec,
+                       int count, off_t offset, int flags, glfs_io_cbk fn,
+                       void *data)
+{
+    return glfs_pwritev_async_common(glfd, iovec, count, offset, flags,
+                                     _gf_false, fn, data);
+}
 
-	gio->iov = iov_dup (iovec, count);
-	if (!gio->iov) {
-		GF_FREE (gio);
-		errno = ENOMEM;
-		return -1;
-	}
+GFAPI_SYMVER_PUBLIC(glfs_write_async34, glfs_write_async, 3.4.0)
+int
+pub_glfs_write_async34(struct glfs_fd *glfd, const void *buf, size_t count,
+                       int flags, glfs_io_cbk34 fn, void *data)
+{
+    struct iovec iov = {
+        0,
+    };
+    ssize_t ret = 0;
 
-	gio->op     = GF_FOP_WRITE;
-	gio->glfd   = glfd;
-	gio->count  = count;
-	gio->offset = offset;
-	gio->flags  = flags;
-	gio->fn     = fn;
-	gio->data   = data;
+    if (glfd == NULL) {
+        errno = EBADF;
+        return -1;
+    }
 
-	ret = synctask_new (pub_glfs_from_glfd (glfd)->ctx->env,
-			    glfs_io_async_task, glfs_io_async_cbk,
-			    NULL, gio);
+    iov.iov_base = (void *)buf;
+    iov.iov_len = count;
 
-	if (ret) {
-		GF_FREE (gio->iov);
-		GF_FREE (gio);
-	}
+    ret = glfs_pwritev_async_common(glfd, &iov, 1, glfd->offset, flags,
+                                    _gf_true, (void *)fn, data);
 
-	return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_pwritev_async, 3.4.0);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_write_async, 6.0)
 int
-pub_glfs_write_async (struct glfs_fd *glfd, const void *buf, size_t count,
-                      int flags, glfs_io_cbk fn, void *data)
+pub_glfs_write_async(struct glfs_fd *glfd, const void *buf, size_t count,
+                     int flags, glfs_io_cbk fn, void *data)
 {
-	struct iovec iov = {0, };
-	ssize_t      ret = 0;
+    struct iovec iov = {
+        0,
+    };
+    ssize_t ret = 0;
 
-	iov.iov_base = (void *) buf;
-	iov.iov_len = count;
+    if (glfd == NULL) {
+        errno = EBADF;
+        return -1;
+    }
 
-	ret = pub_glfs_pwritev_async (glfd, &iov, 1, glfd->offset, flags, fn, data);
-
-	return ret;
-}
+    iov.iov_base = (void *)buf;
+    iov.iov_len = count;
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_write_async, 3.4.0);
+    ret = glfs_pwritev_async_common(glfd, &iov, 1, glfd->offset, flags,
+                                    _gf_false, fn, data);
 
+    return ret;
+}
 
+GFAPI_SYMVER_PUBLIC(glfs_pwrite_async34, glfs_pwrite_async, 3.4.0)
 int
-pub_glfs_pwrite_async (struct glfs_fd *glfd, const void *buf, int count,
-                       off_t offset, int flags, glfs_io_cbk fn, void *data)
+pub_glfs_pwrite_async34(struct glfs_fd *glfd, const void *buf, int count,
+                        off_t offset, int flags, glfs_io_cbk34 fn, void *data)
 {
-	struct iovec iov = {0, };
-	ssize_t      ret = 0;
+    struct iovec iov = {
+        0,
+    };
+    ssize_t ret = 0;
 
-	iov.iov_base = (void *) buf;
-	iov.iov_len = count;
+    iov.iov_base = (void *)buf;
+    iov.iov_len = count;
 
-	ret = pub_glfs_pwritev_async (glfd, &iov, 1, offset, flags, fn, data);
+    ret = glfs_pwritev_async_common(glfd, &iov, 1, offset, flags, _gf_true,
+                                    (void *)fn, data);
 
-	return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_pwrite_async, 3.4.0);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_pwrite_async, 6.0)
 int
-pub_glfs_writev_async (struct glfs_fd *glfd, const struct iovec *iov, int count,
-                       int flags, glfs_io_cbk fn, void *data)
+pub_glfs_pwrite_async(struct glfs_fd *glfd, const void *buf, int count,
+                      off_t offset, int flags, glfs_io_cbk fn, void *data)
 {
-	ssize_t      ret = 0;
+    struct iovec iov = {
+        0,
+    };
+    ssize_t ret = 0;
 
-	ret = pub_glfs_pwritev_async (glfd, iov, count, glfd->offset, flags,
-				   fn, data);
-	return ret;
-}
+    iov.iov_base = (void *)buf;
+    iov.iov_len = count;
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_writev_async, 3.4.0);
+    ret = glfs_pwritev_async_common(glfd, &iov, 1, offset, flags, _gf_false, fn,
+                                    data);
 
+    return ret;
+}
 
+GFAPI_SYMVER_PUBLIC(glfs_writev_async34, glfs_writev_async, 3.4.0)
 int
-pub_glfs_fsync (struct glfs_fd *glfd)
+pub_glfs_writev_async34(struct glfs_fd *glfd, const struct iovec *iov,
+                        int count, int flags, glfs_io_cbk34 fn, void *data)
 {
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	fd_t            *fd = NULL;
+    ssize_t ret = 0;
 
-	__glfs_entry_fd (glfd);
+    if (glfd == NULL) {
+        errno = EBADF;
+        return -1;
+    }
 
-	subvol = priv_glfs_active_subvol (glfd->fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
-
-	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
-	if (!fd) {
-		ret = -1;
-		errno = EBADFD;
-		goto out;
-	}
+    ret = glfs_pwritev_async_common(glfd, iov, count, glfd->offset, flags,
+                                    _gf_true, (void *)fn, data);
+    return ret;
+}
 
-	ret = syncop_fsync (subvol, fd, 0);
-        DECODE_SYNCOP_ERR (ret);
-out:
-	if (fd)
-		fd_unref (fd);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_writev_async, 6.0)
+int
+pub_glfs_writev_async(struct glfs_fd *glfd, const struct iovec *iov, int count,
+                      int flags, glfs_io_cbk fn, void *data)
+{
+    ssize_t ret = 0;
 
-	priv_glfs_subvol_done (glfd->fs, subvol);
+    if (glfd == NULL) {
+        errno = EBADF;
+        return -1;
+    }
 
-	return ret;
+    ret = glfs_pwritev_async_common(glfd, iov, count, glfd->offset, flags,
+                                    _gf_false, fn, data);
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fsync, 3.4.0);
-
-
 static int
-glfs_fsync_async_common (struct glfs_fd *glfd, glfs_io_cbk fn, void *data,
-			 int dataonly)
+glfs_fsync_common(struct glfs_fd *glfd, struct glfs_stat *prestat,
+                  struct glfs_stat *poststat)
 {
-	struct glfs_io *gio = NULL;
-	int             ret = 0;
-
-	gio = GF_CALLOC (1, sizeof (*gio), glfs_mt_glfs_io_t);
-	if (!gio) {
-		errno = ENOMEM;
-		return -1;
-	}
-
-	gio->op     = GF_FOP_FSYNC;
-	gio->glfd   = glfd;
-	gio->flags  = dataonly;
-	gio->fn     = fn;
-	gio->data   = data;
-
-	ret = synctask_new (pub_glfs_from_glfd (glfd)->ctx->env,
-			    glfs_io_async_task, glfs_io_async_cbk,
-			    NULL, gio);
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    fd_t *fd = NULL;
+    struct iatt preiatt =
+                    {
+                        0,
+                    },
+                postiatt = {
+                    0,
+                };
+    dict_t *fop_attr = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    GF_REF_GET(glfd);
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    ret = get_fop_attr_thrd_key(&fop_attr);
+    if (ret)
+        gf_msg_debug("gfapi", 0, "Getting leaseid from thread failed");
+
+    ret = syncop_fsync(subvol, fd, 0, &preiatt, &postiatt, fop_attr, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    if (ret >= 0) {
+        if (prestat)
+            glfs_iatt_to_statx(glfd->fs, &preiatt, prestat);
+        if (poststat)
+            glfs_iatt_to_statx(glfd->fs, &postiatt, poststat);
+    }
+out:
+    if (fd)
+        fd_unref(fd);
+    if (glfd)
+        GF_REF_PUT(glfd);
+    if (fop_attr)
+        dict_unref(fop_attr);
 
-	if (ret) {
-		GF_FREE (gio->iov);
-		GF_FREE (gio);
-	}
+    glfs_subvol_done(glfd->fs, subvol);
 
-	return ret;
+    __GLFS_EXIT_FS;
 
+invalid_fs:
+    return ret;
 }
 
-
+GFAPI_SYMVER_PUBLIC(glfs_fsync34, glfs_fsync, 3.4.0)
 int
-pub_glfs_fsync_async (struct glfs_fd *glfd, glfs_io_cbk fn, void *data)
+pub_glfs_fsync34(struct glfs_fd *glfd)
 {
-	return glfs_fsync_async_common (glfd, fn, data, 0);
+    return glfs_fsync_common(glfd, NULL, NULL);
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fsync_async, 3.4.0);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fsync, 6.0)
 int
-pub_glfs_fdatasync (struct glfs_fd *glfd)
+pub_glfs_fsync(struct glfs_fd *glfd, struct glfs_stat *prestat,
+               struct glfs_stat *poststat)
 {
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	fd_t            *fd = NULL;
+    return glfs_fsync_common(glfd, prestat, poststat);
+}
 
-	__glfs_entry_fd (glfd);
+static int
+glfs_fsync_async_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                     struct iatt *postbuf, dict_t *xdata)
+{
+    glfs_io_async_cbk(op_ret, op_errno, frame, cookie, NULL, 0, prebuf,
+                      postbuf);
 
-	subvol = priv_glfs_active_subvol (glfd->fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+    return 0;
+}
 
-	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
-	if (!fd) {
-		ret = -1;
-		errno = EBADFD;
-		goto out;
-	}
+static int
+glfs_fsync_async_common(struct glfs_fd *glfd, gf_boolean_t oldcb,
+                        glfs_io_cbk fn, void *data, int dataonly)
+{
+    struct glfs_io *gio = NULL;
+    int ret = 0;
+    call_frame_t *frame = NULL;
+    xlator_t *subvol = NULL;
+    fd_t *fd = NULL;
+
+    /* Need to take explicit ref so that the fd
+     * is not destroyed before the fop is complete
+     */
+    GF_REF_GET(glfd);
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    frame = syncop_create_frame(THIS);
+    if (!frame) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gio = GF_CALLOC(1, sizeof(*gio), glfs_mt_glfs_io_t);
+    if (!gio) {
+        errno = ENOMEM;
+        ret = -1;
+        goto out;
+    }
+
+    gio->op = GF_FOP_FSYNC;
+    gio->glfd = glfd;
+    gio->flags = dataonly;
+    gio->oldcb = oldcb;
+    gio->fn = fn;
+    gio->data = data;
+
+    frame->local = gio;
+
+    STACK_WIND_COOKIE(frame, glfs_fsync_async_cbk, subvol, subvol,
+                      subvol->fops->fsync, fd, dataonly, NULL);
 
-	ret = syncop_fsync (subvol, fd, 1);
-        DECODE_SYNCOP_ERR (ret);
 out:
-	if (fd)
-		fd_unref (fd);
-
-	priv_glfs_subvol_done (glfd->fs, subvol);
-
-	return ret;
+    if (ret) {
+        if (fd)
+            fd_unref(fd);
+        GF_REF_PUT(glfd);
+        GF_FREE(gio);
+        if (frame)
+            STACK_DESTROY(frame->root);
+        glfs_subvol_done(glfd->fs, subvol);
+    }
+
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fdatasync, 3.4.0);
-
-
+GFAPI_SYMVER_PUBLIC(glfs_fsync_async34, glfs_fsync_async, 3.4.0)
 int
-pub_glfs_fdatasync_async (struct glfs_fd *glfd, glfs_io_cbk fn, void *data)
+pub_glfs_fsync_async34(struct glfs_fd *glfd, glfs_io_cbk34 fn, void *data)
 {
-	return glfs_fsync_async_common (glfd, fn, data, 1);
-}
+    int ret = -1;
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fdatasync_async, 3.4.0);
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
 
+    ret = glfs_fsync_async_common(glfd, _gf_true, (void *)fn, data, 0);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fsync_async, 6.0)
 int
-pub_glfs_ftruncate (struct glfs_fd *glfd, off_t offset)
+pub_glfs_fsync_async(struct glfs_fd *glfd, glfs_io_cbk fn, void *data)
 {
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	fd_t            *fd = NULL;
+    int ret = -1;
 
-	__glfs_entry_fd (glfd);
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
 
-	subvol = priv_glfs_active_subvol (glfd->fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+    ret = glfs_fsync_async_common(glfd, _gf_false, fn, data, 0);
 
-	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
-	if (!fd) {
-		ret = -1;
-		errno = EBADFD;
-		goto out;
-	}
+    __GLFS_EXIT_FS;
 
-	ret = syncop_ftruncate (subvol, fd, offset);
-        DECODE_SYNCOP_ERR (ret);
+invalid_fs:
+    return ret;
+}
+
+static int
+glfs_fdatasync_common(struct glfs_fd *glfd, struct glfs_stat *prestat,
+                      struct glfs_stat *poststat)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    fd_t *fd = NULL;
+    struct iatt preiatt =
+                    {
+                        0,
+                    },
+                postiatt = {
+                    0,
+                };
+    dict_t *fop_attr = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    GF_REF_GET(glfd);
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    ret = get_fop_attr_thrd_key(&fop_attr);
+    if (ret)
+        gf_msg_debug("gfapi", 0, "Getting leaseid from thread failed");
+
+    ret = syncop_fsync(subvol, fd, 1, &preiatt, &postiatt, fop_attr, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    if (ret >= 0) {
+        if (prestat)
+            glfs_iatt_to_statx(glfd->fs, &preiatt, prestat);
+        if (poststat)
+            glfs_iatt_to_statx(glfd->fs, &postiatt, poststat);
+    }
 out:
-	if (fd)
-		fd_unref (fd);
+    if (fd)
+        fd_unref(fd);
+    if (glfd)
+        GF_REF_PUT(glfd);
+    if (fop_attr)
+        dict_unref(fop_attr);
 
-	priv_glfs_subvol_done (glfd->fs, subvol);
+    glfs_subvol_done(glfd->fs, subvol);
 
-	return ret;
-}
+    __GLFS_EXIT_FS;
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_ftruncate, 3.4.0);
+invalid_fs:
+    return ret;
+}
 
+GFAPI_SYMVER_PUBLIC(glfs_fdatasync34, glfs_fdatasync, 3.4.0)
+int
+pub_glfs_fdatasync34(struct glfs_fd *glfd)
+{
+    return glfs_fdatasync_common(glfd, NULL, NULL);
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fdatasync, 6.0)
 int
-pub_glfs_ftruncate_async (struct glfs_fd *glfd, off_t offset, glfs_io_cbk fn,
-                          void *data)
+pub_glfs_fdatasync(struct glfs_fd *glfd, struct glfs_stat *prestat,
+                   struct glfs_stat *poststat)
 {
-	struct glfs_io *gio = NULL;
-	int             ret = 0;
+    return glfs_fdatasync_common(glfd, prestat, poststat);
+}
 
-	gio = GF_CALLOC (1, sizeof (*gio), glfs_mt_glfs_io_t);
-	if (!gio) {
-		errno = ENOMEM;
-		return -1;
-	}
+GFAPI_SYMVER_PUBLIC(glfs_fdatasync_async34, glfs_fdatasync_async, 3.4.0)
+int
+pub_glfs_fdatasync_async34(struct glfs_fd *glfd, glfs_io_cbk34 fn, void *data)
+{
+    int ret = -1;
 
-	gio->op     = GF_FOP_FTRUNCATE;
-	gio->glfd   = glfd;
-	gio->offset = offset;
-	gio->fn     = fn;
-	gio->data   = data;
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
 
-	ret = synctask_new (pub_glfs_from_glfd (glfd)->ctx->env,
-			    glfs_io_async_task, glfs_io_async_cbk,
-			    NULL, gio);
+    ret = glfs_fsync_async_common(glfd, _gf_true, (void *)fn, data, 1);
 
-	if (ret) {
-		GF_FREE (gio->iov);
-		GF_FREE (gio);
-	}
+    __GLFS_EXIT_FS;
 
-	return ret;
+invalid_fs:
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_ftruncate_async, 3.4.0);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fdatasync_async, 6.0)
 int
-pub_glfs_access (struct glfs *fs, const char *path, int mode)
+pub_glfs_fdatasync_async(struct glfs_fd *glfd, glfs_io_cbk fn, void *data)
 {
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	loc_t            loc = {0, };
-	struct iatt      iatt = {0, };
-	int              reval = 0;
+    int ret = -1;
 
-	__glfs_entry_fs (fs);
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
 
-	subvol = priv_glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
-retry:
-	ret = glfs_resolve (fs, subvol, path, &loc, &iatt, reval);
-
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    ret = glfs_fsync_async_common(glfd, _gf_false, fn, data, 1);
 
-	if (ret)
-		goto out;
+    __GLFS_EXIT_FS;
 
-	ret = syncop_access (subvol, &loc, mode);
-        DECODE_SYNCOP_ERR (ret);
+invalid_fs:
+    return ret;
+}
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+static int
+glfs_ftruncate_common(struct glfs_fd *glfd, off_t offset,
+                      struct glfs_stat *prestat, struct glfs_stat *poststat)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    fd_t *fd = NULL;
+    struct iatt preiatt =
+                    {
+                        0,
+                    },
+                postiatt = {
+                    0,
+                };
+    dict_t *fop_attr = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    GF_REF_GET(glfd);
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    ret = get_fop_attr_thrd_key(&fop_attr);
+    if (ret)
+        gf_msg_debug("gfapi", 0, "Getting leaseid from thread failed");
+
+    ret = syncop_ftruncate(subvol, fd, offset, &preiatt, &postiatt, fop_attr,
+                           NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    if (ret >= 0) {
+        if (prestat)
+            glfs_iatt_to_statx(glfd->fs, &preiatt, prestat);
+        if (poststat)
+            glfs_iatt_to_statx(glfd->fs, &postiatt, poststat);
+    }
 out:
-	loc_wipe (&loc);
-
-	priv_glfs_subvol_done (fs, subvol);
+    if (fd)
+        fd_unref(fd);
+    if (glfd)
+        GF_REF_PUT(glfd);
+    if (fop_attr)
+        dict_unref(fop_attr);
 
-	return ret;
-}
+    glfs_subvol_done(glfd->fs, subvol);
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_access, 3.4.0);
+    __GLFS_EXIT_FS;
 
+invalid_fs:
+    return ret;
+}
 
+GFAPI_SYMVER_PUBLIC(glfs_ftruncate34, glfs_ftruncate, 3.4.0)
 int
-pub_glfs_symlink (struct glfs *fs, const char *data, const char *path)
+pub_glfs_ftruncate34(struct glfs_fd *glfd, off_t offset)
 {
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	loc_t            loc = {0, };
-	struct iatt      iatt = {0, };
-	uuid_t           gfid;
-	dict_t          *xattr_req = NULL;
-	int              reval = 0;
-
-	__glfs_entry_fs (fs);
-
-	subvol = priv_glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+    return glfs_ftruncate_common(glfd, offset, NULL, NULL);
+}
 
-	xattr_req = dict_new ();
-	if (!xattr_req) {
-		ret = -1;
-		errno = ENOMEM;
-		goto out;
-	}
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_ftruncate, 6.0)
+int
+pub_glfs_ftruncate(struct glfs_fd *glfd, off_t offset,
+                   struct glfs_stat *prestat, struct glfs_stat *poststat)
+{
+    return glfs_ftruncate_common(glfd, offset, prestat, poststat);
+}
 
-	uuid_generate (gfid);
-	ret = dict_set_static_bin (xattr_req, "gfid-req", gfid, 16);
-	if (ret) {
-		ret = -1;
-		errno = ENOMEM;
-		goto out;
-	}
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_truncate, 3.7.15)
+int
+pub_glfs_truncate(struct glfs *fs, const char *path, off_t length)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
 retry:
-	ret = glfs_lresolve (fs, subvol, path, &loc, &iatt, reval);
-
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
-
-	if (loc.inode) {
-		errno = EEXIST;
-		ret = -1;
-		goto out;
-	}
+    ret = glfs_resolve(fs, subvol, path, &loc, &iatt, reval);
 
-	if (ret == -1 && errno != ENOENT)
-		/* Any other type of error is fatal */
-		goto out;
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	if (ret == -1 && errno == ENOENT && !loc.parent)
-		/* The parent directory or an ancestor even
-		   higher does not exist
-		*/
-		goto out;
+    if (ret)
+        goto out;
 
-	/* ret == -1 && errno == ENOENT */
-	loc.inode = inode_new (loc.parent->table);
-	if (!loc.inode) {
-		ret = -1;
-		errno = ENOMEM;
-		goto out;
-	}
+    ret = syncop_truncate(subvol, &loc, length, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
 
-	ret = syncop_symlink (subvol, &loc, data, xattr_req, &iatt);
-        DECODE_SYNCOP_ERR (ret);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
+out:
+    loc_wipe(&loc);
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    glfs_subvol_done(fs, subvol);
 
-	if (ret == 0)
-		ret = glfs_loc_link (&loc, &iatt);
-out:
-	loc_wipe (&loc);
+    __GLFS_EXIT_FS;
 
-	if (xattr_req)
-		dict_unref (xattr_req);
+invalid_fs:
+    return ret;
+}
 
-	priv_glfs_subvol_done (fs, subvol);
+static int
+glfs_ftruncate_async_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                         int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                         struct iatt *postbuf, dict_t *xdata)
+{
+    glfs_io_async_cbk(op_ret, op_errno, frame, cookie, NULL, 0, prebuf,
+                      postbuf);
 
-	return ret;
+    return 0;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_symlink, 3.4.0);
+static int
+glfs_ftruncate_async_common(struct glfs_fd *glfd, off_t offset,
+                            gf_boolean_t oldcb, glfs_io_cbk fn, void *data)
+{
+    struct glfs_io *gio = NULL;
+    int ret = -1;
+    call_frame_t *frame = NULL;
+    xlator_t *subvol = NULL;
+    fd_t *fd = NULL;
+    dict_t *fop_attr = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    /* Need to take explicit ref so that the fd
+     * is not destroyed before the fop is complete
+     */
+    GF_REF_GET(glfd);
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        errno = EBADFD;
+        goto out;
+    }
+
+    frame = syncop_create_frame(THIS);
+    if (!frame) {
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gio = GF_CALLOC(1, sizeof(*gio), glfs_mt_glfs_io_t);
+    if (!gio) {
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gio->op = GF_FOP_FTRUNCATE;
+    gio->glfd = glfd;
+    gio->offset = offset;
+    gio->oldcb = oldcb;
+    gio->fn = fn;
+    gio->data = data;
+
+    frame->local = gio;
+
+    ret = get_fop_attr_thrd_key(&fop_attr);
+    if (ret)
+        gf_msg_debug("gfapi", 0, "Getting leaseid from thread failed");
+
+    STACK_WIND_COOKIE(frame, glfs_ftruncate_async_cbk, subvol, subvol,
+                      subvol->fops->ftruncate, fd, offset, fop_attr);
+
+    ret = 0;
 
+out:
+    if (ret) {
+        if (fd)
+            fd_unref(fd);
+        if (glfd)
+            GF_REF_PUT(glfd);
+        GF_FREE(gio);
+        if (frame)
+            STACK_DESTROY(frame->root);
+        glfs_subvol_done(glfd->fs, subvol);
+    }
+    if (fop_attr)
+        dict_unref(fop_attr);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
 
+GFAPI_SYMVER_PUBLIC(glfs_ftruncate_async34, glfs_ftruncate_async, 3.4.0)
 int
-pub_glfs_readlink (struct glfs *fs, const char *path, char *buf, size_t bufsiz)
+pub_glfs_ftruncate_async34(struct glfs_fd *glfd, off_t offset, glfs_io_cbk34 fn,
+                           void *data)
 {
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	loc_t            loc = {0, };
-	struct iatt      iatt = {0, };
-	int              reval = 0;
-	char            *linkval = NULL;
+    return glfs_ftruncate_async_common(glfd, offset, _gf_true, (void *)fn,
+                                       data);
+}
 
-	__glfs_entry_fs (fs);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_ftruncate_async, 6.0)
+int
+pub_glfs_ftruncate_async(struct glfs_fd *glfd, off_t offset, glfs_io_cbk fn,
+                         void *data)
+{
+    return glfs_ftruncate_async_common(glfd, offset, _gf_false, fn, data);
+}
 
-	subvol = priv_glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_access, 3.4.0)
+int
+pub_glfs_access(struct glfs *fs, const char *path, int mode)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
 retry:
-	ret = glfs_lresolve (fs, subvol, path, &loc, &iatt, reval);
-
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    ret = glfs_resolve(fs, subvol, path, &loc, &iatt, reval);
 
-	if (ret)
-		goto out;
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	if (iatt.ia_type != IA_IFLNK) {
-		ret = -1;
-		errno = EINVAL;
-		goto out;
-	}
+    if (ret)
+        goto out;
 
-	ret = syncop_readlink (subvol, &loc, &linkval, bufsiz);
-        DECODE_SYNCOP_ERR (ret);
-	if (ret > 0) {
-		memcpy (buf, linkval, ret);
-		GF_FREE (linkval);
-	}
+    ret = syncop_access(subvol, &loc, mode, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 out:
-	loc_wipe (&loc);
-
-	priv_glfs_subvol_done (fs, subvol);
+    loc_wipe(&loc);
 
-	return ret;
-}
+    glfs_subvol_done(fs, subvol);
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_readlink, 3.4.0);
+    __GLFS_EXIT_FS;
 
+invalid_fs:
+    return ret;
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_symlink, 3.4.0)
 int
-pub_glfs_mknod (struct glfs *fs, const char *path, mode_t mode, dev_t dev)
+pub_glfs_symlink(struct glfs *fs, const char *data, const char *path)
 {
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	loc_t            loc = {0, };
-	struct iatt      iatt = {0, };
-	uuid_t           gfid;
-	dict_t          *xattr_req = NULL;
-	int              reval = 0;
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    uuid_t gfid;
+    dict_t *xattr_req = NULL;
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    xattr_req = dict_new();
+    if (!xattr_req) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gf_uuid_generate(gfid);
+    ret = dict_set_gfuuid(xattr_req, "gfid-req", gfid, true);
+    if (ret) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+retry:
+    ret = glfs_lresolve(fs, subvol, path, &loc, &iatt, reval);
 
-	__glfs_entry_fs (fs);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	subvol = priv_glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+    if (loc.inode) {
+        errno = EEXIST;
+        ret = -1;
+        goto out;
+    }
 
-	xattr_req = dict_new ();
-	if (!xattr_req) {
-		ret = -1;
-		errno = ENOMEM;
-		goto out;
-	}
+    if (ret == -1 && errno != ENOENT)
+        /* Any other type of error is fatal */
+        goto out;
 
-	uuid_generate (gfid);
-	ret = dict_set_static_bin (xattr_req, "gfid-req", gfid, 16);
-	if (ret) {
-		ret = -1;
-		errno = ENOMEM;
-		goto out;
-	}
-retry:
-	ret = glfs_lresolve (fs, subvol, path, &loc, &iatt, reval);
+    if (ret == -1 && errno == ENOENT && !loc.parent)
+        /* The parent directory or an ancestor even
+           higher does not exist
+        */
+        goto out;
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    /* ret == -1 && errno == ENOENT */
+    loc.inode = inode_new(loc.parent->table);
+    if (!loc.inode) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
 
-	if (loc.inode) {
-		errno = EEXIST;
-		ret = -1;
-		goto out;
-	}
+    ret = syncop_symlink(subvol, &loc, data, &iatt, xattr_req, NULL);
+    DECODE_SYNCOP_ERR(ret);
 
-	if (ret == -1 && errno != ENOENT)
-		/* Any other type of error is fatal */
-		goto out;
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	if (ret == -1 && errno == ENOENT && !loc.parent)
-		/* The parent directory or an ancestor even
-		   higher does not exist
-		*/
-		goto out;
+    if (ret == 0)
+        ret = glfs_loc_link(&loc, &iatt);
+out:
+    loc_wipe(&loc);
 
-	/* ret == -1 && errno == ENOENT */
-	loc.inode = inode_new (loc.parent->table);
-	if (!loc.inode) {
-		ret = -1;
-		errno = ENOMEM;
-		goto out;
-	}
+    if (xattr_req)
+        dict_unref(xattr_req);
 
-	ret = syncop_mknod (subvol, &loc, mode, dev, xattr_req, &iatt);
-        DECODE_SYNCOP_ERR (ret);
+    glfs_subvol_done(fs, subvol);
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    __GLFS_EXIT_FS;
 
-	if (ret == 0)
-		ret = glfs_loc_link (&loc, &iatt);
-out:
-	loc_wipe (&loc);
+invalid_fs:
+    return ret;
+}
 
-	if (xattr_req)
-		dict_unref (xattr_req);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_readlink, 3.4.0)
+int
+pub_glfs_readlink(struct glfs *fs, const char *path, char *buf, size_t bufsiz)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    int reval = 0;
+    char *linkval = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+retry:
+    ret = glfs_lresolve(fs, subvol, path, &loc, &iatt, reval);
 
-	priv_glfs_subvol_done (fs, subvol);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	return ret;
-}
+    if (ret)
+        goto out;
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_mknod, 3.4.0);
+    if (iatt.ia_type != IA_IFLNK) {
+        ret = -1;
+        errno = EINVAL;
+        goto out;
+    }
 
+    ret = syncop_readlink(subvol, &loc, &linkval, bufsiz, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+    if (ret > 0) {
+        memcpy(buf, linkval, ret);
+        GF_FREE(linkval);
+    }
 
-int
-pub_glfs_mkdir (struct glfs *fs, const char *path, mode_t mode)
-{
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	loc_t            loc = {0, };
-	struct iatt      iatt = {0, };
-	uuid_t           gfid;
-	dict_t          *xattr_req = NULL;
-	int              reval = 0;
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
+out:
+    loc_wipe(&loc);
 
-	__glfs_entry_fs (fs);
+    glfs_subvol_done(fs, subvol);
 
-	subvol = priv_glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+    __GLFS_EXIT_FS;
 
-	xattr_req = dict_new ();
-	if (!xattr_req) {
-		ret = -1;
-		errno = ENOMEM;
-		goto out;
-	}
+invalid_fs:
+    return ret;
+}
 
-	uuid_generate (gfid);
-	ret = dict_set_static_bin (xattr_req, "gfid-req", gfid, 16);
-	if (ret) {
-		ret = -1;
-		errno = ENOMEM;
-		goto out;
-	}
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_mknod, 3.4.0)
+int
+pub_glfs_mknod(struct glfs *fs, const char *path, mode_t mode, dev_t dev)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    uuid_t gfid;
+    dict_t *xattr_req = NULL;
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    xattr_req = dict_new();
+    if (!xattr_req) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gf_uuid_generate(gfid);
+    ret = dict_set_gfuuid(xattr_req, "gfid-req", gfid, true);
+    if (ret) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
 retry:
-	ret = glfs_lresolve (fs, subvol, path, &loc, &iatt, reval);
+    ret = glfs_lresolve(fs, subvol, path, &loc, &iatt, reval);
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	if (loc.inode) {
-		errno = EEXIST;
-		ret = -1;
-		goto out;
-	}
+    if (loc.inode) {
+        errno = EEXIST;
+        ret = -1;
+        goto out;
+    }
 
-	if (ret == -1 && errno != ENOENT)
-		/* Any other type of error is fatal */
-		goto out;
+    if (ret == -1 && errno != ENOENT)
+        /* Any other type of error is fatal */
+        goto out;
 
-	if (ret == -1 && errno == ENOENT && !loc.parent)
-		/* The parent directory or an ancestor even
-		   higher does not exist
-		*/
-		goto out;
+    if (ret == -1 && errno == ENOENT && !loc.parent)
+        /* The parent directory or an ancestor even
+           higher does not exist
+        */
+        goto out;
 
-	/* ret == -1 && errno == ENOENT */
-	loc.inode = inode_new (loc.parent->table);
-	if (!loc.inode) {
-		ret = -1;
-		errno = ENOMEM;
-		goto out;
-	}
+    /* ret == -1 && errno == ENOENT */
+    loc.inode = inode_new(loc.parent->table);
+    if (!loc.inode) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
 
-	ret = syncop_mkdir (subvol, &loc, mode, xattr_req, &iatt);
-        DECODE_SYNCOP_ERR (ret);
+    ret = syncop_mknod(subvol, &loc, mode, dev, &iatt, xattr_req, NULL);
+    DECODE_SYNCOP_ERR(ret);
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	if (ret == 0)
-		ret = glfs_loc_link (&loc, &iatt);
+    if (ret == 0)
+        ret = glfs_loc_link(&loc, &iatt);
 out:
-	loc_wipe (&loc);
-
-	if (xattr_req)
-		dict_unref (xattr_req);
+    loc_wipe(&loc);
 
-	priv_glfs_subvol_done (fs, subvol);
+    if (xattr_req)
+        dict_unref(xattr_req);
 
-	return ret;
-}
+    glfs_subvol_done(fs, subvol);
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_mkdir, 3.4.0);
+    __GLFS_EXIT_FS;
 
+invalid_fs:
+    return ret;
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_mkdir, 3.4.0)
 int
-pub_glfs_unlink (struct glfs *fs, const char *path)
+pub_glfs_mkdir(struct glfs *fs, const char *path, mode_t mode)
 {
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	loc_t            loc = {0, };
-	struct iatt      iatt = {0, };
-	int              reval = 0;
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    uuid_t gfid;
+    dict_t *xattr_req = NULL;
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    xattr_req = dict_new();
+    if (!xattr_req) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gf_uuid_generate(gfid);
+    ret = dict_set_gfuuid(xattr_req, "gfid-req", gfid, true);
+    if (ret) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+retry:
+    ret = glfs_lresolve(fs, subvol, path, &loc, &iatt, reval);
 
-	__glfs_entry_fs (fs);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	subvol = priv_glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
-retry:
-	ret = glfs_lresolve (fs, subvol, path, &loc, &iatt, reval);
+    if (loc.inode) {
+        errno = EEXIST;
+        ret = -1;
+        goto out;
+    }
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    if (ret == -1 && errno != ENOENT)
+        /* Any other type of error is fatal */
+        goto out;
 
-	if (ret)
-		goto out;
+    if (ret == -1 && errno == ENOENT && !loc.parent)
+        /* The parent directory or an ancestor even
+           higher does not exist
+        */
+        goto out;
 
-	if (iatt.ia_type == IA_IFDIR) {
-		ret = -1;
-		errno = EISDIR;
-		goto out;
-	}
+    /* ret == -1 && errno == ENOENT */
+    loc.inode = inode_new(loc.parent->table);
+    if (!loc.inode) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
 
-	ret = syncop_unlink (subvol, &loc);
-        DECODE_SYNCOP_ERR (ret);
+    ret = syncop_mkdir(subvol, &loc, mode, &iatt, xattr_req, NULL);
+    DECODE_SYNCOP_ERR(ret);
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	if (ret == 0)
-		ret = glfs_loc_unlink (&loc);
+    if (ret == 0)
+        ret = glfs_loc_link(&loc, &iatt);
 out:
-	loc_wipe (&loc);
+    loc_wipe(&loc);
 
-	priv_glfs_subvol_done (fs, subvol);
+    if (xattr_req)
+        dict_unref(xattr_req);
 
-	return ret;
-}
+    glfs_subvol_done(fs, subvol);
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_unlink, 3.4.0);
+    __GLFS_EXIT_FS;
 
+invalid_fs:
+    return ret;
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_unlink, 3.4.0)
 int
-pub_glfs_rmdir (struct glfs *fs, const char *path)
+pub_glfs_unlink(struct glfs *fs, const char *path)
 {
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	loc_t            loc = {0, };
-	struct iatt      iatt = {0, };
-	int              reval = 0;
-
-	__glfs_entry_fs (fs);
-
-	subvol = priv_glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
 retry:
-	ret = glfs_lresolve (fs, subvol, path, &loc, &iatt, reval);
+    ret = glfs_lresolve(fs, subvol, path, &loc, &iatt, reval);
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	if (ret)
-		goto out;
+    if (ret)
+        goto out;
 
-	if (iatt.ia_type != IA_IFDIR) {
-		ret = -1;
-		errno = ENOTDIR;
-		goto out;
-	}
+    if (iatt.ia_type == IA_IFDIR) {
+        ret = -1;
+        errno = EISDIR;
+        goto out;
+    }
 
-	ret = syncop_rmdir (subvol, &loc, 0);
-        DECODE_SYNCOP_ERR (ret);
+    /* TODO: Add leaseid */
+    ret = syncop_unlink(subvol, &loc, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	if (ret == 0)
-		ret = glfs_loc_unlink (&loc);
+    if (ret == 0)
+        ret = glfs_loc_unlink(&loc);
 out:
-	loc_wipe (&loc);
+    loc_wipe(&loc);
 
-	priv_glfs_subvol_done (fs, subvol);
+    glfs_subvol_done(fs, subvol);
 
-	return ret;
-}
-
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_rmdir, 3.4.0);
+    __GLFS_EXIT_FS;
 
+invalid_fs:
+    return ret;
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_rmdir, 3.4.0)
 int
-pub_glfs_rename (struct glfs *fs, const char *oldpath, const char *newpath)
+pub_glfs_rmdir(struct glfs *fs, const char *path)
 {
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	loc_t            oldloc = {0, };
-	loc_t            newloc = {0, };
-	struct iatt      oldiatt = {0, };
-	struct iatt      newiatt = {0, };
-	int              reval = 0;
-
-	__glfs_entry_fs (fs);
-
-	subvol = priv_glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
 retry:
-	ret = glfs_lresolve (fs, subvol, oldpath, &oldloc, &oldiatt, reval);
+    ret = glfs_lresolve(fs, subvol, path, &loc, &iatt, reval);
 
-	ESTALE_RETRY (ret, errno, reval, &oldloc, retry);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	if (ret)
-		goto out;
-retrynew:
-	ret = glfs_lresolve (fs, subvol, newpath, &newloc, &newiatt, reval);
-
-	ESTALE_RETRY (ret, errno, reval, &newloc, retrynew);
-
-	if (ret && errno != ENOENT && newloc.parent)
-		goto out;
-
-	if (newiatt.ia_type != IA_INVAL) {
-                if ((oldiatt.ia_type == IA_IFDIR) !=
-                    (newiatt.ia_type == IA_IFDIR)) {
-                       /* Either both old and new must be dirs,
-                        * or both must be non-dirs. Else, fail.
-                        */
-                       ret = -1;
-                       errno = EISDIR;
-                       goto out;
-                }
-        }
+    if (ret)
+        goto out;
 
-	/* TODO: check if new or old is a prefix of the other, and fail EINVAL */
+    if (iatt.ia_type != IA_IFDIR) {
+        ret = -1;
+        errno = ENOTDIR;
+        goto out;
+    }
 
-	ret = syncop_rename (subvol, &oldloc, &newloc);
-        DECODE_SYNCOP_ERR (ret);
+    ret = syncop_rmdir(subvol, &loc, 0, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
 
-	if (ret == -1 && errno == ESTALE) {
-		if (reval < DEFAULT_REVAL_COUNT) {
-			reval++;
-			loc_wipe (&oldloc);
-			loc_wipe (&newloc);
-			goto retry;
-		}
-	}
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	if (ret == 0)
-		inode_rename (oldloc.parent->table, oldloc.parent, oldloc.name,
-			      newloc.parent, newloc.name, oldloc.inode,
-			      &oldiatt);
+    if (ret == 0)
+        ret = glfs_loc_unlink(&loc);
 out:
-	loc_wipe (&oldloc);
-	loc_wipe (&newloc);
-
-	priv_glfs_subvol_done (fs, subvol);
+    loc_wipe(&loc);
 
-	return ret;
-}
+    glfs_subvol_done(fs, subvol);
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_rename, 3.4.0);
+    __GLFS_EXIT_FS;
 
+invalid_fs:
+    return ret;
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_rename, 3.4.0)
 int
-pub_glfs_link (struct glfs *fs, const char *oldpath, const char *newpath)
+pub_glfs_rename(struct glfs *fs, const char *oldpath, const char *newpath)
 {
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	loc_t            oldloc = {0, };
-	loc_t            newloc = {0, };
-	struct iatt      oldiatt = {0, };
-	struct iatt      newiatt = {0, };
-	int              reval = 0;
-
-	__glfs_entry_fs (fs);
-
-	subvol = priv_glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    loc_t oldloc = {
+        0,
+    };
+    loc_t newloc = {
+        0,
+    };
+    struct iatt oldiatt = {
+        0,
+    };
+    struct iatt newiatt = {
+        0,
+    };
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
 retry:
-	ret = glfs_lresolve (fs, subvol, oldpath, &oldloc, &oldiatt, reval);
+    ret = glfs_lresolve(fs, subvol, oldpath, &oldloc, &oldiatt, reval);
 
-	ESTALE_RETRY (ret, errno, reval, &oldloc, retry);
+    ESTALE_RETRY(ret, errno, reval, &oldloc, retry);
 
-	if (ret)
-		goto out;
+    if (ret)
+        goto out;
 retrynew:
-	ret = glfs_lresolve (fs, subvol, newpath, &newloc, &newiatt, reval);
+    ret = glfs_lresolve(fs, subvol, newpath, &newloc, &newiatt, reval);
 
-	ESTALE_RETRY (ret, errno, reval, &newloc, retrynew);
+    ESTALE_RETRY(ret, errno, reval, &newloc, retrynew);
 
-	if (ret == 0) {
-		ret = -1;
-		errno = EEXIST;
-		goto out;
-	}
+    if (ret && errno != ENOENT && newloc.parent)
+        goto out;
 
-	if (oldiatt.ia_type == IA_IFDIR) {
-		ret = -1;
-		errno = EISDIR;
-		goto out;
-	}
+    if (newiatt.ia_type != IA_INVAL) {
+        if ((oldiatt.ia_type == IA_IFDIR) != (newiatt.ia_type == IA_IFDIR)) {
+            /* Either both old and new must be dirs,
+             * or both must be non-dirs. Else, fail.
+             */
+            ret = -1;
+            errno = EISDIR;
+            goto out;
+        }
+    }
 
-        /* Filling the inode of the hard link to be same as that of the
-           original file
-        */
-	if (newloc.inode) {
-		inode_unref (newloc.inode);
-		newloc.inode = NULL;
-	}
-        newloc.inode = inode_ref (oldloc.inode);
+    /* TODO: - check if new or old is a prefix of the other, and fail EINVAL
+     *       - Add leaseid */
 
-	ret = syncop_link (subvol, &oldloc, &newloc);
-        DECODE_SYNCOP_ERR (ret);
+    ret = syncop_rename(subvol, &oldloc, &newloc, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
 
-	if (ret == -1 && errno == ESTALE) {
-		loc_wipe (&oldloc);
-		loc_wipe (&newloc);
-		if (reval--)
-			goto retry;
-	}
+    if (ret == -1 && errno == ESTALE) {
+        if (reval < DEFAULT_REVAL_COUNT) {
+            reval++;
+            loc_wipe(&oldloc);
+            loc_wipe(&newloc);
+            goto retry;
+        }
+    }
+
+    if (ret == 0) {
+        inode_rename(oldloc.parent->table, oldloc.parent, oldloc.name,
+                     newloc.parent, newloc.name, oldloc.inode, &oldiatt);
 
-	if (ret == 0)
-		ret = glfs_loc_link (&newloc, &oldiatt);
+        if (newloc.inode && !inode_has_dentry(newloc.inode))
+            inode_forget(newloc.inode, 0);
+    }
 out:
-	loc_wipe (&oldloc);
-	loc_wipe (&newloc);
+    loc_wipe(&oldloc);
+    loc_wipe(&newloc);
+
+    glfs_subvol_done(fs, subvol);
 
-	priv_glfs_subvol_done (fs, subvol);
+    __GLFS_EXIT_FS;
 
-	return ret;
+invalid_fs:
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_link, 3.4.0);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_link, 3.4.0)
+int
+pub_glfs_link(struct glfs *fs, const char *oldpath, const char *newpath)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    loc_t oldloc = {
+        0,
+    };
+    loc_t newloc = {
+        0,
+    };
+    struct iatt oldiatt = {
+        0,
+    };
+    struct iatt newiatt = {
+        0,
+    };
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+retry:
+    ret = glfs_lresolve(fs, subvol, oldpath, &oldloc, &oldiatt, reval);
 
+    ESTALE_RETRY(ret, errno, reval, &oldloc, retry);
 
-struct glfs_fd *
-pub_glfs_opendir (struct glfs *fs, const char *path)
-{
-	int              ret = -1;
-	struct glfs_fd  *glfd = NULL;
-	xlator_t        *subvol = NULL;
-	loc_t            loc = {0, };
-	struct iatt      iatt = {0, };
-	int              reval = 0;
+    if (ret)
+        goto out;
+retrynew:
+    ret = glfs_lresolve(fs, subvol, newpath, &newloc, &newiatt, reval);
+
+    ESTALE_RETRY(ret, errno, reval, &newloc, retrynew);
+
+    if (ret == 0) {
+        ret = -1;
+        errno = EEXIST;
+        goto out;
+    }
+
+    if (oldiatt.ia_type == IA_IFDIR) {
+        ret = -1;
+        errno = EISDIR;
+        goto out;
+    }
+
+    /* Filling the inode of the hard link to be same as that of the
+       original file
+    */
+    if (newloc.inode) {
+        inode_unref(newloc.inode);
+        newloc.inode = NULL;
+    }
+    newloc.inode = inode_ref(oldloc.inode);
+
+    ret = syncop_link(subvol, &oldloc, &newloc, &newiatt, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    if (ret == -1 && errno == ESTALE) {
+        loc_wipe(&oldloc);
+        loc_wipe(&newloc);
+        if (reval--)
+            goto retry;
+    }
+
+    if (ret == 0)
+        ret = glfs_loc_link(&newloc, &newiatt);
+out:
+    loc_wipe(&oldloc);
+    loc_wipe(&newloc);
 
-	__glfs_entry_fs (fs);
+    glfs_subvol_done(fs, subvol);
 
-	subvol = priv_glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+    __GLFS_EXIT_FS;
 
-	glfd = glfs_fd_new (fs);
-	if (!glfd)
-		goto out;
+invalid_fs:
+    return ret;
+}
 
-	INIT_LIST_HEAD (&glfd->entries);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_opendir, 3.4.0)
+struct glfs_fd *
+pub_glfs_opendir(struct glfs *fs, const char *path)
+{
+    int ret = -1;
+    struct glfs_fd *glfd = NULL;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    glfd = glfs_fd_new(fs);
+    if (!glfd)
+        goto out;
+
+    INIT_LIST_HEAD(&glfd->entries);
 retry:
-	ret = glfs_resolve (fs, subvol, path, &loc, &iatt, reval);
+    ret = glfs_resolve(fs, subvol, path, &loc, &iatt, reval);
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	if (ret)
-		goto out;
+    if (ret)
+        goto out;
 
-	if (!IA_ISDIR (iatt.ia_type)) {
-		ret = -1;
-		errno = ENOTDIR;
-		goto out;
-	}
+    if (!IA_ISDIR(iatt.ia_type)) {
+        ret = -1;
+        errno = ENOTDIR;
+        goto out;
+    }
 
-	if (glfd->fd) {
-		/* Retry. Safe to touch glfd->fd as we
-		   still have not glfs_fd_bind() yet.
-		*/
-		fd_unref (glfd->fd);
-		glfd->fd = NULL;
-	}
+    if (glfd->fd) {
+        /* Retry. Safe to touch glfd->fd as we
+           still have not glfs_fd_bind() yet.
+        */
+        fd_unref(glfd->fd);
+        glfd->fd = NULL;
+    }
 
-	glfd->fd = fd_create (loc.inode, getpid());
-	if (!glfd->fd) {
-		ret = -1;
-		errno = ENOMEM;
-		goto out;
-	}
+    glfd->fd = fd_create(loc.inode, getpid());
+    if (!glfd->fd) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
 
-	ret = syncop_opendir (subvol, &loc, glfd->fd);
-        DECODE_SYNCOP_ERR (ret);
+    ret = syncop_opendir(subvol, &loc, glfd->fd, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 out:
-	loc_wipe (&loc);
+    loc_wipe(&loc);
 
-	if (ret && glfd) {
-		glfs_fd_destroy (glfd);
-		glfd = NULL;
-	} else if (glfd) {
-		fd_bind (glfd->fd);
-		glfs_fd_bind (glfd);
-	}
+    if (ret && glfd) {
+        GF_REF_PUT(glfd);
+        glfd = NULL;
+    } else if (glfd) {
+        glfd_set_state_bind(glfd);
+    }
 
-	priv_glfs_subvol_done (fs, subvol);
+    glfs_subvol_done(fs, subvol);
 
-	return glfd;
-}
-
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_opendir, 3.4.0);
+    __GLFS_EXIT_FS;
 
+invalid_fs:
+    return glfd;
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_closedir, 3.4.0)
 int
-pub_glfs_closedir (struct glfs_fd *glfd)
+pub_glfs_closedir(struct glfs_fd *glfd)
 {
-	__glfs_entry_fd (glfd);
+    int ret = -1;
 
-	gf_dirent_free (list_entry (&glfd->entries, gf_dirent_t, list));
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
 
-	glfs_fd_destroy (glfd);
+    gf_dirent_free(list_entry(&glfd->entries, gf_dirent_t, list));
 
-	return 0;
-}
+    glfs_mark_glfd_for_deletion(glfd);
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_closedir, 3.4.0);
+    __GLFS_EXIT_FS;
 
+    ret = 0;
 
-long
-pub_glfs_telldir (struct glfs_fd *fd)
-{
-	return fd->offset;
+invalid_fs:
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_telldir, 3.4.0);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_telldir, 3.4.0)
+long
+pub_glfs_telldir(struct glfs_fd *fd)
+{
+    if (fd == NULL) {
+        errno = EBADF;
+        return -1;
+    }
 
+    return fd->offset;
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_seekdir, 3.4.0)
 void
-pub_glfs_seekdir (struct glfs_fd *fd, long offset)
+pub_glfs_seekdir(struct glfs_fd *fd, long offset)
 {
-	gf_dirent_t *entry = NULL;
-	gf_dirent_t *tmp = NULL;
+    gf_dirent_t *entry = NULL;
+    gf_dirent_t *tmp = NULL;
 
-	if (fd->offset == offset)
-		return;
+    if (fd == NULL) {
+        errno = EBADF;
+        return;
+    }
 
-	fd->offset = offset;
-	fd->next = NULL;
+    if (fd->offset == offset)
+        return;
 
-	list_for_each_entry_safe (entry, tmp, &fd->entries, list) {
-		if (entry->d_off != offset)
-			continue;
+    fd->offset = offset;
+    fd->next = NULL;
 
-		if (&tmp->list != &fd->entries) {
-			/* found! */
-			fd->next = tmp;
-			return;
-		}
-	}
-	/* could not find entry at requested offset in the cache.
-	   next readdir_r() will result in glfd_entry_refresh()
-	*/
-}
-
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_seekdir, 3.4.0);
+    list_for_each_entry_safe(entry, tmp, &fd->entries, list)
+    {
+        if (entry->d_off != offset)
+            continue;
 
+        if (&tmp->list != &fd->entries) {
+            /* found! */
+            fd->next = tmp;
+            return;
+        }
+    }
+    /* could not find entry at requested offset in the cache.
+       next readdir_r() will result in glfd_entry_refresh()
+    */
+}
 
-int
-pub_glfs_discard_async (struct glfs_fd *glfd, off_t offset, size_t len,
-                        glfs_io_cbk fn, void *data)
+static int
+glfs_discard_async_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno,
+                       struct iatt *preop_stbuf, struct iatt *postop_stbuf,
+                       dict_t *xdata)
 {
-	struct glfs_io *gio = NULL;
-	int             ret = 0;
-
-	gio = GF_CALLOC (1, sizeof (*gio), glfs_mt_glfs_io_t);
-	if (!gio) {
-		errno = ENOMEM;
-		return -1;
-	}
-
-	gio->op     = GF_FOP_DISCARD;
-	gio->glfd   = glfd;
-	gio->offset = offset;
-	gio->count  = len;
-	gio->fn     = fn;
-	gio->data   = data;
+    glfs_io_async_cbk(op_ret, op_errno, frame, cookie, NULL, 0, preop_stbuf,
+                      postop_stbuf);
 
-	ret = synctask_new (pub_glfs_from_glfd (glfd)->ctx->env,
-			    glfs_io_async_task, glfs_io_async_cbk,
-			    NULL, gio);
-
-	if (ret) {
-		GF_FREE (gio->iov);
-		GF_FREE (gio);
-	}
-
-	return ret;
+    return 0;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_discard_async, 3.5.0);
-
+static int
+glfs_discard_async_common(struct glfs_fd *glfd, off_t offset, size_t len,
+                          gf_boolean_t oldcb, glfs_io_cbk fn, void *data)
+{
+    struct glfs_io *gio = NULL;
+    int ret = -1;
+    call_frame_t *frame = NULL;
+    xlator_t *subvol = NULL;
+    fd_t *fd = NULL;
+    dict_t *fop_attr = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    /* Need to take explicit ref so that the fd
+     * is not destroyed before the fop is complete
+     */
+    GF_REF_GET(glfd);
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        errno = EBADFD;
+        goto out;
+    }
+
+    frame = syncop_create_frame(THIS);
+    if (!frame) {
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gio = GF_CALLOC(1, sizeof(*gio), glfs_mt_glfs_io_t);
+    if (!gio) {
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gio->op = GF_FOP_DISCARD;
+    gio->glfd = glfd;
+    gio->offset = offset;
+    gio->count = len;
+    gio->oldcb = oldcb;
+    gio->fn = fn;
+    gio->data = data;
+
+    frame->local = gio;
+    ret = get_fop_attr_thrd_key(&fop_attr);
+    if (ret)
+        gf_msg_debug("gfapi", 0, "Getting leaseid from thread failed");
+
+    STACK_WIND_COOKIE(frame, glfs_discard_async_cbk, subvol, subvol,
+                      subvol->fops->discard, fd, offset, len, fop_attr);
+
+    ret = 0;
+out:
+    if (ret) {
+        if (fd)
+            fd_unref(fd);
+        if (glfd)
+            GF_REF_PUT(glfd);
+        GF_FREE(gio);
+        if (frame)
+            STACK_DESTROY(frame->root);
+        glfs_subvol_done(glfd->fs, subvol);
+    }
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
 
+GFAPI_SYMVER_PUBLIC(glfs_discard_async35, glfs_discard_async, 3.5.0)
 int
-pub_glfs_zerofill_async (struct glfs_fd *glfd, off_t offset, off_t len,
-                         glfs_io_cbk fn, void *data)
+pub_glfs_discard_async35(struct glfs_fd *glfd, off_t offset, size_t len,
+                         glfs_io_cbk34 fn, void *data)
 {
-        struct glfs_io *gio  = NULL;
-        int             ret  = 0;
-
-        gio = GF_CALLOC (1, sizeof (*gio), glfs_mt_glfs_io_t);
-        if (!gio) {
-                errno = ENOMEM;
-                return -1;
-        }
+    return glfs_discard_async_common(glfd, offset, len, _gf_true, (void *)fn,
+                                     data);
+}
 
-        gio->op     = GF_FOP_ZEROFILL;
-        gio->glfd   = glfd;
-        gio->offset = offset;
-        gio->count  = len;
-        gio->fn     = fn;
-        gio->data   = data;
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_discard_async, 6.0)
+int
+pub_glfs_discard_async(struct glfs_fd *glfd, off_t offset, size_t len,
+                       glfs_io_cbk fn, void *data)
+{
+    return glfs_discard_async_common(glfd, offset, len, _gf_false, fn, data);
+}
 
-        ret = synctask_new (pub_glfs_from_glfd (glfd)->ctx->env,
-                            glfs_io_async_task, glfs_io_async_cbk,
-                            NULL, gio);
+static int
+glfs_zerofill_async_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno,
+                        struct iatt *preop_stbuf, struct iatt *postop_stbuf,
+                        dict_t *xdata)
+{
+    glfs_io_async_cbk(op_ret, op_errno, frame, cookie, NULL, 0, preop_stbuf,
+                      postop_stbuf);
 
-        if (ret) {
-                GF_FREE (gio->iov);
-                GF_FREE (gio);
-        }
+    return 0;
+}
 
-        return ret;
+static int
+glfs_zerofill_async_common(struct glfs_fd *glfd, off_t offset, off_t len,
+                           gf_boolean_t oldcb, glfs_io_cbk fn, void *data)
+{
+    struct glfs_io *gio = NULL;
+    int ret = -1;
+    call_frame_t *frame = NULL;
+    xlator_t *subvol = NULL;
+    fd_t *fd = NULL;
+    dict_t *fop_attr = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    /* Need to take explicit ref so that the fd
+     * is not destroyed before the fop is complete
+     */
+    GF_REF_GET(glfd);
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        errno = EBADFD;
+        goto out;
+    }
+
+    frame = syncop_create_frame(THIS);
+    if (!frame) {
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gio = GF_CALLOC(1, sizeof(*gio), glfs_mt_glfs_io_t);
+    if (!gio) {
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gio->op = GF_FOP_ZEROFILL;
+    gio->glfd = glfd;
+    gio->offset = offset;
+    gio->count = len;
+    gio->oldcb = oldcb;
+    gio->fn = fn;
+    gio->data = data;
+
+    frame->local = gio;
+
+    ret = get_fop_attr_thrd_key(&fop_attr);
+    if (ret)
+        gf_msg_debug("gfapi", 0, "Getting leaseid from thread failed");
+
+    STACK_WIND_COOKIE(frame, glfs_zerofill_async_cbk, subvol, subvol,
+                      subvol->fops->zerofill, fd, offset, len, fop_attr);
+    ret = 0;
+out:
+    if (ret) {
+        if (fd)
+            fd_unref(fd);
+        if (glfd)
+            GF_REF_PUT(glfd);
+        GF_FREE(gio);
+        if (frame)
+            STACK_DESTROY(frame->root);
+        glfs_subvol_done(glfd->fs, subvol);
+    }
+    if (fop_attr)
+        dict_unref(fop_attr);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_zerofill_async, 3.5.0);
+GFAPI_SYMVER_PUBLIC(glfs_zerofill_async35, glfs_zerofill_async, 3.5.0)
+int
+pub_glfs_zerofill_async35(struct glfs_fd *glfd, off_t offset, off_t len,
+                          glfs_io_cbk34 fn, void *data)
+{
+    return glfs_zerofill_async_common(glfd, offset, len, _gf_true, (void *)fn,
+                                      data);
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_zerofill_async, 6.0)
+int
+pub_glfs_zerofill_async(struct glfs_fd *glfd, off_t offset, off_t len,
+                        glfs_io_cbk fn, void *data)
+{
+    return glfs_zerofill_async_common(glfd, offset, len, _gf_false, fn, data);
+}
 
 void
-gf_dirent_to_dirent (gf_dirent_t *gf_dirent, struct dirent *dirent)
+gf_dirent_to_dirent(gf_dirent_t *gf_dirent, struct dirent *dirent)
 {
-	dirent->d_ino = gf_dirent->d_ino;
+    dirent->d_ino = gf_dirent->d_ino;
 
 #ifdef _DIRENT_HAVE_D_OFF
-	dirent->d_off = gf_dirent->d_off;
+    dirent->d_off = gf_dirent->d_off;
 #endif
 
 #ifdef _DIRENT_HAVE_D_TYPE
-	dirent->d_type = gf_dirent->d_type;
+    dirent->d_type = gf_dirent->d_type;
 #endif
 
 #ifdef _DIRENT_HAVE_D_NAMLEN
-	dirent->d_namlen = strlen (gf_dirent->d_name);
+    dirent->d_namlen = strlen(gf_dirent->d_name);
 #endif
 
-	strncpy (dirent->d_name, gf_dirent->d_name, GF_NAME_MAX + 1);
+    snprintf(dirent->d_name, NAME_MAX + 1, "%s", gf_dirent->d_name);
 }
 
-
 int
-glfd_entry_refresh (struct glfs_fd *glfd, int plus)
+glfd_entry_refresh(struct glfs_fd *glfd, int plus)
 {
-	xlator_t        *subvol = NULL;
-	gf_dirent_t      entries;
-	gf_dirent_t      old;
-	int              ret = -1;
-	fd_t            *fd = NULL;
-
-	subvol = priv_glfs_active_subvol (glfd->fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
-
-	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
-	if (!fd) {
-		ret = -1;
-		errno = EBADFD;
-		goto out;
-	}
-
-	if (fd->inode->ia_type != IA_IFDIR) {
-		ret = -1;
-		errno = EBADF;
-		goto out;
-	}
-
-	INIT_LIST_HEAD (&entries.list);
-	INIT_LIST_HEAD (&old.list);
+    xlator_t *subvol = NULL;
+    gf_dirent_t entries;
+    gf_dirent_t old;
+    gf_dirent_t *entry = NULL;
+    int ret = -1;
+    fd_t *fd = NULL;
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    if (fd->inode->ia_type != IA_IFDIR) {
+        ret = -1;
+        errno = EBADF;
+        goto out;
+    }
+
+    INIT_LIST_HEAD(&entries.list);
+    INIT_LIST_HEAD(&old.list);
+
+    if (plus)
+        ret = syncop_readdirp(subvol, fd, 131072, glfd->offset, &entries, NULL,
+                              NULL);
+    else
+        ret = syncop_readdir(subvol, fd, 131072, glfd->offset, &entries, NULL,
+                             NULL);
+    DECODE_SYNCOP_ERR(ret);
+    if (ret >= 0) {
+        if (plus) {
+            list_for_each_entry(entry, &entries.list, list)
+            {
+                if ((!entry->inode && (!IA_ISDIR(entry->d_stat.ia_type))) ||
+                    ((entry->d_stat.ia_ctime == 0) &&
+                     strcmp(entry->d_name, ".") &&
+                     strcmp(entry->d_name, ".."))) {
+                    /* entry->inode for directories will be
+                     * always set to null to force a lookup
+                     * on the dentry. Hence to not degrade
+                     * readdir performance, we skip lookups
+                     * for directory entries. Also we will have
+                     * proper stat if directory present on
+                     * hashed subvolume.
+                     *
+                     * In addition, if the stat is invalid, force
+                     * lookup to fetch proper stat.
+                     */
+                    gf_fill_iatt_for_dirent(entry, fd->inode, subvol);
+                }
+            }
 
-	if (plus)
-		ret = syncop_readdirp (subvol, fd, 131072, glfd->offset,
-				       NULL, &entries);
-	else
-		ret = syncop_readdir (subvol, fd, 131072, glfd->offset,
-				      &entries);
-        DECODE_SYNCOP_ERR (ret);
-	if (ret >= 0) {
-		if (plus)
-			gf_link_inodes_from_dirent (THIS, fd->inode, &entries);
+            gf_link_inodes_from_dirent(THIS, fd->inode, &entries);
+        }
 
-		list_splice_init (&glfd->entries, &old.list);
-		list_splice_init (&entries.list, &glfd->entries);
+        list_splice_init(&glfd->entries, &old.list);
+        list_splice_init(&entries.list, &glfd->entries);
 
-		/* spurious errno is dangerous for glfd_entry_next() */
-		errno = 0;
-	}
+        /* spurious errno is dangerous for glfd_entry_next() */
+        errno = 0;
+    }
 
-	if (ret > 0)
-		glfd->next = list_entry (glfd->entries.next, gf_dirent_t, list);
+    if (ret > 0)
+        glfd->next = list_entry(glfd->entries.next, gf_dirent_t, list);
 
-	gf_dirent_free (&old);
+    gf_dirent_free(&old);
 out:
-	if (fd)
-		fd_unref (fd);
+    if (fd)
+        fd_unref(fd);
 
-	priv_glfs_subvol_done (glfd->fs, subvol);
+    glfs_subvol_done(glfd->fs, subvol);
 
-	return ret;
+    return ret;
 }
 
-
 gf_dirent_t *
-glfd_entry_next (struct glfs_fd *glfd, int plus)
+glfd_entry_next(struct glfs_fd *glfd, int plus)
 {
-	gf_dirent_t     *entry = NULL;
-	int              ret = -1;
+    gf_dirent_t *entry = NULL;
+    int ret = -1;
 
-	if (!glfd->offset || !glfd->next) {
-		ret = glfd_entry_refresh (glfd, plus);
-		if (ret < 0)
-			return NULL;
-	}
+    if (!glfd->offset || !glfd->next) {
+        ret = glfd_entry_refresh(glfd, plus);
+        if (ret < 0)
+            return NULL;
+    }
 
-	entry = glfd->next;
-	if (!entry)
-		return NULL;
+    entry = glfd->next;
+    if (!entry)
+        return NULL;
 
-	if (&entry->next->list == &glfd->entries)
-		glfd->next = NULL;
-	else
-		glfd->next = entry->next;
+    if (&entry->next->list == &glfd->entries)
+        glfd->next = NULL;
+    else
+        glfd->next = entry->next;
 
-	glfd->offset = entry->d_off;
+    glfd->offset = entry->d_off;
 
-	return entry;
+    return entry;
 }
 
-
-static struct dirent *
-glfs_readdirbuf_get (struct glfs_fd *glfd)
+struct dirent *
+glfs_readdirbuf_get(struct glfs_fd *glfd)
 {
-        struct dirent *buf = NULL;
-
-        LOCK (&glfd->fd->lock);
-        {
-                buf = glfd->readdirbuf;
-                if (buf) {
-                        memset (buf, 0, READDIRBUF_SIZE);
-                        goto unlock;
-                }
-
-                buf = GF_CALLOC (1, READDIRBUF_SIZE, glfs_mt_readdirbuf_t);
-                if (!buf) {
-                        errno = ENOMEM;
-                        goto unlock;
-                }
+    struct dirent *buf = NULL;
+
+    LOCK(&glfd->fd->lock);
+    {
+        buf = glfd->readdirbuf;
+        if (buf) {
+            memset(buf, 0, READDIRBUF_SIZE);
+            goto unlock;
+        }
 
-                glfd->readdirbuf = buf;
+        buf = GF_CALLOC(1, READDIRBUF_SIZE, glfs_mt_readdirbuf_t);
+        if (!buf) {
+            errno = ENOMEM;
+            goto unlock;
         }
+
+        glfd->readdirbuf = buf;
+    }
 unlock:
-        UNLOCK (&glfd->fd->lock);
+    UNLOCK(&glfd->fd->lock);
 
-        return buf;
+    return buf;
 }
 
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_readdirplus_r, 3.4.0)
 int
-pub_glfs_readdirplus_r (struct glfs_fd *glfd, struct stat *stat,
-                        struct dirent *ext, struct dirent **res)
+pub_glfs_readdirplus_r(struct glfs_fd *glfd, struct stat *stat,
+                       struct dirent *ext, struct dirent **res)
 {
-	int              ret = 0;
-	gf_dirent_t     *entry = NULL;
-	struct dirent   *buf = NULL;
+    int ret = 0;
+    gf_dirent_t *entry = NULL;
+    struct dirent *buf = NULL;
 
-	__glfs_entry_fd (glfd);
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
 
-	errno = 0;
+    GF_REF_GET(glfd);
 
-	if (ext)
-		buf = ext;
-	else
-		buf = glfs_readdirbuf_get (glfd);
+    errno = 0;
 
-	if (!buf) {
-		errno = ENOMEM;
-		return -1;
-	}
+    if (ext)
+        buf = ext;
+    else
+        buf = glfs_readdirbuf_get(glfd);
 
-	entry = glfd_entry_next (glfd, !!stat);
-	if (errno)
-		ret = -1;
+    if (!buf) {
+        errno = ENOMEM;
+        ret = -1;
+        goto out;
+    }
 
-	if (res) {
-		if (entry)
-			*res = buf;
-		else
-			*res = NULL;
-	}
+    entry = glfd_entry_next(glfd, !!stat);
+    if (errno)
+        ret = -1;
 
-	if (entry) {
-		gf_dirent_to_dirent (entry, buf);
-		if (stat)
-			glfs_iatt_to_stat (glfd->fs, &entry->d_stat, stat);
-	}
+    if (res) {
+        if (entry)
+            *res = buf;
+        else
+            *res = NULL;
+    }
 
-	return ret;
-}
+    if (entry) {
+        gf_dirent_to_dirent(entry, buf);
+        if (stat)
+            glfs_iatt_to_stat(glfd->fs, &entry->d_stat, stat);
+    }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_readdirplus_r, 3.4.0);
+out:
+    if (glfd)
+        GF_REF_PUT(glfd);
+
+    __GLFS_EXIT_FS;
+
+    return ret;
 
+invalid_fs:
+    return -1;
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_readdir_r, 3.4.0)
 int
-pub_glfs_readdir_r (struct glfs_fd *glfd, struct dirent *buf,
-                    struct dirent **res)
+pub_glfs_readdir_r(struct glfs_fd *glfd, struct dirent *buf,
+                   struct dirent **res)
 {
-	return pub_glfs_readdirplus_r (glfd, 0, buf, res);
+    return pub_glfs_readdirplus_r(glfd, 0, buf, res);
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_readdir_r, 3.4.0);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_readdirplus, 3.5.0)
 struct dirent *
-pub_glfs_readdirplus (struct glfs_fd *glfd, struct stat *stat)
+pub_glfs_readdirplus(struct glfs_fd *glfd, struct stat *stat)
 {
-        struct dirent *res = NULL;
-        int ret = -1;
+    struct dirent *res = NULL;
+    int ret = -1;
 
-        ret = pub_glfs_readdirplus_r (glfd, stat, NULL, &res);
-        if (ret)
-                return NULL;
+    ret = pub_glfs_readdirplus_r(glfd, stat, NULL, &res);
+    if (ret)
+        return NULL;
 
-        return res;
+    return res;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_readdirplus, 3.5.0);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_readdir, 3.5.0)
 struct dirent *
-pub_glfs_readdir (struct glfs_fd *glfd)
+pub_glfs_readdir(struct glfs_fd *glfd)
 {
-        return pub_glfs_readdirplus (glfd, NULL);
+    return pub_glfs_readdirplus(glfd, NULL);
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_readdir, 3.5.0);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_statvfs, 3.4.0)
 int
-pub_glfs_statvfs (struct glfs *fs, const char *path, struct statvfs *buf)
+pub_glfs_statvfs(struct glfs *fs, const char *path, struct statvfs *buf)
 {
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	loc_t            loc = {0, };
-	struct iatt      iatt = {0, };
-	int              reval = 0;
-
-	__glfs_entry_fs (fs);
-
-	subvol = priv_glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
 retry:
-	ret = glfs_resolve (fs, subvol, path, &loc, &iatt, reval);
+    ret = glfs_resolve(fs, subvol, path, &loc, &iatt, reval);
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	if (ret)
-		goto out;
+    if (ret)
+        goto out;
 
-	ret = syncop_statfs (subvol, &loc, buf);
-        DECODE_SYNCOP_ERR (ret);
+    ret = syncop_statfs(subvol, &loc, buf, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 out:
-	loc_wipe (&loc);
-
-	priv_glfs_subvol_done (fs, subvol);
+    loc_wipe(&loc);
 
-	return ret;
-}
+    glfs_subvol_done(fs, subvol);
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_statvfs, 3.4.0);
+    __GLFS_EXIT_FS;
 
+invalid_fs:
+    return ret;
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_setattr, 6.0)
 int
-glfs_setattr (struct glfs *fs, const char *path, struct iatt *iatt,
-	      int valid, int follow)
+pub_glfs_setattr(struct glfs *fs, const char *path, struct glfs_stat *stat,
+                 int follow)
 {
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	loc_t            loc = {0, };
-	struct iatt      riatt = {0, };
-	int              reval = 0;
-
-	__glfs_entry_fs (fs);
-
-	subvol = priv_glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+    int ret = -1;
+    int glvalid;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt riatt = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    GF_VALIDATE_OR_GOTO("glfs_setattr", stat, out);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
 retry:
-	if (follow)
-		ret = glfs_resolve (fs, subvol, path, &loc, &riatt, reval);
-	else
-		ret = glfs_lresolve (fs, subvol, path, &loc, &riatt, reval);
+    if (follow)
+        ret = glfs_resolve(fs, subvol, path, &loc, &riatt, reval);
+    else
+        ret = glfs_lresolve(fs, subvol, path, &loc, &riatt, reval);
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	if (ret)
-		goto out;
+    if (ret)
+        goto out;
 
-	ret = syncop_setattr (subvol, &loc, iatt, valid, 0, 0);
-        DECODE_SYNCOP_ERR (ret);
+    glfs_iatt_from_statx(&iatt, stat);
+    glfsflags_from_gfapiflags(stat, &glvalid);
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    /* TODO : Add leaseid */
+    ret = syncop_setattr(subvol, &loc, &iatt, glvalid, 0, 0, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 out:
-	loc_wipe (&loc);
+    loc_wipe(&loc);
 
-	priv_glfs_subvol_done (fs, subvol);
+    glfs_subvol_done(fs, subvol);
 
-	return ret;
-}
+    __GLFS_EXIT_FS;
 
+invalid_fs:
+    return ret;
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fsetattr, 6.0)
 int
-glfs_fsetattr (struct glfs_fd *glfd, struct iatt *iatt, int valid)
+pub_glfs_fsetattr(struct glfs_fd *glfd, struct glfs_stat *stat)
 {
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	fd_t            *fd = NULL;
-
-	__glfs_entry_fd (glfd);
-
-	subvol = priv_glfs_active_subvol (glfd->fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
-
-	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
-	if (!fd) {
-		ret = -1;
-		errno = EBADFD;
-		goto out;
-	}
-
-	ret = syncop_fsetattr (subvol, fd, iatt, valid, 0, 0);
-        DECODE_SYNCOP_ERR (ret);
+    int ret = -1;
+    int glvalid;
+    struct iatt iatt = {
+        0,
+    };
+    xlator_t *subvol = NULL;
+    fd_t *fd = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    GF_REF_GET(glfd);
+
+    GF_VALIDATE_OR_GOTO("glfs_fsetattr", stat, out);
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    glfs_iatt_from_statx(&iatt, stat);
+    glfsflags_from_gfapiflags(stat, &glvalid);
+
+    /* TODO : Add leaseid */
+    ret = syncop_fsetattr(subvol, fd, &iatt, glvalid, 0, 0, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
 out:
-	if (fd)
-		fd_unref (fd);
+    if (fd)
+        fd_unref(fd);
+    if (glfd)
+        GF_REF_PUT(glfd);
 
-	priv_glfs_subvol_done (glfd->fs, subvol);
+    glfs_subvol_done(glfd->fs, subvol);
 
-	return ret;
-}
+    __GLFS_EXIT_FS;
 
+invalid_fs:
+    return ret;
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_chmod, 3.4.0)
 int
-pub_glfs_chmod (struct glfs *fs, const char *path, mode_t mode)
+pub_glfs_chmod(struct glfs *fs, const char *path, mode_t mode)
 {
-	int              ret = -1;
-	struct iatt      iatt = {0, };
-	int              valid = 0;
+    int ret = -1;
+    struct glfs_stat stat = {
+        0,
+    };
 
-	iatt.ia_prot = ia_prot_from_st_mode (mode);
-	valid = GF_SET_ATTR_MODE;
+    stat.glfs_st_mode = mode;
+    stat.glfs_st_mask = GLFS_STAT_MODE;
 
-	ret = glfs_setattr (fs, path, &iatt, valid, 1);
+    ret = glfs_setattr(fs, path, &stat, 1);
 
-	return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_chmod, 3.4.0);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fchmod, 3.4.0)
 int
-pub_glfs_fchmod (struct glfs_fd *glfd, mode_t mode)
+pub_glfs_fchmod(struct glfs_fd *glfd, mode_t mode)
 {
-	int              ret = -1;
-	struct iatt      iatt = {0, };
-	int              valid = 0;
+    int ret = -1;
+    struct glfs_stat stat = {
+        0,
+    };
 
-	iatt.ia_prot = ia_prot_from_st_mode (mode);
-	valid = GF_SET_ATTR_MODE;
+    stat.glfs_st_mode = mode;
+    stat.glfs_st_mask = GLFS_STAT_MODE;
 
-	ret = glfs_fsetattr (glfd, &iatt, valid);
+    ret = glfs_fsetattr(glfd, &stat);
 
-	return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fchmod, 3.4.0);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_chown, 3.4.0)
 int
-pub_glfs_chown (struct glfs *fs, const char *path, uid_t uid, gid_t gid)
+pub_glfs_chown(struct glfs *fs, const char *path, uid_t uid, gid_t gid)
 {
-	int              ret = 0;
-	int              valid = 0;
-	struct iatt      iatt = {0, };
+    int ret = 0;
+    struct glfs_stat stat = {
+        0,
+    };
 
-        if (uid != (uid_t) -1) {
-                iatt.ia_uid = uid;
-                valid = GF_SET_ATTR_UID;
-        }
+    if (uid != (uid_t)-1) {
+        stat.glfs_st_uid = uid;
+        stat.glfs_st_mask = GLFS_STAT_UID;
+    }
 
-        if (gid != (uid_t) -1) {
-                iatt.ia_gid = gid;
-                valid = valid | GF_SET_ATTR_GID;
-        }
+    if (gid != (uid_t)-1) {
+        stat.glfs_st_gid = gid;
+        stat.glfs_st_mask = stat.glfs_st_mask | GLFS_STAT_GID;
+    }
 
-        if (valid)
-	        ret = glfs_setattr (fs, path, &iatt, valid, 1);
+    if (stat.glfs_st_mask)
+        ret = glfs_setattr(fs, path, &stat, 1);
 
-	return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_chown, 3.4.0);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_lchown, 3.4.0)
 int
-pub_glfs_lchown (struct glfs *fs, const char *path, uid_t uid, gid_t gid)
+pub_glfs_lchown(struct glfs *fs, const char *path, uid_t uid, gid_t gid)
 {
-	int              ret = 0;
-	int              valid = 0;
-	struct iatt      iatt = {0, };
+    int ret = 0;
+    struct glfs_stat stat = {
+        0,
+    };
 
-        if (uid != (uid_t) -1) {
-	        iatt.ia_uid = uid;
-	        valid = GF_SET_ATTR_UID;
-        }
+    if (uid != (uid_t)-1) {
+        stat.glfs_st_uid = uid;
+        stat.glfs_st_mask = GLFS_STAT_UID;
+    }
 
-        if (gid != (uid_t) -1) {
-                iatt.ia_gid = gid;
-                valid = valid | GF_SET_ATTR_GID;
-        }
+    if (gid != (uid_t)-1) {
+        stat.glfs_st_gid = gid;
+        stat.glfs_st_mask = stat.glfs_st_mask | GLFS_STAT_GID;
+    }
 
-        if (valid)
-	        ret = glfs_setattr (fs, path, &iatt, valid, 0);
+    if (stat.glfs_st_mask)
+        ret = glfs_setattr(fs, path, &stat, 0);
 
-	return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_lchown, 3.4.0);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fchown, 3.4.0)
 int
-pub_glfs_fchown (struct glfs_fd *glfd, uid_t uid, gid_t gid)
+pub_glfs_fchown(struct glfs_fd *glfd, uid_t uid, gid_t gid)
 {
-	int              ret = 0;
-	int              valid = 0;
-	struct iatt      iatt = {0, };
+    int ret = 0;
+    struct glfs_stat stat = {
+        0,
+    };
 
-        if (uid != (uid_t) -1) {
-                iatt.ia_uid = uid;
-                valid = GF_SET_ATTR_UID;
-        }
+    if (uid != (uid_t)-1) {
+        stat.glfs_st_uid = uid;
+        stat.glfs_st_mask = GLFS_STAT_UID;
+    }
 
-        if (gid != (uid_t) -1) {
-                iatt.ia_gid = gid;
-                valid = valid | GF_SET_ATTR_GID;
-        }
+    if (gid != (uid_t)-1) {
+        stat.glfs_st_gid = gid;
+        stat.glfs_st_mask = stat.glfs_st_mask | GLFS_STAT_GID;
+    }
 
-        if (valid)
-	        ret = glfs_fsetattr (glfd, &iatt, valid);
+    if (stat.glfs_st_mask)
+        ret = glfs_fsetattr(glfd, &stat);
 
-	return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fchown, 3.4.0);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_utimens, 3.4.0)
 int
-pub_glfs_utimens (struct glfs *fs, const char *path, struct timespec times[2])
+pub_glfs_utimens(struct glfs *fs, const char *path,
+                 const struct timespec times[2])
 {
-	int              ret = -1;
-	int              valid = 0;
-	struct iatt      iatt = {0, };
+    int ret = -1;
+    struct glfs_stat stat = {
+        0,
+    };
 
-	iatt.ia_atime = times[0].tv_sec;
-	iatt.ia_atime_nsec = times[0].tv_nsec;
-	iatt.ia_mtime = times[1].tv_sec;
-	iatt.ia_mtime_nsec = times[1].tv_nsec;
+    stat.glfs_st_atime = times[0];
+    stat.glfs_st_mtime = times[1];
 
-	valid = GF_SET_ATTR_ATIME|GF_SET_ATTR_MTIME;
+    stat.glfs_st_mask = GLFS_STAT_ATIME | GLFS_STAT_MTIME;
 
-	ret = glfs_setattr (fs, path, &iatt, valid, 1);
+    ret = glfs_setattr(fs, path, &stat, 1);
 
-	return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_utimens, 3.4.0);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_lutimens, 3.4.0)
 int
-pub_glfs_lutimens (struct glfs *fs, const char *path, struct timespec times[2])
+pub_glfs_lutimens(struct glfs *fs, const char *path,
+                  const struct timespec times[2])
 {
-	int              ret = -1;
-	int              valid = 0;
-	struct iatt      iatt = {0, };
+    int ret = -1;
+    struct glfs_stat stat = {
+        0,
+    };
 
-	iatt.ia_atime = times[0].tv_sec;
-	iatt.ia_atime_nsec = times[0].tv_nsec;
-	iatt.ia_mtime = times[1].tv_sec;
-	iatt.ia_mtime_nsec = times[1].tv_nsec;
+    stat.glfs_st_atime = times[0];
+    stat.glfs_st_mtime = times[1];
 
-	valid = GF_SET_ATTR_ATIME|GF_SET_ATTR_MTIME;
+    stat.glfs_st_mask = GLFS_STAT_ATIME | GLFS_STAT_MTIME;
 
-	ret = glfs_setattr (fs, path, &iatt, valid, 0);
+    ret = glfs_setattr(fs, path, &stat, 0);
 
-	return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_lutimens, 3.4.0);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_futimens, 3.4.0)
 int
-pub_glfs_futimens (struct glfs_fd *glfd, struct timespec times[2])
+pub_glfs_futimens(struct glfs_fd *glfd, const struct timespec times[2])
 {
-	int              ret = -1;
-	int              valid = 0;
-	struct iatt      iatt = {0, };
+    int ret = -1;
+    struct glfs_stat stat = {
+        0,
+    };
 
-	iatt.ia_atime = times[0].tv_sec;
-	iatt.ia_atime_nsec = times[0].tv_nsec;
-	iatt.ia_mtime = times[1].tv_sec;
-	iatt.ia_mtime_nsec = times[1].tv_nsec;
+    stat.glfs_st_atime = times[0];
+    stat.glfs_st_mtime = times[1];
 
-	valid = GF_SET_ATTR_ATIME|GF_SET_ATTR_MTIME;
+    stat.glfs_st_mask = GLFS_STAT_ATIME | GLFS_STAT_MTIME;
 
-	ret = glfs_fsetattr (glfd, &iatt, valid);
+    ret = glfs_fsetattr(glfd, &stat);
 
-	return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_futimens, 3.4.0);
-
-
 int
-glfs_getxattr_process (void *value, size_t size, dict_t *xattr,
-		       const char *name)
+glfs_getxattr_process(void *value, size_t size, dict_t *xattr, const char *name)
 {
-	data_t *data = NULL;
-	int     ret = -1;
-
-	data = dict_get (xattr, (char *)name);
-	if (!data) {
-		errno = ENODATA;
-		ret = -1;
-		goto out;
-	}
-
-	ret = data->len;
-	if (!value || !size)
-		goto out;
-
-	if (size < ret) {
-		ret = -1;
-		errno = ERANGE;
-		goto out;
-	}
-
-	memcpy (value, data->data, ret);
+    data_t *data = NULL;
+    int ret = -1;
+
+    data = dict_get(xattr, (char *)name);
+    if (!data) {
+        errno = ENODATA;
+        ret = -1;
+        goto out;
+    }
+
+    ret = data->len;
+    if (!value || !size)
+        goto out;
+
+    if (size < ret) {
+        ret = -1;
+        errno = ERANGE;
+        goto out;
+    }
+
+    memcpy(value, data->data, ret);
 out:
-	if (xattr)
-		dict_unref (xattr);
-	return ret;
+    return ret;
 }
 
-
 ssize_t
-glfs_getxattr_common (struct glfs *fs, const char *path, const char *name,
-		      void *value, size_t size, int follow)
-{
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	loc_t            loc = {0, };
-	struct iatt      iatt = {0, };
-	dict_t          *xattr = NULL;
-	int              reval = 0;
-
-	__glfs_entry_fs (fs);
-
-	subvol = priv_glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+glfs_getxattr_common(struct glfs *fs, const char *path, const char *name,
+                     void *value, size_t size, int follow)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    dict_t *xattr = NULL;
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    if (!name || *name == '\0') {
+        ret = -1;
+        errno = EINVAL;
+        goto out;
+    }
+
+    if (strlen(name) > GF_XATTR_NAME_MAX) {
+        ret = -1;
+        errno = ENAMETOOLONG;
+        goto out;
+    }
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
 retry:
-	if (follow)
-		ret = glfs_resolve (fs, subvol, path, &loc, &iatt, reval);
-	else
-		ret = glfs_lresolve (fs, subvol, path, &loc, &iatt, reval);
+    if (follow)
+        ret = glfs_resolve(fs, subvol, path, &loc, &iatt, reval);
+    else
+        ret = glfs_lresolve(fs, subvol, path, &loc, &iatt, reval);
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	if (ret)
-		goto out;
+    if (ret)
+        goto out;
 
-	ret = syncop_getxattr (subvol, &loc, &xattr, name);
-        DECODE_SYNCOP_ERR (ret);
+    ret = syncop_getxattr(subvol, &loc, &xattr, name, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	if (ret)
-		goto out;
+    if (ret)
+        goto out;
 
-	ret = glfs_getxattr_process (value, size, xattr, name);
+    ret = glfs_getxattr_process(value, size, xattr, name);
 out:
-	loc_wipe (&loc);
+    loc_wipe(&loc);
 
-	priv_glfs_subvol_done (fs, subvol);
+    if (xattr)
+        dict_unref(xattr);
 
-	return ret;
-}
+    glfs_subvol_done(fs, subvol);
+
+    __GLFS_EXIT_FS;
 
+invalid_fs:
+    return ret;
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_getxattr, 3.4.0)
 ssize_t
-pub_glfs_getxattr (struct glfs *fs, const char *path, const char *name,
-                   void *value, size_t size)
+pub_glfs_getxattr(struct glfs *fs, const char *path, const char *name,
+                  void *value, size_t size)
 {
-	return glfs_getxattr_common (fs, path, name, value, size, 1);
+    return glfs_getxattr_common(fs, path, name, value, size, 1);
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_getxattr, 3.4.0);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_lgetxattr, 3.4.0)
 ssize_t
-pub_glfs_lgetxattr (struct glfs *fs, const char *path, const char *name,
-                    void *value, size_t size)
+pub_glfs_lgetxattr(struct glfs *fs, const char *path, const char *name,
+                   void *value, size_t size)
 {
-	return glfs_getxattr_common (fs, path, name, value, size, 0);
+    return glfs_getxattr_common(fs, path, name, value, size, 0);
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_lgetxattr, 3.4.0);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fgetxattr, 3.4.0)
 ssize_t
-pub_glfs_fgetxattr (struct glfs_fd *glfd, const char *name, void *value,
-                    size_t size)
+pub_glfs_fgetxattr(struct glfs_fd *glfd, const char *name, void *value,
+                   size_t size)
 {
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	dict_t          *xattr = NULL;
-	fd_t            *fd = NULL;
-
-	__glfs_entry_fd (glfd);
-
-	subvol = priv_glfs_active_subvol (glfd->fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
-
-	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
-	if (!fd) {
-		ret = -1;
-		errno = EBADFD;
-		goto out;
-	}
-
-	ret = syncop_fgetxattr (subvol, fd, &xattr, name);
-        DECODE_SYNCOP_ERR (ret);
-	if (ret)
-		goto out;
-
-	ret = glfs_getxattr_process (value, size, xattr, name);
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    dict_t *xattr = NULL;
+    fd_t *fd = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    GF_REF_GET(glfd);
+
+    if (!name || *name == '\0') {
+        ret = -1;
+        errno = EINVAL;
+        goto out;
+    }
+
+    if (strlen(name) > GF_XATTR_NAME_MAX) {
+        ret = -1;
+        errno = ENAMETOOLONG;
+        goto out;
+    }
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    ret = syncop_fgetxattr(subvol, fd, &xattr, name, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+    if (ret)
+        goto out;
+
+    ret = glfs_getxattr_process(value, size, xattr, name);
 out:
-	if (fd)
-		fd_unref (fd);
+    if (fd)
+        fd_unref(fd);
+    if (glfd)
+        GF_REF_PUT(glfd);
+    if (xattr)
+        dict_unref(xattr);
 
-	priv_glfs_subvol_done (glfd->fs, subvol);
+    glfs_subvol_done(glfd->fs, subvol);
 
-	return ret;
-}
-
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fgetxattr, 3.4.0);
+    __GLFS_EXIT_FS;
 
+invalid_fs:
+    return ret;
+}
 
 int
-glfs_listxattr_process (void *value, size_t size, dict_t *xattr)
+glfs_listxattr_process(void *value, size_t size, dict_t *xattr)
 {
-	int ret = -1;
+    int ret = -1;
 
-	if (!xattr)
-		goto out;
+    if (!xattr)
+        goto out;
 
-	ret = dict_keys_join (NULL, 0, xattr, NULL);
+    ret = dict_keys_join(NULL, 0, xattr, NULL);
 
-        if (!value || !size)
-                goto out;
+    if (!value || !size)
+        goto out;
 
-	if (size < ret) {
-		ret = -1;
-		errno = ERANGE;
-	} else {
-		dict_keys_join (value, size, xattr, NULL);
-	}
+    if (size < ret) {
+        ret = -1;
+        errno = ERANGE;
+    } else {
+        dict_keys_join(value, size, xattr, NULL);
+    }
 
 out:
-        if (xattr)
-                dict_unref (xattr);
-
-	return ret;
+    return ret;
 }
 
-
 ssize_t
-glfs_listxattr_common (struct glfs *fs, const char *path, void *value,
-		       size_t size, int follow)
+glfs_listxattr_common(struct glfs *fs, const char *path, void *value,
+                      size_t size, int follow)
 {
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	loc_t            loc = {0, };
-	struct iatt      iatt = {0, };
-	dict_t          *xattr = NULL;
-	int              reval = 0;
-
-	__glfs_entry_fs (fs);
-
-	subvol = priv_glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    dict_t *xattr = NULL;
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
 
 retry:
-	if (follow)
-		ret = glfs_resolve (fs, subvol, path, &loc, &iatt, reval);
-	else
-		ret = glfs_lresolve (fs, subvol, path, &loc, &iatt, reval);
+    if (follow)
+        ret = glfs_resolve(fs, subvol, path, &loc, &iatt, reval);
+    else
+        ret = glfs_lresolve(fs, subvol, path, &loc, &iatt, reval);
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	if (ret)
-		goto out;
+    if (ret)
+        goto out;
 
-	ret = syncop_getxattr (subvol, &loc, &xattr, NULL);
-        DECODE_SYNCOP_ERR (ret);
+    ret = syncop_getxattr(subvol, &loc, &xattr, NULL, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	if (ret)
-		goto out;
+    if (ret)
+        goto out;
 
-	ret = glfs_listxattr_process (value, size, xattr);
+    ret = glfs_listxattr_process(value, size, xattr);
 out:
-	loc_wipe (&loc);
+    loc_wipe(&loc);
 
-	priv_glfs_subvol_done (fs, subvol);
+    if (xattr)
+        dict_unref(xattr);
 
-	return ret;
-}
+    glfs_subvol_done(fs, subvol);
 
+    __GLFS_EXIT_FS;
 
+invalid_fs:
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_listxattr, 3.4.0)
 ssize_t
-pub_glfs_listxattr (struct glfs *fs, const char *path, void *value, size_t size)
+pub_glfs_listxattr(struct glfs *fs, const char *path, void *value, size_t size)
 {
-	return glfs_listxattr_common (fs, path, value, size, 1);
+    return glfs_listxattr_common(fs, path, value, size, 1);
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_listxattr, 3.4.0);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_llistxattr, 3.4.0)
 ssize_t
-pub_glfs_llistxattr (struct glfs *fs, const char *path, void *value, size_t size)
+pub_glfs_llistxattr(struct glfs *fs, const char *path, void *value, size_t size)
 {
-	return glfs_listxattr_common (fs, path, value, size, 0);
+    return glfs_listxattr_common(fs, path, value, size, 0);
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_llistxattr, 3.4.0);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_flistxattr, 3.4.0)
+ssize_t
+pub_glfs_flistxattr(struct glfs_fd *glfd, void *value, size_t size)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    dict_t *xattr = NULL;
+    fd_t *fd = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    GF_REF_GET(glfd);
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    ret = syncop_fgetxattr(subvol, fd, &xattr, NULL, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+    if (ret)
+        goto out;
+
+    ret = glfs_listxattr_process(value, size, xattr);
+out:
+    if (fd)
+        fd_unref(fd);
+    if (glfd)
+        GF_REF_PUT(glfd);
+    if (xattr)
+        dict_unref(xattr);
+
+    glfs_subvol_done(glfd->fs, subvol);
 
+    __GLFS_EXIT_FS;
 
-ssize_t
-pub_glfs_flistxattr (struct glfs_fd *glfd, void *value, size_t size)
+invalid_fs:
+    return ret;
+}
+
+int
+glfs_setxattr_common(struct glfs *fs, const char *path, const char *name,
+                     const void *value, size_t size, int flags, int follow)
 {
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	dict_t          *xattr = NULL;
-	fd_t            *fd = NULL;
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    dict_t *xattr = NULL;
+    int reval = 0;
+    void *value_cp = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    if (!name || *name == '\0') {
+        ret = -1;
+        errno = EINVAL;
+        goto out;
+    }
+
+    if (strlen(name) > GF_XATTR_NAME_MAX) {
+        ret = -1;
+        errno = ENAMETOOLONG;
+        goto out;
+    }
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+retry:
+    if (follow)
+        ret = glfs_resolve(fs, subvol, path, &loc, &iatt, reval);
+    else
+        ret = glfs_lresolve(fs, subvol, path, &loc, &iatt, reval);
+
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	__glfs_entry_fd (glfd);
+    if (ret)
+        goto out;
 
-	subvol = priv_glfs_active_subvol (glfd->fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+    value_cp = gf_memdup(value, size);
+    GF_CHECK_ALLOC_AND_LOG(subvol->name, value_cp, ret,
+                           "Failed to"
+                           " duplicate setxattr value",
+                           out);
 
-	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
-	if (!fd) {
-		ret = -1;
-		errno = EBADFD;
-		goto out;
-	}
+    xattr = dict_for_key_value(name, value_cp, size, _gf_false);
+    if (!xattr) {
+        GF_FREE(value_cp);
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
 
-	ret = syncop_fgetxattr (subvol, fd, &xattr, NULL);
-        DECODE_SYNCOP_ERR (ret);
-	if (ret)
-		goto out;
+    ret = syncop_setxattr(subvol, &loc, xattr, flags, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
 
-	ret = glfs_listxattr_process (value, size, xattr);
 out:
-	if (fd)
-		fd_unref (fd);
+    loc_wipe(&loc);
+    if (xattr)
+        dict_unref(xattr);
 
-	priv_glfs_subvol_done (glfd->fs, subvol);
+    glfs_subvol_done(fs, subvol);
 
-	return ret;
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_flistxattr, 3.4.0);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_setxattr, 3.4.0)
+int
+pub_glfs_setxattr(struct glfs *fs, const char *path, const char *name,
+                  const void *value, size_t size, int flags)
+{
+    return glfs_setxattr_common(fs, path, name, value, size, flags, 1);
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_lsetxattr, 3.4.0)
+int
+pub_glfs_lsetxattr(struct glfs *fs, const char *path, const char *name,
+                   const void *value, size_t size, int flags)
+{
+    return glfs_setxattr_common(fs, path, name, value, size, flags, 0);
+}
 
-dict_t *
-dict_for_key_value (const char *name, const char *value, size_t size)
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fsetxattr, 3.4.0)
+int
+pub_glfs_fsetxattr(struct glfs_fd *glfd, const char *name, const void *value,
+                   size_t size, int flags)
 {
-	dict_t *xattr = NULL;
-	int     ret = 0;
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    dict_t *xattr = NULL;
+    fd_t *fd = NULL;
+    void *value_cp = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    GF_REF_GET(glfd);
+
+    if (!name || *name == '\0') {
+        ret = -1;
+        errno = EINVAL;
+        goto out;
+    }
+
+    if (strlen(name) > GF_XATTR_NAME_MAX) {
+        ret = -1;
+        errno = ENAMETOOLONG;
+        goto out;
+    }
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    value_cp = gf_memdup(value, size);
+    GF_CHECK_ALLOC_AND_LOG(subvol->name, value_cp, ret,
+                           "Failed to"
+                           " duplicate setxattr value",
+                           out);
+
+    xattr = dict_for_key_value(name, value_cp, size, _gf_false);
+    if (!xattr) {
+        GF_FREE(value_cp);
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    ret = syncop_fsetxattr(subvol, fd, xattr, flags, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+out:
+    if (xattr)
+        dict_unref(xattr);
 
-	xattr = dict_new ();
-	if (!xattr)
-		return NULL;
+    if (fd)
+        fd_unref(fd);
+    if (glfd)
+        GF_REF_PUT(glfd);
 
-	ret = dict_set_static_bin (xattr, (char *)name, (void *)value, size);
-	if (ret) {
-		dict_destroy (xattr);
-		xattr = NULL;
-	}
+    glfs_subvol_done(glfd->fs, subvol);
 
-	return xattr;
-}
+    __GLFS_EXIT_FS;
 
+invalid_fs:
+    return ret;
+}
 
 int
-glfs_setxattr_common (struct glfs *fs, const char *path, const char *name,
-		      const void *value, size_t size, int flags, int follow)
+glfs_removexattr_common(struct glfs *fs, const char *path, const char *name,
+                        int follow)
 {
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	loc_t            loc = {0, };
-	struct iatt      iatt = {0, };
-	dict_t          *xattr = NULL;
-	int              reval = 0;
-
-	__glfs_entry_fs (fs);
-
-	subvol = priv_glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
 retry:
-	if (follow)
-		ret = glfs_resolve (fs, subvol, path, &loc, &iatt, reval);
-	else
-		ret = glfs_lresolve (fs, subvol, path, &loc, &iatt, reval);
-
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    if (follow)
+        ret = glfs_resolve(fs, subvol, path, &loc, &iatt, reval);
+    else
+        ret = glfs_lresolve(fs, subvol, path, &loc, &iatt, reval);
 
-	if (ret)
-		goto out;
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	xattr = dict_for_key_value (name, value, size);
-	if (!xattr) {
-		ret = -1;
-		errno = ENOMEM;
-		goto out;
-	}
+    if (ret)
+        goto out;
 
-	ret = syncop_setxattr (subvol, &loc, xattr, flags);
-        DECODE_SYNCOP_ERR (ret);
+    ret = syncop_removexattr(subvol, &loc, name, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
 out:
-	loc_wipe (&loc);
-	if (xattr)
-		dict_unref (xattr);
+    loc_wipe(&loc);
 
-	priv_glfs_subvol_done (fs, subvol);
+    glfs_subvol_done(fs, subvol);
 
-	return ret;
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
 }
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_removexattr, 3.4.0)
+int
+pub_glfs_removexattr(struct glfs *fs, const char *path, const char *name)
+{
+    return glfs_removexattr_common(fs, path, name, 1);
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_lremovexattr, 3.4.0)
 int
-pub_glfs_setxattr (struct glfs *fs, const char *path, const char *name,
-                   const void *value, size_t size, int flags)
+pub_glfs_lremovexattr(struct glfs *fs, const char *path, const char *name)
 {
-	return glfs_setxattr_common (fs, path, name, value, size, flags, 1);
+    return glfs_removexattr_common(fs, path, name, 0);
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_setxattr, 3.4.0);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fremovexattr, 3.4.0)
+int
+pub_glfs_fremovexattr(struct glfs_fd *glfd, const char *name)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    fd_t *fd = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    GF_REF_GET(glfd);
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    ret = syncop_fremovexattr(subvol, fd, name, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+out:
+    if (fd)
+        fd_unref(fd);
+    if (glfd)
+        GF_REF_PUT(glfd);
 
+    glfs_subvol_done(glfd->fs, subvol);
 
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fallocate, 3.5.0)
 int
-pub_glfs_lsetxattr (struct glfs *fs, const char *path, const char *name,
-                    const void *value, size_t size, int flags)
+pub_glfs_fallocate(struct glfs_fd *glfd, int keep_size, off_t offset,
+                   size_t len)
 {
-	return glfs_setxattr_common (fs, path, name, value, size, flags, 0);
-}
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    fd_t *fd = NULL;
+    dict_t *fop_attr = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    GF_REF_GET(glfd);
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    ret = get_fop_attr_thrd_key(&fop_attr);
+    if (ret)
+        gf_msg_debug("gfapi", 0, "Getting leaseid from thread failed");
+
+    ret = syncop_fallocate(subvol, fd, keep_size, offset, len, fop_attr, NULL);
+    DECODE_SYNCOP_ERR(ret);
+out:
+    if (fd)
+        fd_unref(fd);
+    if (glfd)
+        GF_REF_PUT(glfd);
+    if (fop_attr)
+        dict_unref(fop_attr);
+
+    glfs_subvol_done(glfd->fs, subvol);
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_lsetxattr, 3.4.0);
+    __GLFS_EXIT_FS;
 
+invalid_fs:
+    return ret;
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_discard, 3.5.0)
 int
-pub_glfs_fsetxattr (struct glfs_fd *glfd, const char *name, const void *value,
-                    size_t size, int flags)
+pub_glfs_discard(struct glfs_fd *glfd, off_t offset, size_t len)
 {
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	dict_t          *xattr = NULL;
-	fd_t            *fd = NULL;
-
-	__glfs_entry_fd (glfd);
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    fd_t *fd = NULL;
+    dict_t *fop_attr = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    GF_REF_GET(glfd);
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    ret = get_fop_attr_thrd_key(&fop_attr);
+    if (ret)
+        gf_msg_debug("gfapi", 0, "Getting leaseid from thread failed");
+
+    ret = syncop_discard(subvol, fd, offset, len, fop_attr, NULL);
+    DECODE_SYNCOP_ERR(ret);
+out:
+    if (fd)
+        fd_unref(fd);
+    if (glfd)
+        GF_REF_PUT(glfd);
+    if (fop_attr)
+        dict_unref(fop_attr);
 
-	subvol = priv_glfs_active_subvol (glfd->fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+    glfs_subvol_done(glfd->fs, subvol);
 
-	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
-	if (!fd) {
-		ret = -1;
-		errno = EBADFD;
-		goto out;
-	}
+    __GLFS_EXIT_FS;
 
-	xattr = dict_for_key_value (name, value, size);
-	if (!xattr) {
-		ret = -1;
-		errno = ENOMEM;
-		goto out;
-	}
+invalid_fs:
+    return ret;
+}
 
-	ret = syncop_fsetxattr (subvol, fd, xattr, flags);
-        DECODE_SYNCOP_ERR (ret);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_zerofill, 3.5.0)
+int
+pub_glfs_zerofill(struct glfs_fd *glfd, off_t offset, off_t len)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    fd_t *fd = NULL;
+    dict_t *fop_attr = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    GF_REF_GET(glfd);
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        errno = EBADFD;
+        goto out;
+    }
+
+    ret = get_fop_attr_thrd_key(&fop_attr);
+    if (ret)
+        gf_msg_debug("gfapi", 0, "Getting leaseid from thread failed");
+
+    ret = syncop_zerofill(subvol, fd, offset, len, fop_attr, NULL);
+    DECODE_SYNCOP_ERR(ret);
 out:
-	if (xattr)
-		dict_unref (xattr);
+    if (fd)
+        fd_unref(fd);
+    if (glfd)
+        GF_REF_PUT(glfd);
+    if (fop_attr)
+        dict_unref(fop_attr);
 
-	if (fd)
-		fd_unref (fd);
+    glfs_subvol_done(glfd->fs, subvol);
 
-	priv_glfs_subvol_done (glfd->fs, subvol);
+    __GLFS_EXIT_FS;
 
-	return ret;
+invalid_fs:
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fsetxattr, 3.4.0);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_chdir, 3.4.0)
+int
+pub_glfs_chdir(struct glfs *fs, const char *path)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+retry:
+    ret = glfs_resolve(fs, subvol, path, &loc, &iatt, reval);
+
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
+
+    if (ret)
+        goto out;
+
+    if (!IA_ISDIR(iatt.ia_type)) {
+        ret = -1;
+        errno = ENOTDIR;
+        goto out;
+    }
 
+    glfs_cwd_set(fs, loc.inode);
+
+out:
+    loc_wipe(&loc);
+
+    glfs_subvol_done(fs, subvol);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fchdir, 3.4.0)
 int
-glfs_removexattr_common (struct glfs *fs, const char *path, const char *name,
-			 int follow)
+pub_glfs_fchdir(struct glfs_fd *glfd)
 {
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	loc_t            loc = {0, };
-	struct iatt      iatt = {0, };
-	int              reval = 0;
+    int ret = -1;
+    inode_t *inode = NULL;
+    xlator_t *subvol = NULL;
+    fd_t *fd = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    GF_REF_GET(glfd);
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    inode = fd->inode;
+
+    if (!IA_ISDIR(inode->ia_type)) {
+        ret = -1;
+        errno = ENOTDIR;
+        goto out;
+    }
+
+    glfs_cwd_set(glfd->fs, inode);
+    ret = 0;
+out:
+    if (fd)
+        fd_unref(fd);
+    if (glfd)
+        GF_REF_PUT(glfd);
 
-	__glfs_entry_fs (fs);
+    glfs_subvol_done(glfd->fs, subvol);
 
-	subvol = priv_glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
-retry:
-	if (follow)
-		ret = glfs_resolve (fs, subvol, path, &loc, &iatt, reval);
-	else
-		ret = glfs_lresolve (fs, subvol, path, &loc, &iatt, reval);
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
+
+static gf_boolean_t warn_realpath = _gf_true; /* log once */
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+static char *
+glfs_realpath_common(struct glfs *fs, const char *path, char *resolved_path,
+                     gf_boolean_t warn_deprecated)
+{
+    int ret = -1;
+    char *retpath = NULL;
+    char *allocpath = NULL;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    if (resolved_path)
+        retpath = resolved_path;
+    else if (warn_deprecated) {
+        retpath = allocpath = malloc(PATH_MAX + 1);
+        if (warn_realpath) {
+            warn_realpath = _gf_false;
+            gf_log(THIS->name, GF_LOG_WARNING,
+                   "this application "
+                   "is compiled against an old version of "
+                   "libgfapi, it should use glfs_free() to "
+                   "release the path returned by "
+                   "glfs_realpath()");
+        }
+    } else {
+        retpath = allocpath = GLFS_CALLOC(1, PATH_MAX + 1, NULL,
+                                          glfs_mt_realpath_t);
+    }
+
+    if (!retpath) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+retry:
+    ret = glfs_resolve(fs, subvol, path, &loc, &iatt, reval);
 
-	if (ret)
-		goto out;
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	ret = syncop_removexattr (subvol, &loc, name, 0);
-        DECODE_SYNCOP_ERR (ret);
+    if (ret)
+        goto out;
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    if (loc.path) {
+        snprintf(retpath, PATH_MAX + 1, "%s", loc.path);
+    }
 
 out:
-	loc_wipe (&loc);
+    loc_wipe(&loc);
+
+    if (ret == -1) {
+        if (warn_deprecated && allocpath)
+            free(allocpath);
+        else if (allocpath)
+            GLFS_FREE(allocpath);
+        retpath = NULL;
+    }
 
-	priv_glfs_subvol_done (fs, subvol);
+    glfs_subvol_done(fs, subvol);
 
-	return ret;
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return retpath;
 }
 
+GFAPI_SYMVER_PUBLIC(glfs_realpath34, glfs_realpath, 3.4.0)
+char *
+pub_glfs_realpath34(struct glfs *fs, const char *path, char *resolved_path)
+{
+    return glfs_realpath_common(fs, path, resolved_path, _gf_true);
+}
 
-int
-pub_glfs_removexattr (struct glfs *fs, const char *path, const char *name)
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_realpath, 3.7.17)
+char *
+pub_glfs_realpath(struct glfs *fs, const char *path, char *resolved_path)
 {
-	return glfs_removexattr_common (fs, path, name, 1);
+    return glfs_realpath_common(fs, path, resolved_path, _gf_false);
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_removexattr, 3.4.0);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_getcwd, 3.4.0)
+char *
+pub_glfs_getcwd(struct glfs *fs, char *buf, size_t n)
+{
+    int ret = -1;
+    inode_t *inode = NULL;
+    char *path = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    if (!buf || n < 2) {
+        ret = -1;
+        errno = EINVAL;
+        goto out;
+    }
+
+    inode = glfs_cwd_get(fs);
+
+    if (!inode) {
+        strncpy(buf, "/", n);
+        ret = 0;
+        goto out;
+    }
+
+    ret = inode_path(inode, 0, &path);
+    if (n <= ret) {
+        ret = -1;
+        errno = ERANGE;
+        goto out;
+    }
+
+    strncpy(buf, path, n);
+    ret = 0;
+out:
+    GF_FREE(path);
 
+    if (inode)
+        inode_unref(inode);
 
-int
-pub_glfs_lremovexattr (struct glfs *fs, const char *path, const char *name)
-{
-	return glfs_removexattr_common (fs, path, name, 0);
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    if (ret < 0)
+        return NULL;
+
+    return buf;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_lremovexattr, 3.4.0);
+static void
+gf_flock_to_flock(struct gf_flock *gf_flock, struct flock *flock)
+{
+    flock->l_type = gf_flock->l_type;
+    flock->l_whence = gf_flock->l_whence;
+    flock->l_start = gf_flock->l_start;
+    flock->l_len = gf_flock->l_len;
+    flock->l_pid = gf_flock->l_pid;
+}
 
+static void
+gf_flock_from_flock(struct gf_flock *gf_flock, struct flock *flock)
+{
+    gf_flock->l_type = flock->l_type;
+    gf_flock->l_whence = flock->l_whence;
+    gf_flock->l_start = flock->l_start;
+    gf_flock->l_len = flock->l_len;
+    gf_flock->l_pid = flock->l_pid;
+}
 
-int
-pub_glfs_fremovexattr (struct glfs_fd *glfd, const char *name)
+static int
+glfs_lock_common(struct glfs_fd *glfd, int cmd, struct flock *flock,
+                 dict_t *xdata)
 {
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	fd_t            *fd = NULL;
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    struct gf_flock gf_flock = {
+        0,
+    };
+    struct gf_flock saved_flock = {
+        0,
+    };
+    fd_t *fd = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    if (!flock) {
+        errno = EINVAL;
+        goto out;
+    }
+
+    GF_REF_GET(glfd);
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    /* Generate glusterfs flock structure from client flock
+     * structure to be processed by server */
+    gf_flock_from_flock(&gf_flock, flock);
+
+    /* Keep another copy of flock for split/merge of locks
+     * at client side */
+    gf_flock_from_flock(&saved_flock, flock);
+
+    if (glfd->lk_owner.len != 0) {
+        ret = syncopctx_setfslkowner(&glfd->lk_owner);
+
+        if (ret)
+            goto out;
+    }
 
-	__glfs_entry_fd (glfd);
+    ret = get_fop_attr_thrd_key(&xdata);
+    if (ret)
+        gf_msg_debug("gfapi", 0, "Getting leaseid from thread failed");
 
-	subvol = priv_glfs_active_subvol (glfd->fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+    ret = syncop_lk(subvol, fd, cmd, &gf_flock, xdata, NULL);
+    DECODE_SYNCOP_ERR(ret);
 
-	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
-	if (!fd) {
-		ret = -1;
-		errno = EBADFD;
-		goto out;
-	}
+    /* Convert back from gf_flock to flock as expected by application */
+    gf_flock_to_flock(&gf_flock, flock);
+
+    if (ret == 0 && (cmd == F_SETLK || cmd == F_SETLKW)) {
+        ret = fd_lk_insert_and_merge(fd, cmd, &saved_flock);
+        if (ret) {
+            gf_smsg(THIS->name, GF_LOG_ERROR, 0,
+                    API_MSG_LOCK_INSERT_MERGE_FAILED, "gfid=%s",
+                    uuid_utoa(fd->inode->gfid), NULL);
+            ret = 0;
+        }
+    }
 
-	ret = syncop_fremovexattr (subvol, fd, name, 0);
-        DECODE_SYNCOP_ERR (ret);
 out:
-	if (fd)
-		fd_unref (fd);
+    if (fd)
+        fd_unref(fd);
+    if (glfd)
+        GF_REF_PUT(glfd);
 
-	priv_glfs_subvol_done (glfd->fs, subvol);
+    glfs_subvol_done(glfd->fs, subvol);
 
-	return ret;
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fremovexattr, 3.4.0);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_file_lock, 4.0.0)
+int
+pub_glfs_file_lock(struct glfs_fd *glfd, int cmd, struct flock *flock,
+                   glfs_lock_mode_t lk_mode)
+{
+    int ret = -1;
+    dict_t *xdata_in = NULL;
+
+    if (lk_mode == GLFS_LK_MANDATORY) {
+        /* Create a new dictionary */
+        xdata_in = dict_new();
+        if (xdata_in == NULL) {
+            ret = -1;
+            errno = ENOMEM;
+            goto out;
+        }
+
+        /* Set GF_LK_MANDATORY internally within dictionary to map
+         * GLFS_LK_MANDATORY */
+        ret = dict_set_uint32(xdata_in, GF_LOCK_MODE, GF_LK_MANDATORY);
+        if (ret) {
+            gf_smsg(THIS->name, GF_LOG_ERROR, 0,
+                    API_MSG_SETTING_LOCK_TYPE_FAILED, NULL);
+            ret = -1;
+            errno = ENOMEM;
+            goto out;
+        }
+    }
+
+    ret = glfs_lock_common(glfd, cmd, flock, xdata_in);
+out:
+    if (xdata_in)
+        dict_unref(xdata_in);
 
+    return ret;
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_posix_lock, 3.4.0)
 int
-pub_glfs_fallocate (struct glfs_fd *glfd, int keep_size, off_t offset, size_t len)
+pub_glfs_posix_lock(struct glfs_fd *glfd, int cmd, struct flock *flock)
 {
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	fd_t		*fd = NULL;
+    return glfs_lock_common(glfd, cmd, flock, NULL);
+}
 
-	__glfs_entry_fd (glfd);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fd_set_lkowner, 3.10.7)
+int
+pub_glfs_fd_set_lkowner(struct glfs_fd *glfd, void *data, int len)
+{
+    int ret = -1;
 
-	subvol = priv_glfs_active_subvol (glfd->fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
 
-	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
-	if (!fd) {
-		ret = -1;
-		errno = EBADFD;
-		goto out;
-	}
+    if (!GF_REF_GET(glfd)) {
+        goto invalid_fs;
+    }
 
-	ret = syncop_fallocate (subvol, fd, keep_size, offset, len);
-        DECODE_SYNCOP_ERR (ret);
-out:
-	if (fd)
-		fd_unref(fd);
+    GF_VALIDATE_OR_GOTO(THIS->name, data, out);
 
-	priv_glfs_subvol_done (glfd->fs, subvol);
+    if ((len <= 0) || (len > GFAPI_MAX_LOCK_OWNER_LEN)) {
+        errno = EINVAL;
+        gf_smsg(THIS->name, GF_LOG_ERROR, errno, API_MSG_INVALID_ARG,
+                "lk_owner len=%d", len, NULL);
+        goto out;
+    }
 
-	return ret;
-}
+    glfd->lk_owner.len = len;
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fallocate, 3.5.0);
+    memcpy(glfd->lk_owner.data, data, len);
 
+    ret = 0;
+out:
+    if (glfd)
+        GF_REF_PUT(glfd);
 
-int
-pub_glfs_discard (struct glfs_fd *glfd, off_t offset, size_t len)
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_dup, 3.4.0)
+struct glfs_fd *
+pub_glfs_dup(struct glfs_fd *glfd)
 {
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	fd_t		*fd = NULL;
+    xlator_t *subvol = NULL;
+    fd_t *fd = NULL;
+    struct glfs_fd *dupfd = NULL;
+    struct glfs *fs = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    GF_REF_GET(glfd);
+
+    fs = glfd->fs;
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(fs, subvol, glfd);
+    if (!fd) {
+        errno = EBADFD;
+        goto out;
+    }
+
+    dupfd = glfs_fd_new(fs);
+    if (!dupfd) {
+        errno = ENOMEM;
+        goto out;
+    }
+
+    dupfd->fd = fd_ref(fd);
+    dupfd->state = glfd->state;
+out:
+    if (fd)
+        fd_unref(fd);
+    if (dupfd)
+        glfs_fd_bind(dupfd);
+    if (glfd)
+        GF_REF_PUT(glfd);
 
-	__glfs_entry_fd (glfd);
+    glfs_subvol_done(fs, subvol);
 
-	subvol = priv_glfs_active_subvol (glfd->fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+    __GLFS_EXIT_FS;
 
-	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
-	if (!fd) {
-		ret = -1;
-		errno = EBADFD;
-		goto out;
-	}
+invalid_fs:
+    return dupfd;
+}
+
+static void
+glfs_enqueue_upcall_data(struct glfs *fs, struct gf_upcall *upcall_data)
+{
+    int ret = -1;
+    upcall_entry *u_list = NULL;
+
+    if (!fs || !upcall_data)
+        goto out;
+
+    u_list = GF_CALLOC(1, sizeof(*u_list), glfs_mt_upcall_entry_t);
+
+    if (!u_list) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, ENOMEM, API_MSG_ALLOC_FAILED, "entry",
+                NULL);
+        goto out;
+    }
+
+    INIT_LIST_HEAD(&u_list->upcall_list);
+
+    gf_uuid_copy(u_list->upcall_data.gfid, upcall_data->gfid);
+    u_list->upcall_data.event_type = upcall_data->event_type;
+
+    switch (upcall_data->event_type) {
+        case GF_UPCALL_CACHE_INVALIDATION:
+            ret = glfs_get_upcall_cache_invalidation(&u_list->upcall_data,
+                                                     upcall_data);
+            break;
+        case GF_UPCALL_RECALL_LEASE:
+            ret = glfs_get_upcall_lease(&u_list->upcall_data, upcall_data);
+            break;
+        default:
+            break;
+    }
+
+    if (ret) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, errno, API_MSG_INVALID_ENTRY, NULL);
+        goto out;
+    }
+
+    pthread_mutex_lock(&fs->upcall_list_mutex);
+    {
+        list_add_tail(&u_list->upcall_list, &fs->upcall_list);
+    }
+    pthread_mutex_unlock(&fs->upcall_list_mutex);
+
+    ret = 0;
 
-	ret = syncop_discard (subvol, fd, offset, len);
-        DECODE_SYNCOP_ERR (ret);
 out:
-	if (fd)
-		fd_unref(fd);
+    if (ret && u_list) {
+        GF_FREE(u_list->upcall_data.data);
+        GF_FREE(u_list);
+    }
+}
 
-	priv_glfs_subvol_done (glfd->fs, subvol);
+static void
+glfs_free_upcall_lease(void *to_free)
+{
+    struct glfs_upcall_lease *arg = to_free;
 
-	return ret;
-}
+    if (!arg)
+        return;
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_discard, 3.5.0);
+    if (arg->object)
+        glfs_h_close(arg->object);
 
+    GF_FREE(arg);
+}
 
 int
-pub_glfs_zerofill (struct glfs_fd *glfd, off_t offset, off_t len)
+glfs_recall_lease_fd(struct glfs *fs, struct gf_upcall *up_data)
 {
-        int               ret             = -1;
-        xlator_t         *subvol          = NULL;
-        fd_t             *fd              = NULL;
-
-        __glfs_entry_fd (glfd);
-
-        subvol = priv_glfs_active_subvol (glfd->fs);
-        if (!subvol) {
-                errno = EIO;
-                goto out;
+    struct gf_upcall_recall_lease *recall_lease = NULL;
+    xlator_t *subvol = NULL;
+    int ret = 0;
+    inode_t *inode = NULL;
+    struct glfs_fd *glfd = NULL;
+    struct glfs_fd *tmp = NULL;
+    struct list_head glfd_list;
+    fd_t *fd = NULL;
+    uint64_t value = 0;
+    struct glfs_lease lease = {
+        0,
+    };
+
+    GF_VALIDATE_OR_GOTO("gfapi", up_data, out);
+    GF_VALIDATE_OR_GOTO("gfapi", fs, out);
+
+    recall_lease = up_data->data;
+    GF_VALIDATE_OR_GOTO("gfapi", recall_lease, out);
+
+    INIT_LIST_HEAD(&glfd_list);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    gf_msg_debug(THIS->name, 0, "Recall lease received for gfid:%s",
+                 uuid_utoa(up_data->gfid));
+
+    inode = inode_find(subvol->itable, up_data->gfid);
+    if (!inode) {
+        ret = -1;
+        gf_smsg(THIS->name, GF_LOG_ERROR, errno, API_MSG_INODE_FIND_FAILED,
+                "gfid=%s", uuid_utoa(up_data->gfid), "graph_id=%d",
+                subvol->graph->id, NULL);
+        goto out;
+    }
+
+    LOCK(&inode->lock);
+    {
+        list_for_each_entry(fd, &inode->fd_list, inode_list)
+        {
+            ret = fd_ctx_get(fd, subvol, &value);
+            glfd = (struct glfs_fd *)(uintptr_t)value;
+            if (glfd) {
+                gf_msg_trace(THIS->name, 0, "glfd (%p) has held lease", glfd);
+                GF_REF_GET(glfd);
+                list_add_tail(&glfd->list, &glfd_list);
+            }
         }
+    }
+    UNLOCK(&inode->lock);
 
-        fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
-        if (!fd) {
-                errno = EBADFD;
-                goto out;
+    if (!list_empty(&glfd_list)) {
+        list_for_each_entry_safe(glfd, tmp, &glfd_list, list)
+        {
+            LOCK(&glfd->lock);
+            {
+                if (glfd->state != GLFD_CLOSE) {
+                    gf_msg_trace(THIS->name, 0,
+                                 "glfd (%p) has held lease, "
+                                 "calling recall cbk",
+                                 glfd);
+                    glfd->cbk(lease, glfd->cookie);
+                }
+            }
+            UNLOCK(&glfd->lock);
+
+            list_del_init(&glfd->list);
+            GF_REF_PUT(glfd);
         }
+    }
 
-        ret = syncop_zerofill (subvol, fd, offset, len);
-        DECODE_SYNCOP_ERR (ret);
 out:
-        if (fd)
-                fd_unref(fd);
+    return ret;
+}
 
-        priv_glfs_subvol_done (glfd->fs, subvol);
+static int
+glfs_recall_lease_upcall(struct glfs *fs, struct glfs_upcall *up_arg,
+                         struct gf_upcall *up_data)
+{
+    struct gf_upcall_recall_lease *recall_lease = NULL;
+    struct glfs_object *object = NULL;
+    xlator_t *subvol = NULL;
+    int ret = -1;
+    struct glfs_upcall_lease *up_lease_arg = NULL;
+
+    GF_VALIDATE_OR_GOTO("gfapi", up_data, out);
+    GF_VALIDATE_OR_GOTO("gfapi", fs, out);
+
+    recall_lease = up_data->data;
+    GF_VALIDATE_OR_GOTO("gfapi", recall_lease, out);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        errno = EIO;
+        goto out;
+    }
+
+    gf_msg_debug(THIS->name, 0, "Recall lease received for gfid:%s",
+                 uuid_utoa(up_data->gfid));
+
+    object = glfs_h_find_handle(fs, up_data->gfid, GFAPI_HANDLE_LENGTH);
+    if (!object) {
+        /* The reason handle creation will fail is because we
+         * couldn't find the inode in the gfapi inode table.
+         *
+         * But since application would have taken inode_ref, the
+         * only case when this can happen is when it has closed
+         * the handle and hence will no more be interested in
+         * the upcall for this particular gfid.
+         */
+        gf_smsg(THIS->name, GF_LOG_DEBUG, errno, API_MSG_CREATE_HANDLE_FAILED,
+                "gfid=%s", uuid_utoa(up_data->gfid), NULL);
+        errno = ESTALE;
+        goto out;
+    }
+
+    up_lease_arg = GF_CALLOC(1, sizeof(struct glfs_upcall_lease),
+                             glfs_mt_upcall_inode_t);
+    up_lease_arg->object = object;
+
+    GF_VALIDATE_OR_GOTO("glfs_recall_lease", up_lease_arg, out);
+
+    up_lease_arg->lease_type = recall_lease->lease_type;
+
+    up_arg->reason = GLFS_UPCALL_RECALL_LEASE;
+    up_arg->event = up_lease_arg;
+    up_arg->free_event = glfs_free_upcall_lease;
+
+    ret = 0;
 
-        return ret;
+out:
+    if (ret) {
+        /* Close p_object and oldp_object as well if being referenced.*/
+        if (object)
+            glfs_h_close(object);
+
+        /* Set reason to prevent applications from using ->event */
+        up_arg->reason = GF_UPCALL_EVENT_NULL;
+    }
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_zerofill, 3.5.0);
-
+static int
+upcall_syncop_args_free(struct upcall_syncop_args *args)
+{
+    dict_t *dict = NULL;
+    struct gf_upcall *upcall_data = NULL;
+
+    if (args) {
+        upcall_data = &args->upcall_data;
+        switch (upcall_data->event_type) {
+            case GF_UPCALL_CACHE_INVALIDATION:
+                dict = ((struct gf_upcall_cache_invalidation *)(upcall_data
+                                                                    ->data))
+                           ->dict;
+                break;
+            case GF_UPCALL_RECALL_LEASE:
+                dict = ((struct gf_upcall_recall_lease *)(upcall_data->data))
+                           ->dict;
+                break;
+        }
+        if (dict)
+            dict_unref(dict);
+
+        GF_FREE(upcall_data->client_uid);
+        GF_FREE(upcall_data->data);
+    }
+    GF_FREE(args);
+    return 0;
+}
 
-int
-pub_glfs_chdir (struct glfs *fs, const char *path)
+static int
+glfs_upcall_syncop_cbk(int ret, call_frame_t *frame, void *opaque)
 {
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	loc_t            loc = {0, };
-	struct iatt      iatt = {0, };
-	int              reval = 0;
+    struct upcall_syncop_args *args = opaque;
 
-	__glfs_entry_fs (fs);
+    (void)upcall_syncop_args_free(args);
 
-	subvol = priv_glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
-retry:
-	ret = glfs_resolve (fs, subvol, path, &loc, &iatt, reval);
+    return 0;
+}
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+static int
+glfs_cbk_upcall_syncop(void *opaque)
+{
+    struct upcall_syncop_args *args = opaque;
+    struct gf_upcall *upcall_data = NULL;
+    struct glfs_upcall *up_arg = NULL;
+    struct glfs *fs;
+    int ret = -1;
+
+    fs = args->fs;
+    upcall_data = &args->upcall_data;
+
+    if (!upcall_data) {
+        goto out;
+    }
+
+    up_arg = GLFS_CALLOC(1, sizeof(struct gf_upcall), glfs_release_upcall,
+                         glfs_mt_upcall_entry_t);
+    if (!up_arg) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, ENOMEM, API_MSG_ALLOC_FAILED, "entry",
+                NULL);
+        goto out;
+    }
+
+    switch (upcall_data->event_type) {
+        case GF_UPCALL_CACHE_INVALIDATION:
+            ret = glfs_h_poll_cache_invalidation(fs, up_arg, upcall_data);
+            break;
+        case GF_UPCALL_RECALL_LEASE:
+            ret = glfs_recall_lease_upcall(fs, up_arg, upcall_data);
+            break;
+        default:
+            errno = EINVAL;
+    }
+
+    /* It could so happen that the file which got
+     * upcall notification may have got deleted by
+     * the same client. In such cases up_arg->reason
+     * is set to GLFS_UPCALL_EVENT_NULL. No need to
+     * send upcall then
+     */
+    if (up_arg->reason == GLFS_UPCALL_EVENT_NULL) {
+        gf_smsg(THIS->name, GF_LOG_DEBUG, errno,
+                API_MSG_UPCALL_EVENT_NULL_RECEIVED, NULL);
+        ret = 0;
+        GLFS_FREE(up_arg);
+        goto out;
+    } else if (ret) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, errno, API_MSG_INVALID_ENTRY, NULL);
+        GLFS_FREE(up_arg);
+        goto out;
+    }
+
+    if (fs->up_cbk && up_arg)
+        (fs->up_cbk)(up_arg, fs->up_data);
+
+    /* application takes care of calling glfs_free on up_arg post
+     * their processing */
 
-	if (ret)
-		goto out;
+out:
+    return ret;
+}
 
-	if (!IA_ISDIR (iatt.ia_type)) {
-		ret = -1;
-		errno = ENOTDIR;
-		goto out;
-	}
+static struct gf_upcall_cache_invalidation *
+gf_copy_cache_invalidation(struct gf_upcall_cache_invalidation *src)
+{
+    struct gf_upcall_cache_invalidation *dst = NULL;
 
-	glfs_cwd_set (fs, loc.inode);
+    if (!src)
+        goto out;
 
-out:
-	loc_wipe (&loc);
+    dst = GF_CALLOC(1, sizeof(struct gf_upcall_cache_invalidation),
+                    glfs_mt_upcall_entry_t);
 
-	priv_glfs_subvol_done (fs, subvol);
+    if (!dst) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, ENOMEM, API_MSG_ALLOC_FAILED, "entry",
+                NULL);
+        goto out;
+    }
 
-	return ret;
-}
+    dst->flags = src->flags;
+    dst->expire_time_attr = src->expire_time_attr;
+    dst->stat = src->stat;
+    dst->p_stat = src->p_stat;
+    dst->oldp_stat = src->oldp_stat;
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_chdir, 3.4.0);
+    if (src->dict)
+        dst->dict = dict_copy_with_ref(src->dict, NULL);
 
+    return dst;
+out:
+    return NULL;
+}
 
-int
-pub_glfs_fchdir (struct glfs_fd *glfd)
+static struct gf_upcall_recall_lease *
+gf_copy_recall_lease(struct gf_upcall_recall_lease *src)
 {
-	int       ret = -1;
-	inode_t  *inode = NULL;
-	xlator_t *subvol = NULL;
-	fd_t     *fd = NULL;
+    struct gf_upcall_recall_lease *dst = NULL;
 
-	__glfs_entry_fd (glfd);
+    if (!src)
+        goto out;
 
-	subvol = priv_glfs_active_subvol (glfd->fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+    dst = GF_CALLOC(1, sizeof(struct gf_upcall_recall_lease),
+                    glfs_mt_upcall_entry_t);
 
-	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
-	if (!fd) {
-		ret = -1;
-		errno = EBADFD;
-		goto out;
-	}
+    if (!dst) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, ENOMEM, API_MSG_ALLOC_FAILED, "entry",
+                NULL);
+        goto out;
+    }
 
-	inode = fd->inode;
+    dst->lease_type = src->lease_type;
+    memcpy(dst->tid, src->tid, 16);
 
-	if (!IA_ISDIR (inode->ia_type)) {
-		ret = -1;
-		errno = ENOTDIR;
-		goto out;
-	}
+    if (src->dict)
+        dst->dict = dict_copy_with_ref(src->dict, NULL);
 
-	glfs_cwd_set (glfd->fs, inode);
-	ret = 0;
+    return dst;
 out:
-	if (fd)
-		fd_unref (fd);
+    return NULL;
+}
 
-	priv_glfs_subvol_done (glfd->fs, subvol);
+static struct upcall_syncop_args *
+upcall_syncop_args_init(struct glfs *fs, struct gf_upcall *upcall_data)
+{
+    struct upcall_syncop_args *args = NULL;
+    int ret = -1;
+    struct gf_upcall *t_data = NULL;
+
+    if (!fs || !upcall_data)
+        goto out;
+
+    args = GF_CALLOC(1, sizeof(struct upcall_syncop_args),
+                     glfs_mt_upcall_entry_t);
+    if (!args) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, ENOMEM, API_MSG_ALLOC_FAILED,
+                "syncop args", NULL);
+        goto out;
+    }
+
+    /* Note: we are not taking any ref on fs here.
+     * Ideally applications have to unregister for upcall events
+     * or stop polling for upcall events before performing
+     * glfs_fini. And as for outstanding synctasks created, we wait
+     * for all syncenv threads to finish tasks before cleaning up the
+     * fs->ctx. Hence it seems safe to process these callback
+     * notification without taking any lock/ref.
+     */
+    args->fs = fs;
+    t_data = &(args->upcall_data);
+    t_data->client_uid = gf_strdup(upcall_data->client_uid);
+
+    gf_uuid_copy(t_data->gfid, upcall_data->gfid);
+    t_data->event_type = upcall_data->event_type;
+
+    switch (t_data->event_type) {
+        case GF_UPCALL_CACHE_INVALIDATION:
+            t_data->data = gf_copy_cache_invalidation(
+                (struct gf_upcall_cache_invalidation *)upcall_data->data);
+            break;
+        case GF_UPCALL_RECALL_LEASE:
+            t_data->data = gf_copy_recall_lease(
+                (struct gf_upcall_recall_lease *)upcall_data->data);
+            break;
+    }
+
+    if (!t_data->data)
+        goto out;
+
+    return args;
+out:
+    if (ret) {
+        if (args) {
+            GF_FREE(args->upcall_data.client_uid);
+            GF_FREE(args);
+        }
+    }
 
-	return ret;
+    return NULL;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fchdir, 3.4.0);
+static void
+glfs_cbk_upcall_data(struct glfs *fs, struct gf_upcall *upcall_data)
+{
+    struct upcall_syncop_args *args = NULL;
+    int ret = -1;
 
+    if (!fs || !upcall_data)
+        goto out;
 
-char *
-pub_glfs_realpath (struct glfs *fs, const char *path, char *resolved_path)
-{
-	int              ret = -1;
-	char            *retpath = NULL;
-	char            *allocpath = NULL;
-	xlator_t        *subvol = NULL;
-	loc_t            loc = {0, };
-	struct iatt      iatt = {0, };
-	int              reval = 0;
-
-	__glfs_entry_fs (fs);
-
-	if (resolved_path)
-		retpath = resolved_path;
-	else
-		retpath = allocpath = malloc (PATH_MAX + 1);
-
-	if (!retpath) {
-		ret = -1;
-		errno = ENOMEM;
-		goto out;
-	}
-
-	subvol = priv_glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
-retry:
-	ret = glfs_resolve (fs, subvol, path, &loc, &iatt, reval);
+    if (!(fs->upcall_events & upcall_data->event_type)) {
+        /* ignore events which application hasn't registered*/
+        goto out;
+    }
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    args = upcall_syncop_args_init(fs, upcall_data);
 
-	if (ret)
-		goto out;
+    if (!args)
+        goto out;
 
-	if (loc.path) {
-		strncpy (retpath, loc.path, PATH_MAX);
-		retpath[PATH_MAX] = 0;
-	}
+    ret = synctask_new(THIS->ctx->env, glfs_cbk_upcall_syncop,
+                       glfs_upcall_syncop_cbk, NULL, args);
+    /* should we retry incase of failure? */
+    if (ret) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, errno, API_MSG_UPCALL_SYNCOP_FAILED,
+                "event_type=%d", upcall_data->event_type, "gfid=%s",
+                (char *)(upcall_data->gfid), NULL);
+        upcall_syncop_args_free(args);
+    }
 
 out:
-	loc_wipe (&loc);
-
-	if (ret == -1) {
-		if (allocpath)
-			free (allocpath);
-		retpath = NULL;
-	}
+    return;
+}
 
-	priv_glfs_subvol_done (fs, subvol);
+/*
+ * This routine is called in case of any notification received
+ * from the server. All the upcall events are queued up in a list
+ * to be read by the applications.
+ *
+ * In case if the application registers a cbk function, that shall
+ * be called by this routine in case of any event received.
+ * The cbk fn is responsible for notifying the
+ * applications the way it desires for each event queued (for eg.,
+ * can raise a signal or broadcast a cond variable etc.)
+ *
+ * Otherwise all the upcall events are queued up in a list
+ * to be read/polled by the applications.
+ */
+GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_process_upcall_event, 3.7.0)
+void
+priv_glfs_process_upcall_event(struct glfs *fs, void *data)
+{
+    glusterfs_ctx_t *ctx = NULL;
+    struct gf_upcall *upcall_data = NULL;
 
-	return retpath;
-}
+    DECLARE_OLD_THIS;
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_realpath, 3.4.0);
+    gf_msg_debug(THIS->name, 0, "Upcall gfapi callback is called");
 
+    __GLFS_ENTRY_VALIDATE_FS(fs, err);
 
-char *
-pub_glfs_getcwd (struct glfs *fs, char *buf, size_t n)
-{
-	int              ret = -1;
-	inode_t         *inode = NULL;
-	char            *path = NULL;
+    if (!data)
+        goto out;
 
-	__glfs_entry_fs (fs);
+    /* Unlike in I/O path, "glfs_fini" would not have freed
+     * 'fs' by the time we take lock as it waits for all epoll
+     * threads to exit including this
+     */
+    pthread_mutex_lock(&fs->mutex);
+    {
+        ctx = fs->ctx;
 
-	if (!buf || n < 2) {
-		ret = -1;
-		errno = EINVAL;
-		goto out;
-	}
+        /* if we're not interested in upcalls (anymore), skip them */
+        if (ctx->cleanup_started || !fs->cache_upcalls) {
+            pthread_mutex_unlock(&fs->mutex);
+            goto out;
+        }
 
-	inode = glfs_cwd_get (fs);
+        fs->pin_refcnt++;
+    }
+    pthread_mutex_unlock(&fs->mutex);
+
+    upcall_data = (struct gf_upcall *)data;
+
+    gf_msg_trace(THIS->name, 0, "Upcall gfapi gfid = %s",
+                 (char *)(upcall_data->gfid));
+
+    /* *
+     * TODO: RECALL LEASE for each glfd
+     *
+     * In case of RECALL_LEASE, we could associate separate
+     * cbk function for each glfd either by
+     * - extending pub_glfs_lease to accept new args (recall_cbk_fn, cookie)
+     * - or by defining new API "glfs_register_recall_cbk_fn (glfd,
+     * recall_cbk_fn, cookie) . In such cases, flag it and instead of calling
+     * below upcall functions, define a new one to go through the glfd list and
+     * invoke each of theirs recall_cbk_fn.
+     * */
+
+    if (fs->up_cbk) { /* upcall cbk registered */
+        (void)glfs_cbk_upcall_data(fs, upcall_data);
+    } else {
+        (void)glfs_enqueue_upcall_data(fs, upcall_data);
+    }
+
+    pthread_mutex_lock(&fs->mutex);
+    {
+        fs->pin_refcnt--;
+    }
+    pthread_mutex_unlock(&fs->mutex);
 
-	if (!inode) {
-		strncpy (buf, "/", n);
-		ret = 0;
-		goto out;
-	}
+out:
+    __GLFS_EXIT_FS;
+err:
+    return;
+}
 
-	ret = inode_path (inode, 0, &path);
-	if (n <= ret) {
-		ret = -1;
-		errno = ERANGE;
-		goto out;
-	}
+ssize_t
+glfs_anonymous_pwritev(struct glfs *fs, struct glfs_object *object,
+                       const struct iovec *iovec, int iovcnt, off_t offset,
+                       int flags)
+{
+    xlator_t *subvol = NULL;
+    struct iobref *iobref = NULL;
+    struct iobuf *iobuf = NULL;
+    struct iovec iov = {
+        0,
+    };
+    inode_t *inode = NULL;
+    fd_t *fd = NULL;
+    int ret = -1;
+    size_t size = -1;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, object);
+    if (!inode) {
+        ret = -1;
+        errno = ESTALE;
+        goto out;
+    }
+
+    fd = fd_anonymous(inode);
+    if (!fd) {
+        ret = -1;
+        gf_smsg("gfapi", GF_LOG_ERROR, ENOMEM, API_MSG_FDCREATE_FAILED, NULL);
+        errno = ENOMEM;
+        goto out;
+    }
+
+    size = iov_length(iovec, iovcnt);
+
+    iobuf = iobuf_get2(subvol->ctx->iobuf_pool, size);
+    if (!iobuf) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    iobref = iobref_new();
+    if (!iobref) {
+        iobuf_unref(iobuf);
+        errno = ENOMEM;
+        ret = -1;
+        goto out;
+    }
+
+    ret = iobref_add(iobref, iobuf);
+    if (ret) {
+        iobuf_unref(iobuf);
+        iobref_unref(iobref);
+        errno = ENOMEM;
+        ret = -1;
+        goto out;
+    }
+
+    iov_unload(iobuf_ptr(iobuf), iovec, iovcnt);
+
+    iov.iov_base = iobuf_ptr(iobuf);
+    iov.iov_len = size;
+
+    /* TODO : set leaseid */
+    ret = syncop_writev(subvol, fd, &iov, 1, offset, iobref, flags, NULL, NULL,
+                        NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    iobuf_unref(iobuf);
+    iobref_unref(iobref);
+
+    if (ret <= 0)
+        goto out;
 
-	strncpy (buf, path, n);
-	ret = 0;
 out:
-	GF_FREE (path);
 
-	if (inode)
-		inode_unref (inode);
+    if (fd)
+        fd_unref(fd);
 
-	if (ret < 0)
-		return NULL;
+    if (inode)
+        inode_unref(inode);
 
-	return buf;
-}
+    glfs_subvol_done(fs, subvol);
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_getcwd, 3.4.0);
+    __GLFS_EXIT_FS;
 
+invalid_fs:
+    return ret;
+}
 
-static void
-gf_flock_to_flock (struct gf_flock *gf_flock, struct flock *flock)
+ssize_t
+glfs_anonymous_preadv(struct glfs *fs, struct glfs_object *object,
+                      const struct iovec *iovec, int iovcnt, off_t offset,
+                      int flags)
 {
-	flock->l_type   = gf_flock->l_type;
-	flock->l_whence = gf_flock->l_whence;
-	flock->l_start  = gf_flock->l_start;
-	flock->l_len    = gf_flock->l_len;
-	flock->l_pid    = gf_flock->l_pid;
-}
+    xlator_t *subvol = NULL;
+    struct iovec *iov = NULL;
+    struct iobref *iobref = NULL;
+    inode_t *inode = NULL;
+    fd_t *fd = NULL;
+    int cnt = 0;
+    ssize_t ret = -1;
+    ssize_t size = -1;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, object);
+    if (!inode) {
+        ret = -1;
+        errno = ESTALE;
+        goto out;
+    }
+
+    fd = fd_anonymous(inode);
+    if (!fd) {
+        ret = -1;
+        gf_smsg("gfapi", GF_LOG_ERROR, ENOMEM, API_MSG_FDCREATE_FAILED, NULL);
+        errno = ENOMEM;
+        goto out;
+    }
+
+    size = iov_length(iovec, iovcnt);
+
+    /* TODO : set leaseid */
+    ret = syncop_readv(subvol, fd, size, offset, flags, &iov, &cnt, &iobref,
+                       NULL, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+    if (ret <= 0)
+        goto out;
+
+    size = iov_copy(iovec, iovcnt, iov, cnt);
+
+    ret = size;
+out:
+    if (iov)
+        GF_FREE(iov);
+    if (iobref)
+        iobref_unref(iobref);
+    if (fd)
+        fd_unref(fd);
+
+    if (inode)
+        inode_unref(inode);
+
+    glfs_subvol_done(fs, subvol);
 
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
 
 static void
-gf_flock_from_flock (struct gf_flock *gf_flock, struct flock *flock)
+glfs_release_xreaddirp_stat(void *ptr)
 {
-	gf_flock->l_type   = flock->l_type;
-	gf_flock->l_whence = flock->l_whence;
-	gf_flock->l_start  = flock->l_start;
-	gf_flock->l_len    = flock->l_len;
-	gf_flock->l_pid    = flock->l_pid;
-}
+    struct glfs_xreaddirp_stat *to_free = ptr;
 
+    if (to_free->object)
+        glfs_h_close(to_free->object);
+}
 
+/*
+ * Given glfd of a directory, this function does readdirp and returns
+ * xstat along with dirents.
+ */
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_xreaddirplus_r, 3.11.0)
 int
-pub_glfs_posix_lock (struct glfs_fd *glfd, int cmd, struct flock *flock)
+pub_glfs_xreaddirplus_r(struct glfs_fd *glfd, uint32_t flags,
+                        struct glfs_xreaddirp_stat **xstat_p,
+                        struct dirent *ext, struct dirent **res)
 {
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	struct gf_flock  gf_flock = {0, };
-	struct gf_flock  saved_flock = {0, };
-	fd_t            *fd = NULL;
+    int ret = -1;
+    gf_dirent_t *entry = NULL;
+    struct dirent *buf = NULL;
+    struct glfs_xreaddirp_stat *xstat = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    GF_REF_GET(glfd);
+
+    GF_VALIDATE_OR_GOTO(THIS->name, xstat_p, out);
+    GF_VALIDATE_OR_GOTO(THIS->name, res, out);
+
+    errno = 0;
+
+    if (ext)
+        buf = ext;
+    else
+        buf = glfs_readdirbuf_get(glfd);
+
+    if (!buf)
+        goto out;
+
+    xstat = GLFS_CALLOC(1, sizeof(struct glfs_xreaddirp_stat),
+                        glfs_release_xreaddirp_stat, glfs_mt_xreaddirp_stat_t);
+
+    if (!xstat)
+        goto out;
+
+    /* this is readdirplus operation */
+    entry = glfd_entry_next(glfd, 1);
+
+    /* XXX: Ideally when we reach EOD, errno should have been
+     * set to ENOENT. But that doesn't seem to be the case.
+     *
+     * The only way to confirm if its EOD at this point is that
+     * errno == 0 and entry == NULL
+     */
+    if (errno)
+        goto out;
+
+    if (!entry) {
+        /* reached EOD, ret = 0  */
+        ret = 0;
+        *res = NULL;
+        *xstat_p = NULL;
+
+        /* free xstat as applications shall not be using it */
+        GLFS_FREE(xstat);
+
+        goto out;
+    }
+
+    *res = buf;
+    gf_dirent_to_dirent(entry, buf);
+
+    if (flags & GFAPI_XREADDIRP_STAT) {
+        glfs_iatt_to_stat(glfd->fs, &entry->d_stat, &xstat->st);
+        xstat->flags_handled |= GFAPI_XREADDIRP_STAT;
+    }
+
+    if ((flags & GFAPI_XREADDIRP_HANDLE) &&
+        /* skip . and .. */
+        strcmp(buf->d_name, ".") && strcmp(buf->d_name, "..")) {
+        /* Now create object.
+         * We can use "glfs_h_find_handle" as well as inodes would have
+         * already got linked as part of 'gf_link_inodes_from_dirent' */
+        xstat->object = glfs_h_create_from_handle(
+            glfd->fs, entry->d_stat.ia_gfid, GFAPI_HANDLE_LENGTH, NULL);
+
+        if (xstat->object) { /* success */
+            /* note: xstat->object->inode->ref is taken
+             * This shall be unref'ed when application does
+             * glfs_free(xstat) */
+            xstat->flags_handled |= GFAPI_XREADDIRP_HANDLE;
+        }
+    }
 
-	__glfs_entry_fd (glfd);
+    ret = xstat->flags_handled;
+    *xstat_p = xstat;
 
-	subvol = priv_glfs_active_subvol (glfd->fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+    gf_msg_debug(THIS->name, 0,
+                 "xreaddirp- requested_flags (%x) , processed_flags (%x)",
+                 flags, xstat->flags_handled);
 
-	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
-	if (!fd) {
-		ret = -1;
-		errno = EBADFD;
-		goto out;
-	}
+out:
+    GF_REF_PUT(glfd);
 
-	gf_flock_from_flock (&gf_flock, flock);
-	gf_flock_from_flock (&saved_flock, flock);
-	ret = syncop_lk (subvol, fd, cmd, &gf_flock);
-        DECODE_SYNCOP_ERR (ret);
-	gf_flock_to_flock (&gf_flock, flock);
+    if (ret < 0) {
+        gf_smsg(THIS->name, GF_LOG_WARNING, errno, API_MSG_XREADDIRP_R_FAILED,
+                "reason=%s", strerror(errno), NULL);
 
-	if (ret == 0 && (cmd == F_SETLK || cmd == F_SETLKW))
-		fd_lk_insert_and_merge (fd, cmd, &saved_flock);
-out:
-	if (fd)
-		fd_unref (fd);
+        if (xstat)
+            GLFS_FREE(xstat);
+    }
 
-	priv_glfs_subvol_done (glfd->fs, subvol);
+    __GLFS_EXIT_FS;
 
-	return ret;
+    return ret;
+
+invalid_fs:
+    return -1;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_posix_lock, 3.4.0);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_xreaddirplus_get_stat, 3.11.0)
+struct stat *
+pub_glfs_xreaddirplus_get_stat(struct glfs_xreaddirp_stat *xstat)
+{
+    GF_VALIDATE_OR_GOTO("glfs_xreaddirplus_get_stat", xstat, out);
 
+    if (!xstat->flags_handled & GFAPI_XREADDIRP_STAT)
+        gf_smsg(THIS->name, GF_LOG_ERROR, errno, API_MSG_FLAGS_HANDLE,
+                "GFAPI_XREADDIRP_STAT"
+                "xstat=%p",
+                xstat, "handles=%x", xstat->flags_handled, NULL);
+    return &xstat->st;
 
-struct glfs_fd *
-pub_glfs_dup (struct glfs_fd *glfd)
+out:
+    return NULL;
+}
+
+void
+gf_lease_to_glfs_lease(struct gf_lease *gf_lease, struct glfs_lease *lease)
 {
-	xlator_t  *subvol = NULL;
-	fd_t      *fd = NULL;
-	glfs_fd_t *dupfd = NULL;
-	struct glfs *fs = NULL;
+    u_int lease_type = gf_lease->lease_type;
+    lease->cmd = gf_lease->cmd;
+    lease->lease_type = lease_type;
+    memcpy(lease->lease_id, gf_lease->lease_id, LEASE_ID_SIZE);
+}
 
-	__glfs_entry_fd (glfd);
+void
+glfs_lease_to_gf_lease(struct glfs_lease *lease, struct gf_lease *gf_lease)
+{
+    u_int lease_type = lease->lease_type;
+    gf_lease->cmd = lease->cmd;
+    gf_lease->lease_type = lease_type;
+    memcpy(gf_lease->lease_id, lease->lease_id, LEASE_ID_SIZE);
+}
 
-	fs = glfd->fs;
-	subvol = priv_glfs_active_subvol (fs);
-	if (!subvol) {
-		errno = EIO;
-		goto out;
-	}
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_lease, 4.0.0)
+int
+pub_glfs_lease(struct glfs_fd *glfd, struct glfs_lease *lease,
+               glfs_recall_cbk fn, void *data)
+{
+    int ret = -1;
+    loc_t loc = {
+        0,
+    };
+    xlator_t *subvol = NULL;
+    fd_t *fd = NULL;
+    struct gf_lease gf_lease = {
+        0,
+    };
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    GF_REF_GET(glfd);
+
+    if (!is_valid_lease_id(lease->lease_id)) {
+        ret = -1;
+        errno = EINVAL;
+        goto out;
+    }
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    switch (lease->lease_type) {
+        case GLFS_RD_LEASE:
+            if ((fd->flags != O_RDONLY) && !(fd->flags & O_RDWR)) {
+                ret = -1;
+                errno = EINVAL;
+                goto out;
+            }
+            break;
+        case GLFS_RW_LEASE:
+            if (!((fd->flags & O_WRONLY) || (fd->flags & O_RDWR))) {
+                ret = -1;
+                errno = EINVAL;
+                goto out;
+            }
+            break;
+        default:
+            if (lease->cmd != GLFS_GET_LEASE) {
+                ret = -1;
+                errno = EINVAL;
+                goto out;
+            }
+            break;
+    }
+
+    /* populate loc */
+    GLFS_LOC_FILL_INODE(fd->inode, loc, out);
+
+    glfs_lease_to_gf_lease(lease, &gf_lease);
 
-	fd = glfs_resolve_fd (fs, subvol, glfd);
-	if (!fd) {
-		errno = EBADFD;
-		goto out;
-	}
+    ret = syncop_lease(subvol, &loc, &gf_lease, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
 
-	dupfd = glfs_fd_new (fs);
-	if (!dupfd) {
-		errno = ENOMEM;
-		goto out;
-	}
+    gf_lease_to_glfs_lease(&gf_lease, lease);
+
+    /* TODO: Add leases for client replay
+    if (ret == 0 && (cmd == F_SETLK || cmd == F_SETLKW))
+            fd_lk_insert_and_merge (fd, cmd, &saved_flock);
+    */
+    if (ret == 0) {
+        ret = fd_ctx_set(glfd->fd, subvol, (uint64_t)(long)glfd);
+        if (ret) {
+            gf_smsg(subvol->name, GF_LOG_ERROR, ENOMEM,
+                    API_MSG_FDCTX_SET_FAILED, "fd=%p", glfd->fd, NULL);
+            goto out;
+        }
+        glfd->cbk = fn;
+        glfd->cookie = data;
+    }
 
-	dupfd->fd = fd_ref (fd);
 out:
-	if (fd)
-		fd_unref (fd);
-	if (dupfd)
-		glfs_fd_bind (dupfd);
 
-	priv_glfs_subvol_done (fs, subvol);
+    if (glfd)
+        GF_REF_PUT(glfd);
 
-	return dupfd;
-}
+    if (subvol)
+        glfs_subvol_done(glfd->fs, subvol);
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_dup, 3.4.0);
+    __GLFS_EXIT_FS;
 
+invalid_fs:
+    return ret;
+}
diff --git a/api/src/glfs-handleops.c b/api/src/glfs-handleops.c
index 2ee3820ed79..53c2ee896f9 100644
--- a/api/src/glfs-handleops.c
+++ b/api/src/glfs-handleops.c
@@ -1,5 +1,5 @@
 /*
- *  Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+ *  Copyright (c) 2013-2018 Red Hat, Inc. <http://www.redhat.com>
  *  This file is part of GlusterFS.
  *
  *  This file is licensed to you under your choice of the GNU Lesser
@@ -8,1582 +8,2648 @@
  *  cases as published by the Free Software Foundation.
  */
 
-
 #include "glfs-internal.h"
 #include "glfs-mem-types.h"
-#include "syncop.h"
+#include <glusterfs/syncop.h>
 #include "glfs.h"
 #include "glfs-handles.h"
+#include "gfapi-messages.h"
 
 int
-glfs_listxattr_process (void *value, size_t size, dict_t *xattr);
+glfs_listxattr_process(void *value, size_t size, dict_t *xattr);
 
-static void
-glfs_iatt_from_stat (struct stat *stat, int valid, struct iatt *iatt,
-                     int *glvalid)
+void
+glfs_iatt_from_stat(struct stat *stat, int valid, struct iatt *iatt,
+                    int *glvalid)
 {
-        /* validate in args */
-        if ((stat == NULL) || (iatt == NULL) || (glvalid == NULL)) {
-                errno = EINVAL;
-                return;
-        }
-
-        *glvalid = 0;
-
-        if (valid & GFAPI_SET_ATTR_MODE) {
-                iatt->ia_prot = ia_prot_from_st_mode (stat->st_mode);
-                *glvalid |= GF_SET_ATTR_MODE;
-        }
-
-        if (valid & GFAPI_SET_ATTR_UID) {
-                iatt->ia_uid = stat->st_uid;
-                *glvalid |= GF_SET_ATTR_UID;
-        }
-
-        if (valid & GFAPI_SET_ATTR_GID) {
-                iatt->ia_gid = stat->st_gid;
-                *glvalid |= GF_SET_ATTR_GID;
-        }
-
-        if (valid & GFAPI_SET_ATTR_ATIME) {
-                iatt->ia_atime = stat->st_atime;
-                iatt->ia_atime_nsec = ST_ATIM_NSEC (stat);
-                *glvalid |= GF_SET_ATTR_ATIME;
-        }
-
-        if (valid & GFAPI_SET_ATTR_MTIME) {
-                iatt->ia_mtime = stat->st_mtime;
-                iatt->ia_mtime_nsec = ST_MTIM_NSEC (stat);
-                *glvalid |= GF_SET_ATTR_MTIME;
-        }
-
+    /* validate in args */
+    if ((stat == NULL) || (iatt == NULL) || (glvalid == NULL)) {
+        errno = EINVAL;
         return;
+    }
+
+    *glvalid = 0;
+
+    if (valid & GFAPI_SET_ATTR_MODE) {
+        iatt->ia_prot = ia_prot_from_st_mode(stat->st_mode);
+        *glvalid |= GF_SET_ATTR_MODE;
+    }
+
+    if (valid & GFAPI_SET_ATTR_UID) {
+        iatt->ia_uid = stat->st_uid;
+        *glvalid |= GF_SET_ATTR_UID;
+    }
+
+    if (valid & GFAPI_SET_ATTR_GID) {
+        iatt->ia_gid = stat->st_gid;
+        *glvalid |= GF_SET_ATTR_GID;
+    }
+
+    if (valid & GFAPI_SET_ATTR_ATIME) {
+        iatt->ia_atime = stat->st_atime;
+        iatt->ia_atime_nsec = ST_ATIM_NSEC(stat);
+        *glvalid |= GF_SET_ATTR_ATIME;
+    }
+
+    if (valid & GFAPI_SET_ATTR_MTIME) {
+        iatt->ia_mtime = stat->st_mtime;
+        iatt->ia_mtime_nsec = ST_MTIM_NSEC(stat);
+        *glvalid |= GF_SET_ATTR_MTIME;
+    }
+
+    return;
 }
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_lookupat, 3.7.4)
 struct glfs_object *
-pub_glfs_h_lookupat (struct glfs *fs, struct glfs_object *parent,
-                     const char *path, struct stat *stat)
+pub_glfs_h_lookupat(struct glfs *fs, struct glfs_object *parent,
+                    const char *path, struct stat *stat, int follow)
 {
-        int                      ret = 0;
-        xlator_t                *subvol = NULL;
-        inode_t                 *inode = NULL;
-        struct iatt              iatt = {0, };
-        struct glfs_object      *object = NULL;
-        loc_t                    loc = {0, };
-
-        /* validate in args */
-        if ((fs == NULL) || (path == NULL)) {
-                errno = EINVAL;
-                return NULL;
-        }
-
-        __glfs_entry_fs (fs);
-
-        /* get the active volume */
-        subvol = priv_glfs_active_subvol (fs);
-        if (!subvol) {
-                errno = EIO;
-                goto out;
+    int ret = 0;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    struct iatt iatt = {
+        0,
+    };
+    struct glfs_object *object = NULL;
+    loc_t loc = {
+        0,
+    };
+
+    DECLARE_OLD_THIS;
+
+    /* validate in args */
+    if (path == NULL) {
+        errno = EINVAL;
+        return NULL;
+    }
+
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    if (parent) {
+        inode = glfs_resolve_inode(fs, subvol, parent);
+        if (!inode) {
+            errno = ESTALE;
+            goto out;
         }
+    }
 
-        /* get/refresh the in arg objects inode in correlation to the xlator */
-        if (parent) {
-                inode = glfs_resolve_inode (fs, subvol, parent);
-                if (!inode) {
-                        errno = ESTALE;
-                        goto out;
-                }
-        }
+    /* fop/op */
+    ret = glfs_resolve_at(fs, subvol, inode, path, &loc, &iatt, follow, 0);
 
-        /* fop/op */
-        ret = priv_glfs_resolve_at (fs, subvol, inode, path, &loc, &iatt,
-                                    0 /*TODO: links? */, 0);
+    /* populate out args */
+    if (!ret) {
+        if (stat)
+            glfs_iatt_to_stat(fs, &iatt, stat);
 
-        /* populate out args */
-        if (!ret) {
-                if (stat)
-                        glfs_iatt_to_stat (fs, &iatt, stat);
-
-                ret = glfs_create_object (&loc, &object);
-        }
+        ret = glfs_create_object(&loc, &object);
+    }
 
 out:
-        loc_wipe (&loc);
+    loc_wipe(&loc);
 
-        if (inode)
-                inode_unref (inode);
+    if (inode)
+        inode_unref(inode);
 
-        priv_glfs_subvol_done (fs, subvol);
+    glfs_subvol_done(fs, subvol);
 
-        return object;
-}
+    __GLFS_EXIT_FS;
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_lookupat, 3.4.2);
+invalid_fs:
+    return object;
+}
 
+GFAPI_SYMVER_PUBLIC(glfs_h_lookupat34, glfs_h_lookupat, 3.4.2)
+struct glfs_object *
+pub_glfs_h_lookupat34(struct glfs *fs, struct glfs_object *parent,
+                      const char *path, struct stat *stat)
+{
+    return pub_glfs_h_lookupat(fs, parent, path, stat, 0);
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_statfs, 3.7.0)
 int
-pub_glfs_h_stat (struct glfs *fs, struct glfs_object *object, struct stat *stat)
+pub_glfs_h_statfs(struct glfs *fs, struct glfs_object *object,
+                  struct statvfs *statvfs)
 {
-        int              ret = -1;
-        xlator_t        *subvol = NULL;
-        inode_t         *inode = NULL;
-        loc_t            loc = {0, };
-        struct iatt      iatt = {0, };
-
-        /* validate in args */
-        if ((fs == NULL) || (object == NULL)) {
-                errno = EINVAL;
-                return -1;
-        }
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    loc_t loc = {
+        0,
+    };
+
+    DECLARE_OLD_THIS;
+
+    /* validate in args */
+    if ((fs == NULL) || (object == NULL || statvfs == NULL)) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, object);
+    if (!inode) {
+        errno = ESTALE;
+        goto out;
+    }
+
+    /* populate loc */
+    GLFS_LOC_FILL_INODE(inode, loc, out);
+
+    /* fop/op */
+    ret = syncop_statfs(subvol, &loc, statvfs, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    loc_wipe(&loc);
 
-        __glfs_entry_fs (fs);
-
-        /* get the active volume */
-        subvol = priv_glfs_active_subvol (fs);
-        if (!subvol) {
-                ret = -1;
-                errno = EIO;
-                goto out;
-        }
+out:
+    if (inode)
+        inode_unref(inode);
 
-        /* get/refresh the in arg objects inode in correlation to the xlator */
-        inode = glfs_resolve_inode (fs, subvol, object);
-        if (!inode) {
-                errno = ESTALE;
-                goto out;
-        }
+    glfs_subvol_done(fs, subvol);
 
-        /* populate loc */
-        GLFS_LOC_FILL_INODE (inode, loc, out);
+    __GLFS_EXIT_FS;
 
-        /* fop/op */
-        ret = syncop_stat (subvol, &loc, &iatt);
-        DECODE_SYNCOP_ERR (ret);
+invalid_fs:
+    return ret;
+}
 
-        /* populate out args */
-        if (!ret && stat) {
-                glfs_iatt_to_stat (fs, &iatt, stat);
-        }
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_stat, 3.4.2)
+int
+pub_glfs_h_stat(struct glfs *fs, struct glfs_object *object, struct stat *stat)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+
+    DECLARE_OLD_THIS;
+
+    /* validate in args */
+    if ((fs == NULL) || (object == NULL)) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, object);
+    if (!inode) {
+        errno = ESTALE;
+        goto out;
+    }
+
+    /* populate loc */
+    GLFS_LOC_FILL_INODE(inode, loc, out);
+
+    /* fop/op */
+    ret = syncop_stat(subvol, &loc, &iatt, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    /* populate out args */
+    if (!ret && stat) {
+        glfs_iatt_to_stat(fs, &iatt, stat);
+    }
 out:
-        loc_wipe (&loc);
+    loc_wipe(&loc);
 
-        if (inode)
-                inode_unref (inode);
+    if (inode)
+        inode_unref(inode);
 
-        priv_glfs_subvol_done (fs, subvol);
-
-        return ret;
-}
+    glfs_subvol_done(fs, subvol);
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_stat, 3.4.2);
+    __GLFS_EXIT_FS;
 
+invalid_fs:
+    return ret;
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_getattrs, 3.4.2)
 int
-pub_glfs_h_getattrs (struct glfs *fs, struct glfs_object *object,
-                     struct stat *stat)
+pub_glfs_h_getattrs(struct glfs *fs, struct glfs_object *object,
+                    struct stat *stat)
 {
-        int                      ret = 0;
-        xlator_t                *subvol = NULL;
-        inode_t                 *inode = NULL;
-        struct iatt              iatt = {0, };
-
-        /* validate in args */
-        if ((fs == NULL) || (object == NULL)) {
-                errno = EINVAL;
-                return -1;
-        }
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    struct iatt iatt = {
+        0,
+    };
+
+    /* validate in args */
+    if ((fs == NULL) || (object == NULL)) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, object);
+    if (!inode) {
+        ret = 0;
+        errno = ESTALE;
+        goto out;
+    }
+
+    /* fop/op */
+    ret = glfs_resolve_base(fs, subvol, inode, &iatt);
+
+    /* populate out args */
+    if (!ret && stat) {
+        glfs_iatt_to_stat(fs, &iatt, stat);
+    }
 
-        __glfs_entry_fs (fs);
+out:
+    if (inode)
+        inode_unref(inode);
 
-        /* get the active volume */
-        subvol = priv_glfs_active_subvol (fs);
-        if (!subvol) {
-                ret = -1;
-                errno = EIO;
-                goto out;
-        }
+    glfs_subvol_done(fs, subvol);
 
-        /* get/refresh the in arg objects inode in correlation to the xlator */
-        inode = glfs_resolve_inode (fs, subvol, object);
-        if (!inode) {
-                errno = ESTALE;
-                goto out;
-        }
+    __GLFS_EXIT_FS;
 
-        /* fop/op */
-        ret = glfs_resolve_base (fs, subvol, inode, &iatt);
+invalid_fs:
+    return ret;
+}
 
-        /* populate out args */
-        if (!ret && stat) {
-                glfs_iatt_to_stat (fs, &iatt, stat);
-        }
+int
+glfs_h_getxattrs_common(struct glfs *fs, struct glfs_object *object,
+                        dict_t **xattr, const char *name,
+                        gf_boolean_t is_listxattr)
+{
+    int ret = 0;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    loc_t loc = {
+        0,
+    };
+
+    /* validate in args */
+    if ((fs == NULL) || (object == NULL)) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    if (!is_listxattr) {
+        if (!name || *name == '\0') {
+            errno = EINVAL;
+            return -1;
+        }
+
+        if (strlen(name) > GF_XATTR_NAME_MAX) {
+            errno = ENAMETOOLONG;
+            return -1;
+        }
+    }
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, object);
+    if (!inode) {
+        errno = ESTALE;
+        goto out;
+    }
+
+    /* populate loc */
+    GLFS_LOC_FILL_INODE(inode, loc, out);
+
+    ret = syncop_getxattr(subvol, &loc, xattr, name, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
 
 out:
-        if (inode)
-                inode_unref (inode);
+    loc_wipe(&loc);
 
-        priv_glfs_subvol_done (fs, subvol);
+    if (inode)
+        inode_unref(inode);
 
-        return ret;
-}
-
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_getattrs, 3.4.2);
+    glfs_subvol_done(fs, subvol);
 
+    return ret;
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_getxattrs, 3.5.1)
 int
-pub_glfs_h_getxattrs (struct glfs *fs, struct glfs_object *object,
-                      const char *name, void *value, size_t size)
+pub_glfs_h_getxattrs(struct glfs *fs, struct glfs_object *object,
+                     const char *name, void *value, size_t size)
 {
-        int                 ret = 0;
-        xlator_t        *subvol = NULL;
-        inode_t         *inode = NULL;
-        loc_t            loc = {0, };
-        dict_t                *xattr = NULL;
-
-        /* validate in args */
-        if ((fs == NULL) || (object == NULL)) {
-                errno = EINVAL;
-                return -1;
-        }
+    int ret = -1;
+    dict_t *xattr = NULL;
 
-        __glfs_entry_fs (fs);
+    /* validate in args */
+    if ((fs == NULL) || (object == NULL)) {
+        errno = EINVAL;
+        return -1;
+    }
 
-        /* get the active volume */
-        subvol = priv_glfs_active_subvol (fs);
-        if (!subvol) {
-                ret = -1;
-                errno = EIO;
-                goto out;
-        }
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
 
-        /* get/refresh the in arg objects inode in correlation to the xlator */
-        inode = glfs_resolve_inode (fs, subvol, object);
-        if (!inode) {
-                errno = ESTALE;
-                goto out;
-        }
-
-        /* populate loc */
-        GLFS_LOC_FILL_INODE (inode, loc, out);
+    ret = glfs_h_getxattrs_common(fs, object, &xattr, name, (name == NULL));
+    if (ret)
+        goto out;
 
-        ret = syncop_getxattr (subvol, &loc, &xattr, name);
-        DECODE_SYNCOP_ERR (ret);
-
-        if (ret)
-                goto out;
-
-        /* If @name is NULL, means get all the xattrs (i.e listxattr). */
-        if (name)
-                ret = glfs_getxattr_process (value, size, xattr, name);
-        else
-                ret = glfs_listxattr_process (value, size, xattr);
+    /* If @name is NULL, means get all the xattrs (i.e listxattr). */
+    if (name)
+        ret = glfs_getxattr_process(value, size, xattr, name);
+    else
+        ret = glfs_listxattr_process(value, size, xattr);
 
 out:
-        loc_wipe (&loc);
-
-        if (inode)
-                inode_unref (inode);
+    if (xattr)
+        dict_unref(xattr);
 
-        priv_glfs_subvol_done (fs, subvol);
+    __GLFS_EXIT_FS;
 
-        return ret;
+invalid_fs:
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_getxattrs, 3.5.1);
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_setattrs, 3.4.2)
 int
-pub_glfs_h_setattrs (struct glfs *fs, struct glfs_object *object,
-                     struct stat *stat, int valid)
+pub_glfs_h_setattrs(struct glfs *fs, struct glfs_object *object,
+                    struct stat *stat, int valid)
 {
-        int              ret = -1;
-        xlator_t        *subvol = NULL;
-        inode_t         *inode = NULL;
-        loc_t            loc = {0, };
-        struct iatt      iatt = {0, };
-        int              glvalid = 0;
-
-        /* validate in args */
-        if ((fs == NULL) || (object == NULL) || (stat == NULL)) {
-                errno = EINVAL;
-                return -1;
-        }
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    int glvalid = 0;
+
+    /* validate in args */
+    if ((fs == NULL) || (object == NULL) || (stat == NULL)) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, object);
+    if (!inode) {
+        errno = ESTALE;
+        goto out;
+    }
+
+    /* map valid masks from in args */
+    glfs_iatt_from_stat(stat, valid, &iatt, &glvalid);
+
+    /* populate loc */
+    GLFS_LOC_FILL_INODE(inode, loc, out);
+
+    /* fop/op */
+    ret = syncop_setattr(subvol, &loc, &iatt, glvalid, 0, 0, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+out:
+    loc_wipe(&loc);
 
-        __glfs_entry_fs (fs);
+    if (inode)
+        inode_unref(inode);
 
-        /* get the active volume */
-        subvol = priv_glfs_active_subvol (fs);
-        if (!subvol) {
-                ret = -1;
-                errno = EIO;
-                goto out;
-        }
+    glfs_subvol_done(fs, subvol);
 
-        /* get/refresh the in arg objects inode in correlation to the xlator */
-        inode = glfs_resolve_inode (fs, subvol, object);
-        if (!inode) {
-                errno = ESTALE;
-                goto out;
-        }
+    __GLFS_EXIT_FS;
 
-        /* map valid masks from in args */
-        glfs_iatt_from_stat (stat, valid, &iatt, &glvalid);
+invalid_fs:
+    return ret;
+}
 
-        /* populate loc */
-        GLFS_LOC_FILL_INODE (inode, loc, out);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_setxattrs, 3.5.0)
+int
+pub_glfs_h_setxattrs(struct glfs *fs, struct glfs_object *object,
+                     const char *name, const void *value, size_t size,
+                     int flags)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    loc_t loc = {
+        0,
+    };
+    dict_t *xattr = NULL;
+    void *value_cp = NULL;
+
+    /* validate in args */
+    if ((fs == NULL) || (object == NULL) || (name == NULL) || (value == NULL)) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    if (!name || *name == '\0') {
+        errno = EINVAL;
+        return -1;
+    }
+
+    if (strlen(name) > GF_XATTR_NAME_MAX) {
+        errno = ENAMETOOLONG;
+        return -1;
+    }
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, object);
+    if (!inode) {
+        errno = ESTALE;
+        goto out;
+    }
+
+    value_cp = gf_memdup(value, size);
+    GF_CHECK_ALLOC_AND_LOG(subvol->name, value_cp, ret,
+                           "Failed to"
+                           " duplicate setxattr value",
+                           out);
+
+    xattr = dict_for_key_value(name, value_cp, size, _gf_false);
+    if (!xattr) {
+        GF_FREE(value_cp);
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    /* populate loc */
+    GLFS_LOC_FILL_INODE(inode, loc, out);
+
+    /* fop/op */
+    ret = syncop_setxattr(subvol, &loc, xattr, flags, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
 
-        /* fop/op */
-        ret = syncop_setattr (subvol, &loc, &iatt, glvalid, 0, 0);
-        DECODE_SYNCOP_ERR (ret);
 out:
-        loc_wipe (&loc);
+    loc_wipe(&loc);
 
-        if (inode)
-                inode_unref (inode);
+    if (inode)
+        inode_unref(inode);
 
-        priv_glfs_subvol_done (fs, subvol);
+    if (xattr)
+        dict_unref(xattr);
 
-        return ret;
-}
+    glfs_subvol_done(fs, subvol);
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_setattrs, 3.4.2);
+    __GLFS_EXIT_FS;
 
+invalid_fs:
+    return ret;
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_removexattrs, 3.5.1)
 int
-pub_glfs_h_setxattrs (struct glfs *fs, struct glfs_object *object,
-                      const char *name, const void *value, size_t size,
-                      int flags)
+pub_glfs_h_removexattrs(struct glfs *fs, struct glfs_object *object,
+                        const char *name)
 {
-        int              ret = -1;
-        xlator_t        *subvol = NULL;
-        inode_t         *inode = NULL;
-        loc_t            loc = {0, };
-        dict_t          *xattr = NULL;
-
-        /* validate in args */
-        if ((fs == NULL) || (object == NULL) ||
-                 (name == NULL) || (value == NULL)) {
-                errno = EINVAL;
-                return -1;
-        }
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    loc_t loc = {
+        0,
+    };
+
+    /* validate in args */
+    if ((fs == NULL) || (object == NULL) || (name == NULL)) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, object);
+    if (!inode) {
+        errno = ESTALE;
+        goto out;
+    }
+
+    /* populate loc */
+    GLFS_LOC_FILL_INODE(inode, loc, out);
+
+    /* fop/op */
+    ret = syncop_removexattr(subvol, &loc, name, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
 
-        __glfs_entry_fs (fs);
+out:
+    loc_wipe(&loc);
 
-        /* get the active volume */
-        subvol = priv_glfs_active_subvol (fs);
-        if (!subvol) {
-                ret = -1;
-                errno = EIO;
-                goto out;
-        }
+    if (inode)
+        inode_unref(inode);
 
-        /* get/refresh the in arg objects inode in correlation to the xlator */
-        inode = glfs_resolve_inode (fs, subvol, object);
-        if (!inode) {
-                errno = ESTALE;
-                goto out;
-        }
+    glfs_subvol_done(fs, subvol);
 
-        xattr = dict_for_key_value (name, value, size);
-        if (!xattr) {
-                ret = -1;
-                errno = ENOMEM;
-                goto out;
-        }
+    __GLFS_EXIT_FS;
 
-        /* populate loc */
-        GLFS_LOC_FILL_INODE (inode, loc, out);
+invalid_fs:
+    return ret;
+}
 
-        /* fop/op */
-        ret = syncop_setxattr (subvol, &loc, xattr, flags);
-        DECODE_SYNCOP_ERR (ret);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_open, 3.4.2)
+struct glfs_fd *
+pub_glfs_h_open(struct glfs *fs, struct glfs_object *object, int flags)
+{
+    int ret = -1;
+    struct glfs_fd *glfd = NULL;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    loc_t loc = {
+        0,
+    };
+    dict_t *fop_attr = NULL;
+
+    /* validate in args */
+    if ((fs == NULL) || (object == NULL)) {
+        errno = EINVAL;
+        return NULL;
+    }
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, object);
+    if (!inode) {
+        errno = ESTALE;
+        goto out;
+    }
+
+    /* check types to open */
+    if (IA_ISDIR(inode->ia_type)) {
+        ret = -1;
+        errno = EISDIR;
+        goto out;
+    }
+
+    if (!IA_ISREG(inode->ia_type)) {
+        ret = -1;
+        errno = EINVAL;
+        goto out;
+    }
+
+    glfd = glfs_fd_new(fs);
+    if (!glfd) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    glfd->fd = fd_create(inode, getpid());
+    if (!glfd->fd) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+    glfd->fd->flags = flags;
+
+    /* populate loc */
+    GLFS_LOC_FILL_INODE(inode, loc, out);
+
+    /* fop/op */
+    ret = get_fop_attr_thrd_key(&fop_attr);
+    if (ret)
+        gf_msg_debug("gfapi", 0, "Getting leaseid from thread failed");
+
+    ret = syncop_open(subvol, &loc, flags, glfd->fd, fop_attr, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    glfd->fd->flags = flags;
 
 out:
-        loc_wipe (&loc);
+    loc_wipe(&loc);
 
-        if (inode)
-                inode_unref (inode);
+    if (inode)
+        inode_unref(inode);
+    if (fop_attr)
+        dict_unref(fop_attr);
 
-        if (xattr)
-                dict_unref (xattr);
+    if (ret && glfd) {
+        GF_REF_PUT(glfd);
+        glfd = NULL;
+    } else if (glfd) {
+        glfd_set_state_bind(glfd);
+    }
 
-        priv_glfs_subvol_done (fs, subvol);
+    glfs_subvol_done(fs, subvol);
 
-        return ret;
-}
-
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_setxattrs, 3.5.0);
+    __GLFS_EXIT_FS;
 
+invalid_fs:
+    return glfd;
+}
 
-int
-pub_glfs_h_removexattrs (struct glfs *fs, struct glfs_object *object,
-                         const char *name)
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_creat, 3.4.2)
+struct glfs_object *
+pub_glfs_h_creat(struct glfs *fs, struct glfs_object *parent, const char *path,
+                 int flags, mode_t mode, struct stat *stat)
 {
-        int              ret = -1;
-        xlator_t        *subvol = NULL;
-        inode_t         *inode = NULL;
-        loc_t            loc = {0, };
-
-        /* validate in args */
-        if ((fs == NULL) || (object == NULL) || (name == NULL)) {
-                errno = EINVAL;
-                return -1;
-        }
-
-        __glfs_entry_fs (fs);
-
-        /* get the active volume */
-        subvol = priv_glfs_active_subvol (fs);
-        if (!subvol) {
-                ret = -1;
-                errno = EIO;
-                goto out;
-        }
-
-        /* get/refresh the in arg objects inode in correlation to the xlator */
-        inode = glfs_resolve_inode (fs, subvol, object);
-        if (!inode) {
-                errno = ESTALE;
-                goto out;
+    int ret = -1;
+    fd_t *fd = NULL;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    uuid_t gfid;
+    dict_t *xattr_req = NULL;
+    struct glfs_object *object = NULL;
+
+    /* validate in args */
+    if ((fs == NULL) || (parent == NULL) || (path == NULL)) {
+        errno = EINVAL;
+        return NULL;
+    }
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, parent);
+    if (!inode) {
+        ret = -1;
+        errno = ESTALE;
+        goto out;
+    }
+
+    xattr_req = dict_new();
+    if (!xattr_req) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gf_uuid_generate(gfid);
+    ret = dict_set_gfuuid(xattr_req, "gfid-req", gfid, true);
+    if (ret) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    GLFS_LOC_FILL_PINODE(inode, loc, ret, errno, out, path);
+
+    fd = fd_create(loc.inode, getpid());
+    if (!fd) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+    fd->flags = flags;
+
+    /* fop/op */
+    ret = syncop_create(subvol, &loc, flags, mode, fd, &iatt, xattr_req, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    /* populate out args */
+    if (ret == 0) {
+        ret = glfs_loc_link(&loc, &iatt);
+        if (ret != 0) {
+            goto out;
         }
 
-        /* populate loc */
-        GLFS_LOC_FILL_INODE (inode, loc, out);
+        if (stat)
+            glfs_iatt_to_stat(fs, &iatt, stat);
 
-        /* fop/op */
-        ret = syncop_removexattr (subvol, &loc, name, 0);
-        DECODE_SYNCOP_ERR (ret);
+        ret = glfs_create_object(&loc, &object);
+    }
 
 out:
-        loc_wipe (&loc);
+    if (ret && object != NULL) {
+        /* Release the held reference */
+        glfs_h_close(object);
+        object = NULL;
+    }
 
-        if (inode)
-                inode_unref (inode);
+    loc_wipe(&loc);
 
-        priv_glfs_subvol_done (fs, subvol);
+    if (inode)
+        inode_unref(inode);
 
-        return ret;
-}
+    if (xattr_req)
+        dict_unref(xattr_req);
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_removexattrs, 3.5.1);
+    if (fd)
+        fd_unref(fd);
 
+    glfs_subvol_done(fs, subvol);
 
-struct glfs_fd *
-pub_glfs_h_open (struct glfs *fs, struct glfs_object *object, int flags)
-{
-        int              ret = -1;
-        struct glfs_fd  *glfd = NULL;
-        xlator_t        *subvol = NULL;
-        inode_t         *inode = NULL;
-        loc_t            loc = {0, };
-
-        /* validate in args */
-        if ((fs == NULL) || (object == NULL)) {
-                errno = EINVAL;
-                return NULL;
-        }
+    __GLFS_EXIT_FS;
 
-        __glfs_entry_fs (fs);
+invalid_fs:
+    return object;
+}
 
-        /* get the active volume */
-        subvol = priv_glfs_active_subvol (fs);
-        if (!subvol) {
-                errno = EIO;
-                goto out;
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_creat_open, 6.6)
+struct glfs_object *
+pub_glfs_h_creat_open(struct glfs *fs, struct glfs_object *parent,
+                      const char *path, int flags, mode_t mode,
+                      struct stat *stat, struct glfs_fd **out_fd)
+{
+    int ret = -1;
+    struct glfs_fd *glfd = NULL;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    uuid_t gfid;
+    dict_t *xattr_req = NULL;
+    struct glfs_object *object = NULL;
+    dict_t *fop_attr = NULL;
+
+    /* validate in args */
+    if ((fs == NULL) || (parent == NULL) || (path == NULL) ||
+        (out_fd == NULL)) {
+        errno = EINVAL;
+        return NULL;
+    }
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, parent);
+    if (!inode) {
+        ret = -1;
+        goto out;
+    }
+
+    xattr_req = dict_new();
+    if (!xattr_req) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gf_uuid_generate(gfid);
+    ret = dict_set_gfuuid(xattr_req, "gfid-req", gfid, true);
+    if (ret) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    GLFS_LOC_FILL_PINODE(inode, loc, ret, errno, out, path);
+
+    glfd = glfs_fd_new(fs);
+    if (!glfd) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    glfd->fd = fd_create(loc.inode, getpid());
+    if (!glfd->fd) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+    glfd->fd->flags = flags;
+
+    ret = get_fop_attr_thrd_key(&fop_attr);
+    if (ret)
+        gf_msg_debug("gfapi", 0, "Getting leaseid from thread failed");
+
+    /* fop/op */
+    ret = syncop_create(subvol, &loc, flags, mode, glfd->fd, &iatt, xattr_req,
+                        NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    /* populate out args */
+    if (ret == 0) {
+        glfd->fd->flags = flags;
+
+        ret = glfs_loc_link(&loc, &iatt);
+        if (ret != 0) {
+            goto out;
         }
 
-        /* get/refresh the in arg objects inode in correlation to the xlator */
-        inode = glfs_resolve_inode (fs, subvol, object);
-        if (!inode) {
-                errno = ESTALE;
-                goto out;
-        }
+        if (stat)
+            glfs_iatt_to_stat(fs, &iatt, stat);
 
-        /* check types to open */
-        if (IA_ISDIR (inode->ia_type)) {
-                ret = -1;
-                errno = EISDIR;
-                goto out;
-        }
+        ret = glfs_create_object(&loc, &object);
+    }
 
-        if (!IA_ISREG (inode->ia_type)) {
-                ret = -1;
-                errno = EINVAL;
-                goto out;
-        }
+out:
+    if (ret && object != NULL) {
+        /* Release the held reference */
+        glfs_h_close(object);
+        object = NULL;
+    }
 
-        glfd = glfs_fd_new (fs);
-        if (!glfd) {
-                errno = ENOMEM;
-                goto out;
-        }
+    loc_wipe(&loc);
 
-        glfd->fd = fd_create (inode, getpid());
-        if (!glfd->fd) {
-                ret = -1;
-                errno = ENOMEM;
-                goto out;
-        }
-
-        /* populate loc */
-        GLFS_LOC_FILL_INODE (inode, loc, out);
+    if (inode)
+        inode_unref(inode);
 
-        /* fop/op */
-        ret = syncop_open (subvol, &loc, flags, glfd->fd);
-        DECODE_SYNCOP_ERR (ret);
+    if (fop_attr)
+        dict_unref(fop_attr);
 
-out:
-        loc_wipe (&loc);
+    if (xattr_req)
+        dict_unref(xattr_req);
 
-        if (inode)
-                inode_unref (inode);
+    if (ret && glfd) {
+        GF_REF_PUT(glfd);
+    } else if (glfd) {
+        glfd_set_state_bind(glfd);
+        *out_fd = glfd;
+    }
 
-        if (ret && glfd) {
-                glfs_fd_destroy (glfd);
-                glfd = NULL;
-        } else {
-                glfd->fd->flags = flags;
-                fd_bind (glfd->fd);
-                glfs_fd_bind (glfd);
-        }
+    glfs_subvol_done(fs, subvol);
 
-        priv_glfs_subvol_done (fs, subvol);
+    __GLFS_EXIT_FS;
 
-        return glfd;
+invalid_fs:
+    return object;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_open, 3.4.2);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_mkdir, 3.4.2)
 struct glfs_object *
-pub_glfs_h_creat (struct glfs *fs, struct glfs_object *parent, const char *path,
-                  int flags, mode_t mode, struct stat *stat)
+pub_glfs_h_mkdir(struct glfs *fs, struct glfs_object *parent, const char *path,
+                 mode_t mode, struct stat *stat)
 {
-        int                 ret = -1;
-        struct glfs_fd     *glfd = NULL;
-        xlator_t           *subvol = NULL;
-        inode_t            *inode = NULL;
-        loc_t               loc = {0, };
-        struct iatt         iatt = {0, };
-        uuid_t              gfid;
-        dict_t             *xattr_req = NULL;
-        struct glfs_object *object = NULL;
-
-        /* validate in args */
-        if ((fs == NULL) || (parent == NULL) || (path == NULL)) {
-                errno = EINVAL;
-                return NULL;
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    uuid_t gfid;
+    dict_t *xattr_req = NULL;
+    struct glfs_object *object = NULL;
+
+    /* validate in args */
+    if ((fs == NULL) || (parent == NULL) || (path == NULL)) {
+        errno = EINVAL;
+        return NULL;
+    }
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, parent);
+    if (!inode) {
+        errno = ESTALE;
+        goto out;
+    }
+
+    xattr_req = dict_new();
+    if (!xattr_req) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gf_uuid_generate(gfid);
+    ret = dict_set_gfuuid(xattr_req, "gfid-req", gfid, true);
+    if (ret) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    GLFS_LOC_FILL_PINODE(inode, loc, ret, errno, out, path);
+
+    /* fop/op */
+    ret = syncop_mkdir(subvol, &loc, mode, &iatt, xattr_req, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    /* populate out args */
+    if (ret == 0) {
+        ret = glfs_loc_link(&loc, &iatt);
+        if (ret != 0) {
+            goto out;
         }
 
-        __glfs_entry_fs (fs);
+        if (stat)
+            glfs_iatt_to_stat(fs, &iatt, stat);
 
-        /* get the active volume */
-        subvol = priv_glfs_active_subvol (fs);
-        if (!subvol) {
-                ret = -1;
-                errno = EIO;
-                goto out;
-        }
+        ret = glfs_create_object(&loc, &object);
+    }
 
-        /* get/refresh the in arg objects inode in correlation to the xlator */
-        inode = glfs_resolve_inode (fs, subvol, parent);
-        if (!inode) {
-                errno = ESTALE;
-                goto out;
-        }
+out:
+    if (ret && object != NULL) {
+        glfs_h_close(object);
+        object = NULL;
+    }
 
-        xattr_req = dict_new ();
-        if (!xattr_req) {
-                ret = -1;
-                errno = ENOMEM;
-                goto out;
-        }
+    loc_wipe(&loc);
 
-        uuid_generate (gfid);
-        ret = dict_set_static_bin (xattr_req, "gfid-req", gfid, 16);
-        if (ret) {
-                ret = -1;
-                errno = ENOMEM;
-                goto out;
-        }
+    if (inode)
+        inode_unref(inode);
 
-        GLFS_LOC_FILL_PINODE (inode, loc, ret, errno, out, path);
+    if (xattr_req)
+        dict_unref(xattr_req);
 
-        glfd = glfs_fd_new (fs);
-        if (!glfd)
-                goto out;
+    glfs_subvol_done(fs, subvol);
 
-        glfd->fd = fd_create (loc.inode, getpid());
-        if (!glfd->fd) {
-                ret = -1;
-                errno = ENOMEM;
-                goto out;
-        }
+    __GLFS_EXIT_FS;
 
-        /* fop/op */
-        ret = syncop_create (subvol, &loc, flags, mode, glfd->fd,
-                             xattr_req, &iatt);
-        DECODE_SYNCOP_ERR (ret);
-
-        /* populate out args */
-        if (ret == 0) {
-                /* TODO: If the inode existed in the cache (say file already
-                   exists), then the glfs_loc_link will not update the
-                   loc.inode, as a result we will have a 0000 GFID that we
-                   would copy out to the object, this needs to be fixed.
-                */
-                ret = glfs_loc_link (&loc, &iatt);
-                if (ret != 0) {
-                        goto out;
-                }
-
-                if (stat)
-                        glfs_iatt_to_stat (fs, &iatt, stat);
+invalid_fs:
+    return object;
+}
 
-                ret = glfs_create_object (&loc, &object);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_mknod, 3.4.2)
+struct glfs_object *
+pub_glfs_h_mknod(struct glfs *fs, struct glfs_object *parent, const char *path,
+                 mode_t mode, dev_t dev, struct stat *stat)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    uuid_t gfid;
+    dict_t *xattr_req = NULL;
+    struct glfs_object *object = NULL;
+
+    /* validate in args */
+    if ((fs == NULL) || (parent == NULL) || (path == NULL)) {
+        errno = EINVAL;
+        return NULL;
+    }
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, parent);
+    if (!inode) {
+        errno = ESTALE;
+        goto out;
+    }
+
+    xattr_req = dict_new();
+    if (!xattr_req) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gf_uuid_generate(gfid);
+    ret = dict_set_gfuuid(xattr_req, "gfid-req", gfid, true);
+    if (ret) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    GLFS_LOC_FILL_PINODE(inode, loc, ret, errno, out, path);
+
+    /* fop/op */
+    ret = syncop_mknod(subvol, &loc, mode, dev, &iatt, xattr_req, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    /* populate out args */
+    if (ret == 0) {
+        ret = glfs_loc_link(&loc, &iatt);
+        if (ret != 0) {
+            goto out;
         }
 
+        if (stat)
+            glfs_iatt_to_stat(fs, &iatt, stat);
+
+        ret = glfs_create_object(&loc, &object);
+    }
 out:
-        if (ret && object != NULL) {
-                glfs_h_close (object);
-                object = NULL;
-        }
+    if (ret && object != NULL) {
+        glfs_h_close(object);
+        object = NULL;
+    }
 
-        loc_wipe(&loc);
+    loc_wipe(&loc);
 
-        if (inode)
-                inode_unref (inode);
+    if (inode)
+        inode_unref(inode);
 
-        if (xattr_req)
-                dict_unref (xattr_req);
+    if (xattr_req)
+        dict_unref(xattr_req);
 
-        if (glfd) {
-                glfs_fd_destroy (glfd);
-                glfd = NULL;
-        }
+    glfs_subvol_done(fs, subvol);
 
-        priv_glfs_subvol_done (fs, subvol);
+    __GLFS_EXIT_FS;
 
-        return object;
+invalid_fs:
+    return object;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_creat, 3.4.2);
-
-
-struct glfs_object *
-pub_glfs_h_mkdir (struct glfs *fs, struct glfs_object *parent, const char *path,
-                  mode_t mode, struct stat *stat)
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_unlink, 3.4.2)
+int
+pub_glfs_h_unlink(struct glfs *fs, struct glfs_object *parent, const char *path)
 {
-        int                 ret = -1;
-        xlator_t           *subvol = NULL;
-        inode_t            *inode = NULL;
-        loc_t               loc = {0, };
-        struct iatt         iatt = {0, };
-        uuid_t              gfid;
-        dict_t             *xattr_req = NULL;
-        struct glfs_object *object = NULL;
-
-        /* validate in args */
-        if ((fs == NULL) || (parent == NULL) || (path == NULL)) {
-                errno = EINVAL;
-                return NULL;
-        }
-
-        __glfs_entry_fs (fs);
-
-        /* get the active volume */
-        subvol = priv_glfs_active_subvol (fs);
-        if (!subvol) {
-                ret = -1;
-                errno = EIO;
-                goto out;
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    loc_t loc = {
+        0,
+    };
+
+    /* validate in args */
+    if ((fs == NULL) || (parent == NULL) || (path == NULL)) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, parent);
+    if (!inode) {
+        errno = ESTALE;
+        goto out;
+    }
+
+    ret = glfs_resolve_at(fs, subvol, inode, path, &loc, NULL, 0, 0);
+    if (ret != 0) {
+        goto out;
+    }
+
+    if (!IA_ISDIR(loc.inode->ia_type)) {
+        ret = syncop_unlink(subvol, &loc, NULL, NULL);
+        DECODE_SYNCOP_ERR(ret);
+        if (ret != 0) {
+            goto out;
         }
-
-        /* get/refresh the in arg objects inode in correlation to the xlator */
-        inode = glfs_resolve_inode (fs, subvol, parent);
-        if (!inode) {
-                errno = ESTALE;
-                goto out;
+    } else {
+        ret = syncop_rmdir(subvol, &loc, 0, NULL, NULL);
+        DECODE_SYNCOP_ERR(ret);
+        if (ret != 0) {
+            goto out;
         }
+    }
 
-        xattr_req = dict_new ();
-        if (!xattr_req) {
-                ret = -1;
-                errno = ENOMEM;
-                goto out;
-        }
+    if (ret == 0)
+        ret = glfs_loc_unlink(&loc);
 
-        uuid_generate (gfid);
-        ret = dict_set_static_bin (xattr_req, "gfid-req", gfid, 16);
-        if (ret) {
-                ret = -1;
-                errno = ENOMEM;
-                goto out;
-        }
+out:
+    loc_wipe(&loc);
 
-        GLFS_LOC_FILL_PINODE (inode, loc, ret, errno, out, path);
+    if (inode)
+        inode_unref(inode);
 
-        /* fop/op */
-        ret = syncop_mkdir (subvol, &loc, mode, xattr_req, &iatt);
-        DECODE_SYNCOP_ERR (ret);
+    glfs_subvol_done(fs, subvol);
 
-        /* populate out args */
-        if ( ret == 0 )  {
-                ret = glfs_loc_link (&loc, &iatt);
-                if (ret != 0) {
-                        goto out;
-                }
+    __GLFS_EXIT_FS;
 
-                if (stat)
-                        glfs_iatt_to_stat (fs, &iatt, stat);
+invalid_fs:
+    return ret;
+}
 
-                ret = glfs_create_object (&loc, &object);
-        }
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_opendir, 3.4.2)
+struct glfs_fd *
+pub_glfs_h_opendir(struct glfs *fs, struct glfs_object *object)
+{
+    int ret = -1;
+    struct glfs_fd *glfd = NULL;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    loc_t loc = {
+        0,
+    };
+
+    /* validate in args */
+    if ((fs == NULL) || (object == NULL)) {
+        errno = EINVAL;
+        return NULL;
+    }
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, object);
+    if (!inode) {
+        errno = ESTALE;
+        goto out;
+    }
+
+    if (!IA_ISDIR(inode->ia_type)) {
+        ret = -1;
+        errno = ENOTDIR;
+        goto out;
+    }
+
+    glfd = glfs_fd_new(fs);
+    if (!glfd)
+        goto out;
+
+    INIT_LIST_HEAD(&glfd->entries);
+
+    glfd->fd = fd_create(inode, getpid());
+    if (!glfd->fd) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    GLFS_LOC_FILL_INODE(inode, loc, out);
+
+    /* fop/op */
+    ret = syncop_opendir(subvol, &loc, glfd->fd, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
 
 out:
-        if (ret && object != NULL) {
-                glfs_h_close (object);
-                object = NULL;
-        }
+    loc_wipe(&loc);
 
-        loc_wipe(&loc);
+    if (inode)
+        inode_unref(inode);
 
-        if (inode)
-                inode_unref (inode);
+    if (ret && glfd) {
+        GF_REF_PUT(glfd);
+        glfd = NULL;
+    } else if (glfd) {
+        glfd_set_state_bind(glfd);
+    }
 
-        if (xattr_req)
-                dict_unref (xattr_req);
+    glfs_subvol_done(fs, subvol);
 
-        priv_glfs_subvol_done (fs, subvol);
+    __GLFS_EXIT_FS;
 
-        return object;
+invalid_fs:
+    return glfd;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_mkdir, 3.4.2);
-
-
-struct glfs_object *
-pub_glfs_h_mknod (struct glfs *fs, struct glfs_object *parent, const char *path,
-                  mode_t mode, dev_t dev, struct stat *stat)
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_access, 3.6.0)
+int
+pub_glfs_h_access(struct glfs *fs, struct glfs_object *object, int mask)
 {
-        int                 ret = -1;
-        xlator_t           *subvol = NULL;
-        inode_t            *inode = NULL;
-        loc_t               loc = {0, };
-        struct iatt         iatt = {0, };
-        uuid_t              gfid;
-        dict_t             *xattr_req = NULL;
-        struct glfs_object *object = NULL;
-
-        /* validate in args */
-        if ((fs == NULL) || (parent == NULL) || (path == NULL)) {
-                errno = EINVAL;
-                return NULL;
-        }
-
-        __glfs_entry_fs (fs);
-
-        /* get the active volume */
-        subvol = priv_glfs_active_subvol (fs);
-        if (!subvol) {
-                ret = -1;
-                errno = EIO;
-                goto out;
-        }
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    loc_t loc = {
+        0,
+    };
+
+    DECLARE_OLD_THIS;
+
+    /* validate in args */
+    if ((fs == NULL) || (object == NULL)) {
+        errno = EINVAL;
+        return ret;
+    }
 
-        /* get/refresh the in arg objects inode in correlation to the xlator */
-        inode = glfs_resolve_inode (fs, subvol, parent);
-        if (!inode) {
-                errno = ESTALE;
-                goto out;
-        }
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
 
-        xattr_req = dict_new ();
-        if (!xattr_req) {
-                ret = -1;
-                errno = ENOMEM;
-                goto out;
-        }
-
-        uuid_generate (gfid);
-        ret = dict_set_static_bin (xattr_req, "gfid-req", gfid, 16);
-        if (ret) {
-                ret = -1;
-                errno = ENOMEM;
-                goto out;
-        }
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
 
-        GLFS_LOC_FILL_PINODE (inode, loc, ret, errno, out, path);
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, object);
+    if (!inode) {
+        errno = ESTALE;
+        goto out;
+    }
 
-        /* fop/op */
-        ret = syncop_mknod (subvol, &loc, mode, dev, xattr_req, &iatt);
-        DECODE_SYNCOP_ERR (ret);
+    GLFS_LOC_FILL_INODE(inode, loc, out);
 
-        /* populate out args */
-        if (ret == 0) {
-                ret = glfs_loc_link (&loc, &iatt);
-                if (ret != 0) {
-                        goto out;
-                }
+    /* fop/op */
 
-                if (stat)
-                        glfs_iatt_to_stat (fs, &iatt, stat);
+    ret = syncop_access(subvol, &loc, mask, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
 
-                ret = glfs_create_object (&loc, &object);
-        }
 out:
-        if (ret && object != NULL) {
-                glfs_h_close (object);
-                object = NULL;
-        }
+    loc_wipe(&loc);
 
-        loc_wipe(&loc);
+    if (inode)
+        inode_unref(inode);
 
-        if (inode)
-                inode_unref (inode);
+    glfs_subvol_done(fs, subvol);
 
-        if (xattr_req)
-                dict_unref (xattr_req);
+    __GLFS_EXIT_FS;
 
-        priv_glfs_subvol_done (fs, subvol);
-
-        return object;
+invalid_fs:
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_mknod, 3.4.2);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_extract_handle, 3.4.2)
+ssize_t
+pub_glfs_h_extract_handle(struct glfs_object *object, unsigned char *handle,
+                          int len)
+{
+    ssize_t ret = -1;
 
+    /* validate in args */
+    if (object == NULL) {
+        errno = EINVAL;
+        goto out;
+    }
 
-int
-pub_glfs_h_unlink (struct glfs *fs, struct glfs_object *parent, const char *path)
-{
-        int                 ret = -1;
-        xlator_t           *subvol = NULL;
-        inode_t            *inode = NULL;
-        loc_t               loc = {0, };
-
-        /* validate in args */
-        if ((fs == NULL) || (parent == NULL) || (path == NULL)) {
-                errno = EINVAL;
-                return -1;
-        }
+    if (!handle || !len) {
+        ret = GFAPI_HANDLE_LENGTH;
+        goto out;
+    }
 
-        __glfs_entry_fs (fs);
+    if (len < GFAPI_HANDLE_LENGTH) {
+        errno = ERANGE;
+        goto out;
+    }
 
-        /* get the active volume */
-        subvol = priv_glfs_active_subvol (fs);
-        if ( !subvol ) {
-                ret = -1;
-                errno = EIO;
-                goto out;
-        }
+    memcpy(handle, object->gfid, GFAPI_HANDLE_LENGTH);
 
-        /* get/refresh the in arg objects inode in correlation to the xlator */
-        inode = glfs_resolve_inode (fs, subvol, parent);
-        if (!inode) {
-                errno = ESTALE;
-                goto out;
-        }
+    ret = GFAPI_HANDLE_LENGTH;
 
-        ret = priv_glfs_resolve_at (fs, subvol, inode, path, &loc, NULL, 0 , 0);
-        if (ret != 0) {
-                goto out;
-        }
+out:
+    return ret;
+}
 
-        if (!IA_ISDIR(loc.inode->ia_type)) {
-                ret = syncop_unlink (subvol, &loc);
-                DECODE_SYNCOP_ERR (ret);
-                if (ret != 0) {
-                        goto out;
-                }
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_create_from_handle, 3.4.2)
+struct glfs_object *
+pub_glfs_h_create_from_handle(struct glfs *fs, unsigned char *handle, int len,
+                              struct stat *stat)
+{
+    loc_t loc = {
+        0,
+    };
+    int ret = -1;
+    struct iatt iatt = {
+        0,
+    };
+    inode_t *newinode = NULL;
+    xlator_t *subvol = NULL;
+    struct glfs_object *object = NULL;
+    uint64_t ctx_value = LOOKUP_NOT_NEEDED;
+    gf_boolean_t lookup_needed = _gf_false;
+
+    /* validate in args */
+    if ((fs == NULL) || (handle == NULL) || (len != GFAPI_HANDLE_LENGTH)) {
+        errno = EINVAL;
+        return NULL;
+    }
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        errno = EIO;
+        goto out;
+    }
+
+    memcpy(loc.gfid, handle, GFAPI_HANDLE_LENGTH);
+
+    /* make sure the gfid received is valid */
+    GF_VALIDATE_OR_GOTO("glfs_h_create_from_handle",
+                        !(gf_uuid_is_null(loc.gfid)), out);
+
+    newinode = inode_find(subvol->itable, loc.gfid);
+    if (newinode) {
+        if (!stat) /* No need of lookup */
+            goto found;
+
+        lookup_needed = inode_needs_lookup(newinode, THIS);
+        if (lookup_needed) {
+            loc.inode = newinode;
         } else {
-                ret = syncop_rmdir (subvol, &loc, 0);
-                DECODE_SYNCOP_ERR (ret);
-                if (ret != 0) {
-                        goto out;
-                }
-        }
-
-        if (ret == 0)
-                ret = glfs_loc_unlink (&loc);
+            /* populate loc */
+            GLFS_LOC_FILL_INODE(newinode, loc, fill_out);
+
+            /* fop/op */
+            ret = syncop_stat(subvol, &loc, &iatt, NULL, NULL);
+            DECODE_SYNCOP_ERR(ret);
+
+            if (ret) {
+            fill_out:
+                /* Drop the reference hold in inode_find */
+                inode_unref(newinode);
+                goto out;
+            }
+
+            glfs_iatt_to_stat(fs, &iatt, stat);
+            goto found;
+        }
+    } else {
+        loc.inode = inode_new(subvol->itable);
+        if (!loc.inode) {
+            errno = ENOMEM;
+            goto out;
+        }
+    }
+
+    ret = syncop_lookup(subvol, &loc, &iatt, 0, 0, 0);
+    DECODE_SYNCOP_ERR(ret);
+    if (ret) {
+        gf_smsg(subvol->name, GF_LOG_WARNING, errno,
+                API_MSG_INODE_REFRESH_FAILED, "gfid=%s", uuid_utoa(loc.gfid),
+                "error=%s", strerror(errno), NULL);
+        goto out;
+    }
+
+    newinode = inode_link(loc.inode, 0, 0, &iatt);
+    if (newinode) {
+        if (newinode == loc.inode) {
+            inode_ctx_set(newinode, THIS, &ctx_value);
+        }
+        inode_lookup(newinode);
+    } else {
+        gf_smsg(subvol->name, GF_LOG_WARNING, errno, API_MSG_INODE_LINK_FAILED,
+                "gfid=%s", uuid_utoa(loc.gfid), NULL);
+        goto out;
+    }
+
+    /* populate stat */
+    if (stat)
+        glfs_iatt_to_stat(fs, &iatt, stat);
+
+found:
+    object = GF_CALLOC(1, sizeof(struct glfs_object), glfs_mt_glfs_object_t);
+    if (object == NULL) {
+        errno = ENOMEM;
+        ret = -1;
+        goto out;
+    }
+
+    /* populate the return object */
+    object->inode = newinode;
+    gf_uuid_copy(object->gfid, object->inode->gfid);
 
 out:
-        loc_wipe (&loc);
+    /* TODO: Check where the inode ref is being held? */
+    loc_wipe(&loc);
 
-        if (inode)
-                inode_unref (inode);
+    glfs_subvol_done(fs, subvol);
 
-        priv_glfs_subvol_done (fs, subvol);
+    __GLFS_EXIT_FS;
 
-        return ret;
+invalid_fs:
+    return object;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_unlink, 3.4.2);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_close, 3.4.2)
+int
+pub_glfs_h_close(struct glfs_object *object)
+{
+    /* since glfs_h_* objects hold a reference to inode
+     * it is safe to keep lookup count to '0' */
+    inode_forget(object->inode, 0);
+    inode_unref(object->inode);
+    GF_FREE(object);
 
+    return 0;
+}
 
-struct glfs_fd *
-pub_glfs_h_opendir (struct glfs *fs, struct glfs_object *object)
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_truncate, 3.4.2)
+int
+pub_glfs_h_truncate(struct glfs *fs, struct glfs_object *object, off_t offset)
 {
-        int              ret = -1;
-        struct glfs_fd  *glfd = NULL;
-        xlator_t        *subvol = NULL;
-        inode_t         *inode = NULL;
-        loc_t            loc = {0, };
-
-        /* validate in args */
-        if ((fs == NULL) || (object == NULL)) {
-                errno = EINVAL;
-                return NULL;
-        }
+    loc_t loc = {
+        0,
+    };
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+
+    DECLARE_OLD_THIS;
+
+    /* validate in args */
+    if (object == NULL) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, object);
+    if (!inode) {
+        errno = ESTALE;
+        goto out;
+    }
+
+    GLFS_LOC_FILL_INODE(inode, loc, out);
+
+    /* fop/op */
+    ret = syncop_truncate(subvol, &loc, (off_t)offset, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    /* populate out args */
+    if (ret == 0)
+        ret = glfs_loc_unlink(&loc);
 
-        __glfs_entry_fs (fs);
-
-        /* get the active volume */
-        subvol = priv_glfs_active_subvol (fs);
-        if (!subvol) {
-                ret = -1;
-                errno = EIO;
-                goto out;
-        }
+out:
+    loc_wipe(&loc);
 
-        /* get/refresh the in arg objects inode in correlation to the xlator */
-        inode = glfs_resolve_inode (fs, subvol, object);
-        if (!inode) {
-                errno = ESTALE;
-                goto out;
-        }
+    if (inode)
+        inode_unref(inode);
 
-        if (!IA_ISDIR (inode->ia_type)) {
-                ret = -1;
-                errno = ENOTDIR;
-                goto out;
-        }
+    glfs_subvol_done(fs, subvol);
 
-        glfd = glfs_fd_new (fs);
-        if (!glfd)
-                goto out;
+    __GLFS_EXIT_FS;
 
-        INIT_LIST_HEAD (&glfd->entries);
+invalid_fs:
+    return ret;
+}
 
-        glfd->fd = fd_create (inode, getpid());
-        if (!glfd->fd) {
-                ret = -1;
-                errno = ENOMEM;
-                goto out;
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_symlink, 3.4.2)
+struct glfs_object *
+pub_glfs_h_symlink(struct glfs *fs, struct glfs_object *parent,
+                   const char *name, const char *data, struct stat *stat)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    uuid_t gfid;
+    dict_t *xattr_req = NULL;
+    struct glfs_object *object = NULL;
+
+    DECLARE_OLD_THIS;
+
+    /* validate in args */
+    if ((parent == NULL) || (name == NULL) || (data == NULL)) {
+        errno = EINVAL;
+        return NULL;
+    }
+
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, parent);
+    if (!inode) {
+        errno = ESTALE;
+        goto out;
+    }
+
+    xattr_req = dict_new();
+    if (!xattr_req) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gf_uuid_generate(gfid);
+    ret = dict_set_gfuuid(xattr_req, "gfid-req", gfid, true);
+    if (ret) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    GLFS_LOC_FILL_PINODE(inode, loc, ret, errno, out, name);
+
+    /* fop/op */
+    ret = syncop_symlink(subvol, &loc, data, &iatt, xattr_req, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    /* populate out args */
+    if (ret == 0) {
+        ret = glfs_loc_link(&loc, &iatt);
+        if (ret != 0) {
+            goto out;
         }
 
-        GLFS_LOC_FILL_INODE (inode, loc, out);
+        if (stat)
+            glfs_iatt_to_stat(fs, &iatt, stat);
 
-        /* fop/op */
-        ret = syncop_opendir (subvol, &loc, glfd->fd);
-        DECODE_SYNCOP_ERR (ret);
+        ret = glfs_create_object(&loc, &object);
+    }
 
 out:
-        loc_wipe (&loc);
+    if (ret && object != NULL) {
+        pub_glfs_h_close(object);
+        object = NULL;
+    }
 
-        if (inode)
-                inode_unref (inode);
+    loc_wipe(&loc);
 
-        if (ret && glfd) {
-                glfs_fd_destroy (glfd);
-                glfd = NULL;
-        } else {
-                fd_bind (glfd->fd);
-                glfs_fd_bind (glfd);
-        }
+    if (inode)
+        inode_unref(inode);
 
-        priv_glfs_subvol_done (fs, subvol);
+    if (xattr_req)
+        dict_unref(xattr_req);
 
-        return glfd;
-}
+    glfs_subvol_done(fs, subvol);
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_opendir, 3.4.2);
+    __GLFS_EXIT_FS;
 
+invalid_fs:
+    return object;
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_readlink, 3.4.2)
 int
-pub_glfs_h_access (struct glfs *fs, struct glfs_object *object, int mask)
+pub_glfs_h_readlink(struct glfs *fs, struct glfs_object *object, char *buf,
+                    size_t bufsiz)
 {
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	inode_t         *inode = NULL;
-	loc_t            loc = {0, };
-
-	/* validate in args */
-	if ((fs == NULL) || (object == NULL)) {
-		errno = EINVAL;
-		goto out;
-	}
-
-	__glfs_entry_fs (fs);
+    loc_t loc = {
+        0,
+    };
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    char *linkval = NULL;
+
+    DECLARE_OLD_THIS;
+
+    /* validate in args */
+    if ((object == NULL) || (buf == NULL)) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, object);
+    if (!inode) {
+        errno = ESTALE;
+        goto out;
+    }
+
+    GLFS_LOC_FILL_INODE(inode, loc, out);
+
+    /* fop/op */
+    ret = syncop_readlink(subvol, &loc, &linkval, bufsiz, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    /* populate out args */
+    if (ret > 0)
+        memcpy(buf, linkval, ret);
 
-	/* get the active volume */
-	subvol = priv_glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+out:
+    loc_wipe(&loc);
 
-	/* get/refresh the in arg objects inode in correlation to the xlator */
-	inode = glfs_resolve_inode (fs, subvol, object);
-	if (!inode) {
-		errno = ESTALE;
-		goto out;
-	}
+    if (inode)
+        inode_unref(inode);
 
+    if (linkval)
+        GF_FREE(linkval);
 
-	GLFS_LOC_FILL_INODE (inode, loc, out);
+    glfs_subvol_done(fs, subvol);
 
-	/* fop/op */
+    __GLFS_EXIT_FS;
 
-	ret = syncop_access (subvol, &loc, mask);
-        DECODE_SYNCOP_ERR (ret);
+invalid_fs:
+    return ret;
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_link, 3.4.2)
+int
+pub_glfs_h_link(struct glfs *fs, struct glfs_object *linksrc,
+                struct glfs_object *parent, const char *name)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    inode_t *pinode = NULL;
+    loc_t oldloc = {
+        0,
+    };
+    loc_t newloc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+
+    DECLARE_OLD_THIS;
+
+    /* validate in args */
+    if ((linksrc == NULL) || (parent == NULL) || (name == NULL)) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, linksrc);
+    if (!inode) {
+        errno = ESTALE;
+        goto out;
+    }
+
+    if (inode->ia_type == IA_IFDIR) {
+        ret = -1;
+        errno = EISDIR;
+        goto out;
+    }
+
+    GLFS_LOC_FILL_INODE(inode, oldloc, out);
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    pinode = glfs_resolve_inode(fs, subvol, parent);
+    if (!pinode) {
+        errno = ESTALE;
+        goto out;
+    }
+
+    /* setup newloc based on parent */
+    newloc.parent = inode_ref(pinode);
+    newloc.name = name;
+    ret = glfs_loc_touchup(&newloc);
+    if (ret != 0) {
+        errno = EINVAL;
+        goto out;
+    }
+
+    /* Filling the inode of the hard link to be same as that of the
+     * original file
+     */
+    newloc.inode = inode_ref(inode);
+
+    /* fop/op */
+    ret = syncop_link(subvol, &oldloc, &newloc, &iatt, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    if (ret == 0)
+        ret = glfs_loc_link(&newloc, &iatt);
 out:
-	loc_wipe (&loc);
+    loc_wipe(&oldloc);
+    loc_wipe(&newloc);
 
-	if (inode)
-		inode_unref (inode);
+    if (inode)
+        inode_unref(inode);
 
+    if (pinode)
+        inode_unref(pinode);
 
-	priv_glfs_subvol_done (fs, subvol);
+    glfs_subvol_done(fs, subvol);
 
-	return ret;
-}
-
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_access, 3.6.0);
+    __GLFS_EXIT_FS;
 
+invalid_fs:
+    return ret;
+}
 
-ssize_t
-pub_glfs_h_extract_handle (struct glfs_object *object, unsigned char *handle,
-                           int len)
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_rename, 3.4.2)
+int
+pub_glfs_h_rename(struct glfs *fs, struct glfs_object *olddir,
+                  const char *oldname, struct glfs_object *newdir,
+                  const char *newname)
 {
-        ssize_t ret = -1;
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    inode_t *oldpinode = NULL;
+    inode_t *newpinode = NULL;
+    loc_t oldloc = {
+        0,
+    };
+    loc_t newloc = {
+        0,
+    };
+    struct iatt oldiatt = {
+        0,
+    };
+    struct iatt newiatt = {
+        0,
+    };
+
+    DECLARE_OLD_THIS;
+
+    /* validate in args */
+    if ((olddir == NULL) || (oldname == NULL) || (newdir == NULL) ||
+        (newname == NULL)) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    oldpinode = glfs_resolve_inode(fs, subvol, olddir);
+    if (!oldpinode) {
+        errno = ESTALE;
+        goto out;
+    }
+
+    ret = glfs_resolve_at(fs, subvol, oldpinode, oldname, &oldloc, &oldiatt, 0,
+                          0);
+    if (ret != 0) {
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    newpinode = glfs_resolve_inode(fs, subvol, newdir);
+    if (!newpinode) {
+        errno = ESTALE;
+        goto out;
+    }
+
+    ret = glfs_resolve_at(fs, subvol, newpinode, newname, &newloc, &newiatt, 0,
+                          0);
+
+    if (ret && errno != ENOENT && newloc.parent)
+        goto out;
+
+    if (newiatt.ia_type != IA_INVAL) {
+        if ((oldiatt.ia_type == IA_IFDIR) != (newiatt.ia_type == IA_IFDIR)) {
+            /* Either both old and new must be dirs,
+             * or both must be non-dirs. Else, fail.
+             */
+            ret = -1;
+            errno = EEXIST;
+            goto out;
+        }
+    }
+
+    /* TODO: check if new or old is a prefix of the other, and fail EINVAL */
+
+    ret = syncop_rename(subvol, &oldloc, &newloc, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    if (ret == 0) {
+        inode_rename(oldloc.parent->table, oldloc.parent, oldloc.name,
+                     newloc.parent, newloc.name, oldloc.inode, &oldiatt);
+
+        if (newloc.inode && !inode_has_dentry(newloc.inode))
+            inode_forget(newloc.inode, 0);
+    }
 
-        /* validate in args */
-        if (object == NULL) {
-                errno = EINVAL;
-                goto out;
-        }
+out:
+    loc_wipe(&oldloc);
+    loc_wipe(&newloc);
 
-        if (!handle || !len) {
-                ret = GFAPI_HANDLE_LENGTH;
-                goto out;
-        }
+    if (oldpinode)
+        inode_unref(oldpinode);
 
-        if (len < GFAPI_HANDLE_LENGTH)
-        {
-                errno = ERANGE;
-                goto out;
-        }
+    if (newpinode)
+        inode_unref(newpinode);
 
-        memcpy (handle, object->gfid, GFAPI_HANDLE_LENGTH);
+    glfs_subvol_done(fs, subvol);
 
-        ret = GFAPI_HANDLE_LENGTH;
+    __GLFS_EXIT_FS;
 
-out:
-        return ret;
+invalid_fs:
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_extract_handle, 3.4.2);
-
-
+/*
+ * Given a handle/gfid, find if the corresponding inode is present in
+ * the inode table. If yes create and return the corresponding glfs_object.
+ */
 struct glfs_object *
-pub_glfs_h_create_from_handle (struct glfs *fs, unsigned char *handle, int len,
-                               struct stat *stat)
+glfs_h_find_handle(struct glfs *fs, unsigned char *handle, int len)
 {
-        loc_t               loc = {0, };
-        int                 ret = -1;
-        struct iatt         iatt = {0, };
-        inode_t            *newinode = NULL;
-        xlator_t           *subvol = NULL;
-        struct glfs_object *object = NULL;
-
-        /* validate in args */
-        if ((fs == NULL) || (handle == NULL) || (len != GFAPI_HANDLE_LENGTH)) {
-                errno = EINVAL;
-                return NULL;
-        }
+    inode_t *newinode = NULL;
+    xlator_t *subvol = NULL;
+    struct glfs_object *object = NULL;
+    uuid_t gfid;
+
+    /* validate in args */
+    if ((fs == NULL) || (handle == NULL) || (len != GFAPI_HANDLE_LENGTH)) {
+        errno = EINVAL;
+        return NULL;
+    }
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        errno = EIO;
+        goto out;
+    }
+
+    memcpy(gfid, handle, GFAPI_HANDLE_LENGTH);
+
+    /* make sure the gfid received is valid */
+    GF_VALIDATE_OR_GOTO("glfs_h_find_handle", !(gf_uuid_is_null(gfid)), out);
+
+    newinode = inode_find(subvol->itable, gfid);
+    if (!newinode) {
+        goto out;
+    }
+
+    object = GF_CALLOC(1, sizeof(struct glfs_object), glfs_mt_glfs_object_t);
+    if (object == NULL) {
+        errno = ENOMEM;
+        goto out;
+    }
+
+    /* populate the return object. The ref taken here
+     * is un'refed when the application does glfs_h_close() */
+    object->inode = inode_ref(newinode);
+    gf_uuid_copy(object->gfid, object->inode->gfid);
 
-        __glfs_entry_fs (fs);
+out:
+    /* inode_find takes a reference. Unref it. */
+    if (newinode)
+        inode_unref(newinode);
 
-        /* get the active volume */
-        subvol = priv_glfs_active_subvol (fs);
-        if (!subvol) {
-                errno = EIO;
-                goto out;
-        }
+    glfs_subvol_done(fs, subvol);
 
-        memcpy (loc.gfid, handle, GFAPI_HANDLE_LENGTH);
+    __GLFS_EXIT_FS;
 
-        newinode = inode_find (subvol->itable, loc.gfid);
-        if (newinode)
-                loc.inode = newinode;
-        else {
-                loc.inode = inode_new (subvol->itable);
-                if (!loc.inode) {
-                        errno = ENOMEM;
-                        goto out;
-                }
-        }
+invalid_fs:
+    return object;
+}
 
-        ret = syncop_lookup (subvol, &loc, 0, &iatt, 0, 0);
-        DECODE_SYNCOP_ERR (ret);
-        if (ret) {
-                gf_log (subvol->name, GF_LOG_WARNING,
-                        "inode refresh of %s failed: %s",
-                        uuid_utoa (loc.gfid), strerror (errno));
-                goto out;
-        }
+static void
+glfs_free_upcall_inode(void *to_free)
+{
+    struct glfs_upcall_inode *arg = to_free;
 
-        newinode = inode_link (loc.inode, 0, 0, &iatt);
-        if (newinode)
-                inode_lookup (newinode);
-        else {
-                gf_log (subvol->name, GF_LOG_WARNING,
-                        "inode linking of %s failed: %s",
-                        uuid_utoa (loc.gfid), strerror (errno));
-                errno = EINVAL;
-                goto out;
-        }
+    if (!arg)
+        return;
 
-        /* populate stat */
-        if (stat)
-                glfs_iatt_to_stat (fs, &iatt, stat);
+    if (arg->object)
+        glfs_h_close(arg->object);
+    if (arg->p_object)
+        glfs_h_close(arg->p_object);
+    if (arg->oldp_object)
+        glfs_h_close(arg->oldp_object);
 
-        object = GF_CALLOC (1, sizeof(struct glfs_object),
-                            glfs_mt_glfs_object_t);
-        if (object == NULL) {
-                errno = ENOMEM;
-                ret = -1;
-                goto out;
-        }
+    GF_FREE(arg);
+}
 
-        /* populate the return object */
-        object->inode = newinode;
-        uuid_copy (object->gfid, object->inode->gfid);
+int
+glfs_h_poll_cache_invalidation(struct glfs *fs, struct glfs_upcall *up_arg,
+                               struct gf_upcall *upcall_data)
+{
+    int ret = -1;
+    struct glfs_object *p_object = NULL;
+    struct glfs_object *oldp_object = NULL;
+    struct glfs_object *object = NULL;
+    struct gf_upcall_cache_invalidation *ca_data = NULL;
+    struct glfs_upcall_inode *up_inode_arg = NULL;
+
+    ca_data = upcall_data->data;
+    GF_VALIDATE_OR_GOTO("glfs_h_poll_cache_invalidation", ca_data, out);
+
+    object = glfs_h_find_handle(fs, upcall_data->gfid, GFAPI_HANDLE_LENGTH);
+    if (!object) {
+        /* The reason handle creation will fail is because we
+         * couldn't find the inode in the gfapi inode table.
+         *
+         * But since application would have taken inode_ref, the
+         * only case when this can happen is when it has closed
+         * the handle and hence will no more be interested in
+         * the upcall for this particular gfid.
+         */
+        gf_smsg(THIS->name, GF_LOG_DEBUG, errno, API_MSG_CREATE_HANDLE_FAILED,
+                "gfid=%s", uuid_utoa(upcall_data->gfid), NULL);
+        errno = ESTALE;
+        goto out;
+    }
+
+    up_inode_arg = GF_CALLOC(1, sizeof(struct glfs_upcall_inode),
+                             glfs_mt_upcall_inode_t);
+    GF_VALIDATE_OR_GOTO("glfs_h_poll_cache_invalidation", up_inode_arg, out);
+
+    up_inode_arg->object = object;
+    up_inode_arg->flags = ca_data->flags;
+    up_inode_arg->expire_time_attr = ca_data->expire_time_attr;
+
+    /* XXX: Update stat as well in case of UP_*_TIMES.
+     * This will be addressed as part of INODE_UPDATE */
+    if (ca_data->flags & GFAPI_INODE_UPDATE_FLAGS) {
+        glfs_iatt_to_stat(fs, &ca_data->stat, &up_inode_arg->buf);
+    }
+
+    if (ca_data->flags & GFAPI_UP_PARENT_TIMES) {
+        p_object = glfs_h_find_handle(fs, ca_data->p_stat.ia_gfid,
+                                      GFAPI_HANDLE_LENGTH);
+        if (!p_object) {
+            gf_smsg(THIS->name, GF_LOG_DEBUG, errno,
+                    API_MSG_CREATE_HANDLE_FAILED, "gfid=%s",
+                    uuid_utoa(ca_data->p_stat.ia_gfid), NULL);
+            errno = ESTALE;
+            goto out;
+        }
+
+        glfs_iatt_to_stat(fs, &ca_data->p_stat, &up_inode_arg->p_buf);
+    }
+    up_inode_arg->p_object = p_object;
+
+    /* In case of RENAME, update old parent as well */
+    if (ca_data->flags & GFAPI_UP_RENAME) {
+        oldp_object = glfs_h_find_handle(fs, ca_data->oldp_stat.ia_gfid,
+                                         GFAPI_HANDLE_LENGTH);
+        if (!oldp_object) {
+            gf_smsg(THIS->name, GF_LOG_DEBUG, errno,
+                    API_MSG_CREATE_HANDLE_FAILED, "gfid=%s",
+                    uuid_utoa(ca_data->oldp_stat.ia_gfid), NULL);
+            errno = ESTALE;
+            /* By the time we receive upcall old parent_dir may
+             * have got removed. We still need to send upcall
+             * for the file/dir and current parent handles. */
+            up_inode_arg->oldp_object = NULL;
+            ret = 0;
+        }
+
+        glfs_iatt_to_stat(fs, &ca_data->oldp_stat, &up_inode_arg->oldp_buf);
+    }
+    up_inode_arg->oldp_object = oldp_object;
+
+    up_arg->reason = GLFS_UPCALL_INODE_INVALIDATE;
+    up_arg->event = up_inode_arg;
+    up_arg->free_event = glfs_free_upcall_inode;
+
+    ret = 0;
 
 out:
-        /* TODO: Check where the inode ref is being held? */
-        loc_wipe (&loc);
-
-        priv_glfs_subvol_done (fs, subvol);
-
-        return object;
+    if (ret) {
+        /* Close p_object and oldp_object as well if being referenced.*/
+        if (object)
+            glfs_h_close(object);
+
+        /* Set reason to prevent applications from using ->event */
+        up_arg->reason = GLFS_UPCALL_EVENT_NULL;
+        GF_FREE(up_inode_arg);
+    }
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_create_from_handle, 3.4.2);
+void
+glfs_release_upcall(void *ptr)
+{
+    struct glfs_upcall *to_free = ptr;
 
+    if (to_free->event)
+        to_free->free_event(to_free->event);
+}
 
+/*
+ * This API is used to poll for upcall events stored in the upcall list.
+ * Current users of this API is NFS-Ganesha. In case of any event received, it
+ * will be mapped appropriately into 'glfs_upcall' along with the handle object
+ * to be passed to NFS-Ganesha.
+ *
+ * On success, applications need to check if up_arg is not-NULL or errno is not
+ * ENOENT. glfs_upcall_get_reason() can be used to decide what kind of event
+ * has been received.
+ *
+ * Current supported upcall_events:
+ *      GLFS_UPCALL_INODE_INVALIDATE
+ *
+ * After processing the event, applications need to free 'up_arg' by calling
+ * glfs_free().
+ *
+ * Also similar to I/Os, the application should ideally stop polling before
+ * calling glfs_fini(..). Hence making an assumption that 'fs' & ctx structures
+ * cannot be freed while in this routine.
+ */
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_poll_upcall, 3.7.16)
 int
-pub_glfs_h_close (struct glfs_object *object)
+pub_glfs_h_poll_upcall(struct glfs *fs, struct glfs_upcall **up_arg)
 {
-        /* Release the held reference */
-        inode_unref (object->inode);
-        GF_FREE (object);
+    upcall_entry *u_list = NULL;
+    upcall_entry *tmp = NULL;
+    xlator_t *subvol = NULL;
+    glusterfs_ctx_t *ctx = NULL;
+    int ret = -1;
+    struct gf_upcall *upcall_data = NULL;
+
+    DECLARE_OLD_THIS;
+
+    if (!up_arg) {
+        errno = EINVAL;
+        goto err;
+    }
+
+    __GLFS_ENTRY_VALIDATE_FS(fs, err);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        errno = EIO;
+        goto restore;
+    }
+
+    /* Ideally applications should stop polling before calling
+     * 'glfs_fini'. Yet cross check if cleanup has started. */
+    pthread_mutex_lock(&fs->mutex);
+    {
+        ctx = fs->ctx;
+
+        if (ctx->cleanup_started) {
+            pthread_mutex_unlock(&fs->mutex);
+            goto out;
+        }
+
+        fs->pin_refcnt++;
+
+        /* once we call this function, the applications seems to be
+         * interested in events, enable caching them */
+        fs->cache_upcalls = _gf_true;
+    }
+    pthread_mutex_unlock(&fs->mutex);
+
+    pthread_mutex_lock(&fs->upcall_list_mutex);
+    {
+        list_for_each_entry_safe(u_list, tmp, &fs->upcall_list, upcall_list)
+        {
+            list_del_init(&u_list->upcall_list);
+            upcall_data = &u_list->upcall_data;
+            break;
+        }
+    }
+    /* No other thread can delete this entry. So unlock it */
+    pthread_mutex_unlock(&fs->upcall_list_mutex);
+
+    if (upcall_data) {
+        switch (upcall_data->event_type) {
+            case GF_UPCALL_CACHE_INVALIDATION:
+                *up_arg = GLFS_CALLOC(1, sizeof(struct gf_upcall),
+                                      glfs_release_upcall,
+                                      glfs_mt_upcall_entry_t);
+                if (!*up_arg) {
+                    errno = ENOMEM;
+                    break; /* goto free u_list */
+                }
 
-        return 0;
-}
+                /* XXX: Need to revisit this to support
+                 * GLFS_UPCALL_INODE_UPDATE if required. */
+                ret = glfs_h_poll_cache_invalidation(fs, *up_arg, upcall_data);
+                if (ret || (*up_arg)->reason == GLFS_UPCALL_EVENT_NULL) {
+                    /* It could so happen that the file which got
+                     * upcall notification may have got deleted by
+                     * the same client. Irrespective of the error,
+                     * return with an error or success+ENOENT. */
+                    if ((*up_arg)->reason == GLFS_UPCALL_EVENT_NULL)
+                        errno = ENOENT;
+
+                    GLFS_FREE(*up_arg);
+                    *up_arg = NULL;
+                }
+                break;
+            case GF_UPCALL_RECALL_LEASE:
+                gf_log("glfs_h_poll_upcall", GF_LOG_DEBUG,
+                       "UPCALL_RECALL_LEASE is not implemented yet");
+            /* fallthrough till we support leases */
+            case GF_UPCALL_EVENT_NULL:
+                /* no 'default:' label, to force handling all upcall events */
+                errno = ENOENT;
+                break;
+        }
+
+        GF_FREE(u_list->upcall_data.data);
+        GF_FREE(u_list);
+    } else {
+        /* fs->upcall_list was empty, no upcall events cached */
+        errno = ENOENT;
+    }
+
+    ret = 0;
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_close, 3.4.2);
+out:
+    pthread_mutex_lock(&fs->mutex);
+    {
+        fs->pin_refcnt--;
+    }
+    pthread_mutex_unlock(&fs->mutex);
+
+    glfs_subvol_done(fs, subvol);
+
+restore:
+    __GLFS_EXIT_FS;
+err:
+    return ret;
+}
 
+static gf_boolean_t log_upcall370 = _gf_true; /* log once */
 
+/* The old glfs_h_poll_upcall interface requires intimate knowledge of the
+ * structures that are returned to the calling application. This is not
+ * recommended, as the returned structures need to returned correctly (handles
+ * closed, memory free'd with the unavailable GF_FREE(), and possibly more.)
+ *
+ * To the best of our knowledge, only NFS-Ganesha uses the upcall events
+ * through gfapi. We keep this backwards compatibility function around so that
+ * applications using the existing implementation do not break.
+ *
+ * WARNING: this function will be removed in the future.
+ */
+GFAPI_SYMVER_PUBLIC(glfs_h_poll_upcall370, glfs_h_poll_upcall, 3.7.0)
 int
-pub_glfs_h_truncate (struct glfs *fs, struct glfs_object *object, off_t offset)
+pub_glfs_h_poll_upcall370(struct glfs *fs, struct glfs_callback_arg *up_arg)
 {
-        loc_t               loc = {0, };
-        int                 ret = -1;
-        xlator_t           *subvol = NULL;
-        inode_t            *inode = NULL;
-
-        /* validate in args */
-        if ((fs == NULL) || (object == NULL)) {
-                errno = EINVAL;
-                return -1;
-        }
-
-        __glfs_entry_fs (fs);
-
-        /* get the active volume */
-        subvol = priv_glfs_active_subvol (fs);
-        if (!subvol) {
+    struct glfs_upcall *upcall = NULL;
+    int ret = -1;
+
+    if (log_upcall370) {
+        log_upcall370 = _gf_false;
+        gf_log(THIS->name, GF_LOG_WARNING,
+               "this application is "
+               "compiled against an old version of libgfapi, it "
+               "should use glfs_free() to release the structure "
+               "returned by glfs_h_poll_upcall() - for more details, "
+               "see http://review.gluster.org/14701");
+    }
+
+    ret = pub_glfs_h_poll_upcall(fs, &upcall);
+    if (ret == 0) {
+        up_arg->fs = fs;
+        if ((errno == ENOENT) || !upcall || !upcall->event) {
+            up_arg->reason = GLFS_UPCALL_EVENT_NULL;
+            goto out;
+        }
+
+        up_arg->reason = upcall->reason;
+
+        if (upcall->reason == GLFS_UPCALL_INODE_INVALIDATE) {
+            struct glfs_callback_inode_arg *cb_inode = NULL;
+            struct glfs_upcall_inode *up_inode = NULL;
+
+            cb_inode = GF_CALLOC(1, sizeof(struct glfs_callback_inode_arg),
+                                 glfs_mt_upcall_inode_t);
+            if (!cb_inode) {
+                errno = ENOMEM;
                 ret = -1;
-                errno = EIO;
-                goto out;
-        }
-
-        /* get/refresh the in arg objects inode in correlation to the xlator */
-        inode = glfs_resolve_inode (fs, subvol, object);
-        if (!inode) {
-                errno = ESTALE;
                 goto out;
-        }
+            }
 
-        GLFS_LOC_FILL_INODE (inode, loc, out);
+            up_inode = upcall->event;
 
-        /* fop/op */
-        ret = syncop_truncate (subvol, &loc, (off_t)offset);
-        DECODE_SYNCOP_ERR (ret);
+            /* copy attributes one by one, the memory layout might
+             * be different between the old glfs_callback_inode_arg
+             * and new glfs_upcall_inode */
+            cb_inode->object = up_inode->object;
+            cb_inode->flags = up_inode->flags;
+            memcpy(&cb_inode->buf, &up_inode->buf, sizeof(struct stat));
+            cb_inode->expire_time_attr = up_inode->expire_time_attr;
+            cb_inode->p_object = up_inode->p_object;
+            memcpy(&cb_inode->p_buf, &up_inode->p_buf, sizeof(struct stat));
+            cb_inode->oldp_object = up_inode->oldp_object;
+            memcpy(&cb_inode->oldp_buf, &up_inode->oldp_buf,
+                   sizeof(struct stat));
 
-        /* populate out args */
-        if (ret == 0)
-                ret = glfs_loc_unlink (&loc);
+            up_arg->event_arg = cb_inode;
+        }
+    }
 
 out:
-        loc_wipe (&loc);
-
-        if (inode)
-                inode_unref (inode);
+    if (upcall) {
+        /* we can not use glfs_free() here, objects need to stay */
+        GF_FREE(upcall->event);
+        GF_FREE(upcall);
+    }
 
-        priv_glfs_subvol_done (fs, subvol);
-
-        return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_truncate, 3.4.2);
+#ifdef HAVE_ACL_LIBACL_H
+#include <glusterfs/glusterfs-acl.h>
+#include <acl/libacl.h>
 
-
-struct glfs_object *
-pub_glfs_h_symlink (struct glfs *fs, struct glfs_object *parent,
-                    const char *name, const char *data, struct stat *stat)
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_acl_set, 3.7.0)
+int
+pub_glfs_h_acl_set(struct glfs *fs, struct glfs_object *object,
+                   const acl_type_t type, const acl_t acl)
 {
-        int                 ret = -1;
-        xlator_t           *subvol = NULL;
-        inode_t            *inode = NULL;
-        loc_t               loc = {0, };
-        struct iatt         iatt = {0, };
-        uuid_t              gfid;
-        dict_t             *xattr_req = NULL;
-        struct glfs_object *object = NULL;
-
-        /* validate in args */
-        if ((fs == NULL) || (parent == NULL) || (name == NULL) ||
-                (data == NULL)) {
-                errno = EINVAL;
-                return NULL;
-        }
+    int ret = -1;
+    char *acl_s = NULL;
+    const char *acl_key = NULL;
+    struct glfs_object *new_object = NULL;
 
-        __glfs_entry_fs (fs);
+    DECLARE_OLD_THIS;
 
-        /* get the active volume */
-        subvol = priv_glfs_active_subvol (fs);
-        if (!subvol) {
-                ret = -1;
-                errno = EIO;
-                goto out;
-        }
+    if (!object || !acl) {
+        errno = EINVAL;
+        return ret;
+    }
 
-        /* get/refresh the in arg objects inode in correlation to the xlator */
-        inode = glfs_resolve_inode (fs, subvol, parent);
-        if (!inode) {
-                errno = ESTALE;
-                goto out;
-        }
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
 
-        xattr_req = dict_new ();
-        if (!xattr_req) {
-                ret = -1;
-                errno = ENOMEM;
-                goto out;
-        }
+    acl_key = gf_posix_acl_get_key(type);
+    if (!acl_key)
+        goto out;
 
-        uuid_generate (gfid);
-        ret = dict_set_static_bin (xattr_req, "gfid-req", gfid, 16);
-        if (ret) {
-                ret = -1;
-                errno = ENOMEM;
-                goto out;
-        }
+    acl_s = acl_to_any_text(acl, NULL, ',', TEXT_ABBREVIATE | TEXT_NUMERIC_IDS);
+    if (!acl_s)
+        goto out;
 
-        GLFS_LOC_FILL_PINODE (inode, loc, ret, errno, out, name);
-
-        /* fop/op */
-        ret = syncop_symlink (subvol, &loc, data, xattr_req, &iatt);
-        DECODE_SYNCOP_ERR (ret);
-
-        /* populate out args */
-        if (ret == 0) {
-                /* TODO: If the inode existed in the cache (say file already
-                 * exists), then the glfs_loc_link will not update the
-                 * loc.inode, as a result we will have a 0000 GFID that we
-                 * would copy out to the object, this needs to be fixed.
-                 */
-                ret = glfs_loc_link (&loc, &iatt);
-                if (ret != 0) {
-                        goto out;
-                }
+    if (IA_ISLNK(object->inode->ia_type)) {
+        new_object = glfs_h_resolve_symlink(fs, object);
+        if (new_object == NULL)
+            goto out;
+    } else
+        new_object = object;
 
-                if (stat)
-                        glfs_iatt_to_stat (fs, &iatt, stat);
+    ret = pub_glfs_h_setxattrs(fs, new_object, acl_key, acl_s,
+                               strlen(acl_s) + 1, 0);
 
-                ret = glfs_create_object (&loc, &object);
-        }
+    acl_free(acl_s);
 
 out:
-        if (ret && object != NULL) {
-                pub_glfs_h_close (object);
-                object = NULL;
-        }
-
-        loc_wipe(&loc);
+    if (IA_ISLNK(object->inode->ia_type) && new_object)
+        glfs_h_close(new_object);
 
-        if (inode)
-                inode_unref (inode);
+    __GLFS_EXIT_FS;
 
-        if (xattr_req)
-                dict_unref (xattr_req);
-
-        priv_glfs_subvol_done (fs, subvol);
-
-        return object;
+invalid_fs:
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_symlink, 3.4.2);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_acl_get, 3.7.0)
+acl_t
+pub_glfs_h_acl_get(struct glfs *fs, struct glfs_object *object,
+                   const acl_type_t type)
+{
+    int ret = 0;
+    acl_t acl = NULL;
+    char *acl_s = NULL;
+    dict_t *xattr = NULL;
+    const char *acl_key = NULL;
+    struct glfs_object *new_object = NULL;
 
+    DECLARE_OLD_THIS;
 
-int
-pub_glfs_h_readlink (struct glfs *fs, struct glfs_object *object, char *buf,
-                     size_t bufsiz)
-{
-        loc_t               loc = {0, };
-        int                 ret = -1;
-        xlator_t           *subvol = NULL;
-        inode_t            *inode = NULL;
-        char               *linkval = NULL;
-
-        /* validate in args */
-        if ((fs == NULL) || (object == NULL) || (buf == NULL)) {
-                errno = EINVAL;
-                return -1;
-        }
+    if (!object) {
+        errno = EINVAL;
+        return NULL;
+    }
 
-        __glfs_entry_fs (fs);
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
 
-        /* get the active volume */
-        subvol = priv_glfs_active_subvol (fs);
-        if (!subvol) {
-                ret = -1;
-                errno = EIO;
-                goto out;
-        }
+    acl_key = gf_posix_acl_get_key(type);
+    if (!acl_key)
+        goto out;
 
-        /* get/refresh the in arg objects inode in correlation to the xlator */
-        inode = glfs_resolve_inode (fs, subvol, object);
-        if (!inode) {
-                errno = ESTALE;
-                goto out;
-        }
+    if (IA_ISLNK(object->inode->ia_type)) {
+        new_object = glfs_h_resolve_symlink(fs, object);
+        if (new_object == NULL)
+            goto out;
+    } else
+        new_object = object;
 
-        GLFS_LOC_FILL_INODE (inode, loc, out);
+    ret = glfs_h_getxattrs_common(fs, new_object, &xattr, acl_key, _gf_false);
+    if (ret)
+        goto out;
 
-        /* fop/op */
-        ret = syncop_readlink (subvol, &loc, &linkval, bufsiz);
-        DECODE_SYNCOP_ERR (ret);
+    ret = dict_get_str(xattr, (char *)acl_key, &acl_s);
+    if (ret)
+        goto out;
 
-        /* populate out args */
-        if (ret > 0)
-                memcpy (buf, linkval, ret);
+    acl = acl_from_text(acl_s);
 
 out:
-        loc_wipe (&loc);
-
-        if (inode)
-                inode_unref (inode);
+    if (xattr)
+        dict_unref(xattr);
 
-        if (linkval)
-                GF_FREE (linkval);
+    if (IA_ISLNK(object->inode->ia_type) && new_object)
+        glfs_h_close(new_object);
 
-        priv_glfs_subvol_done (fs, subvol);
+    __GLFS_EXIT_FS;
 
-        return ret;
+invalid_fs:
+    return acl;
+}
+#else /* !HAVE_ACL_LIBACL_H */
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_acl_get, 3.7.0)
+acl_t
+pub_glfs_h_acl_get(struct glfs *fs, struct glfs_object *object,
+                   const acl_type_t type)
+{
+    errno = ENOTSUP;
+    return NULL;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_readlink, 3.4.2);
-
-
-extern int
-priv_glfs_loc_touchup (loc_t *);
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_acl_set, 3.7.0)
 int
-pub_glfs_h_link (struct glfs *fs, struct glfs_object *linksrc,
-             struct glfs_object *parent, const char *name)
+pub_glfs_h_acl_set(struct glfs *fs, struct glfs_object *object,
+                   const acl_type_t type, const acl_t acl)
 {
-        int                 ret = -1;
-        xlator_t           *subvol = NULL;
-        inode_t            *inode = NULL;
-        inode_t            *pinode = NULL;
-        loc_t               oldloc = {0, };
-        loc_t               newloc = {0, };
-
-        /* validate in args */
-        if ((fs == NULL) || (linksrc == NULL) || (parent == NULL) ||
-                (name == NULL)) {
-                errno = EINVAL;
-                return -1;
-        }
-
-        __glfs_entry_fs (fs);
-
-        /* get the active volume */
-        subvol = priv_glfs_active_subvol (fs);
-        if (!subvol) {
-                ret = -1;
-                errno = EIO;
-                goto out;
-        }
-
-        /* get/refresh the in arg objects inode in correlation to the xlator */
-        inode = glfs_resolve_inode (fs, subvol, linksrc);
-        if (!inode) {
-                errno = ESTALE;
-                goto out;
-        }
-
-        if (inode->ia_type == IA_IFDIR) {
-                ret = -1;
-                errno = EISDIR;
-                goto out;
-        }
+    errno = ENOTSUP;
+    return -1;
+}
+#endif
 
-        GLFS_LOC_FILL_INODE (inode, oldloc, out);
+/* The API to perform read using anonymous fd */
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_anonymous_read, 3.7.0)
+ssize_t
+pub_glfs_h_anonymous_read(struct glfs *fs, struct glfs_object *object,
+                          const void *buf, size_t count, off_t offset)
+{
+    struct iovec iov = {
+        0,
+    };
+    ssize_t ret = 0;
 
-        /* get/refresh the in arg objects inode in correlation to the xlator */
-        pinode = glfs_resolve_inode (fs, subvol, parent);
-        if (!pinode) {
-                errno = ESTALE;
-                goto out;
-        }
+    /* validate in args */
+    if ((fs == NULL) || (object == NULL)) {
+        errno = EINVAL;
+        return -1;
+    }
 
-        /* setup newloc based on parent */
-        newloc.parent = inode_ref (pinode);
-        newloc.name = name;
-        ret = priv_glfs_loc_touchup (&newloc);
-        if (ret != 0) {
-                errno = EINVAL;
-                goto out;
-        }
+    iov.iov_base = (void *)buf;
+    iov.iov_len = count;
 
-        /* Filling the inode of the hard link to be same as that of the
-         * original file
-         */
-        newloc.inode = inode_ref (inode);
+    ret = glfs_anonymous_preadv(fs, object, &iov, 1, offset, 0);
 
-        /* fop/op */
-        ret = syncop_link (subvol, &oldloc, &newloc);
-        DECODE_SYNCOP_ERR (ret);
+    return ret;
+}
 
-        if (ret == 0)
-                /* TODO: No iatt to pass as there has been no lookup */
-                ret = glfs_loc_link (&newloc, NULL);
-out:
-        loc_wipe (&oldloc);
-        loc_wipe (&newloc);
+/* The API to perform write using anonymous fd */
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_anonymous_write, 3.7.0)
+ssize_t
+pub_glfs_h_anonymous_write(struct glfs *fs, struct glfs_object *object,
+                           const void *buf, size_t count, off_t offset)
+{
+    struct iovec iov = {
+        0,
+    };
+    ssize_t ret = 0;
 
-        if (inode)
-                inode_unref (inode);
+    /* validate in args */
+    if ((fs == NULL) || (object == NULL)) {
+        errno = EINVAL;
+        return -1;
+    }
 
-        if (pinode)
-                inode_unref (pinode);
+    iov.iov_base = (void *)buf;
+    iov.iov_len = count;
 
-        priv_glfs_subvol_done (fs, subvol);
+    ret = glfs_anonymous_pwritev(fs, object, &iov, 1, offset, 0);
 
-        return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_link, 3.4.2);
-
-
-int
-pub_glfs_h_rename (struct glfs *fs, struct glfs_object *olddir,
-                   const char *oldname, struct glfs_object *newdir,
-                   const char *newname)
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_object_copy, 3.11.0)
+struct glfs_object *
+pub_glfs_object_copy(struct glfs_object *src)
 {
-        int                 ret = -1;
-        xlator_t           *subvol = NULL;
-        inode_t            *oldpinode = NULL;
-        inode_t            *newpinode = NULL;
-        loc_t               oldloc = {0, };
-        loc_t               newloc = {0, };
-        struct iatt         oldiatt = {0, };
-        struct iatt         newiatt = {0, };
-
-        /* validate in args */
-        if ((fs == NULL) || (olddir == NULL) || (oldname == NULL) ||
-                (newdir == NULL) || (newname == NULL)) {
-                errno = EINVAL;
-                return -1;
-        }
+    struct glfs_object *object = NULL;
 
-        __glfs_entry_fs (fs);
+    GF_VALIDATE_OR_GOTO("glfs_dup_object", src, out);
 
-        /* get the active volume */
-        subvol = priv_glfs_active_subvol (fs);
-        if ( !subvol ) {
-                ret = -1;
-                errno = EIO;
-                goto out;
-        }
+    object = GF_CALLOC(1, sizeof(struct glfs_object), glfs_mt_glfs_object_t);
+    if (object == NULL) {
+        errno = ENOMEM;
+        gf_smsg(THIS->name, GF_LOG_WARNING, errno, API_MSG_CREATE_HANDLE_FAILED,
+                "glfs_dup_object gfid=%s", uuid_utoa(src->inode->gfid), NULL);
+        return NULL;
+    }
 
-        /* get/refresh the in arg objects inode in correlation to the xlator */
-        oldpinode = glfs_resolve_inode (fs, subvol, olddir);
-        if (!oldpinode) {
-                errno = ESTALE;
-                goto out;
-        }
-
-        ret = priv_glfs_resolve_at (fs, subvol, oldpinode, oldname, &oldloc,
-                                    &oldiatt, 0 , 0);
-        if (ret != 0) {
-                goto out;
-        }
-
-        /* get/refresh the in arg objects inode in correlation to the xlator */
-        newpinode = glfs_resolve_inode (fs, subvol, newdir);
-        if (!newpinode) {
-                errno = ESTALE;
-                goto out;
-        }
+    object->inode = inode_ref(src->inode);
+    gf_uuid_copy(object->gfid, src->inode->gfid);
 
-        ret = priv_glfs_resolve_at (fs, subvol, newpinode, newname, &newloc,
-                                    &newiatt, 0, 0);
+out:
+    return object;
+}
 
-        if (ret && errno != ENOENT && newloc.parent)
-                goto out;
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_xreaddirplus_get_object, 3.11.0)
+struct glfs_object *
+pub_glfs_xreaddirplus_get_object(struct glfs_xreaddirp_stat *xstat)
+{
+    GF_VALIDATE_OR_GOTO("glfs_xreaddirplus_get_object", xstat, out);
 
-        if (newiatt.ia_type != IA_INVAL) {
-                if ((oldiatt.ia_type == IA_IFDIR) !=
-                        (newiatt.ia_type == IA_IFDIR)) {
-                        /* Either both old and new must be dirs,
-                         * or both must be non-dirs. Else, fail.
-                         */
-                        ret = -1;
-                        errno = EEXIST;
-                        goto out;
-                }
-        }
+    if (!(xstat->flags_handled & GFAPI_XREADDIRP_HANDLE))
+        gf_smsg(THIS->name, GF_LOG_ERROR, errno, API_MSG_HANDLE_NOT_SET,
+                "GFAPI_XREADDIRP_HANDLE xstat=%p", xstat, "handle=%x",
+                xstat->flags_handled, NULL);
 
-        /* TODO: check if new or old is a prefix of the other, and fail EINVAL */
+    return xstat->object;
 
-        ret = syncop_rename (subvol, &oldloc, &newloc);
-        DECODE_SYNCOP_ERR (ret);
+out:
+    return NULL;
+}
 
-        if (ret == 0)
-                inode_rename (oldloc.parent->table, oldloc.parent, oldloc.name,
-                              newloc.parent, newloc.name, oldloc.inode,
-                              &oldiatt);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_lease, 4.0.0)
+int
+pub_glfs_h_lease(struct glfs *fs, struct glfs_object *object,
+                 struct glfs_lease *lease)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct gf_lease gf_lease = {
+        0,
+    };
+
+    /* validate in args */
+    if ((fs == NULL) || (object == NULL)) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, object);
+    if (!inode) {
+        errno = ESTALE;
+        goto out;
+    }
+
+    /* populate loc */
+    GLFS_LOC_FILL_INODE(inode, loc, out);
+
+    glfs_lease_to_gf_lease(lease, &gf_lease);
+
+    ret = syncop_lease(subvol, &loc, &gf_lease, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    gf_lease_to_glfs_lease(&gf_lease, lease);
 
 out:
-        loc_wipe (&oldloc);
-        loc_wipe (&newloc);
+    loc_wipe(&loc);
 
-        if (oldpinode)
-                inode_unref (oldpinode);
+    if (inode)
+        inode_unref(inode);
 
-        if (newpinode)
-                inode_unref (newpinode);
+    glfs_subvol_done(fs, subvol);
 
-        priv_glfs_subvol_done (fs, subvol);
+    __GLFS_EXIT_FS;
 
-        return ret;
+invalid_fs:
+    return ret;
 }
-
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_rename, 3.4.2);
-
diff --git a/api/src/glfs-handles.h b/api/src/glfs-handles.h
index ad963455e85..4d039b9c76b 100644
--- a/api/src/glfs-handles.h
+++ b/api/src/glfs-handles.h
@@ -1,5 +1,5 @@
 /*
-  Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+  Copyright (c) 2013-2018 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
   This file is licensed to you under your choice of the GNU Lesser
@@ -46,27 +46,39 @@
  *
  */
 
-/* Values for valid falgs to be used when using XXXsetattr, to set multiple
- attribute values passed via the related stat structure.
- */
-#define GFAPI_SET_ATTR_MODE  0x1
-#define GFAPI_SET_ATTR_UID   0x2
-#define GFAPI_SET_ATTR_GID   0x4
-#define GFAPI_SET_ATTR_SIZE  0x8
-#define GFAPI_SET_ATTR_ATIME 0x10
-#define GFAPI_SET_ATTR_MTIME 0x20
-
 /* Handle length for object handles returned from glfs_h_extract_handle or
  * glfs_h_create_from_handle */
 #define GFAPI_HANDLE_LENGTH 16
 
+/* These flags should be in sync to the ones defined in upcall.h */
+#define GFAPI_UP_NLINK 0x00000001 /* update nlink */
+#define GFAPI_UP_MODE 0x00000002  /* update mode and ctime */
+#define GFAPI_UP_OWN 0x00000004   /* update mode,uid,gid and ctime */
+#define GFAPI_UP_SIZE 0x00000008  /* update fsize */
+#define GFAPI_UP_TIMES 0x00000010 /* update all times */
+#define GFAPI_UP_ATIME 0x00000020 /* update atime only */
+#define GFAPI_UP_PERM                                                          \
+    0x00000040 /* update fields needed for                                     \
+                  permission checking */
+#define GFAPI_UP_RENAME                                                        \
+    0x00000080 /* this is a rename op -                                        \
+                  delete the cache entry */
+#define GFAPI_UP_FORGET                                                        \
+    0x00000100                           /* inode_forget on server side -      \
+                                            invalidate the cache entry */
+#define GFAPI_UP_PARENT_TIMES 0x00000200 /* update parent dir times */
+
+#define GFAPI_INODE_UPDATE_FLAGS                                               \
+    (GFAPI_UP_NLINK | GFAPI_UP_MODE | GFAPI_UP_OWN | GFAPI_UP_SIZE |           \
+     GFAPI_UP_TIMES | GFAPI_UP_ATIME)
+
 /* Portability non glibc c++ build systems */
 #ifndef __THROW
-# if defined __cplusplus
-#  define __THROW       throw ()
-# else
-#  define __THROW
-# endif
+#if defined __cplusplus
+#define __THROW throw()
+#else
+#define __THROW
+#endif
 #endif
 
 __BEGIN_DECLS
@@ -82,88 +94,261 @@ __BEGIN_DECLS
 struct glfs_object;
 typedef struct glfs_object glfs_object_t;
 
-/* Handle based operations */
-/* Operations that generate handles */
-struct glfs_object *glfs_h_lookupat (struct glfs *fs,
-				     struct glfs_object *parent,
-				     const char *path,
-                                     struct stat *stat) __THROW;
+/* Functions for getting details about the glfs_upcall_inode
+ *
+ * None of the pointers returned by the below functions should be free()'d,
+ * glfs_free()'d or glfs_h_close()'d by the application.
+ *
+ * Releasing of the structures is done by passing the glfs_upcall pointer
+ * to glfs_free().
+ */
+struct glfs_upcall_inode;
+typedef struct glfs_upcall_inode glfs_upcall_inode_t;
+
+glfs_object_t *
+glfs_upcall_inode_get_object(glfs_upcall_inode_t *arg) __THROW
+    GFAPI_PUBLIC(glfs_upcall_inode_get_object, 3.7.16);
+
+uint64_t
+glfs_upcall_inode_get_flags(glfs_upcall_inode_t *arg) __THROW
+    GFAPI_PUBLIC(glfs_upcall_inode_get_flags, 3.7.16);
+
+struct stat *
+glfs_upcall_inode_get_stat(glfs_upcall_inode_t *arg) __THROW
+    GFAPI_PUBLIC(glfs_upcall_inode_get_stat, 3.7.16);
 
-struct glfs_object *glfs_h_creat (struct glfs *fs, struct glfs_object *parent,
-				  const char *path, int flags, mode_t mode,
-				  struct stat *sb) __THROW;
+uint64_t
+glfs_upcall_inode_get_expire(glfs_upcall_inode_t *arg) __THROW
+    GFAPI_PUBLIC(glfs_upcall_inode_get_expire, 3.7.16);
 
-struct glfs_object *glfs_h_mkdir (struct glfs *fs, struct glfs_object *parent,
-				  const char *path, mode_t flags,
-				  struct stat *sb) __THROW;
+glfs_object_t *
+glfs_upcall_inode_get_pobject(glfs_upcall_inode_t *arg) __THROW
+    GFAPI_PUBLIC(glfs_upcall_inode_get_pobject, 3.7.16);
 
-struct glfs_object *glfs_h_mknod (struct glfs *fs, struct glfs_object *parent,
-				  const char *path, mode_t mode, dev_t dev,
-				  struct stat *sb) __THROW;
+struct stat *
+glfs_upcall_inode_get_pstat(glfs_upcall_inode_t *arg) __THROW
+    GFAPI_PUBLIC(glfs_upcall_inode_get_pstat, 3.7.16);
 
-struct glfs_object *glfs_h_symlink (struct glfs *fs, struct glfs_object *parent,
-				    const char *name, const char *data,
-				    struct stat *stat) __THROW;
+glfs_object_t *
+glfs_upcall_inode_get_oldpobject(glfs_upcall_inode_t *arg) __THROW
+    GFAPI_PUBLIC(glfs_upcall_inode_get_oldpobject, 3.7.16);
+
+struct stat *
+glfs_upcall_inode_get_oldpstat(glfs_upcall_inode_t *arg) __THROW
+    GFAPI_PUBLIC(glfs_upcall_inode_get_oldpstat, 3.7.16);
+
+/* Handle based operations */
+/* Operations that generate handles */
+glfs_object_t *
+glfs_h_lookupat(glfs_t *fs, glfs_object_t *parent, const char *path,
+                struct stat *stat, int follow) __THROW
+    GFAPI_PUBLIC(glfs_h_lookupat, 3.7.4);
+
+glfs_object_t *
+glfs_h_creat(glfs_t *fs, glfs_object_t *parent, const char *path, int flags,
+             mode_t mode, struct stat *sb) __THROW
+    GFAPI_PUBLIC(glfs_h_creat, 3.4.2);
+
+glfs_object_t *
+glfs_h_mkdir(glfs_t *fs, glfs_object_t *parent, const char *path, mode_t flags,
+             struct stat *sb) __THROW GFAPI_PUBLIC(glfs_h_mkdir, 3.4.2);
+
+glfs_object_t *
+glfs_h_mknod(glfs_t *fs, glfs_object_t *parent, const char *path, mode_t mode,
+             dev_t dev, struct stat *sb) __THROW
+    GFAPI_PUBLIC(glfs_h_mknod, 3.4.2);
+
+glfs_object_t *
+glfs_h_symlink(glfs_t *fs, glfs_object_t *parent, const char *name,
+               const char *data, struct stat *stat) __THROW
+    GFAPI_PUBLIC(glfs_h_symlink, 3.4.2);
 
 /* Operations on the actual objects */
-int glfs_h_unlink (struct glfs *fs, struct glfs_object *parent,
-		   const char *path) __THROW;
+int
+glfs_h_unlink(glfs_t *fs, glfs_object_t *parent, const char *path) __THROW
+    GFAPI_PUBLIC(glfs_h_unlink, 3.4.2);
+
+int
+glfs_h_close(glfs_object_t *object) __THROW GFAPI_PUBLIC(glfs_h_close, 3.4.2);
 
-int glfs_h_close (struct glfs_object *object) __THROW;
+int
+glfs_caller_specific_init(void *uid_caller_key, void *gid_caller_key,
+                          void *future) __THROW
+    GFAPI_PUBLIC(glfs_caller_specific_init, 3.5.0);
 
-int glfs_caller_specific_init (void *uid_caller_key, void *gid_caller_key,
-			       void *future) __THROW;
+int
+glfs_h_truncate(glfs_t *fs, glfs_object_t *object, off_t offset) __THROW
+    GFAPI_PUBLIC(glfs_h_truncate, 3.4.2);
 
-int glfs_h_truncate (struct glfs *fs, struct glfs_object *object,
-                     off_t offset) __THROW;
+int
+glfs_h_stat(glfs_t *fs, glfs_object_t *object, struct stat *stat) __THROW
+    GFAPI_PUBLIC(glfs_h_stat, 3.4.2);
 
-int glfs_h_stat(struct glfs *fs, struct glfs_object *object,
-                struct stat *stat) __THROW;
+int
+glfs_h_statfs(glfs_t *fs, glfs_object_t *object, struct statvfs *stat) __THROW
+    GFAPI_PUBLIC(glfs_h_statfs, 3.7.0);
 
-int glfs_h_getattrs (struct glfs *fs, struct glfs_object *object,
-		     struct stat *stat) __THROW;
+int
+glfs_h_getattrs(glfs_t *fs, glfs_object_t *object, struct stat *stat) __THROW
+    GFAPI_PUBLIC(glfs_h_getattrs, 3.4.2);
 
-int glfs_h_getxattrs (struct glfs *fs, struct glfs_object *object,
-		      const char *name, void *value,
-		      size_t size) __THROW;
+int
+glfs_h_getxattrs(glfs_t *fs, glfs_object_t *object, const char *name,
+                 void *value, size_t size) __THROW
+    GFAPI_PUBLIC(glfs_h_getxattrs, 3.5.1);
 
-int glfs_h_setattrs (struct glfs *fs, struct glfs_object *object,
-		     struct stat *sb, int valid) __THROW;
+int
+glfs_h_setattrs(glfs_t *fs, glfs_object_t *object, struct stat *sb,
+                int valid) __THROW GFAPI_PUBLIC(glfs_h_setattrs, 3.4.2);
 
-int glfs_h_setxattrs (struct glfs *fs, struct glfs_object *object,
-		      const char *name, const void *value,
-		      size_t size, int flags) __THROW;
+int
+glfs_h_setxattrs(glfs_t *fs, glfs_object_t *object, const char *name,
+                 const void *value, size_t size, int flags) __THROW
+    GFAPI_PUBLIC(glfs_h_setxattrs, 3.5.0);
 
-int glfs_h_readlink (struct glfs *fs, struct glfs_object *object, char *buf,
-		     size_t bufsiz) __THROW;
+int
+glfs_h_readlink(glfs_t *fs, glfs_object_t *object, char *buf,
+                size_t bufsiz) __THROW GFAPI_PUBLIC(glfs_h_readlink, 3.4.2);
 
-int glfs_h_link (struct glfs *fs, struct glfs_object *linktgt,
-		 struct glfs_object *parent, const char *name) __THROW;
+int
+glfs_h_link(glfs_t *fs, glfs_object_t *linktgt, glfs_object_t *parent,
+            const char *name) __THROW GFAPI_PUBLIC(glfs_h_link, 3.4.2);
 
-int glfs_h_rename (struct glfs *fs, struct glfs_object *olddir,
-		   const char *oldname, struct glfs_object *newdir,
-		   const char *newname) __THROW;
+int
+glfs_h_rename(glfs_t *fs, glfs_object_t *olddir, const char *oldname,
+              glfs_object_t *newdir, const char *newname) __THROW
+    GFAPI_PUBLIC(glfs_h_rename, 3.4.2);
 
-int glfs_h_removexattrs (struct glfs *fs, struct glfs_object *object,
-			 const char *name) __THROW;
+int
+glfs_h_removexattrs(glfs_t *fs, glfs_object_t *object, const char *name) __THROW
+    GFAPI_PUBLIC(glfs_h_removexattrs, 3.5.1);
 
 /* Operations enabling opaque invariant handle to object transitions */
-ssize_t glfs_h_extract_handle (struct glfs_object *object,
-			       unsigned char *handle, int len) __THROW;
+ssize_t
+glfs_h_extract_handle(glfs_object_t *object, unsigned char *handle,
+                      int len) __THROW
+    GFAPI_PUBLIC(glfs_h_extract_handle, 3.4.2);
 
-struct glfs_object *glfs_h_create_from_handle (struct glfs *fs,
-					       unsigned char *handle, int len,
-					       struct stat *stat) __THROW;
+/* Given a handle, looks up the inode and creates glfs_object.
+ * In addition, if provided 'stat', copies the inode attributes
+ */
+glfs_object_t *
+glfs_h_create_from_handle(glfs_t *fs, unsigned char *handle, int len,
+                          struct stat *stat) __THROW
+    GFAPI_PUBLIC(glfs_h_create_from_handle, 3.4.2);
 
 /* Operations enabling object handles to fd transitions */
-struct glfs_fd *glfs_h_opendir (struct glfs *fs,
-                                struct glfs_object *object) __THROW;
+glfs_fd_t *
+glfs_h_opendir(glfs_t *fs, glfs_object_t *object) __THROW
+    GFAPI_PUBLIC(glfs_h_opendir, 3.4.2);
 
-struct glfs_fd *glfs_h_open (struct glfs *fs, struct glfs_object *object,
-			     int flags) __THROW;
+glfs_fd_t *
+glfs_h_open(glfs_t *fs, glfs_object_t *object, int flags) __THROW
+    GFAPI_PUBLIC(glfs_h_open, 3.4.2);
 
 int
-glfs_h_access (struct glfs *fs, struct glfs_object *object, int mask) __THROW;
+glfs_h_access(glfs_t *fs, glfs_object_t *object, int mask) __THROW
+    GFAPI_PUBLIC(glfs_h_access, 3.6.0);
+
+struct glfs_object *
+glfs_h_creat_open(struct glfs *fs, struct glfs_object *parent, const char *path,
+                  int flags, mode_t mode, struct stat *stat,
+                  struct glfs_fd **out_fd) __THROW
+    GFAPI_PUBLIC(glfs_h_creat_open, 6.6);
+/*
+  SYNOPSIS
+
+  glfs_h_poll_upcall: Poll for upcall events given a 'glfs' object.
+
+  DESCRIPTION
+
+  This API is used to poll for upcall events stored in the
+  upcall list. Current users of this API is NFS-Ganesha.
+  In case of any event received, it will be mapped appropriately
+  into 'glfs_upcall' along with the handle('glfs_object') to be
+  passed to NFS-Ganesha.
+
+  In case of success, applications need to check the value of
+  cbk->handle to be NON NULL before processing the upcall
+  events.
+
+  PARAMETERS
+
+  @fs: glfs object to poll the upcall events for
+  @cbk: Pointer that will contain an upcall event for use by the application.
+        Application is responsible for free'ing the structure with glfs_free().
+
+  RETURN VALUES
+
+  0   : Success.
+  -1  : Error condition, mostly due to out of memory.
+
+*/
+
+int
+glfs_h_poll_upcall(glfs_t *fs, glfs_upcall_t **cbk) __THROW
+    GFAPI_PUBLIC(glfs_h_poll_upcall, 3.7.16);
+
+int
+glfs_h_acl_set(glfs_t *fs, glfs_object_t *object, const acl_type_t type,
+               const acl_t acl) __THROW GFAPI_PUBLIC(glfs_h_acl_set, 3.7.0);
+
+acl_t
+glfs_h_acl_get(glfs_t *fs, glfs_object_t *object, const acl_type_t type) __THROW
+    GFAPI_PUBLIC(glfs_h_acl_get, 3.7.0);
+
+size_t
+glfs_h_anonymous_write(glfs_t *fs, glfs_object_t *object, const void *buf,
+                       size_t count, off_t offset) __THROW
+    GFAPI_PUBLIC(glfs_h_anonymous_write, 3.7.0);
+
+ssize_t
+glfs_h_anonymous_read(glfs_t *fs, glfs_object_t *object, const void *buf,
+                      size_t count, off_t offset) __THROW
+    GFAPI_PUBLIC(glfs_h_anonymous_read, 3.7.0);
+
+/*
+ * Caution: The object returned by this object gets freed as part
+ * of 'glfs_free(xstat)'. Make sure to have a copy using 'glfs_object_copy()'
+ * to use post that.
+ */
+glfs_object_t *
+glfs_xreaddirplus_get_object(struct glfs_xreaddirp_stat *xstat) __THROW
+    GFAPI_PUBLIC(glfs_xreaddirplus_get_object, 3.11.0);
+
+/* Applications should close the object returned by this routine
+ * explicitly using 'glfs_h_close()'
+ */
+glfs_object_t *
+glfs_object_copy(glfs_object_t *src) __THROW
+    GFAPI_PUBLIC(glfs_object_copy, 3.11.0);
+
+int
+glfs_h_lease(glfs_t *fs, glfs_object_t *object, glfs_lease_t *lease) __THROW
+    GFAPI_PUBLIC(glfs_h_lease, 4.0.0);
+
+glfs_object_t *
+glfs_h_find_handle(glfs_t *fs, unsigned char *handle, int len) __THROW
+    GFAPI_PUBLIC(glfs_h_lease, 4.0.0);
+
+/* Functions for getting details about the glfs_upcall_lease
+ *
+ * None of the pointers returned by the below functions should be free()'d,
+ * glfs_free()'d or glfs_h_close()'d by the application.
+ *
+ * Releasing of the structures is done by passing the glfs_upcall pointer
+ * to glfs_free().
+ */
+struct glfs_upcall_lease;
+typedef struct glfs_upcall_lease glfs_upcall_lease_t;
+
+glfs_object_t *
+glfs_upcall_lease_get_object(glfs_upcall_lease_t *arg) __THROW
+    GFAPI_PUBLIC(glfs_upcall_lease_get_object, 4.1.6);
+
+uint32_t
+glfs_upcall_lease_get_lease_type(glfs_upcall_lease_t *arg) __THROW
+    GFAPI_PUBLIC(glfs_upcall_lease_get_lease_type, 4.1.6);
 
 __END_DECLS
 
diff --git a/api/src/glfs-internal.h b/api/src/glfs-internal.h
index bbfd3425f25..7cc3b18a104 100644
--- a/api/src/glfs-internal.h
+++ b/api/src/glfs-internal.h
@@ -1,5 +1,5 @@
 /*
-  Copyright (c) 2012 Red Hat, Inc. <http://www.redhat.com>
+  Copyright (c) 2012-2018 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
   This file is licensed to you under your choice of the GNU Lesser
@@ -8,158 +8,484 @@
   cases as published by the Free Software Foundation.
 */
 
-
 #ifndef _GLFS_INTERNAL_H
 #define _GLFS_INTERNAL_H
 
-#include "xlator.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/upcall-utils.h>
+#include "glfs-handles.h"
+#include <glusterfs/refcount.h>
+#include <glusterfs/syncop.h>
 
 #define GLFS_SYMLINK_MAX_FOLLOW 2048
 
 #define DEFAULT_REVAL_COUNT 1
 
 /*
- * syncop_xxx() calls are executed in two ways, one is inside a synctask where
- * the executing function will do 'swapcontext' and the other is without
- * synctask where the executing thread is made to wait using pthread_cond_wait.
- * Executing thread may change when syncop_xxx() is executed inside a synctask.
- * This leads to errno_location change i.e. errno may give errno of
- * non-executing thread. So errno is not touched inside a synctask execution.
- * All gfapi calls are executed using the second way of executing syncop_xxx()
- * where the executing thread waits using pthread_cond_wait so it is ok to set
- * errno in these cases. The following macro makes syncop_xxx() behave just
- * like a system call, where -1 is returned and errno is set when a failure
- * occurs.
+ * According to  pthread mutex and conditional variable ( cond,
+ * child_down_count, upcall mutex and mutex) initialization of struct glfs
+ * members, below GLFS_INIT_* flags are set in 'pthread_flags' member of struct
+ * glfs. The flags are set from glfs_init() and  glfs_new_from_ctx() functions
+ * as part of fs inititialization.
+ *
+ * These flag bits are validated in glfs_fini() to destroy all or partially
+ * initialized mutex and conditional variables of glfs object.
+ * If you introduce new pthread mutex or conditional variable in glfs object,
+ * please make sure you have a flag bit intorduced here for proper cleanup
+ * in glfs_fini().
+ *
  */
-#define DECODE_SYNCOP_ERR(ret) do {  \
-        if (ret < 0) {          \
-                errno = -ret;   \
-                ret = -1;       \
-        }                       \
-        } while (0)
-
-#define ESTALE_RETRY(ret,errno,reval,loc,label) do {	\
-	if (ret == -1 && errno == ESTALE) {	        \
-		if (reval < DEFAULT_REVAL_COUNT) {	\
-			reval++;			\
-			loc_wipe (loc);			\
-			goto label;			\
-		}					\
-	}						\
-	} while (0)
-
-#define GLFS_LOC_FILL_INODE(oinode, loc, label) do {   \
-	loc.inode = inode_ref (oinode);                \
-	uuid_copy (loc.gfid, oinode->gfid);            \
-	ret = priv_glfs_loc_touchup (&loc);            \
-	if (ret != 0) {                                \
-		errno = EINVAL;                        \
-		goto label;                            \
-	}                                              \
-	} while (0)
-
-#define GLFS_LOC_FILL_PINODE(pinode, loc, ret, errno, label, path) do {   \
-	loc.inode = inode_new (pinode->table);                            \
-	if (!loc.inode) {                                                 \
-		ret = -1;                                                 \
-		errno = ENOMEM;                                           \
-		goto label;                                               \
-	}                                                                 \
-	loc.parent = inode_ref (pinode);                                  \
-	loc.name = path;                                                  \
-	ret = priv_glfs_loc_touchup (&loc);                               \
-	if (ret != 0) {                                                   \
-		errno = EINVAL;                                           \
-		goto label;                                               \
-	}                                                                 \
-	} while (0)
 
-struct glfs;
+#define PTHREAD_MUTEX_INIT(mutex, attr, flags, mask, label)                    \
+    do {                                                                       \
+        int __ret = -1;                                                        \
+        __ret = pthread_mutex_init(mutex, attr);                               \
+        if (__ret == 0)                                                        \
+            flags |= mask;                                                     \
+        else                                                                   \
+            goto label;                                                        \
+    } while (0)
+
+#define PTHREAD_MUTEX_DESTROY(mutex, flags, mask)                              \
+    do {                                                                       \
+        if (flags & mask)                                                      \
+            (void)pthread_mutex_destroy(mutex);                                \
+    } while (0)
+
+#define PTHREAD_COND_INIT(cond, attr, flags, mask, label)                      \
+    do {                                                                       \
+        int __ret = -1;                                                        \
+        __ret = pthread_cond_init(cond, attr);                                 \
+        if (__ret == 0)                                                        \
+            flags |= mask;                                                     \
+        else                                                                   \
+            goto label;                                                        \
+    } while (0)
+
+#define PTHREAD_COND_DESTROY(cond, flags, mask)                                \
+    do {                                                                       \
+        if (flags & mask)                                                      \
+            (void)pthread_cond_destroy(cond);                                  \
+    } while (0)
+
+#define GLFS_INIT_MUTEX 0x00000001        /* pthread_mutex_flag */
+#define GLFS_INIT_COND 0x00000002         /* pthread_cond_flag */
+#define GLFS_INIT_COND_CHILD 0x00000004   /* pthread_cond_child_down_flag */
+#define GLFS_INIT_MUTEX_UPCALL 0x00000008 /* pthread_mutex_upcall_flag */
+
+#ifndef GF_DARWIN_HOST_OS
+#ifndef GFAPI_PUBLIC
+#define GFAPI_PUBLIC(sym, ver) /**/
+#endif
+#ifndef GFAPI_PRIVATE
+#define GFAPI_PRIVATE(sym, ver) /**/
+#endif
+#if __GNUC__ >= 10
+#define GFAPI_SYMVER_PUBLIC_DEFAULT(fn, ver)                                   \
+    __attribute__((__symver__(STR(fn) "@@GFAPI_" STR(ver))))
+
+#define GFAPI_SYMVER_PRIVATE_DEFAULT(fn, ver)                                  \
+    __attribute__((__symver__(STR(fn) "@@GFAPI_PRIVATE_" STR(ver))))
+
+#define GFAPI_SYMVER_PUBLIC(fn1, fn2, ver)                                     \
+    __attribute__((__symver__(STR(fn2) "@GFAPI_" STR(ver))))
+
+#define GFAPI_SYMVER_PRIVATE(fn1, fn2, ver)                                    \
+    __attribute__((__symver__(STR(fn2) "@GFAPI_PRIVATE_" STR(ver))))
+
+#else
+#define GFAPI_SYMVER_PUBLIC_DEFAULT(fn, ver)                                   \
+    asm(".symver pub_" STR(fn) ", " STR(fn) "@@GFAPI_" STR(ver));
+
+#define GFAPI_SYMVER_PRIVATE_DEFAULT(fn, ver)                                  \
+    asm(".symver priv_" STR(fn) ", " STR(fn) "@@GFAPI_PRIVATE_" STR(ver));
+
+#define GFAPI_SYMVER_PUBLIC(fn1, fn2, ver)                                     \
+    asm(".symver pub_" STR(fn1) ", " STR(fn2) "@GFAPI_" STR(ver));
+
+#define GFAPI_SYMVER_PRIVATE(fn1, fn2, ver)                                    \
+    asm(".symver priv_" STR(fn1) ", " STR(fn2) "@GFAPI_PRIVATE_" STR(ver));
+#endif
+#define STR(str) #str
+#else
+#ifndef GFAPI_PUBLIC
+#define GFAPI_PUBLIC(sym, ver) __asm("_" __STRING(sym) "$GFAPI_" __STRING(ver));
+#endif
+#ifndef GFAPI_PRIVATE
+#define GFAPI_PRIVATE(sym, ver)                                                \
+    __asm("_" __STRING(sym) "$GFAPI_PRIVATE_" __STRING(ver));
+#endif
+#define GFAPI_SYMVER_PUBLIC_DEFAULT(fn, dotver)  /**/
+#define GFAPI_SYMVER_PRIVATE_DEFAULT(fn, dotver) /**/
+#define GFAPI_SYMVER_PUBLIC(fn1, fn2, dotver)    /**/
+#define GFAPI_SYMVER_PRIVATE(fn1, fn2, dotver)   /**/
+#endif
+
+#define ESTALE_RETRY(ret, errno, reval, loc, label)                            \
+    do {                                                                       \
+        if (ret == -1 && errno == ESTALE) {                                    \
+            if (reval < DEFAULT_REVAL_COUNT) {                                 \
+                reval++;                                                       \
+                loc_wipe(loc);                                                 \
+                goto label;                                                    \
+            }                                                                  \
+        }                                                                      \
+    } while (0)
+
+#define GLFS_LOC_FILL_INODE(oinode, loc, label)                                \
+    do {                                                                       \
+        loc.inode = inode_ref(oinode);                                         \
+        gf_uuid_copy(loc.gfid, oinode->gfid);                                  \
+        ret = glfs_loc_touchup(&loc);                                          \
+        if (ret != 0) {                                                        \
+            errno = EINVAL;                                                    \
+            goto label;                                                        \
+        }                                                                      \
+    } while (0)
+
+#define GLFS_LOC_FILL_PINODE(pinode, loc, ret, errno, label, path)             \
+    do {                                                                       \
+        loc.inode = inode_new(pinode->table);                                  \
+        if (!loc.inode) {                                                      \
+            ret = -1;                                                          \
+            errno = ENOMEM;                                                    \
+            goto label;                                                        \
+        }                                                                      \
+        loc.parent = inode_ref(pinode);                                        \
+        loc.name = path;                                                       \
+        ret = glfs_loc_touchup(&loc);                                          \
+        if (ret != 0) {                                                        \
+            errno = EINVAL;                                                    \
+            goto label;                                                        \
+        }                                                                      \
+    } while (0)
 
-typedef int (*glfs_init_cbk) (struct glfs *fs, int ret);
+struct glfs;
 
-struct glfs {
-	char               *volname;
-        uuid_t              vol_uuid;
+struct _upcall_entry {
+    struct list_head upcall_list;
+    struct gf_upcall upcall_data;
+};
+typedef struct _upcall_entry upcall_entry;
 
-	glusterfs_ctx_t    *ctx;
+typedef int (*glfs_init_cbk)(struct glfs *fs, int ret);
 
-	pthread_t           poller;
+struct glfs {
+    char *volname;
+    uuid_t vol_uuid;
+
+    glusterfs_ctx_t *ctx;
+
+    pthread_t poller;
+
+    glfs_init_cbk init_cbk;
+    pthread_mutex_t mutex;
+    pthread_cond_t cond;
+    pthread_cond_t child_down_cond; /* for broadcasting CHILD_DOWN */
+    int init;
+    int ret;
+    int err;
+
+    xlator_t *active_subvol; /* active graph */
+    xlator_t *mip_subvol;    /* graph for which migration is in
+                              * progress */
+    xlator_t *next_subvol;   /* Any new graph is put to
+                              * next_subvol, the graph in
+                              * next_subvol can either be moved
+                              * to mip_subvol (if any IO picks it
+                              * up for migration), or be
+                              * destroyed (if there is a new
+                              * graph, and this was never picked
+                              * for migration) */
+    xlator_t *old_subvol;
+
+    char *oldvolfile;
+    ssize_t oldvollen;
+
+    inode_t *cwd;
+
+    uint32_t dev_id; /* Used to fill st_dev in struct stat */
+
+    struct list_head openfds;
+
+    gf_boolean_t migration_in_progress;
+
+    gf_boolean_t cache_upcalls; /* add upcalls to the upcall_list? */
+    struct list_head upcall_list;
+    pthread_mutex_t upcall_list_mutex; /* mutex for upcall entry list */
+
+    uint32_t pin_refcnt;
+    uint32_t pthread_flags; /* GLFS_INIT_* # defines set this flag */
+
+    uint32_t upcall_events; /* Mask of upcall events application
+                             * is interested in */
+    glfs_upcall_cbk up_cbk; /* upcall cbk function to be registered */
+    void *up_data;          /* Opaque data provided by application
+                             * during upcall registration */
+    struct list_head waitq; /* waiting synctasks */
+};
 
-	glfs_init_cbk       init_cbk;
-	pthread_mutex_t     mutex;
-	pthread_cond_t      cond;
-	int                 init;
-	int                 ret;
-	int                 err;
+/* This enum is used to maintain the state of glfd. In case of async fops
+ * fd might be closed before the actual fop is complete. Therefore we need
+ * to track whether the fd is closed or not, instead actually closing it.*/
+enum glfs_fd_state { GLFD_INIT, GLFD_OPEN, GLFD_CLOSE };
 
-	xlator_t           *active_subvol;
-	xlator_t           *next_subvol;
-	xlator_t           *old_subvol;
+struct glfs_fd {
+    struct list_head openfds;
+    struct list_head list;
+    GF_REF_DECL;
+    struct glfs *fs;
+    enum glfs_fd_state state;
+    off_t offset;
+    fd_t *fd; /* Currently guared by @fs->mutex. TODO: per-glfd lock */
+    struct list_head entries;
+    gf_dirent_t *next;
+    struct dirent *readdirbuf;
+    gf_lkowner_t lk_owner;
+    glfs_leaseid_t lease_id; /* Stores lease_id of client in glfd */
+    gf_lock_t lock;          /* lock taken before updating fd state */
+    glfs_recall_cbk cbk;
+    void *cookie;
+};
 
-	char               *oldvolfile;
-	ssize_t             oldvollen;
+/* glfs object handle introduced for the alternate gfapi implementation based
+   on glfs handles/gfid/inode
+*/
+struct glfs_object {
+    inode_t *inode;
+    uuid_t gfid;
+};
 
-	inode_t            *cwd;
+struct glfs_upcall {
+    struct glfs *fs;                /* glfs object */
+    enum glfs_upcall_reason reason; /* Upcall event type */
+    void *event;                    /* changes based in the event type */
+    void (*free_event)(void *);     /* free event after the usage */
+};
 
-	uint32_t            dev_id; /* Used to fill st_dev in struct stat */
+struct glfs_upcall_inode {
+    struct glfs_object *object;      /* Object which need to be acted upon */
+    int flags;                       /* Cache UPDATE/INVALIDATE flags */
+    struct stat buf;                 /* Latest stat of this entry */
+    unsigned int expire_time_attr;   /* the amount of time for which
+                                      * the application need to cache
+                                      * this entry */
+    struct glfs_object *p_object;    /* parent Object to be updated */
+    struct stat p_buf;               /* Latest stat of parent dir handle */
+    struct glfs_object *oldp_object; /* Old parent Object to be updated */
+    struct stat oldp_buf;            /* Latest stat of old parent dir handle */
+};
 
-	struct list_head    openfds;
+struct glfs_upcall_lease {
+    struct glfs_object *object; /* Object which need to be acted upon */
+    uint32_t lease_type;        /* Lease type to which client can downgrade to*/
+};
 
-	gf_boolean_t        migration_in_progress;
+struct glfs_upcall_lease_fd {
+    uint32_t lease_type; /* Lease type to which client can downgrade to*/
+    void *fd_cookie;     /* Object which need to be acted upon */
 };
 
-struct glfs_fd {
-	struct list_head   openfds;
-	struct glfs       *fs;
-	off_t              offset;
-	fd_t              *fd; /* Currently guared by @fs->mutex. TODO: per-glfd lock */
-	struct list_head   entries;
-	gf_dirent_t       *next;
-	struct dirent     *readdirbuf;
+struct glfs_xreaddirp_stat {
+    struct stat
+        st; /* Stat for that dirent - corresponds to GFAPI_XREADDIRP_STAT */
+    struct glfs_object *object; /* handled for GFAPI_XREADDIRP_HANDLE */
+    uint32_t flags_handled;     /* final set of flags successfulyy handled */
 };
 
-/* glfs object handle introduced for the alternate gfapi implementation based
-   on glfs handles/gfid/inode
-*/
-struct glfs_object {
-        inode_t         *inode;
-        uuid_t          gfid;
+#define DEFAULT_EVENT_POOL_SIZE 16384
+#define GF_MEMPOOL_COUNT_OF_DICT_T 4096
+#define GF_MEMPOOL_COUNT_OF_DATA_T (GF_MEMPOOL_COUNT_OF_DICT_T * 4)
+#define GF_MEMPOOL_COUNT_OF_DATA_PAIR_T (GF_MEMPOOL_COUNT_OF_DICT_T * 4)
+
+#define GF_MEMPOOL_COUNT_OF_LRU_BUF_T 256
+
+typedef void(glfs_mem_release_t)(void *ptr);
+
+struct glfs_mem_header {
+    uint32_t magic;
+    size_t nmemb;
+    size_t size;
+    glfs_mem_release_t *release;
 };
 
-#define DEFAULT_EVENT_POOL_SIZE           16384
-#define GF_MEMPOOL_COUNT_OF_DICT_T        4096
-#define GF_MEMPOOL_COUNT_OF_DATA_T        (GF_MEMPOOL_COUNT_OF_DICT_T * 4)
-#define GF_MEMPOOL_COUNT_OF_DATA_PAIR_T   (GF_MEMPOOL_COUNT_OF_DICT_T * 4)
+#define GLFS_MEM_HEADER_SIZE (sizeof(struct glfs_mem_header))
+#define GLFS_MEM_HEADER_MAGIC 0x20170830
 
-#define GF_MEMPOOL_COUNT_OF_LRU_BUF_T     256
+static inline void *
+__glfs_calloc(size_t nmemb, size_t size, glfs_mem_release_t release,
+              uint32_t type, const char *typestr)
+{
+    struct glfs_mem_header *header = NULL;
 
-int glfs_mgmt_init (struct glfs *fs);
-void priv_glfs_init_done (struct glfs *fs, int ret);
-int glfs_process_volfp (struct glfs *fs, FILE *fp);
-int glfs_resolve (struct glfs *fs, xlator_t *subvol, const char *path, loc_t *loc,
-		  struct iatt *iatt, int reval);
-int glfs_lresolve (struct glfs *fs, xlator_t *subvol, const char *path, loc_t *loc,
-		   struct iatt *iatt, int reval);
-fd_t *glfs_resolve_fd (struct glfs *fs, xlator_t *subvol, struct glfs_fd *glfd);
+    header = __gf_calloc(nmemb, (size + GLFS_MEM_HEADER_SIZE), type, typestr);
+    if (!header)
+        return NULL;
 
-fd_t *__glfs_migrate_fd (struct glfs *fs, xlator_t *subvol, struct glfs_fd *glfd);
+    header->magic = GLFS_MEM_HEADER_MAGIC;
+    header->nmemb = nmemb;
+    header->size = size;
+    header->release = release;
 
-int glfs_first_lookup (xlator_t *subvol);
+    return header + 1;
+}
 
-static inline void
-__glfs_entry_fs (struct glfs *fs)
+static inline void *
+__glfs_malloc(size_t size, glfs_mem_release_t release, uint32_t type,
+              const char *typestr)
 {
-	THIS = fs->ctx->master;
+    struct glfs_mem_header *header = NULL;
+
+    header = __gf_malloc((size + GLFS_MEM_HEADER_SIZE), type, typestr);
+    if (!header)
+        return NULL;
+
+    header->magic = GLFS_MEM_HEADER_MAGIC;
+    header->nmemb = 1;
+    header->size = size;
+    header->release = release;
+
+    return header + 1;
 }
 
+static inline void *
+__glfs_realloc(void *ptr, size_t size)
+{
+    struct glfs_mem_header *old_header = NULL;
+    struct glfs_mem_header *new_header = NULL;
+    struct glfs_mem_header tmp_header;
+    void *new_ptr = NULL;
+
+    GF_ASSERT(NULL != ptr);
+
+    old_header = (struct glfs_mem_header *)(ptr - GLFS_MEM_HEADER_SIZE);
+    GF_ASSERT(old_header->magic == GLFS_MEM_HEADER_MAGIC);
+    tmp_header = *old_header;
+
+    new_ptr = __gf_realloc(old_header, (size + GLFS_MEM_HEADER_SIZE));
+    if (!new_ptr)
+        return NULL;
+
+    new_header = (struct glfs_mem_header *)new_ptr;
+    *new_header = tmp_header;
+    new_header->size = size;
+
+    return new_header + 1;
+}
 
 static inline void
-__glfs_entry_fd (struct glfs_fd *fd)
+__glfs_free(void *free_ptr)
 {
-	THIS = fd->fd->inode->table->xl->ctx->master;
+    struct glfs_mem_header *header = NULL;
+    void *release_ptr = NULL;
+    int i = 0;
+
+    if (!free_ptr)
+        return;
+
+    header = (struct glfs_mem_header *)(free_ptr - GLFS_MEM_HEADER_SIZE);
+    GF_ASSERT(header->magic == GLFS_MEM_HEADER_MAGIC);
+
+    if (header->release) {
+        release_ptr = free_ptr;
+        for (i = 0; i < header->nmemb; i++) {
+            header->release(release_ptr);
+            release_ptr += header->size;
+        }
+    }
+
+    __gf_free(header);
 }
 
+#define GLFS_CALLOC(nmemb, size, release, type)                                \
+    __glfs_calloc(nmemb, size, release, type, #type)
+
+#define GLFS_MALLOC(size, release, type)                                       \
+    __glfs_malloc(size, release, type, #type)
+
+#define GLFS_REALLOC(ptr, size) __glfs_realloc(ptr, size)
+
+#define GLFS_FREE(free_ptr) __glfs_free(free_ptr)
+
+int
+glfs_mgmt_init(struct glfs *fs);
+void
+glfs_init_done(struct glfs *fs, int ret) GFAPI_PRIVATE(glfs_init_done, 3.4.0);
+int
+glfs_process_volfp(struct glfs *fs, FILE *fp);
+int
+glfs_resolve(struct glfs *fs, xlator_t *subvol, const char *path, loc_t *loc,
+             struct iatt *iatt, int reval) GFAPI_PRIVATE(glfs_resolve, 3.7.0);
+int
+glfs_lresolve(struct glfs *fs, xlator_t *subvol, const char *path, loc_t *loc,
+              struct iatt *iatt, int reval);
+fd_t *
+glfs_resolve_fd(struct glfs *fs, xlator_t *subvol, struct glfs_fd *glfd);
+
+fd_t *
+__glfs_migrate_fd(struct glfs *fs, xlator_t *subvol, struct glfs_fd *glfd);
+
+int
+glfs_first_lookup(xlator_t *subvol);
+
+void
+glfs_process_upcall_event(struct glfs *fs, void *data)
+    GFAPI_PRIVATE(glfs_process_upcall_event, 3.7.0);
+
+#define __GLFS_ENTRY_VALIDATE_FS(fs, label)                                    \
+    do {                                                                       \
+        if (!fs) {                                                             \
+            errno = EINVAL;                                                    \
+            goto label;                                                        \
+        }                                                                      \
+        old_THIS = THIS;                                                       \
+        THIS = fs->ctx->master;                                                \
+    } while (0)
+
+#define __GLFS_EXIT_FS                                                         \
+    do {                                                                       \
+        THIS = old_THIS;                                                       \
+    } while (0)
+
+#define __GLFS_ENTRY_VALIDATE_FD(glfd, label)                                  \
+    do {                                                                       \
+        if (!glfd || !glfd->fd || !glfd->fd->inode ||                          \
+            glfd->state != GLFD_OPEN) {                                        \
+            errno = EBADF;                                                     \
+            goto label;                                                        \
+        }                                                                      \
+        old_THIS = THIS;                                                       \
+        THIS = glfd->fd->inode->table->xl->ctx->master;                        \
+    } while (0)
+
+#define __GLFS_LOCK_WAIT(fs)                                                   \
+    do {                                                                       \
+        struct synctask *task = NULL;                                          \
+                                                                               \
+        task = synctask_get();                                                 \
+                                                                               \
+        if (task) {                                                            \
+            list_add_tail(&task->waitq, &fs->waitq);                           \
+            pthread_mutex_unlock(&fs->mutex);                                  \
+            synctask_yield(task, NULL);                                        \
+            pthread_mutex_lock(&fs->mutex);                                    \
+        } else {                                                               \
+            /* non-synctask */                                                 \
+            pthread_cond_wait(&fs->cond, &fs->mutex);                          \
+        }                                                                      \
+    } while (0)
+
+#define __GLFS_SYNCTASK_WAKE(fs)                                               \
+    do {                                                                       \
+        struct synctask *waittask = NULL;                                      \
+                                                                               \
+        while (!list_empty(&fs->waitq)) {                                      \
+            waittask = list_entry(fs->waitq.next, struct synctask, waitq);     \
+            list_del_init(&waittask->waitq);                                   \
+            synctask_wake(waittask);                                           \
+        }                                                                      \
+    } while (0)
 
 /*
   By default all lock attempts from user context must
@@ -168,69 +494,263 @@ __glfs_entry_fd (struct glfs_fd *fd)
   we can give up the mutex during syncop calls so
   that bottom up calls (particularly CHILD_UP notify)
   can do a mutex_lock() on @glfs without deadlocking
-  the filesystem
+  the filesystem.
+
+  All the fops should wait for graph migration to finish
+  before starting the fops. Therefore these functions should
+  call glfs_lock with wait_for_migration as true. But waiting
+  for migration to finish in call-back path can result thread
+  dead-locks. The reason for this is we only have finite
+  number of epoll threads. so if we wait on epoll threads
+  there will not be any thread left to handle outstanding
+  rpc replies.
 */
 static inline int
-glfs_lock (struct glfs *fs)
+glfs_lock(struct glfs *fs, gf_boolean_t wait_for_migration)
 {
-	pthread_mutex_lock (&fs->mutex);
+    pthread_mutex_lock(&fs->mutex);
 
-	while (!fs->init)
-		pthread_cond_wait (&fs->cond, &fs->mutex);
+    while (!fs->init)
+        __GLFS_LOCK_WAIT(fs);
 
-	while (fs->migration_in_progress)
-		pthread_cond_wait (&fs->cond, &fs->mutex);
+    while (wait_for_migration && fs->migration_in_progress)
+        __GLFS_LOCK_WAIT(fs);
 
-	return 0;
+    return 0;
 }
 
-
 static inline void
-glfs_unlock (struct glfs *fs)
+glfs_unlock(struct glfs *fs)
 {
-	pthread_mutex_unlock (&fs->mutex);
+    pthread_mutex_unlock(&fs->mutex);
 }
 
+struct glfs_fd *
+glfs_fd_new(struct glfs *fs);
+void
+glfs_fd_bind(struct glfs_fd *glfd);
+void
+glfd_set_state_bind(struct glfs_fd *glfd);
+
+xlator_t *
+glfs_active_subvol(struct glfs *fs) GFAPI_PRIVATE(glfs_active_subvol, 3.4.0);
+xlator_t *
+__glfs_active_subvol(struct glfs *fs);
+void
+glfs_subvol_done(struct glfs *fs, xlator_t *subvol)
+    GFAPI_PRIVATE(glfs_subvol_done, 3.4.0);
+
+inode_t *
+glfs_refresh_inode(xlator_t *subvol, inode_t *inode);
+
+inode_t *
+glfs_cwd_get(struct glfs *fs);
+int
+glfs_cwd_set(struct glfs *fs, inode_t *inode);
+inode_t *
+glfs_resolve_inode(struct glfs *fs, xlator_t *subvol,
+                   struct glfs_object *object);
+int
+glfs_create_object(loc_t *loc, struct glfs_object **retobject);
+int
+__glfs_cwd_set(struct glfs *fs, inode_t *inode);
+
+int
+glfs_resolve_base(struct glfs *fs, xlator_t *subvol, inode_t *inode,
+                  struct iatt *iatt);
+
+int
+glfs_resolve_at(struct glfs *fs, xlator_t *subvol, inode_t *at,
+                const char *origpath, loc_t *loc, struct iatt *iatt, int follow,
+                int reval) GFAPI_PRIVATE(glfs_resolve_at, 3.4.0);
+int
+glfs_loc_touchup(loc_t *loc) GFAPI_PRIVATE(glfs_loc_touchup, 3.4.0);
+void
+glfs_iatt_to_stat(struct glfs *fs, struct iatt *iatt, struct stat *stat);
+void
+glfs_iatt_from_stat(struct stat *stat, int valid, struct iatt *iatt,
+                    int *gvalid);
+int
+glfs_loc_link(loc_t *loc, struct iatt *iatt);
+int
+glfs_loc_unlink(loc_t *loc);
+int
+glfs_getxattr_process(void *value, size_t size, dict_t *xattr,
+                      const char *name);
+
+/* Sends RPC call to glusterd to fetch required volume info */
+int
+glfs_get_volume_info(struct glfs *fs);
+
+/*
+  SYNOPSIS
 
-void glfs_fd_destroy (struct glfs_fd *glfd);
+       glfs_new_from_ctx: Creates a virtual mount object by taking a
+       glusterfs_ctx_t object.
 
-struct glfs_fd *glfs_fd_new (struct glfs *fs);
-void glfs_fd_bind (struct glfs_fd *glfd);
+  DESCRIPTION
 
-xlator_t *priv_glfs_active_subvol (struct glfs *fs);
-xlator_t *__glfs_active_subvol (struct glfs *fs);
-void priv_glfs_subvol_done (struct glfs *fs, xlator_t *subvol);
+       glfs_new_from_ctx() is not same as glfs_new(). It takes the
+       glusterfs_ctx_t object instead of creating one by glusterfs_ctx_new().
+       Again the usage is restricted to NFS MOUNT over UDP i.e. in
+       glfs_resolve_at() which would take fs object as input but never use
+       (purpose is not to change the ABI of glfs_resolve_at()).
 
-inode_t *glfs_refresh_inode (xlator_t *subvol, inode_t *inode);
+  PARAMETERS
 
-inode_t *glfs_cwd_get (struct glfs *fs);
-int glfs_cwd_set (struct glfs *fs, inode_t *inode);
-inode_t *glfs_resolve_inode (struct glfs *fs, xlator_t *subvol,
-			     struct glfs_object *object);
-int glfs_create_object (loc_t *loc, struct glfs_object **retobject);
-int __glfs_cwd_set (struct glfs *fs, inode_t *inode);
+       @ctx: glusterfs_ctx_t object
 
-int glfs_resolve_base (struct glfs *fs, xlator_t *subvol, inode_t *inode,
-		       struct iatt *iatt);
-int priv_glfs_resolve_at (struct glfs *fs, xlator_t *subvol, inode_t *at,
-                          const char *origpath, loc_t *loc, struct iatt *iatt,
-                          int follow, int reval);
-int priv_glfs_loc_touchup (loc_t *loc);
-void glfs_iatt_to_stat (struct glfs *fs, struct iatt *iatt, struct stat *stat);
-int glfs_loc_link (loc_t *loc, struct iatt *iatt);
-int glfs_loc_unlink (loc_t *loc);
-dict_t *dict_for_key_value (const char *name, const char *value, size_t size);
-int glfs_getxattr_process (void *value, size_t size, dict_t *xattr,
-			   const char *name);
+  RETURN VALUES
 
-/* Sends RPC call to glusterd to fetch required volume info */
-int glfs_get_volume_info (struct glfs *fs);
+       fs     : Pointer to the newly created glfs_t object.
+       NULL   : Otherwise.
+*/
 
-#define GFAPI_SYMVER_PUBLIC_DEFAULT(fn, dotver) \
-        asm(".symver pub_"STR(fn)", "STR(fn)"@@GFAPI_"STR(dotver))
+struct glfs *
+glfs_new_from_ctx(glusterfs_ctx_t *ctx) GFAPI_PRIVATE(glfs_new_from_ctx, 3.7.0);
 
-#define GFAPI_SYMVER_PRIVATE_DEFAULT(fn, dotver) \
-        asm(".symver priv_"STR(fn)", "STR(fn)"@@GFAPI_PRIVATE_"STR(dotver))
-#define STR(str) #str
+/*
+  SYNOPSIS
+
+       glfs_free_from_ctx: Free up the memory occupied by glfs_t object
+       created by glfs_new_from_ctx().
+
+  DESCRIPTION
+
+       The glfs_t object allocated by glfs_new_from_ctx() must be released
+       by the caller using this routine. The usage can be found
+       at glfs_fini() or NFS, MOUNT over UDP i.e.
+                        __mnt3udp_get_export_subdir_inode ()
+                                => glfs_resolve_at().
+
+  PARAMETERS
+
+       @fs: The glfs_t object to be deallocated.
+
+  RETURN VALUES
+
+       void
+*/
+
+void
+glfs_free_from_ctx(struct glfs *fs) GFAPI_PRIVATE(glfs_free_from_ctx, 3.7.0);
+
+int
+glfs_recall_lease_fd(struct glfs *fs, struct gf_upcall *up_data);
+
+int
+glfs_get_upcall_cache_invalidation(struct gf_upcall *to_up_data,
+                                   struct gf_upcall *from_up_data);
+int
+glfs_h_poll_cache_invalidation(struct glfs *fs, struct glfs_upcall *up_arg,
+                               struct gf_upcall *upcall_data);
+
+ssize_t
+glfs_anonymous_preadv(struct glfs *fs, struct glfs_object *object,
+                      const struct iovec *iovec, int iovcnt, off_t offset,
+                      int flags);
+ssize_t
+glfs_anonymous_pwritev(struct glfs *fs, struct glfs_object *object,
+                       const struct iovec *iovec, int iovcnt, off_t offset,
+                       int flags);
+
+struct glfs_object *
+glfs_h_resolve_symlink(struct glfs *fs, struct glfs_object *object);
+
+/* Deprecated structures that were passed to client applications, replaced by
+ * accessor functions. Do not use these in new applications, and update older
+ * usage.
+ *
+ * See http://review.gluster.org/14701 for more details.
+ *
+ * WARNING: These structures will be removed in the future.
+ */
+struct glfs_callback_arg {
+    struct glfs *fs;
+    enum glfs_upcall_reason reason;
+    void *event_arg;
+};
+
+struct glfs_callback_inode_arg {
+    struct glfs_object *object;      /* Object which need to be acted upon */
+    int flags;                       /* Cache UPDATE/INVALIDATE flags */
+    struct stat buf;                 /* Latest stat of this entry */
+    unsigned int expire_time_attr;   /* the amount of time for which
+                                      * the application need to cache
+                                      * this entry
+                                      */
+    struct glfs_object *p_object;    /* parent Object to be updated */
+    struct stat p_buf;               /* Latest stat of parent dir handle */
+    struct glfs_object *oldp_object; /* Old parent Object
+                                      * to be updated */
+    struct stat oldp_buf;            /* Latest stat of old parent
+                                      * dir handle */
+};
+struct dirent *
+glfs_readdirbuf_get(struct glfs_fd *glfd);
+
+gf_dirent_t *
+glfd_entry_next(struct glfs_fd *glfd, int plus);
+
+void
+gf_dirent_to_dirent(gf_dirent_t *gf_dirent, struct dirent *dirent);
+
+void
+gf_lease_to_glfs_lease(struct gf_lease *gf_lease, struct glfs_lease *lease);
+
+void
+glfs_lease_to_gf_lease(struct glfs_lease *lease, struct gf_lease *gf_lease);
+
+void
+glfs_release_upcall(void *ptr);
+
+int
+get_fop_attr_glfd(dict_t **fop_attr, struct glfs_fd *glfd);
 
+int
+set_fop_attr_glfd(struct glfs_fd *glfd);
+
+int
+get_fop_attr_thrd_key(dict_t **fop_attr);
+
+void
+unset_fop_attr(dict_t **fop_attr);
+
+/*
+  SYNOPSIS
+  glfs_statx: Fetch extended file attributes for the given path.
+
+  DESCRIPTION
+  This function fetches extended file attributes for the given path.
+
+  PARAMETERS
+  @fs: The 'virtual mount' object referencing a volume, under which file exists.
+  @path: Path of the file within the virtual mount.
+  @mask: Requested extended file attributes mask, (See mask defines above)
+
+  RETURN VALUES
+  -1 : Failure. @errno will be set with the type of failure.
+  0  : Filled in statxbuf with appropriate masks for valid items in the
+       structure.
+
+  ERRNO VALUES
+    EINVAL: fs is invalid
+    EINVAL: mask has unsupported bits set
+    Other errors as returned by stat(2)
+ */
+
+int
+glfs_statx(struct glfs *fs, const char *path, unsigned int mask,
+           struct glfs_stat *statxbuf) GFAPI_PRIVATE(glfs_statx, 6.0);
+
+void
+glfs_iatt_from_statx(struct iatt *, const struct glfs_stat *)
+    GFAPI_PRIVATE(glfs_iatt_from_statx, 6.0);
+
+/*
+ * This API is a per thread setting, similar to glfs_setfs{u/g}id, because of
+ * the call to syncopctx_setfspid.
+ */
+int
+glfs_setfspid(struct glfs *, pid_t) GFAPI_PRIVATE(glfs_setfspid, 6.1);
 #endif /* !_GLFS_INTERNAL_H */
diff --git a/api/src/glfs-master.c b/api/src/glfs-master.c
index 89017bab84d..100dcc16cc0 100644
--- a/api/src/glfs-master.c
+++ b/api/src/glfs-master.c
@@ -1,5 +1,5 @@
 /*
-  Copyright (c) 2012 Red Hat, Inc. <http://www.redhat.com>
+  Copyright (c) 2012-2016 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
   This file is licensed to you under your choice of the GNU Lesser
@@ -8,153 +8,176 @@
   cases as published by the Free Software Foundation.
 */
 
-#include <unistd.h>
-#include <string.h>
-#include <stdlib.h>
 #include <stdio.h>
-#include <inttypes.h>
-#include <limits.h>
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "xlator.h"
-#include "glusterfs.h"
+#include <glusterfs/glusterfs.h>
 
 #include "glfs-internal.h"
 #include "glfs-mem-types.h"
-
-
-extern void
-glfs_subvol_done (struct glfs *, xlator_t *);
+#include "gfapi-messages.h"
 
 int
-graph_setup (struct glfs *fs, glusterfs_graph_t *graph)
+graph_setup(struct glfs *fs, glusterfs_graph_t *graph)
 {
-	xlator_t      *new_subvol = NULL;
-	xlator_t      *old_subvol = NULL;
-	inode_table_t *itable = NULL;
-	int            ret = -1;
-
-	new_subvol = graph->top;
-
-	/* This is called in a bottom-up context, it should specifically
-	   NOT be glfs_lock()
-	*/
-	pthread_mutex_lock (&fs->mutex);
-	{
-		if (new_subvol->switched ||
-		    new_subvol == fs->active_subvol ||
-		    new_subvol == fs->next_subvol) {
-			/* Spurious CHILD_UP event on old graph */
-			ret = 0;
-			goto unlock;
-		}
-
-		if (!new_subvol->itable) {
-			itable = inode_table_new (131072, new_subvol);
-			if (!itable) {
-				errno = ENOMEM;
-				ret = -1;
-				goto unlock;
-			}
-
-			new_subvol->itable = itable;
-		}
-
-		old_subvol = fs->next_subvol;
-		fs->next_subvol = new_subvol;
-		fs->next_subvol->winds++; /* first ref */
-		ret = 0;
-	}
+    xlator_t *new_subvol = NULL;
+    xlator_t *old_subvol = NULL;
+    inode_table_t *itable = NULL;
+    int ret = -1;
+
+    new_subvol = graph->top;
+
+    /* This is called in a bottom-up context, it should specifically
+       NOT be glfs_lock()
+    */
+    pthread_mutex_lock(&fs->mutex);
+    {
+        if (new_subvol->switched || new_subvol == fs->active_subvol ||
+            new_subvol == fs->next_subvol || new_subvol == fs->mip_subvol) {
+            /* Spurious CHILD_UP event on old graph */
+            ret = 0;
+            goto unlock;
+        }
+
+        if (!new_subvol->itable) {
+            itable = inode_table_new(131072, new_subvol);
+            if (!itable) {
+                errno = ENOMEM;
+                ret = -1;
+                goto unlock;
+            }
+
+            new_subvol->itable = itable;
+        }
+
+        old_subvol = fs->next_subvol;
+        fs->next_subvol = new_subvol;
+        fs->next_subvol->winds++; /* first ref */
+        ret = 0;
+    }
 unlock:
-	pthread_mutex_unlock (&fs->mutex);
+    pthread_mutex_unlock(&fs->mutex);
 
-	if (old_subvol)
-		/* wasn't picked up so far, skip */
-		glfs_subvol_done (fs, old_subvol);
+    if (old_subvol)
+        /* wasn't picked up so far, skip */
+        glfs_subvol_done(fs, old_subvol);
 
-	return ret;
+    return ret;
 }
 
-
-extern void
-glfs_init_done (struct glfs *fs, int ret);
-
 int
-notify (xlator_t *this, int event, void *data, ...)
+notify(xlator_t *this, int event, void *data, ...)
 {
-	glusterfs_graph_t   *graph = NULL;
-	struct glfs	    *fs = NULL;
-
-	graph = data;
-	fs = this->private;
-
-	switch (event) {
-	case GF_EVENT_GRAPH_NEW:
-		gf_log (this->name, GF_LOG_INFO, "New graph %s (%d) coming up",
-			uuid_utoa ((unsigned char *)graph->graph_uuid),
-			graph->id);
-		break;
-	case GF_EVENT_CHILD_UP:
-		graph_setup (fs, graph);
-		glfs_init_done (fs, 0);
-		break;
-	case GF_EVENT_CHILD_DOWN:
-		graph_setup (fs, graph);
-		glfs_init_done (fs, 1);
-		break;
-	case GF_EVENT_CHILD_CONNECTING:
-		break;
-	default:
-		gf_log (this->name, GF_LOG_DEBUG,
-			"got notify event %d", event);
-		break;
-	}
-
-	return 0;
+    glusterfs_graph_t *graph = NULL;
+    struct glfs *fs = NULL;
+
+    graph = data;
+    fs = this->private;
+
+    switch (event) {
+        case GF_EVENT_GRAPH_NEW:
+            gf_smsg(this->name, GF_LOG_INFO, 0, API_MSG_NEW_GRAPH,
+                    "graph-uuid=%s",
+                    uuid_utoa((unsigned char *)graph->graph_uuid), "id=%d",
+                    graph->id, NULL);
+            break;
+        case GF_EVENT_CHILD_UP:
+            pthread_mutex_lock(&fs->mutex);
+            {
+                graph->used = 1;
+            }
+            pthread_mutex_unlock(&fs->mutex);
+            graph_setup(fs, graph);
+            glfs_init_done(fs, 0);
+            break;
+        case GF_EVENT_CHILD_DOWN:
+            pthread_mutex_lock(&fs->mutex);
+            {
+                graph->used = 0;
+                pthread_cond_broadcast(&fs->child_down_cond);
+            }
+            pthread_mutex_unlock(&fs->mutex);
+            glfs_init_done(fs, 1);
+            break;
+        case GF_EVENT_CHILD_CONNECTING:
+            break;
+        case GF_EVENT_UPCALL:
+            glfs_process_upcall_event(fs, data);
+            break;
+        default:
+            gf_msg_debug(this->name, 0, "got notify event %d", event);
+            break;
+    }
+
+    return 0;
 }
 
-
 int
-mem_acct_init (xlator_t *this)
+mem_acct_init(xlator_t *this)
 {
-	int ret = -1;
+    int ret = -1;
 
-	if (!this)
-		return ret;
+    if (!this)
+        return ret;
 
-	ret = xlator_mem_acct_init (this, glfs_mt_end + 1);
-	if (ret) {
-		gf_log (this->name, GF_LOG_ERROR, "Failed to initialise "
-                        "memory accounting");
-		return ret;
-	}
+    ret = xlator_mem_acct_init(this, glfs_mt_end + 1);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, ENOMEM, API_MSG_MEM_ACCT_INIT_FAILED,
+                NULL);
+        return ret;
+    }
 
-	return 0;
+    return 0;
 }
 
-
 int
-init (xlator_t *this)
+init(xlator_t *this)
 {
-	return 0;
+    return 0;
 }
 
-
 void
-fini (xlator_t *this)
+fini(xlator_t *this)
+{
+}
+
+/* place-holder fops */
+int
+glfs_forget(xlator_t *this, inode_t *inode)
 {
+    return 0;
+}
 
+int
+glfs_release(xlator_t *this, fd_t *fd)
+{
+    return 0;
 }
 
+int
+glfs_releasedir(xlator_t *this, fd_t *fd)
+{
+    return 0;
+}
 
 struct xlator_dumpops dumpops;
 
-
 struct xlator_fops fops;
 
-
-struct xlator_cbks cbks;
+struct xlator_cbks cbks = {
+    .forget = glfs_forget,
+    .release = glfs_release,
+    .releasedir = glfs_releasedir,
+};
+
+xlator_api_t xlator_api = {
+    .init = init,
+    .fini = fini,
+    .notify = notify,
+    .mem_acct_init = mem_acct_init,
+    .op_version = {1},
+    .dumpops = &dumpops,
+    .fops = &fops,
+    .cbks = &cbks,
+    .identifier = "glfs-api",
+    .category = GF_MAINTAINED,
+};
diff --git a/api/src/glfs-mem-types.h b/api/src/glfs-mem-types.h
index 76f4fc774cc..bfa325a3ad9 100644
--- a/api/src/glfs-mem-types.h
+++ b/api/src/glfs-mem-types.h
@@ -1,5 +1,5 @@
 /*
-  Copyright (c) 2012 Red Hat, Inc. <http://www.redhat.com>
+  Copyright (c) 2012-2017 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
   This file is licensed to you under your choice of the GNU Lesser
@@ -11,22 +11,25 @@
 #ifndef _GLFS_MEM_TYPES_H
 #define _GLFS_MEM_TYPES_H
 
-#include "mem-types.h"
+#include <glusterfs/mem-types.h>
 
 #define GF_MEM_TYPE_START (gf_common_mt_end + 1)
 
 enum glfs_mem_types_ {
-        glfs_mt_glfs_t = GF_MEM_TYPE_START,
-        glfs_mt_call_pool_t,
-        glfs_mt_xlator_t,
-	glfs_mt_glfs_fd_t,
-	glfs_mt_glfs_io_t,
-	glfs_mt_volfile_t,
-	glfs_mt_xlator_cmdline_option_t,
-        glfs_mt_server_cmdline_t,
-	glfs_mt_glfs_object_t,
-	glfs_mt_readdirbuf_t,
-	glfs_mt_end
-
+    glfs_mt_call_pool_t = GF_MEM_TYPE_START,
+    glfs_mt_xlator_t,
+    glfs_mt_glfs_fd_t,
+    glfs_mt_glfs_io_t,
+    glfs_mt_volfile_t,
+    glfs_mt_xlator_cmdline_option_t,
+    glfs_mt_server_cmdline_t,
+    glfs_mt_glfs_object_t,
+    glfs_mt_readdirbuf_t,
+    glfs_mt_upcall_entry_t,
+    glfs_mt_acl_t,
+    glfs_mt_upcall_inode_t,
+    glfs_mt_realpath_t,
+    glfs_mt_xreaddirp_stat_t,
+    glfs_mt_end
 };
 #endif
diff --git a/api/src/glfs-mgmt.c b/api/src/glfs-mgmt.c
index ea017f31af6..7c82b8cd162 100644
--- a/api/src/glfs-mgmt.c
+++ b/api/src/glfs-mgmt.c
@@ -1,5 +1,5 @@
 /*
-  Copyright (c) 2012 Red Hat, Inc. <http://www.redhat.com>
+  Copyright (c) 2012-2018 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
   This file is licensed to you under your choice of the GNU Lesser
@@ -8,7 +8,6 @@
   cases as published by the Free Software Foundation.
 */
 
-
 #include <stdio.h>
 #include <sys/types.h>
 #include <sys/wait.h>
@@ -16,839 +15,1035 @@
 #include <signal.h>
 #include <pthread.h>
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif /* _CONFIG_H */
-
-#include "glusterfs.h"
+#include <glusterfs/glusterfs.h>
 #include "glfs.h"
-#include "stack.h"
-#include "dict.h"
-#include "event.h"
-#include "defaults.h"
+#include <glusterfs/dict.h>
 
 #include "rpc-clnt.h"
 #include "protocol-common.h"
-#include "glusterfs3.h"
-#include "portmap-xdr.h"
-#include "xdr-common.h"
 #include "xdr-generic.h"
+#include "rpc-common-xdr.h"
 
-#include "syncop.h"
-#include "xlator.h"
+#include <glusterfs/syncop.h>
 
 #include "glfs-internal.h"
-#include "glfs-mem-types.h"
+#include "gfapi-messages.h"
+#include <glusterfs/syscall.h>
 
-int glfs_volfile_fetch (struct glfs *fs);
-int32_t glfs_get_volume_info_rpc (call_frame_t *frame, xlator_t *this,
-                                  struct glfs *fs);
+int
+glfs_volfile_fetch(struct glfs *fs);
+int32_t
+glfs_get_volume_info_rpc(call_frame_t *frame, xlator_t *this, struct glfs *fs);
 
 int
-glfs_process_volfp (struct glfs *fs, FILE *fp)
+glfs_process_volfp(struct glfs *fs, FILE *fp)
 {
-	glusterfs_graph_t  *graph = NULL;
-	int		    ret = -1;
-	xlator_t	   *trav = NULL;
-	glusterfs_ctx_t	   *ctx = NULL;
-
-	ctx = fs->ctx;
-	graph = glusterfs_graph_construct (fp);
-	if (!graph) {
-		gf_log ("glfs", GF_LOG_ERROR, "failed to construct the graph");
-		goto out;
-	}
-
-	for (trav = graph->first; trav; trav = trav->next) {
-		if (strcmp (trav->type, "mount/fuse") == 0) {
-			gf_log ("glfs", GF_LOG_ERROR,
-				"fuse xlator cannot be specified "
-				"in volume file");
-			goto out;
-		}
-	}
-
-	ret = glusterfs_graph_prepare (graph, ctx);
-	if (ret) {
-		glusterfs_graph_destroy (graph);
-		goto out;
-	}
-
-	ret = glusterfs_graph_activate (graph, ctx);
-
-	if (ret) {
-		glusterfs_graph_destroy (graph);
-		goto out;
-	}
-
-	ret = 0;
+    glusterfs_graph_t *graph = NULL;
+    int ret = -1;
+    xlator_t *trav = NULL;
+    glusterfs_ctx_t *ctx = NULL;
+
+    ctx = fs->ctx;
+    graph = glusterfs_graph_construct(fp);
+    if (!graph) {
+        gf_smsg("glfs", GF_LOG_ERROR, errno, API_MSG_GRAPH_CONSTRUCT_FAILED,
+                NULL);
+        goto out;
+    }
+
+    for (trav = graph->first; trav; trav = trav->next) {
+        if (strcmp(trav->type, "mount/api") == 0) {
+            gf_smsg("glfs", GF_LOG_ERROR, EINVAL, API_MSG_API_XLATOR_ERROR,
+                    NULL);
+            goto out;
+        }
+    }
+
+    ret = glusterfs_graph_prepare(graph, ctx, fs->volname);
+    if (ret) {
+        glusterfs_graph_destroy(graph);
+        goto out;
+    }
+
+    ret = glusterfs_graph_activate(graph, ctx);
+
+    if (ret) {
+        glusterfs_graph_destroy(graph);
+        goto out;
+    }
+
+    gf_log_dump_graph(fp, graph);
+
+    ret = 0;
 out:
-	if (fp)
-		fclose (fp);
+    if (fp)
+        fclose(fp);
 
-	if (!ctx->active) {
-		ret = -1;
-	}
+    if (!ctx->active) {
+        ret = -1;
+    }
 
-	return ret;
+    return ret;
 }
 
-
 int
-mgmt_cbk_spec (struct rpc_clnt *rpc, void *mydata, void *data)
+mgmt_cbk_spec(struct rpc_clnt *rpc, void *mydata, void *data)
 {
-	struct glfs *fs = NULL;
-	xlator_t    *this = NULL;
+    struct glfs *fs = NULL;
+    xlator_t *this = NULL;
 
-	this = mydata;
-	fs = this->private;
+    this = mydata;
+    fs = this->private;
 
-	glfs_volfile_fetch (fs);
+    glfs_volfile_fetch(fs);
 
-	return 0;
+    return 0;
 }
 
-
 int
-mgmt_cbk_event (struct rpc_clnt *rpc, void *mydata, void *data)
+mgmt_cbk_event(struct rpc_clnt *rpc, void *mydata, void *data)
 {
-	return 0;
+    return 0;
 }
 
+static int
+mgmt_cbk_statedump(struct rpc_clnt *rpc, void *mydata, void *data)
+{
+    struct glfs *fs = NULL;
+    xlator_t *this = NULL;
+    gf_statedump target_pid = {
+        0,
+    };
+    struct iovec *iov = NULL;
+    int ret = -1;
+
+    this = mydata;
+    if (!this) {
+        gf_smsg("glfs", GF_LOG_ERROR, EINVAL, API_MSG_NULL, "mydata", NULL);
+        errno = EINVAL;
+        goto out;
+    }
+
+    fs = this->private;
+    if (!fs) {
+        gf_smsg("glfs", GF_LOG_ERROR, EINVAL, API_MSG_NULL, "glfs", NULL);
+        errno = EINVAL;
+        goto out;
+    }
+
+    iov = (struct iovec *)data;
+    if (!iov) {
+        gf_smsg("glfs", GF_LOG_ERROR, EINVAL, API_MSG_NULL, "iovec data", NULL);
+        errno = EINVAL;
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &target_pid, (xdrproc_t)xdr_gf_statedump);
+    if (ret < 0) {
+        gf_smsg("glfs", GF_LOG_ERROR, EINVAL, API_MSG_DECODE_XDR_FAILED, NULL);
+        goto out;
+    }
+
+    gf_msg_trace("glfs", 0, "statedump requested for pid: %d", target_pid.pid);
+
+    if ((uint64_t)getpid() == target_pid.pid) {
+        gf_msg_debug("glfs", 0, "Taking statedump for pid: %d", target_pid.pid);
+
+        ret = glfs_sysrq(fs, GLFS_SYSRQ_STATEDUMP);
+        if (ret < 0) {
+            gf_smsg("glfs", GF_LOG_INFO, 0, API_MSG_STATEDUMP_FAILED, NULL);
+        }
+    }
+out:
+    return ret;
+}
 
-rpcclnt_cb_actor_t mgmt_cbk_actors[GF_CBK_MAXVALUE] = {
-	[GF_CBK_FETCHSPEC] = {"FETCHSPEC", GF_CBK_FETCHSPEC, mgmt_cbk_spec },
-	[GF_CBK_EVENT_NOTIFY] = {"EVENTNOTIFY", GF_CBK_EVENT_NOTIFY,
-				 mgmt_cbk_event},
+static rpcclnt_cb_actor_t mgmt_cbk_actors[GF_CBK_MAXVALUE] = {
+    [GF_CBK_FETCHSPEC] = {"FETCHSPEC", mgmt_cbk_spec, GF_CBK_FETCHSPEC},
+    [GF_CBK_EVENT_NOTIFY] = {"EVENTNOTIFY", mgmt_cbk_event,
+                             GF_CBK_EVENT_NOTIFY},
+    [GF_CBK_STATEDUMP] = {"STATEDUMP", mgmt_cbk_statedump, GF_CBK_STATEDUMP},
 };
 
-
-struct rpcclnt_cb_program mgmt_cbk_prog = {
-	.progname  = "GlusterFS Callback",
-	.prognum   = GLUSTER_CBK_PROGRAM,
-	.progver   = GLUSTER_CBK_VERSION,
-	.actors	   = mgmt_cbk_actors,
-	.numactors = GF_CBK_MAXVALUE,
+static struct rpcclnt_cb_program mgmt_cbk_prog = {
+    .progname = "GlusterFS Callback",
+    .prognum = GLUSTER_CBK_PROGRAM,
+    .progver = GLUSTER_CBK_VERSION,
+    .actors = mgmt_cbk_actors,
+    .numactors = GF_CBK_MAXVALUE,
 };
 
-char *clnt_handshake_procs[GF_HNDSK_MAXVALUE] = {
-	[GF_HNDSK_NULL]		= "NULL",
-	[GF_HNDSK_SETVOLUME]	= "SETVOLUME",
-	[GF_HNDSK_GETSPEC]	= "GETSPEC",
-	[GF_HNDSK_PING]		= "PING",
-	[GF_HNDSK_EVENT_NOTIFY] = "EVENTNOTIFY",
-        [GF_HNDSK_GET_VOLUME_INFO] = "GETVOLUMEINFO",
+static char *clnt_handshake_procs[GF_HNDSK_MAXVALUE] = {
+    [GF_HNDSK_NULL] = "NULL",
+    [GF_HNDSK_SETVOLUME] = "SETVOLUME",
+    [GF_HNDSK_GETSPEC] = "GETSPEC",
+    [GF_HNDSK_PING] = "PING",
+    [GF_HNDSK_EVENT_NOTIFY] = "EVENTNOTIFY",
+    [GF_HNDSK_GET_VOLUME_INFO] = "GETVOLUMEINFO",
 };
 
-rpc_clnt_prog_t clnt_handshake_prog = {
-	.progname  = "GlusterFS Handshake",
-	.prognum   = GLUSTER_HNDSK_PROGRAM,
-	.progver   = GLUSTER_HNDSK_VERSION,
-	.procnames = clnt_handshake_procs,
+static rpc_clnt_prog_t clnt_handshake_prog = {
+    .progname = "GlusterFS Handshake",
+    .prognum = GLUSTER_HNDSK_PROGRAM,
+    .progver = GLUSTER_HNDSK_VERSION,
+    .procnames = clnt_handshake_procs,
 };
 
-
 int
-mgmt_submit_request (void *req, call_frame_t *frame,
-		     glusterfs_ctx_t *ctx,
-		     rpc_clnt_prog_t *prog, int procnum,
-		     fop_cbk_fn_t cbkfn, xdrproc_t xdrproc)
+mgmt_submit_request(void *req, call_frame_t *frame, glusterfs_ctx_t *ctx,
+                    rpc_clnt_prog_t *prog, int procnum, fop_cbk_fn_t cbkfn,
+                    xdrproc_t xdrproc)
 {
-	int			ret	    = -1;
-	int			count	   = 0;
-	struct iovec		iov	    = {0, };
-	struct iobuf		*iobuf = NULL;
-	struct iobref		*iobref = NULL;
-	ssize_t			xdr_size = 0;
-
-	iobref = iobref_new ();
-	if (!iobref) {
-		goto out;
-	}
-
-	if (req) {
-		xdr_size = xdr_sizeof (xdrproc, req);
-
-		iobuf = iobuf_get2 (ctx->iobuf_pool, xdr_size);
-		if (!iobuf) {
-			goto out;
-		};
-
-		iobref_add (iobref, iobuf);
-
-		iov.iov_base = iobuf->ptr;
-		iov.iov_len  = iobuf_pagesize (iobuf);
-
-		/* Create the xdr payload */
-		ret = xdr_serialize_generic (iov, req, xdrproc);
-		if (ret == -1) {
-			gf_log (THIS->name, GF_LOG_WARNING,
-				"failed to create XDR payload");
-			goto out;
-		}
-		iov.iov_len = ret;
-		count = 1;
-	}
-
-	/* Send the msg */
-	ret = rpc_clnt_submit (ctx->mgmt, prog, procnum, cbkfn,
-			       &iov, count,
-			       NULL, 0, iobref, frame, NULL, 0, NULL, 0, NULL);
+    int ret = -1;
+    int count = 0;
+    struct iovec iov = {
+        0,
+    };
+    struct iobuf *iobuf = NULL;
+    struct iobref *iobref = NULL;
+    ssize_t xdr_size = 0;
+
+    iobref = iobref_new();
+    if (!iobref) {
+        goto out;
+    }
+
+    if (req) {
+        xdr_size = xdr_sizeof(xdrproc, req);
+
+        iobuf = iobuf_get2(ctx->iobuf_pool, xdr_size);
+        if (!iobuf) {
+            goto out;
+        };
+
+        iobref_add(iobref, iobuf);
+
+        iov.iov_base = iobuf->ptr;
+        iov.iov_len = iobuf_pagesize(iobuf);
+
+        /* Create the xdr payload */
+        ret = xdr_serialize_generic(iov, req, xdrproc);
+        if (ret == -1) {
+            gf_smsg(THIS->name, GF_LOG_WARNING, 0, API_MSG_XDR_PAYLOAD_FAILED,
+                    NULL);
+            goto out;
+        }
+        iov.iov_len = ret;
+        count = 1;
+    }
+
+    /* Send the msg */
+    ret = rpc_clnt_submit(ctx->mgmt, prog, procnum, cbkfn, &iov, count, NULL, 0,
+                          iobref, frame, NULL, 0, NULL, 0, NULL);
 
 out:
-	if (iobref)
-		iobref_unref (iobref);
+    if (iobref)
+        iobref_unref(iobref);
 
-	if (iobuf)
-		iobuf_unref (iobuf);
-	return ret;
+    if (iobuf)
+        iobuf_unref(iobuf);
+    return ret;
 }
 
 /*
  * Callback routine for 'GF_HNDSK_GET_VOLUME_INFO' rpc request
  */
 int
-mgmt_get_volinfo_cbk (struct rpc_req *req, struct iovec *iov,
-                      int count, void *myframe)
+mgmt_get_volinfo_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                     void *myframe)
 {
-        int                        ret                  = 0;
-        char                       *volume_id_str       = NULL;
-        dict_t                     *dict                = NULL;
-        char                       key[1024]            = {0};
-        gf_get_volume_info_rsp     rsp                  = {0,};
-        call_frame_t               *frame               = NULL;
-        glusterfs_ctx_t            *ctx                 = NULL;
-        struct glfs                *fs                  = NULL;
-        struct syncargs            *args;
-
-        frame = myframe;
-        ctx = frame->this->ctx;
-        args = frame->local;
-
-        if (!ctx) {
-                gf_log (frame->this->name, GF_LOG_ERROR, "NULL context");
-                errno = EINVAL;
-                ret = -1;
-                goto out;
-        }
-
-        fs = ((xlator_t *)ctx->master)->private;
-
-        if (-1 == req->rpc_status) {
-                gf_log (frame->this->name, GF_LOG_ERROR,
-                        "GET_VOLUME_INFO RPC call is not successfull");
-                errno = EINVAL;
-                ret = -1;
-                goto out;
-        }
-
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_get_volume_info_rsp);
-
-        if (ret < 0) {
-                gf_log (frame->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response for GET_VOLUME_INFO");
-                goto out;
-        }
-
-        gf_log (frame->this->name, GF_LOG_DEBUG,
-                "Received resp to GET_VOLUME_INFO RPC: %d", rsp.op_ret);
-
-        if (rsp.op_ret == -1) {
-                errno = rsp.op_errno;
-                ret = -1;
-                goto out;
-        }
-
-        if (!rsp.dict.dict_len) {
-                gf_log (frame->this->name, GF_LOG_ERROR,
-                        "Response received for GET_VOLUME_INFO RPC call is not valid");
-                ret = -1;
-                errno = EINVAL;
-                goto out;
-        }
-
-        dict = dict_new ();
-
-        if (!dict) {
-                ret = -1;
-                errno = ENOMEM;
-                goto out;
-        }
-
-        ret = dict_unserialize (rsp.dict.dict_val,
-                                rsp.dict.dict_len,
-                                &dict);
-
-        if (ret) {
-                errno = ENOMEM;
-                goto out;
-        }
-
-        snprintf (key, sizeof (key), "volume_id");
-        ret = dict_get_str (dict, key, &volume_id_str);
-        if (ret) {
-                errno = EINVAL;
-                goto out;
-        }
-
-        ret = 0;
+    int ret = 0;
+    char *volume_id_str = NULL;
+    dict_t *dict = NULL;
+    gf_get_volume_info_rsp rsp = {
+        0,
+    };
+    call_frame_t *frame = NULL;
+    glusterfs_ctx_t *ctx = NULL;
+    struct glfs *fs = NULL;
+    struct syncargs *args;
+
+    frame = myframe;
+    ctx = frame->this->ctx;
+    args = frame->local;
+
+    if (!ctx) {
+        gf_smsg(frame->this->name, GF_LOG_ERROR, EINVAL, API_MSG_NULL,
+                "context", NULL);
+        errno = EINVAL;
+        ret = -1;
+        goto out;
+    }
+
+    fs = ((xlator_t *)ctx->master)->private;
+
+    if (-1 == req->rpc_status) {
+        gf_smsg(frame->this->name, GF_LOG_ERROR, EINVAL,
+                API_MSG_CALL_NOT_SUCCESSFUL, NULL);
+        errno = EINVAL;
+        ret = -1;
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_get_volume_info_rsp);
+
+    if (ret < 0) {
+        gf_smsg(frame->this->name, GF_LOG_ERROR, 0,
+                API_MSG_XDR_RESPONSE_DECODE_FAILED, NULL);
+        goto out;
+    }
+
+    gf_msg_debug(frame->this->name, 0,
+                 "Received resp to GET_VOLUME_INFO "
+                 "RPC: %d",
+                 rsp.op_ret);
+
+    if (rsp.op_ret == -1) {
+        errno = rsp.op_errno;
+        ret = -1;
+        goto out;
+    }
+
+    if (!rsp.dict.dict_len) {
+        gf_smsg(frame->this->name, GF_LOG_ERROR, EINVAL, API_MSG_CALL_NOT_VALID,
+                NULL);
+        ret = -1;
+        errno = EINVAL;
+        goto out;
+    }
+
+    dict = dict_new();
+
+    if (!dict) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len, &dict);
+
+    if (ret) {
+        errno = ENOMEM;
+        goto out;
+    }
+
+    ret = dict_get_str_sizen(dict, "volume_id", &volume_id_str);
+    if (ret) {
+        errno = EINVAL;
+        goto out;
+    }
+
+    ret = 0;
 out:
-        if (volume_id_str) {
-                gf_log (frame->this->name, GF_LOG_DEBUG,
-                        "Volume Id: %s", volume_id_str);
-                pthread_mutex_lock (&fs->mutex);
-                uuid_parse (volume_id_str, fs->vol_uuid);
-                pthread_mutex_unlock (&fs->mutex);
-        }
+    if (volume_id_str) {
+        gf_msg_debug(frame->this->name, 0, "Volume Id: %s", volume_id_str);
+        pthread_mutex_lock(&fs->mutex);
+        gf_uuid_parse(volume_id_str, fs->vol_uuid);
+        pthread_mutex_unlock(&fs->mutex);
+    }
 
-        if (ret) {
-               gf_log (frame->this->name, GF_LOG_ERROR,
-                       "In GET_VOLUME_INFO cbk, received error: %s",
-                       strerror(errno));
-        }
+    if (ret) {
+        gf_smsg(frame->this->name, GF_LOG_ERROR, errno,
+                API_MSG_GET_VOLINFO_CBK_FAILED, "error=%s", strerror(errno),
+                NULL);
+    }
 
-        if (dict)
-                dict_destroy (dict);
+    if (dict)
+        dict_unref(dict);
 
-        if (rsp.dict.dict_val)
-                free (rsp.dict.dict_val);
+    if (rsp.dict.dict_val)
+        free(rsp.dict.dict_val);
 
-        if (rsp.op_errstr && *rsp.op_errstr)
-                free (rsp.op_errstr);
+    if (rsp.op_errstr)
+        free(rsp.op_errstr);
 
-        gf_log (frame->this->name, GF_LOG_DEBUG, "Returning: %d", ret);
+    gf_msg_debug(frame->this->name, 0, "Returning: %d", ret);
 
-        __wake (args);
+    __wake(args);
 
-        return ret;
+    return ret;
 }
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_get_volumeid, 3.5.0)
 int
-pub_glfs_get_volumeid (struct glfs *fs, char *volid, size_t size)
+pub_glfs_get_volumeid(struct glfs *fs, char *volid, size_t size)
 {
-        /* TODO: Define a global macro to store UUID size */
-        size_t uuid_size = 16;
-
-        pthread_mutex_lock (&fs->mutex);
-        {
-                /* check if the volume uuid is initialized */
-                if (!uuid_is_null (fs->vol_uuid)) {
-                        pthread_mutex_unlock (&fs->mutex);
-                        goto done;
-                }
+    /* TODO: Define a global macro to store UUID size */
+    size_t uuid_size = 16;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    pthread_mutex_lock(&fs->mutex);
+    {
+        /* check if the volume uuid is initialized */
+        if (!gf_uuid_is_null(fs->vol_uuid)) {
+            pthread_mutex_unlock(&fs->mutex);
+            goto done;
         }
-        pthread_mutex_unlock (&fs->mutex);
+    }
+    pthread_mutex_unlock(&fs->mutex);
 
-        /* Need to fetch volume_uuid */
-        glfs_get_volume_info (fs);
+    /* Need to fetch volume_uuid */
+    glfs_get_volume_info(fs);
 
-        if (uuid_is_null (fs->vol_uuid)) {
-                gf_log (THIS->name, GF_LOG_ERROR, "Unable to fetch volume UUID");
-                return -1;
-        }
+    if (gf_uuid_is_null(fs->vol_uuid)) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, EINVAL, API_MSG_FETCH_VOLUUID_FAILED,
+                NULL);
+        goto out;
+    }
 
 done:
-        if (!volid || !size) {
-                gf_log (THIS->name, GF_LOG_DEBUG, "volumeid/size is null");
-                return uuid_size;
-        }
+    if (!volid || !size) {
+        gf_msg_debug(THIS->name, 0, "volumeid/size is null");
+        __GLFS_EXIT_FS;
+        return uuid_size;
+    }
 
-        if (size < uuid_size) {
-                gf_log (THIS->name, GF_LOG_ERROR, "Insufficient size passed");
-                errno = ERANGE;
-                return -1;
-        }
+    if (size < uuid_size) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, ERANGE, API_MSG_INSUFF_SIZE, NULL);
+        errno = ERANGE;
+        goto out;
+    }
 
-        memcpy (volid, fs->vol_uuid, uuid_size);
+    memcpy(volid, fs->vol_uuid, uuid_size);
 
-        return uuid_size;
-}
+    __GLFS_EXIT_FS;
+
+    return uuid_size;
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_get_volumeid, 3.5.0);
+out:
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return -1;
+}
 
 int
-glfs_get_volume_info (struct glfs *fs)
+glfs_get_volume_info(struct glfs *fs)
 {
-        call_frame_t     *frame = NULL;
-        glusterfs_ctx_t  *ctx   = NULL;
-        struct syncargs  args   = {0, };
-        int              ret    = 0;
+    call_frame_t *frame = NULL;
+    glusterfs_ctx_t *ctx = NULL;
+    struct syncargs args = {
+        0,
+    };
+    int ret = 0;
 
-        ctx = fs->ctx;
-        frame = create_frame (THIS, ctx->pool);
-        frame->local = &args;
+    ctx = fs->ctx;
+    frame = create_frame(THIS, ctx->pool);
+    if (!frame) {
+        gf_smsg("glfs", GF_LOG_ERROR, ENOMEM, API_MSG_FRAME_CREAT_FAILED, NULL);
+        ret = -1;
+        goto out;
+    }
 
-        __yawn ((&args));
+    frame->local = &args;
 
-        ret = glfs_get_volume_info_rpc (frame, THIS, fs);
-        if (ret)
-                goto out;
+    __yawn((&args));
+
+    ret = glfs_get_volume_info_rpc(frame, THIS, fs);
+    if (ret)
+        goto out;
 
-        __yield ((&args));
+    __yield((&args));
 
-        frame->local = NULL;
-        STACK_DESTROY (frame->root);
+    frame->local = NULL;
+    STACK_DESTROY(frame->root);
 
 out:
-        return ret;
+    return ret;
 }
 
 int32_t
-glfs_get_volume_info_rpc (call_frame_t *frame, xlator_t *this,
-                          struct glfs *fs)
+glfs_get_volume_info_rpc(call_frame_t *frame, xlator_t *this, struct glfs *fs)
 {
-        gf_get_volume_info_req  req       = {{0,}};
-        int                     ret       = 0;
-        glusterfs_ctx_t         *ctx      = NULL;
-        dict_t                  *dict     = NULL;
-        int32_t                 flags     = 0;
-
-        if (!frame || !this || !fs) {
-                ret = -1;
-                goto out;
-        }
-
-        ctx = fs->ctx;
-
-        dict = dict_new ();
-        if (!dict) {
-                ret = -1;
-                goto out;
-        }
-
-        if (fs->volname) {
-                ret = dict_set_str (dict, "volname", fs->volname);
-                if (ret)
-                        goto out;
-        }
-
-        // Set the flags for the fields which we are interested in
-        flags = (int32_t)GF_GET_VOLUME_UUID; //ctx->flags;
-        ret = dict_set_int32 (dict, "flags", flags);
-        if (ret) {
-                gf_log (frame->this->name, GF_LOG_ERROR, "failed to set flags");
-                goto out;
-        }
-
-        ret = dict_allocate_and_serialize (dict, &req.dict.dict_val,
-                                           &req.dict.dict_len);
-
-
-        ret = mgmt_submit_request (&req, frame, ctx, &clnt_handshake_prog,
-                                   GF_HNDSK_GET_VOLUME_INFO,
-                                   mgmt_get_volinfo_cbk,
-                                   (xdrproc_t)xdr_gf_get_volume_info_req);
+    gf_get_volume_info_req req = {{
+        0,
+    }};
+    int ret = 0;
+    glusterfs_ctx_t *ctx = NULL;
+    dict_t *dict = NULL;
+    int32_t flags = 0;
+
+    if (!frame || !this || !fs) {
+        ret = -1;
+        goto out;
+    }
+
+    ctx = fs->ctx;
+
+    dict = dict_new();
+    if (!dict) {
+        ret = -1;
+        goto out;
+    }
+
+    if (fs->volname) {
+        ret = dict_set_str(dict, "volname", fs->volname);
+        if (ret)
+            goto out;
+    }
+
+    // Set the flags for the fields which we are interested in
+    flags = (int32_t)GF_GET_VOLUME_UUID;  // ctx->flags;
+    ret = dict_set_int32(dict, "flags", flags);
+    if (ret) {
+        gf_smsg(frame->this->name, GF_LOG_ERROR, EINVAL,
+                API_MSG_DICT_SET_FAILED, "flags", NULL);
+        goto out;
+    }
+
+    ret = dict_allocate_and_serialize(dict, &req.dict.dict_val,
+                                      &req.dict.dict_len);
+
+    ret = mgmt_submit_request(&req, frame, ctx, &clnt_handshake_prog,
+                              GF_HNDSK_GET_VOLUME_INFO, mgmt_get_volinfo_cbk,
+                              (xdrproc_t)xdr_gf_get_volume_info_req);
 out:
-        if (dict) {
-                dict_unref (dict);
-        }
+    if (dict) {
+        dict_unref(dict);
+    }
 
-        GF_FREE (req.dict.dict_val);
+    GF_FREE(req.dict.dict_val);
 
-        return ret;
+    return ret;
 }
 
 static int
-glusterfs_oldvolfile_update (struct glfs *fs, char *volfile, ssize_t size)
+glusterfs_oldvolfile_update(struct glfs *fs, char *volfile, ssize_t size)
 {
-	int ret = -1;
-
-        pthread_mutex_lock (&fs->mutex);
-
-	fs->oldvollen = size;
-	if (!fs->oldvolfile) {
-		fs->oldvolfile = GF_CALLOC (1, size+1, glfs_mt_volfile_t);
-	} else {
-		fs->oldvolfile = GF_REALLOC (fs->oldvolfile, size+1);
-	}
-
-	if (!fs->oldvolfile) {
-		fs->oldvollen = 0;
-	} else {
-		memcpy (fs->oldvolfile, volfile, size);
-		fs->oldvollen = size;
-		ret = 0;
-	}
+    int ret = -1;
+
+    pthread_mutex_lock(&fs->mutex);
+
+    fs->oldvollen = size;
+    if (!fs->oldvolfile) {
+        fs->oldvolfile = CALLOC(1, size + 1);
+    } else {
+        fs->oldvolfile = REALLOC(fs->oldvolfile, size + 1);
+    }
+
+    if (!fs->oldvolfile) {
+        fs->oldvollen = 0;
+    } else {
+        memcpy(fs->oldvolfile, volfile, size);
+        fs->oldvollen = size;
+        ret = 0;
+    }
 
-        pthread_mutex_unlock (&fs->mutex);
+    pthread_mutex_unlock(&fs->mutex);
 
-	return ret;
+    return ret;
 }
 
-
 int
-mgmt_getspec_cbk (struct rpc_req *req, struct iovec *iov, int count,
-		  void *myframe)
+glfs_mgmt_getspec_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                      void *myframe)
 {
-	gf_getspec_rsp		 rsp   = {0,};
-	call_frame_t		*frame = NULL;
-	glusterfs_ctx_t		*ctx = NULL;
-	int			 ret   = 0;
-	ssize_t			 size = 0;
-	FILE			*tmpfp = NULL;
-	int			 need_retry = 0;
-	struct glfs		*fs = NULL;
-
-	frame = myframe;
-	ctx = frame->this->ctx;
-
-	if (!ctx) {
-		gf_log (frame->this->name, GF_LOG_ERROR, "NULL context");
-		errno = EINVAL;
-		ret = -1;
-		goto out;
-	}
-
-	fs = ((xlator_t *)ctx->master)->private;
-
-	if (-1 == req->rpc_status) {
-		ret = -1;
-		need_retry = 1;
-		goto out;
-	}
-
-	ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_getspec_rsp);
-	if (ret < 0) {
-		gf_log (frame->this->name, GF_LOG_ERROR, "XDR decoding error");
-		ret   = -1;
-		goto out;
-	}
-
-	if (-1 == rsp.op_ret) {
-		gf_log (frame->this->name, GF_LOG_ERROR,
-			"failed to get the 'volume file' from server");
-		ret = -1;
-		errno = rsp.op_errno;
-		goto out;
-	}
-
-	ret = 0;
-	size = rsp.op_ret;
-
-	if ((size == fs->oldvollen) &&
-	    (memcmp (fs->oldvolfile, rsp.spec, size) == 0)) {
-		gf_log (frame->this->name, GF_LOG_INFO,
-			"No change in volfile, continuing");
-		goto out;
-	}
-
-	tmpfp = tmpfile ();
-	if (!tmpfp) {
-		ret = -1;
-		goto out;
-	}
-
-	fwrite (rsp.spec, size, 1, tmpfp);
-	fflush (tmpfp);
-	if (ferror (tmpfp)) {
-		ret = -1;
-		goto out;
-	}
-
-	/*  Check if only options have changed. No need to reload the
-	*  volfile if topology hasn't changed.
-	*  glusterfs_volfile_reconfigure returns 3 possible return states
-	*  return 0	     =======> reconfiguration of options has succeeded
-	*  return 1	     =======> the graph has to be reconstructed and all the xlators should be inited
-	*  return -1(or -ve) =======> Some Internal Error occurred during the operation
-	*/
-
-	ret = glusterfs_volfile_reconfigure (fs->oldvollen, tmpfp, fs->ctx,
-					     fs->oldvolfile);
-	if (ret == 0) {
-		gf_log ("glusterfsd-mgmt", GF_LOG_DEBUG,
-			"No need to re-load volfile, reconfigure done");
-		ret = glusterfs_oldvolfile_update (fs, rsp.spec, size);
-		goto out;
-	}
-
-	if (ret < 0) {
-		gf_log ("glusterfsd-mgmt", GF_LOG_DEBUG,
-			"Reconfigure failed !!");
-		goto out;
-	}
-
-	ret = glfs_process_volfp (fs, tmpfp);
-	/* tmpfp closed */
-	tmpfp = NULL;
-	if (ret)
-		goto out;
-
-	ret = glusterfs_oldvolfile_update (fs, rsp.spec, size);
+    gf_getspec_rsp rsp = {
+        0,
+    };
+    call_frame_t *frame = NULL;
+    glusterfs_ctx_t *ctx = NULL;
+    int ret = 0;
+    ssize_t size = 0;
+    FILE *tmpfp = NULL;
+    int need_retry = 0;
+    struct glfs *fs = NULL;
+    dict_t *dict = NULL;
+    char *servers_list = NULL;
+    int tmp_fd = -1;
+    char template[] = "/tmp/gfapi.volfile.XXXXXX";
+
+    frame = myframe;
+    ctx = frame->this->ctx;
+
+    if (!ctx) {
+        gf_smsg(frame->this->name, GF_LOG_ERROR, EINVAL, API_MSG_NULL,
+                "context", NULL);
+        errno = EINVAL;
+        ret = -1;
+        goto out;
+    }
+
+    fs = ((xlator_t *)ctx->master)->private;
+
+    if (-1 == req->rpc_status) {
+        ret = -1;
+        need_retry = 1;
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_getspec_rsp);
+    if (ret < 0) {
+        gf_smsg(frame->this->name, GF_LOG_ERROR, 0, API_MSG_XDR_DECODE_FAILED,
+                NULL);
+        ret = -1;
+        goto out;
+    }
+
+    if (-1 == rsp.op_ret) {
+        gf_smsg(frame->this->name, GF_LOG_ERROR, rsp.op_errno,
+                API_MSG_GET_VOLFILE_FAILED, "from server", NULL);
+        ret = -1;
+        errno = rsp.op_errno;
+        goto out;
+    }
+
+    if (!rsp.xdata.xdata_len) {
+        goto volfile;
+    }
+
+    dict = dict_new();
+    if (!dict) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    ret = dict_unserialize(rsp.xdata.xdata_val, rsp.xdata.xdata_len, &dict);
+    if (ret) {
+        gf_log(frame->this->name, GF_LOG_ERROR,
+               "failed to unserialize xdata to dictionary");
+        goto out;
+    }
+    dict->extra_stdfree = rsp.xdata.xdata_val;
+
+    /* glusterd2 only */
+    ret = dict_get_str(dict, "servers-list", &servers_list);
+    if (ret) {
+        goto volfile;
+    }
+
+    gf_log(frame->this->name, GF_LOG_INFO,
+           "Received list of available volfile servers: %s", servers_list);
+
+    ret = gf_process_getspec_servers_list(&ctx->cmd_args, servers_list);
+    if (ret) {
+        gf_log(frame->this->name, GF_LOG_ERROR,
+               "Failed (%s) to process servers list: %s", strerror(errno),
+               servers_list);
+    }
+
+volfile:
+    ret = 0;
+    size = rsp.op_ret;
+
+    pthread_mutex_lock(&fs->mutex);
+    if ((size == fs->oldvollen) &&
+        (memcmp(fs->oldvolfile, rsp.spec, size) == 0)) {
+        pthread_mutex_unlock(&fs->mutex);
+        gf_smsg(frame->this->name, GF_LOG_INFO, 0, API_MSG_VOLFILE_INFO, NULL);
+        goto out;
+    }
+    pthread_mutex_unlock(&fs->mutex);
+
+    /* coverity[secure_temp] mkstemp uses 0600 as the mode and is safe */
+    tmp_fd = mkstemp(template);
+    if (-1 == tmp_fd) {
+        ret = -1;
+        goto out;
+    }
+
+    /* Calling unlink so that when the file is closed or program
+     * terminates the temporary file is deleted.
+     */
+    ret = sys_unlink(template);
+    if (ret < 0) {
+        gf_smsg(frame->this->name, GF_LOG_INFO, 0, API_MSG_UNABLE_TO_DEL,
+                "template=%s", template, NULL);
+        ret = 0;
+    }
+
+    tmpfp = fdopen(tmp_fd, "w+b");
+    if (!tmpfp) {
+        ret = -1;
+        goto out;
+    }
+
+    fwrite(rsp.spec, size, 1, tmpfp);
+    fflush(tmpfp);
+    if (ferror(tmpfp)) {
+        ret = -1;
+        goto out;
+    }
+
+    /*  Check if only options have changed. No need to reload the
+     *  volfile if topology hasn't changed.
+     *  glusterfs_volfile_reconfigure returns 3 possible return states
+     *  return 0	     =======> reconfiguration of options has succeeded
+     *  return 1	     =======> the graph has to be reconstructed and all
+     * the xlators should be inited return -1(or -ve) =======> Some Internal
+     * Error occurred during the operation
+     */
+
+    pthread_mutex_lock(&fs->mutex);
+    ret = gf_volfile_reconfigure(fs->oldvollen, tmpfp, fs->ctx, fs->oldvolfile);
+    pthread_mutex_unlock(&fs->mutex);
+
+    if (ret == 0) {
+        gf_msg_debug("glusterfsd-mgmt", 0,
+                     "No need to re-load "
+                     "volfile, reconfigure done");
+        ret = glusterfs_oldvolfile_update(fs, rsp.spec, size);
+        goto out;
+    }
+
+    if (ret < 0) {
+        gf_msg_debug("glusterfsd-mgmt", 0, "Reconfigure failed !!");
+        goto out;
+    }
+
+    ret = glfs_process_volfp(fs, tmpfp);
+    /* tmpfp closed */
+    tmpfp = NULL;
+    tmp_fd = -1;
+    if (ret)
+        goto out;
+
+    ret = glusterfs_oldvolfile_update(fs, rsp.spec, size);
 out:
-	STACK_DESTROY (frame->root);
-
-	if (rsp.spec)
-		free (rsp.spec);
-
-	// Stop if server is running at an unsupported op-version
-	if (ENOTSUP == ret) {
-		gf_log ("mgmt", GF_LOG_ERROR, "Server is operating at an "
-			"op-version which is not supported");
-		errno = ENOTSUP;
-		priv_glfs_init_done (fs, -1);
-	}
-
-	if (ret && ctx && !ctx->active) {
-		/* Do it only for the first time */
-		/* Failed to get the volume file, something wrong,
-		   restart the process */
-		gf_log ("glfs-mgmt", GF_LOG_ERROR,
-			"failed to fetch volume file (key:%s)",
-			ctx->cmd_args.volfile_id);
-		if (!need_retry) {
-			if (!errno)
-				errno = EINVAL;
-			priv_glfs_init_done (fs, -1);
-		}
-	}
-
-	if (tmpfp)
-		fclose (tmpfp);
-
-	return 0;
-}
-
-
-int
-glfs_volfile_fetch (struct glfs *fs)
-{
-	cmd_args_t	 *cmd_args = NULL;
-	gf_getspec_req	  req = {0, };
-	int		  ret = 0;
-	call_frame_t	 *frame = NULL;
-	glusterfs_ctx_t	 *ctx = NULL;
-        dict_t           *dict = NULL;
-
-	ctx = fs->ctx;
-	cmd_args = &ctx->cmd_args;
-
-	frame = create_frame (THIS, ctx->pool);
-
-	req.key = cmd_args->volfile_id;
-	req.flags = 0;
-
-        dict = dict_new ();
-        if (!dict) {
-                ret = -1;
-                goto out;
-        }
-
-        // Set the supported min and max op-versions, so glusterd can make a
-        // decision
-        ret = dict_set_int32 (dict, "min-op-version", GD_OP_VERSION_MIN);
-        if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "Failed to set min-op-version"
-                        " in request dict");
-                goto out;
+    STACK_DESTROY(frame->root);
+
+    if (rsp.spec)
+        free(rsp.spec);
+
+    if (dict)
+        dict_unref(dict);
+
+    // Stop if server is running at an unsupported op-version
+    if (ENOTSUP == ret) {
+        gf_smsg("mgmt", GF_LOG_ERROR, ENOTSUP, API_MSG_WRONG_OPVERSION, NULL);
+        errno = ENOTSUP;
+        glfs_init_done(fs, -1);
+    }
+
+    if (ret && ctx && !ctx->active) {
+        /* Do it only for the first time */
+        /* Failed to get the volume file, something wrong,
+           restart the process */
+        gf_smsg("glfs-mgmt", GF_LOG_ERROR, EINVAL, API_MSG_GET_VOLFILE_FAILED,
+                "key=%s", ctx->cmd_args.volfile_id, NULL);
+        if (!need_retry) {
+            if (!errno)
+                errno = EINVAL;
+            glfs_init_done(fs, -1);
         }
+    }
 
-        ret = dict_set_int32 (dict, "max-op-version", GD_OP_VERSION_MAX);
-        if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "Failed to set max-op-version"
-                        " in request dict");
-                goto out;
-        }
+    if (tmpfp)
+        fclose(tmpfp);
+    else if (tmp_fd != -1)
+        sys_close(tmp_fd);
 
-        ret = dict_allocate_and_serialize (dict, &req.xdata.xdata_val,
-                                           &req.xdata.xdata_len);
-        if (ret < 0) {
-                gf_log (THIS->name, GF_LOG_ERROR,
-                        "Failed to serialize dictionary");
-                goto out;
-        }
+    return 0;
+}
 
-	ret = mgmt_submit_request (&req, frame, ctx, &clnt_handshake_prog,
-				   GF_HNDSK_GETSPEC, mgmt_getspec_cbk,
-				   (xdrproc_t)xdr_gf_getspec_req);
+int
+glfs_volfile_fetch(struct glfs *fs)
+{
+    cmd_args_t *cmd_args = NULL;
+    gf_getspec_req req = {
+        0,
+    };
+    int ret = -1;
+    call_frame_t *frame = NULL;
+    glusterfs_ctx_t *ctx = NULL;
+    dict_t *dict = NULL;
+
+    ctx = fs->ctx;
+    cmd_args = &ctx->cmd_args;
+
+    req.key = cmd_args->volfile_id;
+    req.flags = 0;
+
+    dict = dict_new();
+    if (!dict) {
+        goto out;
+    }
+
+    // Set the supported min and max op-versions, so glusterd can make a
+    // decision
+    ret = dict_set_int32(dict, "min-op-version", GD_OP_VERSION_MIN);
+    if (ret) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, EINVAL, API_MSG_DICT_SET_FAILED,
+                "min-op-version", NULL);
+        goto out;
+    }
+
+    ret = dict_set_int32(dict, "max-op-version", GD_OP_VERSION_MAX);
+    if (ret) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, EINVAL, API_MSG_DICT_SET_FAILED,
+                "max-op-version", NULL);
+        goto out;
+    }
+
+    /* Ask for a list of volfile (glusterd2 only) servers */
+    if (GF_CLIENT_PROCESS == ctx->process_mode) {
+        req.flags = req.flags | GF_GETSPEC_FLAG_SERVERS_LIST;
+    }
+
+    ret = dict_allocate_and_serialize(dict, &req.xdata.xdata_val,
+                                      &req.xdata.xdata_len);
+    if (ret < 0) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, 0, API_MSG_DICT_SERIALIZE_FAILED,
+                NULL);
+        goto out;
+    }
+
+    frame = create_frame(THIS, ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = mgmt_submit_request(&req, frame, ctx, &clnt_handshake_prog,
+                              GF_HNDSK_GETSPEC, glfs_mgmt_getspec_cbk,
+                              (xdrproc_t)xdr_gf_getspec_req);
 out:
-        return ret;
-}
+    if (req.xdata.xdata_val)
+        GF_FREE(req.xdata.xdata_val);
+    if (dict)
+        dict_unref(dict);
 
+    return ret;
+}
 
 static int
-mgmt_rpc_notify (struct rpc_clnt *rpc, void *mydata, rpc_clnt_event_t event,
-		 void *data)
+mgmt_rpc_notify(struct rpc_clnt *rpc, void *mydata, rpc_clnt_event_t event,
+                void *data)
 {
-	xlator_t	*this = NULL;
-	glusterfs_ctx_t *ctx = NULL;
-        server_cmdline_t *server = NULL;
-        rpc_transport_t  *rpc_trans = NULL;
-	struct glfs	 *fs = NULL;
-	int		 ret = 0;
-
-	this = mydata;
-        rpc_trans = rpc->conn.trans;
-
-	ctx = this->ctx;
-	if (!ctx)
-		goto out;
-
-	fs = ((xlator_t *)ctx->master)->private;
-
-	switch (event) {
-	case RPC_CLNT_DISCONNECT:
-		if (!ctx->active) {
-                        gf_log ("glfs-mgmt", GF_LOG_ERROR,
-                                "failed to connect with remote-host: %s (%s)",
-                                ctx->cmd_args.volfile_server,
-                                strerror (errno));
-                        server = ctx->cmd_args.curr_server;
-                        if (server->list.next == &ctx->cmd_args.volfile_servers) {
-                                errno = ENOTCONN;
-                                gf_log("glfs-mgmt", GF_LOG_INFO,
-                                       "Exhausted all volfile servers");
-                                priv_glfs_init_done (fs, -1);
-                                break;
-                        }
-                        server = list_entry (server->list.next, typeof(*server),
-                                             list);
-                        ctx->cmd_args.curr_server = server;
-                        ctx->cmd_args.volfile_server_port = server->port;
-                        ctx->cmd_args.volfile_server = server->volfile_server;
-                        ctx->cmd_args.volfile_server_transport = server->transport;
-
-                        ret = dict_set_int32 (rpc_trans->options,
-                                              "remote-port",
-                                              server->port);
-                        if (ret != 0) {
-                                gf_log ("glfs-mgmt", GF_LOG_ERROR,
-                                        "failed to set remote-port: %d",
-                                        server->port);
-                                errno = ENOTCONN;
-                                priv_glfs_init_done (fs, -1);
-                                break;
-                        }
-
-                        ret = dict_set_str (rpc_trans->options,
-                                            "remote-host",
-                                            server->volfile_server);
-                        if (ret != 0) {
-                                gf_log ("glfs-mgmt", GF_LOG_ERROR,
-                                        "failed to set remote-host: %s",
-                                        server->volfile_server);
-                                errno = ENOTCONN;
-                                priv_glfs_init_done (fs, -1);
-                                break;
-                        }
-
-                        ret = dict_set_str (rpc_trans->options,
-                                            "transport-type",
-                                            server->transport);
-                        if (ret != 0) {
-                                gf_log ("glfs-mgmt", GF_LOG_ERROR,
-                                        "failed to set transport-type: %s",
-                                        server->transport);
-                                errno = ENOTCONN;
-                                priv_glfs_init_done (fs, -1);
-                                break;
-                        }
-                        gf_log ("glfs-mgmt", GF_LOG_INFO,
-                                "connecting to next volfile server %s"
-                                " at port %d with transport: %s",
-                                server->volfile_server, server->port,
-                                server->transport);
+    xlator_t *this = NULL;
+    glusterfs_ctx_t *ctx = NULL;
+    server_cmdline_t *server = NULL;
+    rpc_transport_t *rpc_trans = NULL;
+    struct glfs *fs = NULL;
+    int ret = 0;
+    struct dnscache6 *dnscache = NULL;
+
+    this = mydata;
+    rpc_trans = rpc->conn.trans;
+
+    ctx = this->ctx;
+    if (!ctx)
+        goto out;
+
+    fs = ((xlator_t *)ctx->master)->private;
+
+    switch (event) {
+        case RPC_CLNT_DISCONNECT:
+            if (!ctx->active) {
+                if (rpc_trans->connect_failed)
+                    gf_smsg("glfs-mgmt", GF_LOG_ERROR, 0,
+                            API_MSG_REMOTE_HOST_CONN_FAILED, "server=%s",
+                            ctx->cmd_args.volfile_server, NULL);
+                else
+                    gf_smsg("glfs-mgmt", GF_LOG_INFO, 0,
+                            API_MSG_REMOTE_HOST_CONN_FAILED, "server=%s",
+                            ctx->cmd_args.volfile_server, NULL);
+
+                if (!rpc->disabled) {
+                    /*
+                     * Check if dnscache is exhausted for current server
+                     * and continue until cache is exhausted
+                     */
+                    dnscache = rpc_trans->dnscache;
+                    if (dnscache && dnscache->next) {
+                        break;
+                    }
+                }
+                server = ctx->cmd_args.curr_server;
+                if (server->list.next == &ctx->cmd_args.volfile_servers) {
+                    errno = ENOTCONN;
+                    gf_smsg("glfs-mgmt", GF_LOG_INFO, ENOTCONN,
+                            API_MSG_VOLFILE_SERVER_EXHAUST, NULL);
+                    glfs_init_done(fs, -1);
+                    break;
                 }
-                break;
-	case RPC_CLNT_CONNECT:
-		rpc_clnt_set_connected (&((struct rpc_clnt*)ctx->mgmt)->conn);
-
-		ret = glfs_volfile_fetch (fs);
-		if (ret && (ctx->active == NULL)) {
-			/* Do it only for the first time */
-			/* Exit the process.. there are some wrong options */
-			gf_log ("glfs-mgmt", GF_LOG_ERROR,
-				"failed to fetch volume file (key:%s)",
-                                ctx->cmd_args.volfile_id);
-                        errno = EINVAL;
-                        priv_glfs_init_done (fs, -1);
+                server = list_entry(server->list.next, typeof(*server), list);
+                ctx->cmd_args.curr_server = server;
+                ctx->cmd_args.volfile_server_port = server->port;
+                ctx->cmd_args.volfile_server = server->volfile_server;
+                ctx->cmd_args.volfile_server_transport = server->transport;
+
+                ret = dict_set_str(rpc_trans->options, "transport-type",
+                                   server->transport);
+                if (ret != 0) {
+                    gf_smsg("glfs-mgmt", GF_LOG_ERROR, ENOTCONN,
+                            API_MSG_DICT_SET_FAILED, "transport-type=%s",
+                            server->transport, NULL);
+                    errno = ENOTCONN;
+                    glfs_init_done(fs, -1);
+                    break;
                 }
 
-                break;
-	default:
-		break;
-	}
+                if (strcmp(server->transport, "unix") == 0) {
+                    ret = dict_set_str(rpc_trans->options,
+                                       "transport.socket.connect-path",
+                                       server->volfile_server);
+                    if (ret != 0) {
+                        gf_smsg("glfs-mgmt", GF_LOG_ERROR, ENOTCONN,
+                                API_MSG_DICT_SET_FAILED,
+                                "socket.connect-path=%s",
+                                server->volfile_server, NULL);
+                        errno = ENOTCONN;
+                        glfs_init_done(fs, -1);
+                        break;
+                    }
+                    /* delete the remote-host and remote-port keys
+                     * in case they were set while looping through
+                     * list of volfile servers previously
+                     */
+                    dict_del(rpc_trans->options, "remote-host");
+                    dict_del(rpc_trans->options, "remote-port");
+                } else {
+                    ret = dict_set_int32(rpc_trans->options, "remote-port",
+                                         server->port);
+                    if (ret != 0) {
+                        gf_smsg("glfs-mgmt", GF_LOG_ERROR, ENOTCONN,
+                                API_MSG_DICT_SET_FAILED, "remote-port=%d",
+                                server->port, NULL);
+                        errno = ENOTCONN;
+                        glfs_init_done(fs, -1);
+                        break;
+                    }
+
+                    ret = dict_set_str(rpc_trans->options, "remote-host",
+                                       server->volfile_server);
+                    if (ret != 0) {
+                        gf_smsg("glfs-mgmt", GF_LOG_ERROR, ENOTCONN,
+                                API_MSG_DICT_SET_FAILED, "remote-host=%s",
+                                server->volfile_server, NULL);
+                        errno = ENOTCONN;
+                        glfs_init_done(fs, -1);
+                        break;
+                    }
+                    /* delete the "transport.socket.connect-path"
+                     * key in case if it was set while looping
+                     * through list of volfile servers previously
+                     */
+                    dict_del(rpc_trans->options,
+                             "transport.socket.connect-path");
+                }
+
+                gf_smsg("glfs-mgmt", GF_LOG_INFO, 0, API_MSG_VOLFILE_CONNECTING,
+                        "server=%s", server->volfile_server, "port=%d",
+                        server->port, "transport=%s", server->transport, NULL);
+            }
+            break;
+        case RPC_CLNT_CONNECT:
+            ret = glfs_volfile_fetch(fs);
+            if (ret && (ctx->active == NULL)) {
+                /* Do it only for the first time */
+                /* Exit the process.. there are some wrong options */
+                gf_smsg("glfs-mgmt", GF_LOG_ERROR, EINVAL,
+                        API_MSG_GET_VOLFILE_FAILED, "key=%s",
+                        ctx->cmd_args.volfile_id, NULL);
+                errno = EINVAL;
+                glfs_init_done(fs, -1);
+            }
+
+            break;
+        default:
+            break;
+    }
 out:
-	return 0;
+    return 0;
 }
 
-
 int
-glusterfs_mgmt_notify (int32_t op, void *data, ...)
+glusterfs_mgmt_notify(int32_t op, void *data, ...)
 {
-	int ret = 0;
+    int ret = 0;
 
-	switch (op)
-	{
-		case GF_EN_DEFRAG_STATUS:
-			break;
+    switch (op) {
+        case GF_EN_DEFRAG_STATUS:
+            break;
 
-		default:
-			break;
-	}
+        default:
+            break;
+    }
 
-	return ret;
+    return ret;
 }
 
-
 int
-glfs_mgmt_init (struct glfs *fs)
+glfs_mgmt_init(struct glfs *fs)
 {
-	cmd_args_t		*cmd_args = NULL;
-	struct rpc_clnt		*rpc = NULL;
-	dict_t			*options = NULL;
-	int			ret = -1;
-	int			port = GF_DEFAULT_BASE_PORT;
-	char			*host = NULL;
-	glusterfs_ctx_t		*ctx = NULL;
-
-	ctx = fs->ctx;
-	cmd_args = &ctx->cmd_args;
-
-	if (ctx->mgmt)
-		return 0;
-
-	if (cmd_args->volfile_server_port)
-		port = cmd_args->volfile_server_port;
-
-	host = "localhost";
-	if (cmd_args->volfile_server)
-		host = cmd_args->volfile_server;
-
-	ret = rpc_transport_inet_options_build (&options, host, port);
-	if (ret)
-		goto out;
-
-	rpc = rpc_clnt_new (options, THIS->ctx, THIS->name, 8);
-	if (!rpc) {
-		ret = -1;
-		gf_log (THIS->name, GF_LOG_WARNING,
-			"failed to create rpc clnt");
-		goto out;
-	}
-
-	ret = rpc_clnt_register_notify (rpc, mgmt_rpc_notify, THIS);
-	if (ret) {
-		gf_log (THIS->name, GF_LOG_WARNING,
-			"failed to register notify function");
-		goto out;
-	}
-
-	ret = rpcclnt_cbk_program_register (rpc, &mgmt_cbk_prog, THIS);
-	if (ret) {
-		gf_log (THIS->name, GF_LOG_WARNING,
-			"failed to register callback function");
-		goto out;
-	}
-
-	ctx->notify = glusterfs_mgmt_notify;
-
-	/* This value should be set before doing the 'rpc_clnt_start()' as
-	   the notify function uses this variable */
-	ctx->mgmt = rpc;
-
-	ret = rpc_clnt_start (rpc);
+    cmd_args_t *cmd_args = NULL;
+    struct rpc_clnt *rpc = NULL;
+    dict_t *options = NULL;
+    int ret = -1;
+    int port = GF_DEFAULT_BASE_PORT;
+    char *host = NULL;
+    glusterfs_ctx_t *ctx = NULL;
+
+    ctx = fs->ctx;
+    cmd_args = &ctx->cmd_args;
+
+    if (ctx->mgmt)
+        return 0;
+
+    options = dict_new();
+    if (!options)
+        goto out;
+
+    if (cmd_args->volfile_server_port)
+        port = cmd_args->volfile_server_port;
+
+    if (cmd_args->volfile_server) {
+        host = cmd_args->volfile_server;
+    } else if (cmd_args->volfile_server_transport &&
+               !strcmp(cmd_args->volfile_server_transport, "unix")) {
+        host = DEFAULT_GLUSTERD_SOCKFILE;
+    } else {
+        host = "localhost";
+    }
+
+    if (cmd_args->volfile_server_transport &&
+        !strcmp(cmd_args->volfile_server_transport, "unix")) {
+        ret = rpc_transport_unix_options_build(options, host, 0);
+    } else {
+        xlator_cmdline_option_t *opt = find_xlator_option_in_cmd_args_t(
+            "address-family", cmd_args);
+        ret = rpc_transport_inet_options_build(options, host, port,
+                                               (opt ? opt->value : NULL));
+    }
+
+    if (ret)
+        goto out;
+
+    rpc = rpc_clnt_new(options, THIS, THIS->name, 8);
+    if (!rpc) {
+        ret = -1;
+        gf_smsg(THIS->name, GF_LOG_WARNING, 0, API_MSG_CREATE_RPC_CLIENT_FAILED,
+                NULL);
+        goto out;
+    }
+
+    ret = rpc_clnt_register_notify(rpc, mgmt_rpc_notify, THIS);
+    if (ret) {
+        gf_smsg(THIS->name, GF_LOG_WARNING, 0, API_MSG_REG_NOTIFY_FUNC_FAILED,
+                NULL);
+        goto out;
+    }
+
+    ret = rpcclnt_cbk_program_register(rpc, &mgmt_cbk_prog, THIS);
+    if (ret) {
+        gf_smsg(THIS->name, GF_LOG_WARNING, 0, API_MSG_REG_CBK_FUNC_FAILED,
+                NULL);
+        goto out;
+    }
+
+    ctx->notify = glusterfs_mgmt_notify;
+
+    /* This value should be set before doing the 'rpc_clnt_start()' as
+       the notify function uses this variable */
+    ctx->mgmt = rpc;
+
+    ret = rpc_clnt_start(rpc);
 out:
-	return ret;
+    if (options)
+        dict_unref(options);
+    return ret;
 }
diff --git a/api/src/glfs-resolve.c b/api/src/glfs-resolve.c
index 7aa5cc6158e..8a393ecb464 100644
--- a/api/src/glfs-resolve.c
+++ b/api/src/glfs-resolve.c
@@ -1,5 +1,5 @@
 /*
-  Copyright (c) 2012 Red Hat, Inc. <http://www.redhat.com>
+  Copyright (c) 2012-2018 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
   This file is licensed to you under your choice of the GNU Lesser
@@ -8,7 +8,6 @@
   cases as published by the Free Software Foundation.
 */
 
-
 #include <unistd.h>
 #include <string.h>
 #include <stdlib.h>
@@ -16,1003 +15,1185 @@
 #include <inttypes.h>
 #include <limits.h>
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "glusterfs.h"
-#include "logging.h"
-#include "stack.h"
-#include "event.h"
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/stack.h>
+#include <glusterfs/gf-event.h>
 #include "glfs-mem-types.h"
-#include "common-utils.h"
-#include "syncop.h"
-#include "call-stub.h"
-
+#include <glusterfs/common-utils.h>
+#include <glusterfs/syncop.h>
+#include <glusterfs/call-stub.h>
+#include "gfapi-messages.h"
+#include <glusterfs/inode.h>
 #include "glfs-internal.h"
 
-#define graphid_str(subvol) (uuid_utoa((unsigned char *)subvol->graph->graph_uuid))
-
-
+#define graphid_str(subvol)                                                    \
+    (uuid_utoa((unsigned char *)subvol->graph->graph_uuid))
 int
-glfs_first_lookup_safe (xlator_t *subvol)
+glfs_first_lookup_safe(xlator_t *subvol)
 {
-	loc_t  loc = {0, };
-	int    ret = -1;
+    loc_t loc = {
+        0,
+    };
+    int ret = -1;
 
-	loc.inode = subvol->itable->root;
-	memset (loc.gfid, 0, 16);
-	loc.gfid[15] = 1;
-	loc.path = "/";
-	loc.name = "";
+    loc.inode = subvol->itable->root;
+    memset(loc.gfid, 0, 16);
+    loc.gfid[15] = 1;
+    loc.path = "/";
+    loc.name = "";
 
-	ret = syncop_lookup (subvol, &loc, 0, 0, 0, 0);
-        DECODE_SYNCOP_ERR (ret);
+    ret = syncop_lookup(subvol, &loc, 0, 0, 0, 0);
+    DECODE_SYNCOP_ERR(ret);
 
-	gf_log (subvol->name, GF_LOG_DEBUG, "first lookup complete %d", ret);
+    gf_msg_debug(subvol->name, 0, "first lookup complete %d", ret);
 
-	return ret;
+    return ret;
 }
 
-
 int
-__glfs_first_lookup (struct glfs *fs, xlator_t *subvol)
-{
-	int ret = -1;
-
-	fs->migration_in_progress = 1;
-	pthread_mutex_unlock (&fs->mutex);
-	{
-		ret = glfs_first_lookup_safe (subvol);
-	}
-	pthread_mutex_lock (&fs->mutex);
-	fs->migration_in_progress = 0;
-	pthread_cond_broadcast (&fs->cond);
-
-	return ret;
-}
-
-
-inode_t *
-glfs_refresh_inode_safe (xlator_t *subvol, inode_t *oldinode)
+__glfs_first_lookup(struct glfs *fs, xlator_t *subvol)
 {
-	loc_t        loc = {0, };
-	int          ret = -1;
-	struct iatt  iatt = {0, };
-	inode_t     *newinode = NULL;
+    int ret = -1;
 
+    fs->migration_in_progress = 1;
+    pthread_mutex_unlock(&fs->mutex);
+    {
+        ret = glfs_first_lookup_safe(subvol);
+    }
+    pthread_mutex_lock(&fs->mutex);
+    fs->migration_in_progress = 0;
+    pthread_cond_broadcast(&fs->cond);
 
-	if (!oldinode)
-		return NULL;
+    /* wake up other waiting tasks */
+    __GLFS_SYNCTASK_WAKE(fs);
 
-	if (oldinode->table->xl == subvol)
-		return inode_ref (oldinode);
-
-	newinode = inode_find (subvol->itable, oldinode->gfid);
-	if (newinode)
-		return newinode;
-
-	uuid_copy (loc.gfid, oldinode->gfid);
-	loc.inode = inode_new (subvol->itable);
-	if (!loc.inode)
-		return NULL;
+    return ret;
+}
 
-	ret = syncop_lookup (subvol, &loc, 0, &iatt, 0, 0);
-        DECODE_SYNCOP_ERR (ret);
+/**
+ * We have to check if need_lookup flag is set in both old and the new inodes.
+ * If its set in oldinode, then directly go ahead and do an explicit lookup.
+ * But if its not set in the oldinode, then check if the newinode is linked
+ * via readdirp. If so an explicit lookup is needed on the new inode, so that
+ * below xlators can set their respective contexts.
+ */
+inode_t *
+glfs_refresh_inode_safe(xlator_t *subvol, inode_t *oldinode,
+                        gf_boolean_t need_lookup)
+{
+    loc_t loc = {
+        0,
+    };
+    int ret = -1;
+    struct iatt iatt = {
+        0,
+    };
+    inode_t *newinode = NULL;
+    gf_boolean_t lookup_needed = _gf_false;
+    uint64_t ctx_value = LOOKUP_NOT_NEEDED;
+
+    if (!oldinode)
+        return NULL;
+
+    if (!need_lookup && oldinode->table->xl == subvol)
+        return inode_ref(oldinode);
+
+    newinode = inode_find(subvol->itable, oldinode->gfid);
+    if (!need_lookup && newinode) {
+        lookup_needed = inode_needs_lookup(newinode, THIS);
+        if (!lookup_needed)
+            return newinode;
+    }
+
+    gf_uuid_copy(loc.gfid, oldinode->gfid);
+    if (!newinode)
+        loc.inode = inode_new(subvol->itable);
+    else
+        loc.inode = newinode;
+
+    if (!loc.inode)
+        return NULL;
+
+    ret = syncop_lookup(subvol, &loc, &iatt, 0, 0, 0);
+    DECODE_SYNCOP_ERR(ret);
+
+    if (ret) {
+        gf_smsg(subvol->name, GF_LOG_WARNING, errno,
+                API_MSG_INODE_REFRESH_FAILED, "gfid=%s",
+                uuid_utoa(oldinode->gfid), "err=%s", strerror(errno), NULL);
+        loc_wipe(&loc);
+        return NULL;
+    }
+
+    newinode = inode_link(loc.inode, 0, 0, &iatt);
+    if (newinode) {
+        if (newinode == loc.inode)
+            inode_ctx_set(newinode, THIS, &ctx_value);
+        inode_lookup(newinode);
+    } else {
+        gf_smsg(subvol->name, GF_LOG_WARNING, errno, API_MSG_INODE_LINK_FAILED,
+                "gfid=%s", uuid_utoa((unsigned char *)&iatt.ia_gfid), NULL);
+    }
+
+    loc_wipe(&loc);
+
+    return newinode;
+}
 
-	if (ret) {
-		gf_log (subvol->name, GF_LOG_WARNING,
-			"inode refresh of %s failed: %s",
-			uuid_utoa (oldinode->gfid), strerror (errno));
-		loc_wipe (&loc);
-		return NULL;
-	}
+inode_t *
+__glfs_refresh_inode(struct glfs *fs, xlator_t *subvol, inode_t *inode,
+                     gf_boolean_t need_lookup)
+{
+    inode_t *newinode = NULL;
 
-	newinode = inode_link (loc.inode, 0, 0, &iatt);
-	if (newinode)
-		inode_lookup (newinode);
+    fs->migration_in_progress = 1;
+    pthread_mutex_unlock(&fs->mutex);
+    {
+        newinode = glfs_refresh_inode_safe(subvol, inode, need_lookup);
+    }
+    pthread_mutex_lock(&fs->mutex);
+    fs->migration_in_progress = 0;
+    pthread_cond_broadcast(&fs->cond);
 
-	loc_wipe (&loc);
+    /* wake up other waiting tasks */
+    __GLFS_SYNCTASK_WAKE(fs);
 
-	return newinode;
+    return newinode;
 }
 
-
-inode_t *
-__glfs_refresh_inode (struct glfs *fs, xlator_t *subvol, inode_t *inode)
+GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_loc_touchup, 3.4.0)
+int
+priv_glfs_loc_touchup(loc_t *loc)
 {
-	inode_t *newinode = NULL;
-
-	fs->migration_in_progress = 1;
-	pthread_mutex_unlock (&fs->mutex);
-	{
-		newinode = glfs_refresh_inode_safe (subvol, inode);
-	}
-	pthread_mutex_lock (&fs->mutex);
-	fs->migration_in_progress = 0;
-	pthread_cond_broadcast (&fs->cond);
-
-	return newinode;
+    int ret = 0;
+
+    ret = loc_touchup(loc, loc->name);
+    if (ret < 0) {
+        errno = -ret;
+        ret = -1;
+    }
+
+    return ret;
 }
 
 int
-priv_glfs_loc_touchup (loc_t *loc)
+glfs_resolve_symlink(struct glfs *fs, xlator_t *subvol, inode_t *inode,
+                     char **lpath)
 {
-	char *path = NULL;
-	int   ret = -1;
-	char *bn = NULL;
-
-	if (loc->parent)
-		ret = inode_path (loc->parent, loc->name, &path);
-	else
-		ret = inode_path (loc->inode, 0, &path);
-
-	loc->path = path;
-
-	if (ret < 0 || !path) {
-		ret = -1;
-		errno = ENOMEM;
-		goto out;
-	}
-
-	bn = strrchr (path, '/');
-	if (bn)
-		bn++;
-	loc->name = bn;
-	ret = 0;
+    loc_t loc = {
+        0,
+    };
+    char *path = NULL;
+    char *rpath = NULL;
+    int ret = -1;
+
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
+    ret = inode_path(inode, NULL, &rpath);
+    if (ret < 0)
+        goto out;
+    loc.path = rpath;
+
+    ret = syncop_readlink(subvol, &loc, &path, 4096, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    if (ret < 0)
+        goto out;
+
+    if (lpath)
+        *lpath = path;
 out:
-	return ret;
+    loc_wipe(&loc);
+    return ret;
 }
 
-GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_loc_touchup, 3.4.0);
-
 int
-glfs_resolve_symlink (struct glfs *fs, xlator_t *subvol, inode_t *inode,
-		      char **lpath)
+glfs_resolve_base(struct glfs *fs, xlator_t *subvol, inode_t *inode,
+                  struct iatt *iatt)
 {
-	loc_t  loc = {0, };
-	char  *path = NULL;
-	char  *rpath = NULL;
-	int    ret = -1;
-
-	loc.inode = inode_ref (inode);
-	uuid_copy (loc.gfid, inode->gfid);
-	ret = inode_path (inode, NULL, &rpath);
-	if (ret < 0)
-		goto out;
-	loc.path = rpath;
-
-	ret = syncop_readlink (subvol, &loc, &path, 4096);
-        DECODE_SYNCOP_ERR (ret);
-
-	if (ret < 0)
-		goto out;
-
-	if (lpath)
-		*lpath = path;
+    loc_t loc = {
+        0,
+    };
+    int ret = -1;
+    char *path = NULL;
+
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
+
+    ret = inode_path(loc.inode, NULL, &path);
+    loc.path = path;
+    if (ret < 0)
+        goto out;
+
+    ret = syncop_lookup(subvol, &loc, iatt, NULL, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
 out:
-	loc_wipe (&loc);
-	return ret;
+    loc_wipe(&loc);
+
+    return ret;
 }
+/*
+ * This function can be used to call named lookup on root.
+ * If you use glfs_resolve_base, that will be a nameless lookup.
+ */
+static int
+glfs_resolve_root(struct glfs *fs, xlator_t *subvol, inode_t *inode,
+                  struct iatt *iatt)
+{
+    loc_t loc = {
+        0,
+    };
+    int ret = -1;
+    char *path = NULL;
+
+    loc.inode = inode_ref(inode);
+
+    ret = inode_path(loc.inode, ".", &path);
+    loc.path = path;
+    loc.name = ".";
+    /* Having a value in loc.name will help to bypass md-cache check for
+     * nameless lookup.
+     * TODO: Re-visit on nameless lookup and md-cache.
+     * Github issue : https://github.com/gluster/glusterfs/issues/232
+     */
+    loc.parent = inode_ref(inode);
+    if (ret < 0)
+        goto out;
+
+    ret = syncop_lookup(subvol, &loc, iatt, NULL, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+out:
+    loc_wipe(&loc);
 
+    return ret;
+}
 
-int
-glfs_resolve_base (struct glfs *fs, xlator_t *subvol, inode_t *inode,
-		   struct iatt *iatt)
+inode_t *
+glfs_resolve_component(struct glfs *fs, xlator_t *subvol, inode_t *parent,
+                       const char *component, struct iatt *iatt,
+                       int force_lookup)
 {
-	loc_t       loc = {0, };
-	int         ret = -1;
-	char       *path = NULL;
+    loc_t loc = {
+        0,
+    };
+    inode_t *inode = NULL;
+    inode_t *temp_parent = NULL;
+    int reval = 0;
+    int ret = -1;
+    int glret = -1;
+    struct iatt ciatt = {
+        0,
+    };
+    uuid_t gfid;
+    dict_t *xattr_req = NULL;
+    uint64_t ctx_value = LOOKUP_NOT_NEEDED;
+
+    loc.parent = inode_ref(parent);
+    gf_uuid_copy(loc.pargfid, parent->gfid);
+
+    if (__is_root_gfid(parent->gfid) &&
+        ((strcmp(component, ".") == 0) || (strcmp(component, "..") == 0) ||
+         (strcmp(component, "") == 0))) {
+        if (!force_lookup) {
+            inode = inode_ref(parent);
+        } else {
+            ret = glfs_resolve_root(fs, subvol, parent, &ciatt);
+            if (!ret)
+                inode = inode_ref(parent);
+        }
+        goto found;
+    }
+    /* *
+     * if the component name is either "." or "..", it will try to
+     * resolve that if inode has a proper parent (named lookup).
+     *
+     * Below condition works like this
+     *
+     * Example 1 :
+     * Path /out_dir/dir/in_dir/.
+     *     In put values :
+     *         parent = in_dir
+     *         component : "."
+     *
+     *      Out put values:
+     *         parent : dir
+     *         component : "in_dir"
+     *
+     * Example 2 :
+     * Path /out_dir/dir/in_dir/..
+     *     In put values :
+     *         parent = in_dir
+     *         component : ".."
+     *
+     *     Out put values:
+     *         parent : output_dir
+     *         component : "dir"
+     *
+     * In case of nameless lookup, both "." and ".." retained
+     */
+
+    if (strcmp(component, ".") == 0) {
+        loc.inode = inode_ref(parent);
+        temp_parent = inode_parent(loc.inode, 0, 0);
+        if (temp_parent) {
+            inode_unref(loc.parent);
+            loc.parent = temp_parent;
+            gf_uuid_copy(loc.pargfid, temp_parent->gfid);
+            inode_find_directory_name(loc.inode, &loc.name);
+        }
 
-	loc.inode = inode_ref (inode);
-	uuid_copy (loc.gfid, inode->gfid);
+    } else if (strcmp(component, "..") == 0) {
+        loc.inode = inode_parent(parent, 0, 0);
+        if (loc.inode) {
+            temp_parent = inode_parent(loc.inode, 0, 0);
+            if (temp_parent) {
+                inode_unref(loc.parent);
+                loc.parent = temp_parent;
+                gf_uuid_copy(loc.pargfid, temp_parent->gfid);
+                inode_find_directory_name(loc.inode, &loc.name);
+            } else if (__is_root_gfid(loc.inode->gfid)) {
+                inode_unref(loc.parent);
+                loc.parent = inode_ref(loc.inode);
+                gf_uuid_copy(loc.pargfid, loc.inode->gfid);
+                loc.name = ".";
+            } else {
+                inode_unref(loc.inode);
+                loc.inode = NULL;
+            }
+        }
+    } else
+        loc.inode = inode_grep(parent->table, parent, component);
 
-	ret = inode_path (loc.inode, NULL, &path);
-	loc.path = path;
-	if (ret < 0)
-		goto out;
+    if (!loc.name)
+        loc.name = component;
 
-	ret = syncop_lookup (subvol, &loc, NULL, iatt, NULL, NULL);
-        DECODE_SYNCOP_ERR (ret);
-out:
-	loc_wipe (&loc);
+    if (loc.inode) {
+        gf_uuid_copy(loc.gfid, loc.inode->gfid);
+        reval = 1;
 
-	return ret;
-}
+        if (!(force_lookup || inode_needs_lookup(loc.inode, THIS))) {
+            inode = inode_ref(loc.inode);
+            goto found;
+        }
+    } else {
+        gf_uuid_generate(gfid);
+        loc.inode = inode_new(parent->table);
+        if (!loc.inode) {
+            errno = ENOMEM;
+            goto out;
+        }
 
+        xattr_req = dict_new();
+        if (!xattr_req) {
+            errno = ENOMEM;
+            goto out;
+        }
 
-inode_t *
-glfs_resolve_component (struct glfs *fs, xlator_t *subvol, inode_t *parent,
-			const char *component, struct iatt *iatt,
-			int force_lookup)
-{
-	loc_t        loc = {0, };
-	inode_t     *inode = NULL;
-	int          reval = 0;
-	int          ret = -1;
-	int          glret = -1;
-	struct iatt  ciatt = {0, };
-	uuid_t       gfid;
-	dict_t      *xattr_req = NULL;
-
-	loc.name = component;
-
-	loc.parent = inode_ref (parent);
-	uuid_copy (loc.pargfid, parent->gfid);
-
-        /* /.. and /. should point back to /
-           we lookup using inode and gfid of root
-           Fill loc.name so that we make use md-cache.
-           md-cache is not valid for nameless lookups.
-        */
-        if (__is_root_gfid (parent->gfid) &&
-            (strcmp (component, "..") == 0)) {
-                loc.inode = inode_ref (parent);
-                loc.name = ".";
-        } else {
-                if (strcmp (component, ".") == 0)
-                        loc.inode = inode_ref (parent);
-                else if (strcmp (component, "..") == 0)
-                        loc.inode = inode_parent (parent, 0, 0);
-                else
-                        loc.inode = inode_grep (parent->table, parent,
-                                                component);
+        ret = dict_set_gfuuid(xattr_req, "gfid-req", gfid, true);
+        if (ret) {
+            errno = ENOMEM;
+            goto out;
+        }
+    }
+
+    glret = priv_glfs_loc_touchup(&loc);
+    if (glret < 0) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = syncop_lookup(subvol, &loc, &ciatt, NULL, xattr_req, NULL);
+    if (ret && reval) {
+        /*
+         * A stale mapping might exist for a dentry/inode that has been
+         * removed from another client.
+         */
+        if (-ret == ENOENT) {
+            inode_unlink(loc.inode, loc.parent, loc.name);
+            if (!inode_has_dentry(loc.inode))
+                inode_forget(loc.inode, 0);
         }
 
+        inode_unref(loc.inode);
+        gf_uuid_clear(loc.gfid);
+        loc.inode = inode_new(parent->table);
+        if (!loc.inode) {
+            errno = ENOMEM;
+            goto out;
+        }
 
-	if (loc.inode) {
-		uuid_copy (loc.gfid, loc.inode->gfid);
-		reval = 1;
-
-		if (!force_lookup) {
-			inode = inode_ref (loc.inode);
-			ciatt.ia_type = inode->ia_type;
-			goto found;
-		}
-	} else {
-		uuid_generate (gfid);
-		loc.inode = inode_new (parent->table);
-                if (!loc.inode) {
-                        errno = ENOMEM;
-                        goto out;
-                }
+        xattr_req = dict_new();
+        if (!xattr_req) {
+            errno = ENOMEM;
+            goto out;
+        }
 
-                xattr_req = dict_new ();
-                if (!xattr_req) {
-                        errno = ENOMEM;
-                        goto out;
-                }
+        gf_uuid_generate(gfid);
 
-                ret = dict_set_static_bin (xattr_req, "gfid-req", gfid, 16);
-                if (ret) {
-                        errno = ENOMEM;
-                        goto out;
-                }
+        ret = dict_set_gfuuid(xattr_req, "gfid-req", gfid, true);
+        if (ret) {
+            errno = ENOMEM;
+            goto out;
+        }
+
+        ret = syncop_lookup(subvol, &loc, &ciatt, NULL, xattr_req, NULL);
+    }
+    DECODE_SYNCOP_ERR(ret);
+    if (ret)
+        goto out;
 
-	}
-
-	glret = priv_glfs_loc_touchup (&loc);
-	if (glret < 0) {
-		ret = -1;
-		goto out;
-	}
-
-        ret = syncop_lookup (subvol, &loc, xattr_req, &ciatt, NULL, NULL);
-        if (ret && reval) {
-                /*
-                 * A stale mapping might exist for a dentry/inode that has been
-                 * removed from another client.
-                 */
-                if (-ret == ENOENT)
-                        inode_unlink(loc.inode, loc.parent,
-                                     loc.name);
-		inode_unref (loc.inode);
-		loc.inode = inode_new (parent->table);
-		if (!loc.inode) {
-			errno = ENOMEM;
-			goto out;
-		}
-
-		xattr_req = dict_new ();
-		if (!xattr_req) {
-			errno = ENOMEM;
-			goto out;
-		}
-
-		uuid_generate (gfid);
-
-		ret = dict_set_static_bin (xattr_req, "gfid-req", gfid, 16);
-		if (ret) {
-			errno = ENOMEM;
-			goto out;
-		}
-
-		ret = syncop_lookup (subvol, &loc, xattr_req, &ciatt,
-				     NULL, NULL);
-	}
-        DECODE_SYNCOP_ERR (ret);
-	if (ret)
-		goto out;
-
-	inode = inode_link (loc.inode, loc.parent, component, &ciatt);
+    inode = inode_link(loc.inode, loc.parent, component, &ciatt);
+
+    if (!inode) {
+        gf_smsg(subvol->name, GF_LOG_WARNING, errno, API_MSG_INODE_LINK_FAILED,
+                "gfid=%s", uuid_utoa((unsigned char *)&ciatt.ia_gfid), NULL);
+        goto out;
+    } else if (inode == loc.inode)
+        inode_ctx_set(inode, THIS, &ctx_value);
 found:
-	if (inode)
-		inode_lookup (inode);
-	if (iatt)
-		*iatt = ciatt;
+    if (inode) {
+        ciatt.ia_type = inode->ia_type;
+        inode_lookup(inode);
+    }
+    if (iatt)
+        *iatt = ciatt;
 out:
-	if (xattr_req)
-		dict_unref (xattr_req);
-
-	loc_wipe (&loc);
+    if (xattr_req)
+        dict_unref(xattr_req);
+    loc_wipe(&loc);
 
-	return inode;
+    return inode;
 }
 
-
+GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_resolve_at, 3.4.0)
 int
-priv_glfs_resolve_at (struct glfs *fs, xlator_t *subvol, inode_t *at,
-		 const char *origpath, loc_t *loc, struct iatt *iatt,
-		 int follow, int reval)
+priv_glfs_resolve_at(struct glfs *fs, xlator_t *subvol, inode_t *at,
+                     const char *origpath, loc_t *loc, struct iatt *iatt,
+                     int follow, int reval)
 {
-	inode_t    *inode = NULL;
-	inode_t    *parent = NULL;
-	char       *saveptr = NULL;
-	char       *path = NULL;
-	char       *component = NULL;
-	char       *next_component = NULL;
-	int         ret = -1;
-	struct iatt ciatt = {0, };
-
-	path = gf_strdup (origpath);
-	if (!path) {
-		errno = ENOMEM;
-		return -1;
-	}
-
-	parent = NULL;
-	if (at && path[0] != '/') {
-		/* A relative resolution of a path which starts with '/'
-		   is equal to an absolute path resolution.
-		*/
-		inode = inode_ref (at);
-	} else {
-		inode = inode_ref (subvol->itable->root);
-
-		if (strcmp (path, "/") == 0)
-			glfs_resolve_base (fs, subvol, inode, &ciatt);
-	}
-
-	for (component = strtok_r (path, "/", &saveptr);
-	     component; component = next_component) {
-
-		next_component = strtok_r (NULL, "/", &saveptr);
-
-		if (parent)
-			inode_unref (parent);
-
-		parent = inode;
-
-		inode = glfs_resolve_component (fs, subvol, parent,
-						component, &ciatt,
-						/* force hard lookup on the last
-						   component, as the caller
-						   wants proper iatt filled
-						*/
-						(reval || (!next_component &&
-						iatt)));
-		if (!inode)
-			break;
-
-		if (IA_ISLNK (ciatt.ia_type) && (next_component || follow)) {
-			/* If the component is not the last piece,
-			   then following it is necessary even if
-			   not requested by the caller
-			*/
-			char *lpath = NULL;
-			loc_t sym_loc = {0,};
-
-			if (follow > GLFS_SYMLINK_MAX_FOLLOW) {
-				errno = ELOOP;
-				ret = -1;
-				if (inode) {
-					inode_unref (inode);
-					inode = NULL;
-				}
-				break;
-			}
-
-			ret = glfs_resolve_symlink (fs, subvol, inode, &lpath);
-			inode_unref (inode);
-			inode = NULL;
-			if (ret < 0)
-				break;
-
-			ret = priv_glfs_resolve_at (fs, subvol, parent, lpath,
-					       &sym_loc,
-					       /* followed iatt becomes the
-						  component iatt
-					       */
-					       &ciatt,
-					       /* always recurisvely follow while
-						  following symlink
-					       */
-					       follow + 1, reval);
-			if (ret == 0)
-				inode = inode_ref (sym_loc.inode);
-			loc_wipe (&sym_loc);
-			GF_FREE (lpath);
-		}
-
-		if (!next_component)
-			break;
-
-		if (!IA_ISDIR (ciatt.ia_type)) {
-			/* next_component exists and this component is
-			   not a directory
-			*/
-			inode_unref (inode);
-			inode = NULL;
-			ret = -1;
-			errno = ENOTDIR;
-			break;
-		}
-	}
-
-	if (parent && next_component)
-		/* resolution failed mid-way */
-		goto out;
-
-	/* At this point, all components up to the last parent directory
-	   have been resolved successfully (@parent). Resolution of basename
-	   might have failed (@inode) if at all.
-	*/
-
-	loc->parent = parent;
-	if (parent) {
-		uuid_copy (loc->pargfid, parent->gfid);
-		loc->name = component;
-	}
-
-	loc->inode = inode;
-	if (inode) {
-		uuid_copy (loc->gfid, inode->gfid);
-		if (iatt)
-			*iatt = ciatt;
-		ret = 0;
-	}
-
-        if (priv_glfs_loc_touchup (loc) < 0) {
+    inode_t *inode = NULL;
+    inode_t *parent = NULL;
+    char *saveptr = NULL;
+    char *path = NULL;
+    char *component = NULL;
+    char *next_component = NULL;
+    int ret = -1;
+    struct iatt ciatt = {
+        0,
+    };
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    if (origpath[0] == '\0') {
+        errno = EINVAL;
+        goto invalid_fs;
+    }
+
+    parent = NULL;
+    if (at && origpath[0] != '/') {
+        /* A relative resolution of a path which starts with '/'
+           is equal to an absolute path resolution.
+        */
+        inode = inode_ref(at);
+    } else {
+        inode = inode_ref(subvol->itable->root);
+
+        if (strcmp(origpath, "/") == 0)
+            glfs_resolve_root(fs, subvol, inode, &ciatt);
+    }
+
+    path = gf_strdup(origpath);
+    if (!path)
+        goto invalid_fs;
+
+    for (component = strtok_r(path, "/", &saveptr); component;
+         component = next_component) {
+        next_component = strtok_r(NULL, "/", &saveptr);
+
+        if (parent)
+            inode_unref(parent);
+        parent = inode;
+        inode = glfs_resolve_component(fs, subvol, parent, component, &ciatt,
+                                       /* force hard lookup on the last
+                                          component, as the caller
+                                          wants proper iatt filled
+                                       */
+                                       (reval || (!next_component && iatt)));
+        if (!inode) {
+            ret = -1;
+            break;
+        }
+
+        if (IA_ISLNK(ciatt.ia_type) && (next_component || follow)) {
+            /* If the component is not the last piece,
+               then following it is necessary even if
+               not requested by the caller
+            */
+            char *lpath = NULL;
+            loc_t sym_loc = {
+                0,
+            };
+
+            if (follow > GLFS_SYMLINK_MAX_FOLLOW) {
+                errno = ELOOP;
                 ret = -1;
+                if (inode) {
+                    inode_unref(inode);
+                    inode = NULL;
+                }
+                break;
+            }
+
+            ret = glfs_resolve_symlink(fs, subvol, inode, &lpath);
+            inode_unref(inode);
+            inode = NULL;
+            if (ret < 0)
+                break;
+
+            ret = priv_glfs_resolve_at(fs, subvol, parent, lpath, &sym_loc,
+                                       /* followed iatt becomes the
+                                          component iatt
+                                       */
+                                       &ciatt,
+                                       /* always recurisvely follow while
+                                          following symlink
+                                       */
+                                       follow + 1, reval);
+            if (ret == 0)
+                inode = inode_ref(sym_loc.inode);
+            loc_wipe(&sym_loc);
+            GF_FREE(lpath);
         }
-out:
-	GF_FREE (path);
 
-	/* do NOT loc_wipe here as only last component might be missing */
+        if (!next_component)
+            break;
+
+        if (!IA_ISDIR(ciatt.ia_type)) {
+            /* next_component exists and this component is
+               not a directory
+            */
+            inode_unref(inode);
+            inode = NULL;
+            ret = -1;
+            errno = ENOTDIR;
+            break;
+        }
+    }
+
+    if (parent && next_component)
+        /* resolution failed mid-way */
+        goto out;
+
+    /* At this point, all components up to the last parent directory
+       have been resolved successfully (@parent). Resolution of basename
+       might have failed (@inode) if at all.
+    */
+
+    loc->parent = parent;
+    if (parent) {
+        gf_uuid_copy(loc->pargfid, parent->gfid);
+        loc->name = component;
+    }
+
+    loc->inode = inode;
+    if (inode) {
+        gf_uuid_copy(loc->gfid, inode->gfid);
+        if (iatt)
+            *iatt = ciatt;
+        ret = 0;
+    }
+
+    if (priv_glfs_loc_touchup(loc) < 0) {
+        ret = -1;
+    }
+out:
+    GF_FREE(path);
+    __GLFS_EXIT_FS;
 
-	return ret;
+    /* do NOT loc_wipe here as only last component might be missing */
+invalid_fs:
+    return ret;
 }
 
-GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_resolve_at, 3.4.0);
-
-
 int
-glfs_resolve_path (struct glfs *fs, xlator_t *subvol, const char *origpath,
-		   loc_t *loc, struct iatt *iatt, int follow, int reval)
+glfs_resolve_path(struct glfs *fs, xlator_t *subvol, const char *origpath,
+                  loc_t *loc, struct iatt *iatt, int follow, int reval)
 {
-	int ret = -1;
-	inode_t *cwd = NULL;
+    int ret = -1;
+    inode_t *cwd = NULL;
 
-	if (origpath[0] == '/')
-		return priv_glfs_resolve_at (fs, subvol, NULL, origpath, loc,
-                                             iatt, follow, reval);
+    if (origpath[0] == '/')
+        return priv_glfs_resolve_at(fs, subvol, NULL, origpath, loc, iatt,
+                                    follow, reval);
 
-	cwd = glfs_cwd_get (fs);
+    cwd = glfs_cwd_get(fs);
+    if (NULL == cwd) {
+        gf_smsg(subvol->name, GF_LOG_WARNING, EIO, API_MSG_GET_CWD_FAILED,
+                NULL);
+        errno = EIO;
+        goto out;
+    }
 
-	ret = priv_glfs_resolve_at (fs, subvol, cwd, origpath, loc, iatt,
-                                    follow, reval);
-	if (cwd)
-		inode_unref (cwd);
+    ret = priv_glfs_resolve_at(fs, subvol, cwd, origpath, loc, iatt, follow,
+                               reval);
+    if (cwd)
+        inode_unref(cwd);
 
-	return ret;
+out:
+    return ret;
 }
 
-
+GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_resolve, 3.7.0)
 int
-glfs_resolve (struct glfs *fs, xlator_t *subvol, const char *origpath,
-	      loc_t *loc, struct iatt *iatt, int reval)
+priv_glfs_resolve(struct glfs *fs, xlator_t *subvol, const char *origpath,
+                  loc_t *loc, struct iatt *iatt, int reval)
 {
-	int ret = -1;
+    int ret = -1;
 
-	ret = glfs_resolve_path (fs, subvol, origpath, loc, iatt, 1, reval);
+    ret = glfs_resolve_path(fs, subvol, origpath, loc, iatt, 1, reval);
 
-	return ret;
+    return ret;
 }
 
-
 int
-glfs_lresolve (struct glfs *fs, xlator_t *subvol, const char *origpath,
-	       loc_t *loc, struct iatt *iatt, int reval)
+glfs_lresolve(struct glfs *fs, xlator_t *subvol, const char *origpath,
+              loc_t *loc, struct iatt *iatt, int reval)
 {
-	int ret = -1;
+    int ret = -1;
 
-	ret = glfs_resolve_path (fs, subvol, origpath, loc, iatt, 0, reval);
+    ret = glfs_resolve_path(fs, subvol, origpath, loc, iatt, 0, reval);
 
-	return ret;
+    return ret;
 }
 
-
 int
-glfs_migrate_fd_locks_safe (struct glfs *fs, xlator_t *oldsubvol, fd_t *oldfd,
-			    xlator_t *newsubvol, fd_t *newfd)
+glfs_migrate_fd_locks_safe(struct glfs *fs, xlator_t *oldsubvol, fd_t *oldfd,
+                           xlator_t *newsubvol, fd_t *newfd)
 {
-	dict_t *lockinfo = NULL;
-	int ret = 0;
-	char uuid1[64];
-
-	if (!oldfd->lk_ctx || fd_lk_ctx_empty (oldfd->lk_ctx))
-		return 0;
-
-	newfd->lk_ctx = fd_lk_ctx_ref (oldfd->lk_ctx);
-
-	ret = syncop_fgetxattr (oldsubvol, oldfd, &lockinfo,
-				GF_XATTR_LOCKINFO_KEY);
-        DECODE_SYNCOP_ERR (ret);
-	if (ret < 0) {
-		gf_log (fs->volname, GF_LOG_WARNING,
-			"fgetxattr (%s) failed (%s) on graph %s (%d)",
-			uuid_utoa_r (oldfd->inode->gfid, uuid1),
-			strerror (errno),
-			graphid_str (oldsubvol), oldsubvol->graph->id);
-		goto out;
-	}
-
-	if (!dict_get (lockinfo, GF_XATTR_LOCKINFO_KEY)) {
-		gf_log (fs->volname, GF_LOG_WARNING,
-			"missing lokinfo key (%s) on graph %s (%d)",
-			uuid_utoa_r (oldfd->inode->gfid, uuid1),
-			graphid_str (oldsubvol), oldsubvol->graph->id);
-		goto out;
-	}
-
-	ret = syncop_fsetxattr (newsubvol, newfd, lockinfo, 0);
-        DECODE_SYNCOP_ERR (ret);
-	if (ret < 0) {
-		gf_log (fs->volname, GF_LOG_WARNING,
-			"fsetxattr (%s) failed (%s) on graph %s (%d)",
-			uuid_utoa_r (newfd->inode->gfid, uuid1),
-			strerror (errno),
-			graphid_str (newsubvol), newsubvol->graph->id);
-		goto out;
-	}
+    dict_t *lockinfo = NULL;
+    int ret = 0;
+    char uuid1[64];
+
+    if (!oldfd->lk_ctx || fd_lk_ctx_empty(oldfd->lk_ctx))
+        return 0;
+
+    newfd->lk_ctx = fd_lk_ctx_ref(oldfd->lk_ctx);
+
+    ret = syncop_fgetxattr(oldsubvol, oldfd, &lockinfo, GF_XATTR_LOCKINFO_KEY,
+                           NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+    if (ret < 0) {
+        gf_smsg(fs->volname, GF_LOG_WARNING, errno, API_MSG_FGETXATTR_FAILED,
+                "gfid=%s", uuid_utoa_r(oldfd->inode->gfid, uuid1), "err=%s",
+                strerror(errno), "subvol=%s", graphid_str(oldsubvol), "id=%d",
+                oldsubvol->graph->id, NULL);
+        goto out;
+    }
+
+    if (!dict_get(lockinfo, GF_XATTR_LOCKINFO_KEY)) {
+        gf_smsg(fs->volname, GF_LOG_WARNING, 0, API_MSG_LOCKINFO_KEY_MISSING,
+                "gfid=%s", uuid_utoa_r(oldfd->inode->gfid, uuid1), "subvol=%s",
+                graphid_str(oldsubvol), "id=%d", oldsubvol->graph->id, NULL);
+        goto out;
+    }
+
+    ret = syncop_fsetxattr(newsubvol, newfd, lockinfo, 0, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+    if (ret < 0) {
+        gf_smsg(fs->volname, GF_LOG_WARNING, 0, API_MSG_FSETXATTR_FAILED,
+                "gfid=%s", uuid_utoa_r(newfd->inode->gfid, uuid1), "err=%s",
+                strerror(errno), "subvol=%s", graphid_str(newsubvol), "id=%d",
+                newsubvol->graph->id, NULL);
+        goto out;
+    }
 out:
-	if (lockinfo)
-		dict_unref (lockinfo);
-	return ret;
+    if (lockinfo)
+        dict_unref(lockinfo);
+    return ret;
 }
 
-
 fd_t *
-glfs_migrate_fd_safe (struct glfs *fs, xlator_t *newsubvol, fd_t *oldfd)
+glfs_migrate_fd_safe(struct glfs *fs, xlator_t *newsubvol, fd_t *oldfd)
 {
-	fd_t *newfd = NULL;
-	inode_t *oldinode = NULL;
-	inode_t *newinode = NULL;
-	xlator_t *oldsubvol = NULL;
-	int ret = -1;
-	loc_t loc = {0, };
-	char uuid1[64];
-
-
-	oldinode = oldfd->inode;
-	oldsubvol = oldinode->table->xl;
-
-	if (oldsubvol == newsubvol)
-		return fd_ref (oldfd);
-
-	if (!oldsubvol->switched) {
-		ret = syncop_fsync (oldsubvol, oldfd, 0);
-                DECODE_SYNCOP_ERR (ret);
-		if (ret) {
-			gf_log (fs->volname, GF_LOG_WARNING,
-				"fsync() failed (%s) on %s graph %s (%d)",
-				strerror (errno),
-				uuid_utoa_r (oldfd->inode->gfid, uuid1),
-				graphid_str (oldsubvol), oldsubvol->graph->id);
-		}
-	}
-
-	newinode = glfs_refresh_inode_safe (newsubvol, oldinode);
-	if (!newinode) {
-		gf_log (fs->volname, GF_LOG_WARNING,
-			"inode (%s) refresh failed (%s) on graph %s (%d)",
-			uuid_utoa_r (oldinode->gfid, uuid1),
-			strerror (errno),
-			graphid_str (newsubvol), newsubvol->graph->id);
-		goto out;
-	}
-
-	newfd = fd_create (newinode, getpid());
-	if (!newfd) {
-		gf_log (fs->volname, GF_LOG_WARNING,
-			"fd_create (%s) failed (%s) on graph %s (%d)",
-			uuid_utoa_r (newinode->gfid, uuid1),
-			strerror (errno),
-			graphid_str (newsubvol), newsubvol->graph->id);
-		goto out;
-	}
-
-	loc.inode = inode_ref (newinode);
-
-        ret = inode_path (oldfd->inode, NULL, (char **)&loc.path);
-        if (ret < 0) {
-                gf_log (fs->volname, GF_LOG_INFO, "inode_path failed");
-                goto out;
+    fd_t *newfd = NULL;
+    inode_t *oldinode = NULL;
+    inode_t *newinode = NULL;
+    xlator_t *oldsubvol = NULL;
+    int ret = -1;
+    loc_t loc = {
+        0,
+    };
+    char uuid1[64];
+    dict_t *xdata = NULL;
+
+    oldinode = oldfd->inode;
+    oldsubvol = oldinode->table->xl;
+
+    if (oldsubvol == newsubvol)
+        return fd_ref(oldfd);
+
+    if (!oldsubvol->switched) {
+        xdata = dict_new();
+        if (!xdata || dict_set_int8(xdata, "last-fsync", 1)) {
+            gf_smsg(fs->volname, GF_LOG_WARNING, ENOMEM, API_MSG_FSYNC_FAILED,
+                    "err=%s", "last-fsync set failed", "gfid=%s",
+                    uuid_utoa_r(oldfd->inode->gfid, uuid1), "subvol=%s",
+                    graphid_str(oldsubvol), "id=%d", oldsubvol->graph->id,
+                    NULL);
         }
 
-        uuid_copy (loc.gfid, oldinode->gfid);
-
-
-	if (IA_ISDIR (oldinode->ia_type))
-		ret = syncop_opendir (newsubvol, &loc, newfd);
-	else
-		ret = syncop_open (newsubvol, &loc,
-				   oldfd->flags & ~(O_TRUNC|O_EXCL|O_CREAT),
-				   newfd);
-        DECODE_SYNCOP_ERR (ret);
-	loc_wipe (&loc);
-
-	if (ret) {
-		gf_log (fs->volname, GF_LOG_WARNING,
-			"syncop_open%s (%s) failed (%s) on graph %s (%d)",
-			IA_ISDIR (oldinode->ia_type) ? "dir" : "",
-			uuid_utoa_r (newinode->gfid, uuid1),
-			strerror (errno),
-			graphid_str (newsubvol), newsubvol->graph->id);
-		goto out;
-	}
-
-	ret = glfs_migrate_fd_locks_safe (fs, oldsubvol, oldfd, newsubvol,
-					  newfd);
-
-	if (ret) {
-		gf_log (fs->volname, GF_LOG_WARNING,
-			"lock migration (%s) failed (%s) on graph %s (%d)",
-			uuid_utoa_r (newinode->gfid, uuid1),
-			strerror (errno),
-			graphid_str (newsubvol), newsubvol->graph->id);
-		goto out;
-	}
-
-        newfd->flags = oldfd->flags;
-	fd_bind (newfd);
+        ret = syncop_fsync(oldsubvol, oldfd, 0, NULL, NULL, xdata, NULL);
+        DECODE_SYNCOP_ERR(ret);
+        if (ret) {
+            gf_smsg(fs->volname, GF_LOG_WARNING, errno, API_MSG_FSYNC_FAILED,
+                    "err=%s", strerror(errno), "gfid=%s",
+                    uuid_utoa_r(oldfd->inode->gfid, uuid1), "subvol=%s",
+                    graphid_str(oldsubvol), "id=%d", oldsubvol->graph->id,
+                    NULL);
+        }
+    }
+
+    newinode = glfs_refresh_inode_safe(newsubvol, oldinode, _gf_false);
+    if (!newinode) {
+        gf_smsg(fs->volname, GF_LOG_WARNING, errno,
+                API_MSG_INODE_REFRESH_FAILED, "gfid=%s",
+                uuid_utoa_r(oldinode->gfid, uuid1), "err=%s", strerror(errno),
+                "subvol=%s", graphid_str(newsubvol), "id=%d",
+                newsubvol->graph->id, NULL);
+        goto out;
+    }
+
+    newfd = fd_create(newinode, getpid());
+    if (!newfd) {
+        gf_smsg(fs->volname, GF_LOG_WARNING, errno,
+                API_MSG_FDCREATE_FAILED_ON_GRAPH, "gfid=%s",
+                uuid_utoa_r(newinode->gfid, uuid1), "err=%s", strerror(errno),
+                "subvol=%s", graphid_str(newsubvol), "id=%d",
+                newsubvol->graph->id, NULL);
+        goto out;
+    }
+
+    loc.inode = inode_ref(newinode);
+
+    ret = inode_path(oldfd->inode, NULL, (char **)&loc.path);
+    if (ret < 0) {
+        gf_smsg(fs->volname, GF_LOG_INFO, 0, API_MSG_INODE_PATH_FAILED, NULL);
+        goto out;
+    }
+
+    gf_uuid_copy(loc.gfid, oldinode->gfid);
+
+    if (IA_ISDIR(oldinode->ia_type))
+        ret = syncop_opendir(newsubvol, &loc, newfd, NULL, NULL);
+    else
+        ret = syncop_open(newsubvol, &loc,
+                          oldfd->flags & ~(O_TRUNC | O_EXCL | O_CREAT), newfd,
+                          NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+    loc_wipe(&loc);
+
+    if (ret) {
+        gf_smsg(fs->volname, GF_LOG_WARNING, errno, API_MSG_SYNCOP_OPEN_FAILED,
+                "type=%s", IA_ISDIR(oldinode->ia_type) ? "dir" : "", "gfid=%s",
+                uuid_utoa_r(newinode->gfid, uuid1), "err=%s", strerror(errno),
+                "subvol=%s", graphid_str(newsubvol), "id=%d",
+                newsubvol->graph->id, NULL);
+        goto out;
+    }
+
+    ret = glfs_migrate_fd_locks_safe(fs, oldsubvol, oldfd, newsubvol, newfd);
+
+    if (ret) {
+        gf_smsg(fs->volname, GF_LOG_WARNING, errno, API_MSG_LOCK_MIGRATE_FAILED,
+                "gfid=%s", uuid_utoa_r(newinode->gfid, uuid1), "err=%s",
+                strerror(errno), "subvol=%s", graphid_str(newsubvol), "id=%d",
+                newsubvol->graph->id, NULL);
+        goto out;
+    }
+
+    newfd->flags = oldfd->flags;
+    fd_bind(newfd);
 out:
-	if (newinode)
-		inode_unref (newinode);
+    if (newinode)
+        inode_unref(newinode);
 
-	if (ret) {
-		fd_unref (newfd);
-		newfd = NULL;
-	}
+    if (ret) {
+        fd_unref(newfd);
+        newfd = NULL;
+    }
 
-	return newfd;
-}
+    if (xdata)
+        dict_unref(xdata);
 
+    return newfd;
+}
 
 fd_t *
-__glfs_migrate_fd (struct glfs *fs, xlator_t *newsubvol, struct glfs_fd *glfd)
+__glfs_migrate_fd(struct glfs *fs, xlator_t *newsubvol, struct glfs_fd *glfd)
 {
-	fd_t *oldfd = NULL;
-	fd_t *newfd = NULL;
+    fd_t *oldfd = NULL;
+    fd_t *newfd = NULL;
 
-	oldfd = glfd->fd;
+    oldfd = glfd->fd;
 
-	fs->migration_in_progress = 1;
-	pthread_mutex_unlock (&fs->mutex);
-	{
-		newfd = glfs_migrate_fd_safe (fs, newsubvol, oldfd);
-	}
-	pthread_mutex_lock (&fs->mutex);
-	fs->migration_in_progress = 0;
-	pthread_cond_broadcast (&fs->cond);
+    fs->migration_in_progress = 1;
+    pthread_mutex_unlock(&fs->mutex);
+    {
+        newfd = glfs_migrate_fd_safe(fs, newsubvol, oldfd);
+    }
+    pthread_mutex_lock(&fs->mutex);
+    fs->migration_in_progress = 0;
+    pthread_cond_broadcast(&fs->cond);
 
-	return newfd;
-}
+    /* wake up other waiting tasks */
+    __GLFS_SYNCTASK_WAKE(fs);
 
+    return newfd;
+}
 
 fd_t *
-__glfs_resolve_fd (struct glfs *fs, xlator_t *subvol, struct glfs_fd *glfd)
+__glfs_resolve_fd(struct glfs *fs, xlator_t *subvol, struct glfs_fd *glfd)
 {
-	fd_t *fd = NULL;
+    fd_t *fd = NULL;
 
-	if (glfd->fd->inode->table->xl == subvol)
-		return fd_ref (glfd->fd);
+    if (glfd->fd->inode->table->xl == subvol)
+        return fd_ref(glfd->fd);
 
-	fd = __glfs_migrate_fd (fs, subvol, glfd);
-	if (!fd)
-		return NULL;
+    fd = __glfs_migrate_fd(fs, subvol, glfd);
+    if (!fd)
+        return NULL;
 
-	if (subvol == fs->active_subvol) {
-		fd_unref (glfd->fd);
-		glfd->fd = fd_ref (fd);
-	}
+    if (subvol == fs->active_subvol) {
+        fd_unref(glfd->fd);
+        glfd->fd = fd_ref(fd);
+    }
 
-	return fd;
+    return fd;
 }
 
-
 fd_t *
-glfs_resolve_fd (struct glfs *fs, xlator_t *subvol, struct glfs_fd *glfd)
+glfs_resolve_fd(struct glfs *fs, xlator_t *subvol, struct glfs_fd *glfd)
 {
-	fd_t *fd = NULL;
+    fd_t *fd = NULL;
 
-	glfs_lock (fs);
-	{
-		fd = __glfs_resolve_fd (fs, subvol, glfd);
-	}
-	glfs_unlock (fs);
+    glfs_lock(fs, _gf_true);
+    {
+        fd = __glfs_resolve_fd(fs, subvol, glfd);
+    }
+    glfs_unlock(fs);
 
-	return fd;
+    return fd;
 }
 
-
 void
-__glfs_migrate_openfds (struct glfs *fs, xlator_t *subvol)
+__glfs_migrate_openfds(struct glfs *fs, xlator_t *subvol)
 {
-	struct glfs_fd *glfd = NULL;
-	fd_t *fd = NULL;
-
-	list_for_each_entry (glfd, &fs->openfds, openfds) {
-		if (uuid_is_null (glfd->fd->inode->gfid)) {
-			gf_log (fs->volname, GF_LOG_INFO,
-				"skipping openfd %p/%p in graph %s (%d)",
-				glfd, glfd->fd,	graphid_str(subvol),
-				subvol->graph->id);
-			/* create in progress, defer */
-			continue;
-		}
-
-		fd = __glfs_migrate_fd (fs, subvol, glfd);
-		if (fd) {
-			fd_unref (glfd->fd);
-			glfd->fd = fd;
-		}
-	}
-}
-
+    struct glfs_fd *glfd = NULL;
+    fd_t *fd = NULL;
+
+    list_for_each_entry(glfd, &fs->openfds, openfds)
+    {
+        if (gf_uuid_is_null(glfd->fd->inode->gfid)) {
+            gf_smsg(fs->volname, GF_LOG_INFO, 0, API_MSG_OPENFD_SKIPPED,
+                    "glfd=%p", glfd, "glfd->fd=%p", glfd->fd, "subvol=%s",
+                    graphid_str(subvol), "id=%d", subvol->graph->id, NULL);
+            /* create in progress, defer */
+            continue;
+        }
 
-xlator_t *
-__glfs_active_subvol (struct glfs *fs)
-{
-	xlator_t      *new_subvol = NULL;
-	int            ret = -1;
-	inode_t       *new_cwd = NULL;
-
-	if (!fs->next_subvol)
-		return fs->active_subvol;
-
-	new_subvol = fs->next_subvol;
-
-	ret = __glfs_first_lookup (fs, new_subvol);
-	if (ret) {
-		gf_log (fs->volname, GF_LOG_INFO,
-			"first lookup on graph %s (%d) failed (%s)",
-			graphid_str (new_subvol), new_subvol->graph->id,
-			strerror (errno));
-		return NULL;
-	}
-
-	if (fs->cwd) {
-		new_cwd = __glfs_refresh_inode (fs, new_subvol, fs->cwd);
-
-		if (!new_cwd) {
-			char buf1[64];
-			gf_log (fs->volname, GF_LOG_INFO,
-				"cwd refresh of %s graph %s (%d) failed (%s)",
-				uuid_utoa_r (fs->cwd->gfid, buf1),
-				graphid_str (new_subvol),
-				new_subvol->graph->id, strerror (errno));
-			return NULL;
-		}
-	}
-
-	__glfs_migrate_openfds (fs, new_subvol);
-
-	/* switching @active_subvol and @cwd
-	   should be atomic
-	*/
-	fs->old_subvol = fs->active_subvol;
-	fs->active_subvol = fs->next_subvol;
-	fs->next_subvol = NULL;
-
-	if (new_cwd) {
-		__glfs_cwd_set (fs, new_cwd);
-		inode_unref (new_cwd);
-	}
-
-	gf_log (fs->volname, GF_LOG_INFO, "switched to graph %s (%d)",
-		graphid_str (new_subvol), new_subvol->graph->id);
-
-	return new_subvol;
+        fd = __glfs_migrate_fd(fs, subvol, glfd);
+        if (fd) {
+            fd_unref(glfd->fd);
+            glfd->fd = fd;
+        }
+    }
 }
 
+/* Note that though it appears that this function executes under fs->mutex,
+ * it is not fully executed under fs->mutex. i.e. there are functions like
+ * __glfs_first_lookup, __glfs_refresh_inode, __glfs_migrate_openfds which
+ * unlocks fs->mutex before sending any network fop, and reacquire fs->mutex
+ * once the fop is complete. Hence the variable read from fs at the start of the
+ * function need not have the same value by the end of the function.
+ */
 xlator_t *
-priv_glfs_active_subvol (struct glfs *fs)
+__glfs_active_subvol(struct glfs *fs)
 {
-	xlator_t      *subvol = NULL;
-	xlator_t      *old_subvol = NULL;
+    xlator_t *new_subvol = NULL;
+    int ret = -1;
+    inode_t *new_cwd = NULL;
+
+    if (!fs->next_subvol)
+        return fs->active_subvol;
+
+    new_subvol = fs->mip_subvol = fs->next_subvol;
+    fs->next_subvol = NULL;
+
+    ret = __glfs_first_lookup(fs, new_subvol);
+    if (ret) {
+        gf_smsg(fs->volname, GF_LOG_INFO, errno,
+                API_MSG_FIRST_LOOKUP_GRAPH_FAILED, "subvol=%s",
+                graphid_str(new_subvol), "id=%d", new_subvol->graph->id,
+                "err=%s", strerror(errno), NULL);
+        return NULL;
+    }
+
+    if (fs->cwd) {
+        new_cwd = __glfs_refresh_inode(fs, new_subvol, fs->cwd, _gf_false);
+
+        if (!new_cwd) {
+            char buf1[64];
+            gf_smsg(fs->volname, GF_LOG_INFO, errno,
+                    API_MSG_CWD_GRAPH_REF_FAILED, "buf=%s",
+                    uuid_utoa_r(fs->cwd->gfid, buf1), "subvol=%s",
+                    graphid_str(new_subvol), "id=%d", new_subvol->graph->id,
+                    "err=%s", strerror(errno), NULL);
+            return NULL;
+        }
+    }
 
-	glfs_lock (fs);
-	{
-		subvol = __glfs_active_subvol (fs);
+    __glfs_migrate_openfds(fs, new_subvol);
+    /* TODO: Migrate the fds and inodes which have leases to the new graph
+     * (issue #350)*/
 
-		if (subvol)
-			subvol->winds++;
+    /* switching @active_subvol and @cwd
+       should be atomic
+    */
+    fs->old_subvol = fs->active_subvol;
+    fs->active_subvol = fs->mip_subvol;
+    fs->mip_subvol = NULL;
 
-		if (fs->old_subvol) {
-			old_subvol = fs->old_subvol;
-			fs->old_subvol = NULL;
-			old_subvol->switched = 1;
-		}
-	}
-	glfs_unlock (fs);
+    if (new_cwd) {
+        __glfs_cwd_set(fs, new_cwd);
+        inode_unref(new_cwd);
+    }
 
-	if (old_subvol)
-		priv_glfs_subvol_done (fs, old_subvol);
+    gf_smsg(fs->volname, GF_LOG_INFO, 0, API_MSG_SWITCHED_GRAPH, "subvol=%s",
+            graphid_str(new_subvol), "id=%d", new_subvol->graph->id, NULL);
 
-	return subvol;
+    return new_subvol;
 }
 
-GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_active_subvol, 3.4.0);
-
+GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_subvol_done, 3.4.0)
 void
-priv_glfs_subvol_done (struct glfs *fs, xlator_t *subvol)
+priv_glfs_subvol_done(struct glfs *fs, xlator_t *subvol)
 {
-	int ref = 0;
-	xlator_t *active_subvol = NULL;
-
-	if (!subvol)
-		return;
-
-	glfs_lock (fs);
-	{
-		ref = (--subvol->winds);
-		active_subvol = fs->active_subvol;
-	}
-	glfs_unlock (fs);
-
-	if (ref == 0) {
-		assert (subvol != active_subvol);
-		xlator_notify (subvol, GF_EVENT_PARENT_DOWN, subvol, NULL);
-	}
+    int ref = 0;
+    xlator_t *active_subvol = NULL;
+
+    if (!subvol)
+        return;
+
+    /* For decrementing subvol->wind ref count we need not check/wait for
+     * migration-in-progress flag.
+     * Also glfs_subvol_done is called in call-back path therefore waiting
+     * for migration-in-progress flag can lead to dead-lock.
+     */
+    glfs_lock(fs, _gf_false);
+    {
+        ref = (--subvol->winds);
+        active_subvol = fs->active_subvol;
+    }
+    glfs_unlock(fs);
+
+    if (ref == 0) {
+        assert(subvol != active_subvol);
+        xlator_notify(subvol, GF_EVENT_PARENT_DOWN, subvol, NULL);
+    }
 }
 
-GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_subvol_done, 3.4.0);
+GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_active_subvol, 3.4.0)
+xlator_t *
+priv_glfs_active_subvol(struct glfs *fs)
+{
+    xlator_t *subvol = NULL;
+    xlator_t *old_subvol = NULL;
+
+    glfs_lock(fs, _gf_true);
+    {
+        subvol = __glfs_active_subvol(fs);
+
+        if (subvol)
+            subvol->winds++;
+
+        if (fs->old_subvol) {
+            old_subvol = fs->old_subvol;
+            fs->old_subvol = NULL;
+            old_subvol->switched = 1;
+        }
+    }
+    glfs_unlock(fs);
+
+    if (old_subvol)
+        priv_glfs_subvol_done(fs, old_subvol);
+
+    return subvol;
+}
 
 int
-__glfs_cwd_set (struct glfs *fs, inode_t *inode)
+__glfs_cwd_set(struct glfs *fs, inode_t *inode)
 {
-	if (inode->table->xl != fs->active_subvol) {
-		inode = __glfs_refresh_inode (fs, fs->active_subvol, inode);
-		if (!inode)
-			return -1;
-	} else {
-		inode_ref (inode);
-	}
+    if (inode->table->xl != fs->active_subvol) {
+        inode = __glfs_refresh_inode(fs, fs->active_subvol, inode, _gf_false);
+        if (!inode)
+            return -1;
+    } else {
+        inode_ref(inode);
+    }
 
-	if (fs->cwd)
-		inode_unref (fs->cwd);
+    if (fs->cwd)
+        inode_unref(fs->cwd);
 
-	fs->cwd = inode;
+    fs->cwd = inode;
 
-	return 0;
+    return 0;
 }
 
-
 int
-glfs_cwd_set (struct glfs *fs, inode_t *inode)
+glfs_cwd_set(struct glfs *fs, inode_t *inode)
 {
-	int ret = 0;
+    int ret = 0;
 
-	glfs_lock (fs);
-	{
-		ret = __glfs_cwd_set (fs, inode);
-	}
-	glfs_unlock (fs);
+    glfs_lock(fs, _gf_true);
+    {
+        ret = __glfs_cwd_set(fs, inode);
+    }
+    glfs_unlock(fs);
 
-	return ret;
+    return ret;
 }
 
-
 inode_t *
-__glfs_cwd_get (struct glfs *fs)
+__glfs_cwd_get(struct glfs *fs)
 {
-	inode_t *cwd = NULL;
+    inode_t *cwd = NULL;
 
-	if (!fs->cwd)
-		return NULL;
+    if (!fs->cwd)
+        return NULL;
 
-	if (fs->cwd->table->xl == fs->active_subvol) {
-		cwd = inode_ref (fs->cwd);
-		return cwd;
-	}
+    if (fs->cwd->table->xl == fs->active_subvol) {
+        cwd = inode_ref(fs->cwd);
+        return cwd;
+    }
 
-	cwd = __glfs_refresh_inode (fs, fs->active_subvol, fs->cwd);
+    cwd = __glfs_refresh_inode(fs, fs->active_subvol, fs->cwd, _gf_false);
 
-	return cwd;
+    return cwd;
 }
 
 inode_t *
-glfs_cwd_get (struct glfs *fs)
+glfs_cwd_get(struct glfs *fs)
 {
-	inode_t *cwd = NULL;
+    inode_t *cwd = NULL;
 
-	glfs_lock (fs);
-	{
-		cwd = __glfs_cwd_get (fs);
-	}
-	glfs_unlock (fs);
+    glfs_lock(fs, _gf_true);
+    {
+        cwd = __glfs_cwd_get(fs);
+    }
+    glfs_unlock(fs);
 
-	return cwd;
+    return cwd;
 }
 
 inode_t *
-__glfs_resolve_inode (struct glfs *fs, xlator_t *subvol,
-		    struct glfs_object *object)
+__glfs_resolve_inode(struct glfs *fs, xlator_t *subvol,
+                     struct glfs_object *object)
 {
-	inode_t *inode = NULL;
+    inode_t *inode = NULL;
+    gf_boolean_t lookup_needed = _gf_false;
 
-	if (object->inode->table->xl == subvol)
-		return inode_ref (object->inode);
+    lookup_needed = inode_needs_lookup(object->inode, THIS);
 
-	inode = __glfs_refresh_inode (fs, fs->active_subvol,
-					object->inode);
-	if (!inode)
-		return NULL;
+    if (!lookup_needed && object->inode->table->xl == subvol)
+        return inode_ref(object->inode);
 
-	if (subvol == fs->active_subvol) {
-		inode_unref (object->inode);
-		object->inode = inode_ref (inode);
-	}
+    inode = __glfs_refresh_inode(fs, fs->active_subvol, object->inode,
+                                 lookup_needed);
+    if (!inode)
+        return NULL;
 
-	return inode;
+    if (subvol == fs->active_subvol) {
+        inode_unref(object->inode);
+        object->inode = inode_ref(inode);
+    }
+
+    return inode;
 }
 
 inode_t *
-glfs_resolve_inode (struct glfs *fs, xlator_t *subvol,
-		    struct glfs_object *object)
+glfs_resolve_inode(struct glfs *fs, xlator_t *subvol,
+                   struct glfs_object *object)
 {
-	inode_t *inode = NULL;
+    inode_t *inode = NULL;
 
-	glfs_lock (fs);
-	{
-		inode = __glfs_resolve_inode(fs, subvol, object);
-	}
-	glfs_unlock (fs);
+    glfs_lock(fs, _gf_true);
+    {
+        inode = __glfs_resolve_inode(fs, subvol, object);
+    }
+    glfs_unlock(fs);
 
-	return inode;
+    return inode;
 }
 
 int
-glfs_create_object (loc_t *loc, struct glfs_object **retobject)
+glfs_create_object(loc_t *loc, struct glfs_object **retobject)
 {
-	struct glfs_object *object = NULL;
+    struct glfs_object *object = NULL;
+
+    object = GF_CALLOC(1, sizeof(struct glfs_object), glfs_mt_glfs_object_t);
+    if (object == NULL) {
+        errno = ENOMEM;
+        return -1;
+    }
+
+    object->inode = loc->inode;
+    gf_uuid_copy(object->gfid, object->inode->gfid);
 
-	object = GF_CALLOC (1, sizeof(struct glfs_object),
-			    glfs_mt_glfs_object_t);
-	if (object == NULL) {
-		errno = ENOMEM;
-		return -1;
-	}
+    /* we hold the reference */
+    loc->inode = NULL;
 
-	object->inode = loc->inode;
-	uuid_copy (object->gfid, object->inode->gfid);
+    *retobject = object;
 
-	/* we hold the reference */
-	loc->inode = NULL;
+    return 0;
+}
 
-	*retobject = object;
+struct glfs_object *
+glfs_h_resolve_symlink(struct glfs *fs, struct glfs_object *object)
+{
+    xlator_t *subvol = NULL;
+    loc_t sym_loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    char *lpath = NULL;
+    int ret = 0;
+    struct glfs_object *target_object = NULL;
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    ret = glfs_resolve_symlink(fs, subvol, object->inode, &lpath);
+    if (ret < 0)
+        goto out;
+
+    ret = glfs_resolve_at(fs, subvol, NULL, lpath, &sym_loc, &iatt,
+                          /* always recurisvely follow while
+                            following symlink
+                          */
+                          1, 0);
+    if (ret == 0)
+        ret = glfs_create_object(&sym_loc, &target_object);
 
-	return 0;
+out:
+    loc_wipe(&sym_loc);
+    GF_FREE(lpath);
+    return target_object;
 }
diff --git a/api/src/glfs.c b/api/src/glfs.c
index 41c6da576a7..b4bf1423f6d 100644
--- a/api/src/glfs.c
+++ b/api/src/glfs.c
@@ -1,5 +1,5 @@
 /*
-  Copyright (c) 2012 Red Hat, Inc. <http://www.redhat.com>
+  Copyright (c) 2012-2018 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
   This file is licensed to you under your choice of the GNU Lesser
@@ -8,10 +8,8 @@
   cases as published by the Free Software Foundation.
 */
 
-
 /*
   TODO:
-  - merge locks in glfs_posix_lock for lock self-healing
   - set proper pid/lk_owner to call frames (currently buried in syncop)
   - fix logging.c/h to store logfp and loglevel in glusterfs_ctx_t and
     reach it via THIS.
@@ -32,813 +30,1777 @@
 #include <sys/types.h>
 #include <unistd.h>
 #include <limits.h>
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
+#ifdef GF_LINUX_HOST_OS
+#include <sys/prctl.h>
 #endif
 
-#include "glusterfs.h"
-#include "logging.h"
-#include "stack.h"
-#include "event.h"
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/stack.h>
+#include <glusterfs/gf-event.h>
 #include "glfs-mem-types.h"
-#include "common-utils.h"
-#include "syncop.h"
-#include "call-stub.h"
+#include <glusterfs/common-utils.h>
+#include <glusterfs/syncop.h>
+#include <glusterfs/call-stub.h>
+#include <glusterfs/hashfn.h>
+#include "rpc-clnt.h"
+#include <glusterfs/statedump.h>
+#include <glusterfs/syscall.h>
 
+#include "gfapi-messages.h"
 #include "glfs.h"
 #include "glfs-internal.h"
-#include "hashfn.h"
-#include "rpc-clnt.h"
-
 
 static gf_boolean_t
-vol_assigned (cmd_args_t *args)
+vol_assigned(cmd_args_t *args)
 {
-	return args->volfile || args->volfile_server;
+    return args->volfile || args->volfile_server;
 }
 
-
 static int
-glusterfs_ctx_defaults_init (glusterfs_ctx_t *ctx)
-{
-	call_pool_t   *pool = NULL;
-	int	       ret = -1;
-
-	xlator_mem_acct_init (THIS, glfs_mt_end + 1);
-
-	if (!ctx) {
-		goto err;
-	}
-
-	ctx->process_uuid = generate_glusterfs_ctx_id ();
-	if (!ctx->process_uuid) {
-		goto err;
-	}
-
-	ctx->page_size	= 128 * GF_UNIT_KB;
-
-	ctx->iobuf_pool = iobuf_pool_new ();
-	if (!ctx->iobuf_pool) {
-		goto err;
-	}
-
-	ctx->event_pool = event_pool_new (DEFAULT_EVENT_POOL_SIZE);
-	if (!ctx->event_pool) {
-		goto err;
-	}
-
-	ctx->env = syncenv_new (0, 0, 0);
-	if (!ctx->env) {
-		goto err;
-	}
-
-	pool = GF_CALLOC (1, sizeof (call_pool_t),
-			  glfs_mt_call_pool_t);
-	if (!pool) {
-		goto err;
-	}
-
-	/* frame_mem_pool size 112 * 4k */
-	pool->frame_mem_pool = mem_pool_new (call_frame_t, 4096);
-	if (!pool->frame_mem_pool) {
-		goto err;
-	}
-	/* stack_mem_pool size 256 * 1024 */
-	pool->stack_mem_pool = mem_pool_new (call_stack_t, 1024);
-	if (!pool->stack_mem_pool) {
-		goto err;
-	}
-
-	ctx->stub_mem_pool = mem_pool_new (call_stub_t, 1024);
-	if (!ctx->stub_mem_pool) {
-		goto err;
-	}
-
-	ctx->dict_pool = mem_pool_new (dict_t, GF_MEMPOOL_COUNT_OF_DICT_T);
-	if (!ctx->dict_pool)
-		goto err;
-
-	ctx->dict_pair_pool = mem_pool_new (data_pair_t,
-					    GF_MEMPOOL_COUNT_OF_DATA_PAIR_T);
-	if (!ctx->dict_pair_pool)
-		goto err;
-
-	ctx->dict_data_pool = mem_pool_new (data_t, GF_MEMPOOL_COUNT_OF_DATA_T);
-	if (!ctx->dict_data_pool)
-		goto err;
-
-        ctx->logbuf_pool = mem_pool_new (log_buf_t,
-                                         GF_MEMPOOL_COUNT_OF_LRU_BUF_T);
-        if (!ctx->logbuf_pool)
-                goto err;
-
-	INIT_LIST_HEAD (&pool->all_frames);
-	INIT_LIST_HEAD (&ctx->cmd_args.xlator_options);
-        INIT_LIST_HEAD (&ctx->cmd_args.volfile_servers);
-
-	LOCK_INIT (&pool->lock);
-	ctx->pool = pool;
-
-	pthread_mutex_init (&(ctx->lock), NULL);
-
-	ret = 0;
+glusterfs_ctx_defaults_init(glusterfs_ctx_t *ctx)
+{
+    call_pool_t *pool = NULL;
+    int ret = -1;
+
+    if (!ctx) {
+        goto err;
+    }
+
+    ret = xlator_mem_acct_init(THIS, glfs_mt_end + 1);
+    if (ret != 0) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, ENOMEM, API_MSG_MEM_ACCT_INIT_FAILED,
+                NULL);
+        return ret;
+    }
+
+    /* reset ret to -1 so that we don't need to explicitly
+     * set it in all error paths before "goto err"
+     */
+
+    ret = -1;
+
+    ctx->process_uuid = generate_glusterfs_ctx_id();
+    if (!ctx->process_uuid) {
+        goto err;
+    }
+
+    ctx->page_size = 128 * GF_UNIT_KB;
+
+    ctx->iobuf_pool = iobuf_pool_new();
+    if (!ctx->iobuf_pool) {
+        goto err;
+    }
+
+    ctx->event_pool = gf_event_pool_new(DEFAULT_EVENT_POOL_SIZE,
+                                        STARTING_EVENT_THREADS);
+    if (!ctx->event_pool) {
+        goto err;
+    }
+
+    ctx->env = syncenv_new(0, 0, 0);
+    if (!ctx->env) {
+        goto err;
+    }
+
+    pool = GF_CALLOC(1, sizeof(call_pool_t), glfs_mt_call_pool_t);
+    if (!pool) {
+        goto err;
+    }
+
+    /* frame_mem_pool size 112 * 4k */
+    pool->frame_mem_pool = mem_pool_new(call_frame_t, 4096);
+    if (!pool->frame_mem_pool) {
+        goto err;
+    }
+    /* stack_mem_pool size 256 * 1024 */
+    pool->stack_mem_pool = mem_pool_new(call_stack_t, 1024);
+    if (!pool->stack_mem_pool) {
+        goto err;
+    }
+
+    ctx->stub_mem_pool = mem_pool_new(call_stub_t, 1024);
+    if (!ctx->stub_mem_pool) {
+        goto err;
+    }
+
+    ctx->dict_pool = mem_pool_new(dict_t, GF_MEMPOOL_COUNT_OF_DICT_T);
+    if (!ctx->dict_pool)
+        goto err;
+
+    ctx->dict_pair_pool = mem_pool_new(data_pair_t,
+                                       GF_MEMPOOL_COUNT_OF_DATA_PAIR_T);
+    if (!ctx->dict_pair_pool)
+        goto err;
+
+    ctx->dict_data_pool = mem_pool_new(data_t, GF_MEMPOOL_COUNT_OF_DATA_T);
+    if (!ctx->dict_data_pool)
+        goto err;
+
+    ctx->logbuf_pool = mem_pool_new(log_buf_t, GF_MEMPOOL_COUNT_OF_LRU_BUF_T);
+    if (!ctx->logbuf_pool)
+        goto err;
+
+    INIT_LIST_HEAD(&pool->all_frames);
+    INIT_LIST_HEAD(&ctx->cmd_args.xlator_options);
+    INIT_LIST_HEAD(&ctx->cmd_args.volfile_servers);
+
+    LOCK_INIT(&pool->lock);
+    ctx->pool = pool;
+
+    ret = 0;
 err:
-	if (ret && pool) {
-		if (pool->frame_mem_pool)
-			mem_pool_destroy (pool->frame_mem_pool);
-		if (pool->stack_mem_pool)
-			mem_pool_destroy (pool->stack_mem_pool);
-		GF_FREE (pool);
-	}
-
-	if (ret && ctx) {
-		if (ctx->stub_mem_pool)
-			mem_pool_destroy (ctx->stub_mem_pool);
-		if (ctx->dict_pool)
-			mem_pool_destroy (ctx->dict_pool);
-		if (ctx->dict_data_pool)
-			mem_pool_destroy (ctx->dict_data_pool);
-		if (ctx->dict_pair_pool)
-			mem_pool_destroy (ctx->dict_pair_pool);
-                if (ctx->logbuf_pool)
-                        mem_pool_destroy (ctx->logbuf_pool);
-	}
-
-	return ret;
+    if (ret && pool) {
+        if (pool->frame_mem_pool)
+            mem_pool_destroy(pool->frame_mem_pool);
+        if (pool->stack_mem_pool)
+            mem_pool_destroy(pool->stack_mem_pool);
+        GF_FREE(pool);
+    }
+
+    if (ret && ctx) {
+        if (ctx->stub_mem_pool)
+            mem_pool_destroy(ctx->stub_mem_pool);
+        if (ctx->dict_pool)
+            mem_pool_destroy(ctx->dict_pool);
+        if (ctx->dict_data_pool)
+            mem_pool_destroy(ctx->dict_data_pool);
+        if (ctx->dict_pair_pool)
+            mem_pool_destroy(ctx->dict_pair_pool);
+        if (ctx->logbuf_pool)
+            mem_pool_destroy(ctx->logbuf_pool);
+    }
+
+    return ret;
 }
 
-
 static int
-create_master (struct glfs *fs)
+create_master(struct glfs *fs)
 {
-	int		 ret = 0;
-	xlator_t	*master = NULL;
-
-	master = GF_CALLOC (1, sizeof (*master),
-			    glfs_mt_xlator_t);
-	if (!master)
-		goto err;
+    int ret = 0;
+    xlator_t *master = NULL;
 
-	master->name = gf_strdup ("gfapi");
-	if (!master->name)
-		goto err;
+    master = GF_CALLOC(1, sizeof(*master), glfs_mt_xlator_t);
+    if (!master)
+        goto err;
 
-	if (xlator_set_type (master, "mount/api") == -1) {
-		gf_log ("glfs", GF_LOG_ERROR,
-			"master xlator for %s initialization failed",
-			fs->volname);
-		goto err;
-	}
+    master->name = gf_strdup("gfapi");
+    if (!master->name)
+        goto err;
 
-	master->ctx	 = fs->ctx;
-	master->private	 = fs;
-	master->options	 = get_new_dict ();
-	if (!master->options)
-		goto err;
+    if (xlator_set_type(master, "mount/api") == -1) {
+        gf_smsg("glfs", GF_LOG_ERROR, 0, API_MSG_MASTER_XLATOR_INIT_FAILED,
+                "name=%s", fs->volname, NULL);
+        goto err;
+    }
 
+    master->ctx = fs->ctx;
+    master->private = fs;
+    master->options = dict_new();
+    if (!master->options)
+        goto err;
 
-	ret = xlator_init (master);
-	if (ret) {
-		gf_log ("glfs", GF_LOG_ERROR,
-			"failed to initialize gfapi translator");
-		goto err;
-	}
+    ret = xlator_init(master);
+    if (ret) {
+        gf_smsg("glfs", GF_LOG_ERROR, 0, API_MSG_GFAPI_XLATOR_INIT_FAILED,
+                NULL);
+        goto err;
+    }
 
-	fs->ctx->master = master;
-	THIS = master;
+    fs->ctx->master = master;
+    THIS = master;
 
-	return 0;
+    return 0;
 
 err:
-	if (master) {
-		xlator_destroy (master);
-	}
+    if (master) {
+        xlator_destroy(master);
+    }
 
-	return -1;
+    return -1;
 }
 
-
 static FILE *
-get_volfp (struct glfs *fs)
+get_volfp(struct glfs *fs)
 {
-	cmd_args_t  *cmd_args = NULL;
-	FILE	    *specfp = NULL;
+    cmd_args_t *cmd_args = NULL;
+    FILE *specfp = NULL;
 
-	cmd_args = &fs->ctx->cmd_args;
+    cmd_args = &fs->ctx->cmd_args;
 
-	if ((specfp = fopen (cmd_args->volfile, "r")) == NULL) {
-		gf_log ("glfs", GF_LOG_ERROR,
-			"volume file %s: %s",
-			cmd_args->volfile,
-			strerror (errno));
-		return NULL;
-	}
+    if ((specfp = fopen(cmd_args->volfile, "r")) == NULL) {
+        gf_smsg("glfs", GF_LOG_ERROR, errno, API_MSG_VOLFILE_OPEN_FAILED,
+                "file=%s", cmd_args->volfile, "err=%s", strerror(errno), NULL);
+        return NULL;
+    }
 
-	gf_log ("glfs", GF_LOG_DEBUG,
-		"loading volume file %s", cmd_args->volfile);
+    gf_msg_debug("glfs", 0, "loading volume file %s", cmd_args->volfile);
 
-	return specfp;
+    return specfp;
 }
 
-
 int
-glfs_volumes_init (struct glfs *fs)
+glfs_volumes_init(struct glfs *fs)
 {
-	FILE		   *fp = NULL;
-	cmd_args_t	   *cmd_args = NULL;
-	int		    ret = 0;
+    FILE *fp = NULL;
+    cmd_args_t *cmd_args = NULL;
+    int ret = 0;
 
-	cmd_args = &fs->ctx->cmd_args;
+    cmd_args = &fs->ctx->cmd_args;
 
-	if (!vol_assigned (cmd_args))
-		return -1;
+    if (!vol_assigned(cmd_args))
+        return -1;
 
-	if (cmd_args->volfile_server) {
-		ret = glfs_mgmt_init (fs);
-		goto out;
-	}
+    if (sys_access(SECURE_ACCESS_FILE, F_OK) == 0) {
+        fs->ctx->secure_mgmt = 1;
+        fs->ctx->ssl_cert_depth = glusterfs_read_secure_access_file();
+    }
 
-	fp = get_volfp (fs);
+    if (cmd_args->volfile_server) {
+        ret = glfs_mgmt_init(fs);
+        goto out;
+    }
 
-	if (!fp) {
-		gf_log ("glfs", GF_LOG_ERROR,
-			"Cannot reach volume specification file");
-		ret = -1;
-		goto out;
-	}
+    fp = get_volfp(fs);
 
-	ret = glfs_process_volfp (fs, fp);
-	if (ret)
-		goto out;
+    if (!fp) {
+        gf_smsg("glfs", GF_LOG_ERROR, ENOENT, API_MSG_VOL_SPEC_FILE_ERROR,
+                NULL);
+        ret = -1;
+        goto out;
+    }
+
+    ret = glfs_process_volfp(fs, fp);
+    if (ret)
+        goto out;
 
 out:
-	return ret;
+    return ret;
 }
 
-
 ///////////////////////////////////////////////////////////////////////////////
 
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_set_xlator_option, 3.4.0)
 int
-pub_glfs_set_xlator_option (struct glfs *fs, const char *xlator,
-                            const char *key, const char *value)
+pub_glfs_set_xlator_option(struct glfs *fs, const char *xlator, const char *key,
+                           const char *value)
 {
-	xlator_cmdline_option_t *option = NULL;
+    xlator_cmdline_option_t *option = NULL;
 
-	option = GF_CALLOC (1, sizeof (*option),
-			    glfs_mt_xlator_cmdline_option_t);
-	if (!option)
-		goto enomem;
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
 
-	INIT_LIST_HEAD (&option->cmd_args);
+    option = GF_CALLOC(1, sizeof(*option), glfs_mt_xlator_cmdline_option_t);
+    if (!option)
+        goto enomem;
 
-	option->volume = gf_strdup (xlator);
-	if (!option->volume)
-		goto enomem;
-	option->key = gf_strdup (key);
-	if (!option->key)
-		goto enomem;
-	option->value = gf_strdup (value);
-	if (!option->value)
-		goto enomem;
+    INIT_LIST_HEAD(&option->cmd_args);
 
-	list_add (&option->cmd_args, &fs->ctx->cmd_args.xlator_options);
+    option->volume = gf_strdup(xlator);
+    if (!option->volume)
+        goto enomem;
+    option->key = gf_strdup(key);
+    if (!option->key)
+        goto enomem;
+    option->value = gf_strdup(value);
+    if (!option->value)
+        goto enomem;
 
-	return 0;
-enomem:
-	errno = ENOMEM;
+    list_add(&option->cmd_args, &fs->ctx->cmd_args.xlator_options);
 
-	if (!option)
-		return -1;
+    __GLFS_EXIT_FS;
 
-	GF_FREE (option->volume);
-	GF_FREE (option->key);
-	GF_FREE (option->value);
-	GF_FREE (option);
+    return 0;
+enomem:
+    errno = ENOMEM;
 
-	return -1;
-}
+    if (!option) {
+        __GLFS_EXIT_FS;
+        return -1;
+    }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_set_xlator_option, 3.4.0);
+    GF_FREE(option->volume);
+    GF_FREE(option->key);
+    GF_FREE(option->value);
+    GF_FREE(option);
 
+    __GLFS_EXIT_FS;
 
+invalid_fs:
+    return -1;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_unset_volfile_server, 3.5.1)
 int
-pub_glfs_unset_volfile_server (struct glfs *fs, const char *transport,
-                               const char *host, const int port)
+pub_glfs_unset_volfile_server(struct glfs *fs, const char *transport,
+                              const char *host, const int port)
 {
-        cmd_args_t       *cmd_args = NULL;
-        server_cmdline_t *server = NULL;
-        int               ret = -1;
-
-        if (!transport || !host || !port) {
-                errno = EINVAL;
-                return ret;
-        }
-
-        cmd_args = &fs->ctx->cmd_args;
-        list_for_each_entry(server, &cmd_args->curr_server->list, list) {
-                if ((!strcmp(server->volfile_server, host) &&
-                     !strcmp(server->transport, transport) &&
-                     (server->port == port))) {
-                        list_del (&server->list);
-                        ret = 0;
-                        goto out;
-                }
+    cmd_args_t *cmd_args = NULL;
+    server_cmdline_t *server = NULL;
+    server_cmdline_t *tmp = NULL;
+    char *transport_val = NULL;
+    int port_val = 0;
+    int ret = -1;
+
+    if (!fs || !host) {
+        errno = EINVAL;
+        return ret;
+    }
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    cmd_args = &fs->ctx->cmd_args;
+
+    if (transport) {
+        transport_val = gf_strdup(transport);
+    } else {
+        transport_val = gf_strdup(GF_DEFAULT_VOLFILE_TRANSPORT);
+    }
+
+    if (!transport_val) {
+        errno = ENOMEM;
+        goto out;
+    }
+
+    if (port) {
+        port_val = port;
+    } else {
+        port_val = GF_DEFAULT_BASE_PORT;
+    }
+
+    list_for_each_entry_safe(server, tmp, &cmd_args->curr_server->list, list)
+    {
+        if (!server->volfile_server || !server->transport)
+            continue;
+        if ((!strcmp(server->volfile_server, host) &&
+             !strcmp(server->transport, transport_val) &&
+             (server->port == port_val))) {
+            list_del(&server->list);
+            ret = 0;
+            goto out;
         }
+    }
 
 out:
-        return ret;
-}
-
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_unset_volfile_server, 3.5.1);
+    GF_FREE(transport_val);
+    __GLFS_EXIT_FS;
 
+invalid_fs:
+    return ret;
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_set_volfile_server, 3.4.0)
 int
-pub_glfs_set_volfile_server (struct glfs *fs, const char *transport,
-                             const char *host, int port)
+pub_glfs_set_volfile_server(struct glfs *fs, const char *transport,
+                            const char *host, int port)
 {
-        cmd_args_t            *cmd_args = NULL;
-        server_cmdline_t      *server = NULL;
-        server_cmdline_t      *tmp = NULL;
-        int                    ret = -1;
+    cmd_args_t *cmd_args = NULL;
+    int ret = -1;
+    char *server_host = NULL;
+    char *server_transport = NULL;
 
-        if (!transport || !host) {
-                errno = EINVAL;
-                return ret;
-        }
-
-        cmd_args = &fs->ctx->cmd_args;
-
-        cmd_args->max_connect_attempts = 1;
-
-        server = GF_CALLOC (1, sizeof (server_cmdline_t),
-                            glfs_mt_server_cmdline_t);
-
-        if (!server) {
-                errno = ENOMEM;
-                goto out;
-        }
-
-        INIT_LIST_HEAD (&server->list);
-
-        server->volfile_server = gf_strdup (host);
-        if (!server->volfile_server) {
-                errno = ENOMEM;
-                goto out;
-        }
-
-        server->transport = gf_strdup (transport);
-        if (!server->transport) {
-                errno = ENOMEM;
-                goto out;
+    if (!fs || !host) {
+        errno = EINVAL;
+        return ret;
+    }
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    cmd_args = &fs->ctx->cmd_args;
+    cmd_args->max_connect_attempts = 1;
+
+    server_host = gf_strdup(host);
+    if (!server_host) {
+        errno = ENOMEM;
+        goto out;
+    }
+
+    if (transport) {
+        /* volfile fetch support over tcp|unix only */
+        if (!strcmp(transport, "tcp") || !strcmp(transport, "unix")) {
+            server_transport = gf_strdup(transport);
+        } else if (!strcmp(transport, "rdma")) {
+            server_transport = gf_strdup(GF_DEFAULT_VOLFILE_TRANSPORT);
+            gf_smsg("glfs", GF_LOG_WARNING, EINVAL, API_MSG_TRANS_RDMA_DEP,
+                    NULL);
+        } else {
+            gf_smsg("glfs", GF_LOG_TRACE, EINVAL, API_MSG_TRANS_NOT_SUPPORTED,
+                    "transport=%s", transport, NULL);
+            goto out;
         }
+    } else {
+        server_transport = gf_strdup(GF_DEFAULT_VOLFILE_TRANSPORT);
+    }
+
+    if (!server_transport) {
+        errno = ENOMEM;
+        goto out;
+    }
+
+    if (!port) {
+        port = GF_DEFAULT_BASE_PORT;
+    }
+
+    if (!strcmp(server_transport, "unix")) {
+        port = 0;
+    }
+
+    ret = gf_set_volfile_server_common(cmd_args, server_host, server_transport,
+                                       port);
+    if (ret) {
+        gf_log("glfs", GF_LOG_ERROR, "failed to set volfile server: %s",
+               strerror(errno));
+    }
 
-        server->port = port;
+out:
+    if (server_host) {
+        GF_FREE(server_host);
+    }
 
-        if (!cmd_args->volfile_server) {
-                cmd_args->volfile_server = server->volfile_server;
-                cmd_args->volfile_server_transport = server->transport;
-                cmd_args->volfile_server_port = server->port;
-                cmd_args->curr_server = server;
-        }
+    if (server_transport) {
+        GF_FREE(server_transport);
+    }
 
-        list_for_each_entry(tmp, &cmd_args->volfile_servers, list) {
-                if ((!strcmp(tmp->volfile_server, host) &&
-                     !strcmp(tmp->transport, transport) &&
-                     (tmp->port == port))) {
-                        errno = EEXIST;
-                        ret = -1;
-                        goto out;
-                }
-        }
+    __GLFS_EXIT_FS;
 
-        list_add_tail (&server->list, &cmd_args->volfile_servers);
+invalid_fs:
+    return ret;
+}
 
-        ret = 0;
+/* *
+ * Used to free the arguments allocated by glfs_set_volfile_server()
+ */
+static void
+glfs_free_volfile_servers(cmd_args_t *cmd_args)
+{
+    server_cmdline_t *server = NULL;
+    server_cmdline_t *tmp = NULL;
+
+    GF_VALIDATE_OR_GOTO(THIS->name, cmd_args, out);
+
+    list_for_each_entry_safe(server, tmp, &cmd_args->volfile_servers, list)
+    {
+        list_del_init(&server->list);
+        GF_FREE(server->volfile_server);
+        GF_FREE(server->transport);
+        GF_FREE(server);
+    }
+    cmd_args->curr_server = NULL;
 out:
-        if (ret == -1) {
-                if (server) {
-                        GF_FREE (server->volfile_server);
-                        GF_FREE (server->transport);
-                        GF_FREE (server);
-                }
-        }
+    return;
+}
 
-        return ret;
+static void
+glfs_free_xlator_options(cmd_args_t *cmd_args)
+{
+    xlator_cmdline_option_t *xo = NULL;
+    xlator_cmdline_option_t *tmp_xo = NULL;
+
+    if (!&(cmd_args->xlator_options))
+        return;
+
+    list_for_each_entry_safe(xo, tmp_xo, &cmd_args->xlator_options, cmd_args)
+    {
+        list_del_init(&xo->cmd_args);
+        GF_FREE(xo->volume);
+        GF_FREE(xo->key);
+        GF_FREE(xo->value);
+        GF_FREE(xo);
+    }
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_set_volfile_server, 3.4.0);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_setfsuid, 3.4.2)
+int
+pub_glfs_setfsuid(uid_t fsuid)
+{
+    /* TODO:
+     * - Set the THIS and restore it appropriately
+     */
+    return syncopctx_setfsuid(&fsuid);
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_setfsgid, 3.4.2)
+int
+pub_glfs_setfsgid(gid_t fsgid)
+{
+    /* TODO:
+     * - Set the THIS and restore it appropriately
+     */
+    return syncopctx_setfsgid(&fsgid);
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_setfsgroups, 3.4.2)
 int
-pub_glfs_setfsuid (uid_t fsuid)
+pub_glfs_setfsgroups(size_t size, const gid_t *list)
 {
-	return syncopctx_setfsuid (&fsuid);
+    /* TODO:
+     * - Set the THIS and restore it appropriately
+     */
+    return syncopctx_setfsgroups(size, list);
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_setfsuid, 3.4.2);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_setfsleaseid, 4.0.0)
+int
+pub_glfs_setfsleaseid(glfs_leaseid_t leaseid)
+{
+    int ret = -1;
+    char *gleaseid = NULL;
+
+    gleaseid = gf_leaseid_get();
+    if (gleaseid) {
+        if (leaseid)
+            memcpy(gleaseid, leaseid, LEASE_ID_SIZE);
+        else /* reset leaseid */
+            memset(gleaseid, 0, LEASE_ID_SIZE);
+        ret = 0;
+    }
 
+    if (ret)
+        gf_log("glfs", GF_LOG_ERROR, "failed to set leaseid: %s",
+               strerror(errno));
+    return ret;
+}
 
 int
-pub_glfs_setfsgid (gid_t fsgid)
+get_fop_attr_glfd(dict_t **fop_attr, struct glfs_fd *glfd)
 {
-	return syncopctx_setfsgid (&fsgid);
+    char *leaseid = NULL;
+    int ret = 0;
+    gf_boolean_t dict_create = _gf_false;
+
+    leaseid = GF_MALLOC(LEASE_ID_SIZE, gf_common_mt_char);
+    GF_CHECK_ALLOC_AND_LOG("gfapi", leaseid, ret, "lease id alloc failed", out);
+    memcpy(leaseid, glfd->lease_id, LEASE_ID_SIZE);
+    if (*fop_attr == NULL) {
+        *fop_attr = dict_new();
+        dict_create = _gf_true;
+    }
+    GF_CHECK_ALLOC_AND_LOG("gfapi", *fop_attr, ret, "dict_new failed", out);
+    ret = dict_set_bin(*fop_attr, "lease-id", leaseid, LEASE_ID_SIZE);
+out:
+    if (ret) {
+        GF_FREE(leaseid);
+        if (dict_create) {
+            if (*fop_attr)
+                dict_unref(*fop_attr);
+            *fop_attr = NULL;
+        }
+    }
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_setfsgid, 3.4.2);
+int
+set_fop_attr_glfd(struct glfs_fd *glfd)
+{
+    char *lease_id = NULL;
+    int ret = -1;
 
+    lease_id = gf_existing_leaseid();
+    if (lease_id) {
+        memcpy(glfd->lease_id, lease_id, LEASE_ID_SIZE);
+        ret = 0;
+    }
+    return ret;
+}
 
 int
-pub_glfs_setfsgroups (size_t size, const gid_t *list)
+get_fop_attr_thrd_key(dict_t **fop_attr)
 {
-	return syncopctx_setfsgroups(size, list);
-}
+    char *existing_leaseid = NULL, *leaseid = NULL;
+    int ret = 0;
+    gf_boolean_t dict_create = _gf_false;
+
+    existing_leaseid = gf_existing_leaseid();
+    if (existing_leaseid) {
+        leaseid = GF_MALLOC(LEASE_ID_SIZE, gf_common_mt_char);
+        GF_CHECK_ALLOC_AND_LOG("gfapi", leaseid, ret, "lease id alloc failed",
+                               out);
+        memcpy(leaseid, existing_leaseid, LEASE_ID_SIZE);
+        if (*fop_attr == NULL) {
+            *fop_attr = dict_new();
+            dict_create = _gf_true;
+        }
+        GF_CHECK_ALLOC_AND_LOG("gfapi", *fop_attr, ret, "dict_new failed", out);
+        ret = dict_set_bin(*fop_attr, "lease-id", leaseid, LEASE_ID_SIZE);
+    }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_setfsgroups, 3.4.2);
+out:
+    if (ret) {
+        GF_FREE(leaseid);
+        if (dict_create) {
+            if (*fop_attr)
+                dict_unref(*fop_attr);
+            *fop_attr = NULL;
+        }
+    }
+    return ret;
+}
 
+void
+unset_fop_attr(dict_t **fop_attr)
+{
+    char *lease_id = NULL;
+    lease_id = gf_existing_leaseid();
+    if (lease_id)
+        memset(lease_id, 0, LEASE_ID_SIZE);
+    if (*fop_attr) {
+        dict_unref(*fop_attr);
+        *fop_attr = NULL;
+    }
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_from_glfd, 3.4.0)
 struct glfs *
-pub_glfs_from_glfd (struct glfs_fd *glfd)
+pub_glfs_from_glfd(struct glfs_fd *glfd)
 {
-	return glfd->fs;
+    if (glfd == NULL) {
+        errno = EBADF;
+        return NULL;
+    }
+
+    return glfd->fs;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_from_glfd, 3.4.0);
+static void
+glfs_fd_destroy(struct glfs_fd *glfd)
+{
+    if (!glfd)
+        return;
+
+    glfs_lock(glfd->fs, _gf_true);
+    {
+        list_del_init(&glfd->openfds);
+    }
+    glfs_unlock(glfd->fs);
+
+    if (glfd->fd) {
+        fd_unref(glfd->fd);
+        glfd->fd = NULL;
+    }
 
+    GF_FREE(glfd->readdirbuf);
+
+    GF_FREE(glfd);
+}
 
 struct glfs_fd *
-glfs_fd_new (struct glfs *fs)
+glfs_fd_new(struct glfs *fs)
 {
-	struct glfs_fd  *glfd = NULL;
+    struct glfs_fd *glfd = NULL;
 
-	glfd = GF_CALLOC (1, sizeof (*glfd), glfs_mt_glfs_fd_t);
-	if (!glfd)
-		return NULL;
+    glfd = GF_CALLOC(1, sizeof(*glfd), glfs_mt_glfs_fd_t);
+    if (!glfd)
+        return NULL;
 
-	glfd->fs = fs;
+    glfd->fs = fs;
 
-	INIT_LIST_HEAD (&glfd->openfds);
+    INIT_LIST_HEAD(&glfd->openfds);
 
-	return glfd;
-}
+    GF_REF_INIT(glfd, glfs_fd_destroy);
 
+    return glfd;
+}
 
 void
-glfs_fd_bind (struct glfs_fd *glfd)
+glfs_fd_bind(struct glfs_fd *glfd)
 {
-	struct glfs *fs = NULL;
+    struct glfs *fs = NULL;
 
-	fs = glfd->fs;
+    fs = glfd->fs;
 
-	glfs_lock (fs);
-	{
-		list_add_tail (&glfd->openfds, &fs->openfds);
-	}
-	glfs_unlock (fs);
+    glfs_lock(fs, _gf_true);
+    {
+        list_add_tail(&glfd->openfds, &fs->openfds);
+    }
+    glfs_unlock(fs);
 }
 
-void
-glfs_fd_destroy (struct glfs_fd *glfd)
+static void *
+glfs_poller(void *data)
+{
+    struct glfs *fs = NULL;
+
+    fs = data;
+
+    gf_event_dispatch(fs->ctx->event_pool);
+
+    return NULL;
+}
+
+static struct glfs *
+glfs_new_fs(const char *volname)
 {
-	if (!glfd)
-		return;
+    struct glfs *fs = NULL;
+
+    fs = CALLOC(1, sizeof(*fs));
+    if (!fs)
+        return NULL;
+
+    INIT_LIST_HEAD(&fs->openfds);
+    INIT_LIST_HEAD(&fs->upcall_list);
+    INIT_LIST_HEAD(&fs->waitq);
+
+    PTHREAD_MUTEX_INIT(&fs->mutex, NULL, fs->pthread_flags, GLFS_INIT_MUTEX,
+                       err);
 
-	glfs_lock (glfd->fs);
-	{
-		list_del_init (&glfd->openfds);
-	}
-	glfs_unlock (glfd->fs);
+    PTHREAD_COND_INIT(&fs->cond, NULL, fs->pthread_flags, GLFS_INIT_COND, err);
 
-	if (glfd->fd)
-		fd_unref (glfd->fd);
+    PTHREAD_COND_INIT(&fs->child_down_cond, NULL, fs->pthread_flags,
+                      GLFS_INIT_COND_CHILD, err);
 
-	GF_FREE (glfd->readdirbuf);
+    PTHREAD_MUTEX_INIT(&fs->upcall_list_mutex, NULL, fs->pthread_flags,
+                       GLFS_INIT_MUTEX_UPCALL, err);
 
-	GF_FREE (glfd);
+    fs->volname = strdup(volname);
+    if (!fs->volname)
+        goto err;
+
+    fs->pin_refcnt = 0;
+    fs->upcall_events = 0;
+    fs->up_cbk = NULL;
+    fs->up_data = NULL;
+
+    return fs;
+
+err:
+    glfs_free_from_ctx(fs);
+    return NULL;
 }
 
+extern xlator_t global_xlator;
+extern glusterfs_ctx_t *global_ctx;
+extern pthread_mutex_t global_ctx_mutex;
 
-static void *
-glfs_poller (void *data)
+static int
+glfs_init_global_ctx()
 {
-	struct glfs  *fs = NULL;
+    int ret = 0;
+    glusterfs_ctx_t *ctx = NULL;
+
+    pthread_mutex_lock(&global_ctx_mutex);
+    {
+        if (global_xlator.ctx)
+            goto unlock;
+
+        ctx = glusterfs_ctx_new();
+        if (!ctx) {
+            ret = -1;
+            goto unlock;
+        }
 
-	fs = data;
+        gf_log_globals_init(ctx, GF_LOG_NONE);
 
-	event_dispatch (fs->ctx->event_pool);
+        global_ctx = ctx;
+        global_xlator.ctx = global_ctx;
 
-	return NULL;
-}
+        ret = glusterfs_ctx_defaults_init(ctx);
+        if (ret) {
+            global_ctx = NULL;
+            global_xlator.ctx = NULL;
+            goto unlock;
+        }
+    }
+unlock:
+    pthread_mutex_unlock(&global_ctx_mutex);
+
+    if (ret)
+        FREE(ctx);
 
+    return ret;
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_new, 3.4.0)
 struct glfs *
-pub_glfs_new (const char *volname)
+pub_glfs_new(const char *volname)
 {
-	struct glfs     *fs = NULL;
-	int              ret = -1;
-	glusterfs_ctx_t *ctx = NULL;
+    if (!volname) {
+        errno = EINVAL;
+        return NULL;
+    }
+
+    struct glfs *fs = NULL;
+    int i = 0;
+    int ret = -1;
+    glusterfs_ctx_t *ctx = NULL;
+    xlator_t *old_THIS = NULL;
+    char pname[16] = "";
+    char msg[32] = "";
+
+    if (volname[0] == '/' || volname[0] == '-') {
+        if (strncmp(volname, "/snaps/", 7) == 0) {
+            goto label;
+        }
+        errno = EINVAL;
+        return NULL;
+    }
+
+    for (i = 0; i < strlen(volname); i++) {
+        if (!isalnum(volname[i]) && (volname[i] != '_') &&
+            (volname[i] != '-')) {
+            errno = EINVAL;
+            return NULL;
+        }
+    }
 
-	ctx = glusterfs_ctx_new ();
-	if (!ctx) {
-		return NULL;
-	}
+label:
+    /*
+     * Do this as soon as possible in case something else depends on
+     * pool allocations.
+     */
+    mem_pools_init();
 
-	/* first globals init, for gf_mem_acct_enable_set () */
-	ret = glusterfs_globals_init (ctx);
-	if (ret)
-		return NULL;
+    fs = glfs_new_fs(volname);
+    if (!fs)
+        goto out;
 
-	THIS->ctx = ctx;
+    ctx = glusterfs_ctx_new();
+    if (!ctx)
+        goto out;
 
-	/* then ctx_defaults_init, for xlator_mem_acct_init(THIS) */
-	ret = glusterfs_ctx_defaults_init (ctx);
-	if (ret)
-		return NULL;
+    /* first globals init, for gf_mem_acct_enable_set () */
 
-	fs = GF_CALLOC (1, sizeof (*fs), glfs_mt_glfs_t);
-	if (!fs)
-		return NULL;
-	fs->ctx = ctx;
+    ret = glusterfs_globals_init(ctx);
+    if (ret)
+        goto out;
 
-	glfs_set_logging (fs, "/dev/null", 0);
+    old_THIS = THIS;
+    ret = glfs_init_global_ctx();
+    if (ret)
+        goto out;
 
-	fs->ctx->cmd_args.volfile_id = gf_strdup (volname);
+    /* then ctx_defaults_init, for xlator_mem_acct_init(THIS) */
 
-	fs->volname = gf_strdup (volname);
+    ret = glusterfs_ctx_defaults_init(ctx);
+    if (ret)
+        goto out;
 
-	pthread_mutex_init (&fs->mutex, NULL);
-	pthread_cond_init (&fs->cond, NULL);
+    fs->ctx = ctx;
+    fs->ctx->process_mode = GF_CLIENT_PROCESS;
 
-	INIT_LIST_HEAD (&fs->openfds);
+    ret = glfs_set_logging(fs, "/dev/null", 0);
+    if (ret)
+        goto out;
 
-	return fs;
-}
+    fs->ctx->cmd_args.volfile_id = gf_strdup(volname);
+    if (!(fs->ctx->cmd_args.volfile_id)) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = -1;
+#ifdef GF_LINUX_HOST_OS
+    ret = prctl(PR_GET_NAME, (unsigned long)pname, 0, 0, 0);
+#endif
+    if (ret)
+        fs->ctx->cmd_args.process_name = gf_strdup("gfapi");
+    else {
+        snprintf(msg, sizeof(msg), "gfapi.%s", pname);
+        fs->ctx->cmd_args.process_name = gf_strdup(msg);
+    }
+    ret = 0;
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_new, 3.4.0);
+out:
+    if (ret) {
+        if (fs) {
+            glfs_fini(fs);
+            fs = NULL;
+        } else {
+            /* glfs_fini() calls mem_pools_fini() too */
+            mem_pools_fini();
+        }
+    }
 
+    if (old_THIS)
+        THIS = old_THIS;
 
-int
-pub_glfs_set_volfile (struct glfs *fs, const char *volfile)
+    return fs;
+}
+
+GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_new_from_ctx, 3.7.0)
+struct glfs *
+priv_glfs_new_from_ctx(glusterfs_ctx_t *ctx)
 {
-	cmd_args_t  *cmd_args = NULL;
+    struct glfs *fs = NULL;
 
-	cmd_args = &fs->ctx->cmd_args;
+    if (!ctx)
+        goto out;
 
-	if (vol_assigned (cmd_args))
-		return -1;
+    fs = glfs_new_fs("");
+    if (!fs)
+        goto out;
 
-	cmd_args->volfile = gf_strdup (volfile);
+    fs->ctx = ctx;
 
-	return 0;
+out:
+    return fs;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_set_volfile, 3.4.0);
+GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_free_from_ctx, 3.7.0)
+void
+priv_glfs_free_from_ctx(struct glfs *fs)
+{
+    upcall_entry *u_list = NULL;
+    upcall_entry *tmp = NULL;
+
+    if (!fs)
+        return;
 
+    /* cleanup upcall structures */
+    list_for_each_entry_safe(u_list, tmp, &fs->upcall_list, upcall_list)
+    {
+        list_del_init(&u_list->upcall_list);
+        GF_FREE(u_list->upcall_data.data);
+        GF_FREE(u_list);
+    }
 
+    PTHREAD_MUTEX_DESTROY(&fs->mutex, fs->pthread_flags, GLFS_INIT_MUTEX);
+
+    PTHREAD_COND_DESTROY(&fs->cond, fs->pthread_flags, GLFS_INIT_COND);
+
+    PTHREAD_COND_DESTROY(&fs->child_down_cond, fs->pthread_flags,
+                         GLFS_INIT_COND_CHILD);
+
+    PTHREAD_MUTEX_DESTROY(&fs->upcall_list_mutex, fs->pthread_flags,
+                          GLFS_INIT_MUTEX_UPCALL);
+
+    if (fs->oldvolfile)
+        FREE(fs->oldvolfile);
+
+    FREE(fs->volname);
+
+    FREE(fs);
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_set_volfile, 3.4.0)
 int
-pub_glfs_set_logging (struct glfs *fs, const char *logfile, int loglevel)
+pub_glfs_set_volfile(struct glfs *fs, const char *volfile)
 {
-        int  ret = 0;
-        char *tmplog = NULL;
+    cmd_args_t *cmd_args = NULL;
 
-        if (!logfile) {
-                ret = gf_set_log_file_path (&fs->ctx->cmd_args);
-                if (ret)
-                        goto out;
-                tmplog = fs->ctx->cmd_args.log_file;
-        } else {
-                tmplog = (char *)logfile;
-        }
+    cmd_args = &fs->ctx->cmd_args;
 
-        /* finish log set parameters before init */
-        if (loglevel >= 0)
-                gf_log_set_loglevel (loglevel);
+    if (vol_assigned(cmd_args))
+        return -1;
 
-        ret = gf_log_init (fs->ctx, tmplog, NULL);
-        if (ret)
-                goto out;
+    cmd_args->volfile = gf_strdup(volfile);
+    if (!cmd_args->volfile)
+        return -1;
+    return 0;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_set_logging, 3.4.0)
+int
+pub_glfs_set_logging(struct glfs *fs, const char *logfile, int loglevel)
+{
+    int ret = -1;
+    char *tmplog = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
 
-        ret = gf_log_inject_timer_event (fs->ctx);
+    if (!logfile) {
+        ret = gf_set_log_file_path(&fs->ctx->cmd_args, fs->ctx);
         if (ret)
-                goto out;
+            goto out;
+        tmplog = fs->ctx->cmd_args.log_file;
+    } else {
+        tmplog = (char *)logfile;
+    }
+
+    /* finish log set parameters before init */
+    if (loglevel >= 0)
+        gf_log_set_loglevel(fs->ctx, loglevel);
+
+    ret = gf_log_init(fs->ctx, tmplog, NULL);
+    if (ret)
+        goto out;
+
+    ret = gf_log_inject_timer_event(fs->ctx);
+    if (ret)
+        goto out;
 
 out:
-        return ret;
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_set_logging, 3.4.0);
+int
+glfs_init_wait(struct glfs *fs)
+{
+    int ret = -1;
+
+    /* Always a top-down call, use glfs_lock() */
+    glfs_lock(fs, _gf_true);
+    {
+        while (!fs->init)
+            pthread_cond_wait(&fs->cond, &fs->mutex);
+        ret = fs->ret;
+        errno = fs->err;
+    }
+    glfs_unlock(fs);
+
+    return ret;
+}
+
+GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_init_done, 3.4.0)
+void
+priv_glfs_init_done(struct glfs *fs, int ret)
+{
+    glfs_init_cbk init_cbk;
+
+    if (!fs) {
+        gf_smsg("glfs", GF_LOG_ERROR, EINVAL, API_MSG_GLFS_FSOBJ_NULL, NULL);
+        goto out;
+    }
+
+    init_cbk = fs->init_cbk;
+
+    /* Always a bottom-up call, use mutex_lock() */
+    pthread_mutex_lock(&fs->mutex);
+    {
+        fs->init = 1;
+        fs->ret = ret;
+        fs->err = errno;
+
+        if (!init_cbk)
+            pthread_cond_broadcast(&fs->cond);
+    }
+    pthread_mutex_unlock(&fs->mutex);
 
+    if (init_cbk)
+        init_cbk(fs, ret);
+out:
+    return;
+}
 
 int
-glfs_init_wait (struct glfs *fs)
+glfs_init_common(struct glfs *fs)
 {
-	int   ret = -1;
+    int ret = -1;
+
+    ret = create_master(fs);
+    if (ret)
+        return ret;
 
-	/* Always a top-down call, use glfs_lock() */
-	glfs_lock (fs);
-	{
-		while (!fs->init)
-			pthread_cond_wait (&fs->cond,
-					   &fs->mutex);
-		ret = fs->ret;
-		errno = fs->err;
-	}
-	glfs_unlock (fs);
+    ret = gf_thread_create(&fs->poller, NULL, glfs_poller, fs, "glfspoll");
+    if (ret)
+        return ret;
+
+    ret = glfs_volumes_init(fs);
+    if (ret)
+        return ret;
 
-	return ret;
+    fs->dev_id = gf_dm_hashfn(fs->volname, strlen(fs->volname));
+    return ret;
 }
 
+int
+glfs_init_async(struct glfs *fs, glfs_init_cbk cbk)
+{
+    int ret = -1;
+
+    if (!fs || !fs->ctx) {
+        gf_smsg("glfs", GF_LOG_ERROR, EINVAL, API_MSG_FS_NOT_INIT, NULL);
+        errno = EINVAL;
+        return ret;
+    }
 
-void
-priv_glfs_init_done (struct glfs *fs, int ret)
+    fs->init_cbk = cbk;
+
+    ret = glfs_init_common(fs);
+
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_init, 3.4.0)
+int
+pub_glfs_init(struct glfs *fs)
 {
-	glfs_init_cbk init_cbk;
+    int ret = -1;
 
-	if (!fs) {
-		gf_log ("glfs", GF_LOG_ERROR,
-			"fs is NULL");
-		goto out;
-	}
+    DECLARE_OLD_THIS;
 
-	init_cbk = fs->init_cbk;
+    if (!fs || !fs->ctx) {
+        gf_smsg("glfs", GF_LOG_ERROR, EINVAL, API_MSG_FS_NOT_INIT, NULL);
+        errno = EINVAL;
+        return ret;
+    }
 
-	/* Always a bottom-up call, use mutex_lock() */
-	pthread_mutex_lock (&fs->mutex);
-	{
-		fs->init = 1;
-		fs->ret = ret;
-		fs->err = errno;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
 
-		if (!init_cbk)
-			pthread_cond_broadcast (&fs->cond);
-	}
-	pthread_mutex_unlock (&fs->mutex);
+    ret = glfs_init_common(fs);
+    if (ret)
+        goto out;
 
-	if (init_cbk)
-		init_cbk (fs, ret);
+    ret = glfs_init_wait(fs);
 out:
-	return;
+    __GLFS_EXIT_FS;
+
+    /* Set the initial current working directory to "/" */
+    if (ret >= 0) {
+        ret = glfs_chdir(fs, "/");
+    }
+
+invalid_fs:
+    return ret;
+}
+
+static int
+glusterfs_ctx_destroy(glusterfs_ctx_t *ctx)
+{
+    call_pool_t *pool = NULL;
+    int ret = 0;
+    glusterfs_graph_t *trav_graph = NULL;
+    glusterfs_graph_t *tmp = NULL;
+
+    if (ctx == NULL)
+        return 0;
+
+    if (ctx->cmd_args.curr_server)
+        glfs_free_volfile_servers(&ctx->cmd_args);
+
+    glfs_free_xlator_options(&ctx->cmd_args);
+
+    /* For all the graphs, crawl through the xlator_t structs and free
+     * all its members except for the mem_acct member,
+     * as GF_FREE will be referencing it.
+     */
+    list_for_each_entry_safe(trav_graph, tmp, &ctx->graphs, list)
+    {
+        xlator_tree_free_members(trav_graph->first);
+    }
+
+    /* Free the memory pool */
+    if (ctx->stub_mem_pool)
+        mem_pool_destroy(ctx->stub_mem_pool);
+    if (ctx->dict_pool)
+        mem_pool_destroy(ctx->dict_pool);
+    if (ctx->dict_data_pool)
+        mem_pool_destroy(ctx->dict_data_pool);
+    if (ctx->dict_pair_pool)
+        mem_pool_destroy(ctx->dict_pair_pool);
+    if (ctx->logbuf_pool)
+        mem_pool_destroy(ctx->logbuf_pool);
+
+    pool = ctx->pool;
+    if (pool) {
+        if (pool->frame_mem_pool)
+            mem_pool_destroy(pool->frame_mem_pool);
+        if (pool->stack_mem_pool)
+            mem_pool_destroy(pool->stack_mem_pool);
+        LOCK_DESTROY(&pool->lock);
+        GF_FREE(pool);
+    }
+
+    /* Free the event pool */
+    ret = gf_event_pool_destroy(ctx->event_pool);
+
+    /* Free the iobuf pool */
+    iobuf_pool_destroy(ctx->iobuf_pool);
+
+    GF_FREE(ctx->process_uuid);
+    GF_FREE(ctx->cmd_args.volfile_id);
+    GF_FREE(ctx->cmd_args.process_name);
+
+    LOCK_DESTROY(&ctx->lock);
+    pthread_mutex_destroy(&ctx->notify_lock);
+    pthread_cond_destroy(&ctx->notify_cond);
+
+    /* Free all the graph structs and its containing xlator_t structs
+     * from this point there should be no reference to GF_FREE/GF_CALLOC
+     * as it will try to access mem_acct and the below function would
+     * have freed the same.
+     */
+    list_for_each_entry_safe(trav_graph, tmp, &ctx->graphs, list)
+    {
+        glusterfs_graph_destroy_residual(trav_graph);
+    }
+
+    GF_FREE(ctx->statedump_path);
+    FREE(ctx);
+
+    return ret;
 }
 
-GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_init_done, 3.4.0);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fini, 3.4.0)
+int
+pub_glfs_fini(struct glfs *fs)
+{
+    int ret = -1;
+    int countdown = 100;
+    xlator_t *subvol = NULL;
+    glusterfs_ctx_t *ctx = NULL;
+    glusterfs_graph_t *graph = NULL;
+    call_pool_t *call_pool = NULL;
+    int fs_init = 0;
+    int err = -1;
+    struct synctask *waittask = NULL;
+
+    DECLARE_OLD_THIS;
+
+    if (!fs) {
+        errno = EINVAL;
+        goto invalid_fs;
+    }
+
+    ctx = fs->ctx;
+    if (!ctx) {
+        goto free_fs;
+    }
+
+    THIS = fs->ctx->master;
+
+    if (ctx->mgmt) {
+        rpc_clnt_disable(ctx->mgmt);
+    }
+
+    call_pool = fs->ctx->pool;
+
+    /* Wake up any suspended synctasks */
+    while (!list_empty(&fs->waitq)) {
+        waittask = list_entry(fs->waitq.next, struct synctask, waitq);
+        list_del_init(&waittask->waitq);
+        synctask_wake(waittask);
+    }
+
+    while (countdown--) {
+        /* give some time for background frames to finish */
+        pthread_mutex_lock(&fs->mutex);
+        {
+            /* Do we need to increase countdown? */
+            if ((!call_pool->cnt) && (!fs->pin_refcnt)) {
+                gf_msg_trace("glfs", 0,
+                             "call_pool_cnt - %" PRId64
+                             ","
+                             "pin_refcnt - %d",
+                             call_pool->cnt, fs->pin_refcnt);
+
+                ctx->cleanup_started = 1;
+                pthread_mutex_unlock(&fs->mutex);
+                break;
+            }
+        }
+        pthread_mutex_unlock(&fs->mutex);
+        gf_nanosleep(100000 * GF_US_IN_NS);
+    }
+
+    /* leaked frames may exist, we ignore */
+
+    /*We deem glfs_fini as successful if there are no pending frames in the call
+     *pool*/
+    ret = (call_pool->cnt == 0) ? 0 : -1;
+
+    pthread_mutex_lock(&fs->mutex);
+    {
+        fs_init = fs->init;
+    }
+    pthread_mutex_unlock(&fs->mutex);
+
+    if (fs_init != 0) {
+        subvol = glfs_active_subvol(fs);
+        if (subvol) {
+            /* PARENT_DOWN within glfs_subvol_done() is issued
+               only on graph switch (new graph should activiate
+               and decrement the extra @winds count taken in
+               glfs_graph_setup()
+
+               Since we are explicitly destroying,
+               PARENT_DOWN is necessary
+            */
+            xlator_notify(subvol, GF_EVENT_PARENT_DOWN, subvol, 0);
+            /* Here we wait for GF_EVENT_CHILD_DOWN before exiting,
+               in case of asynchrnous cleanup
+            */
+            graph = subvol->graph;
+            err = pthread_mutex_lock(&fs->mutex);
+            if (err != 0) {
+                gf_smsg("glfs", GF_LOG_ERROR, err, API_MSG_FSMUTEX_LOCK_FAILED,
+                        "error=%s", strerror(err), NULL);
+                goto fail;
+            }
+            /* check and wait for CHILD_DOWN for active subvol*/
+            {
+                while (graph->used) {
+                    err = pthread_cond_wait(&fs->child_down_cond, &fs->mutex);
+                    if (err != 0)
+                        gf_smsg("glfs", GF_LOG_INFO, err,
+                                API_MSG_COND_WAIT_FAILED, "name=%s",
+                                subvol->name, "err=%s", strerror(err), NULL);
+                }
+            }
+
+            err = pthread_mutex_unlock(&fs->mutex);
+            if (err != 0) {
+                gf_smsg("glfs", GF_LOG_ERROR, err,
+                        API_MSG_FSMUTEX_UNLOCK_FAILED, "error=%s",
+                        strerror(err), NULL);
+                goto fail;
+            }
+        }
+        glfs_subvol_done(fs, subvol);
+    }
+
+    ctx->cleanup_started = 1;
+
+    if (fs_init != 0) {
+        /* Destroy all the inode tables of all the graphs.
+         * NOTE:
+         * - inode objects should be destroyed before calling fini()
+         *   of each xlator, as fini() and forget() of the xlators
+         *   can share few common locks or data structures, calling
+         *   fini first might destroy those required by forget
+         *   ( eg: in quick-read)
+         * - The call to inode_table_destroy_all is not required when
+         *   the cleanup during graph switch is implemented to perform
+         *   inode table destroy.
+         */
+        inode_table_destroy_all(ctx);
+
+        /* Call fini() of all the xlators in the active graph
+         * NOTE:
+         * - xlator fini() should be called before destroying any of
+         *   the threads. (eg: fini() in protocol-client uses timer
+         *   thread) */
+        glusterfs_graph_deactivate(ctx->active);
+
+        /* Join the syncenv_processor threads and cleanup
+         * syncenv resources*/
+        syncenv_destroy(ctx->env);
+
+        /* Join the poller thread */
+        if (gf_event_dispatch_destroy(ctx->event_pool) < 0)
+            ret = -1;
+    }
+
+    /* Avoid dispatching events to mgmt after freed,
+     * unreference mgmt after the event_dispatch_destroy */
+    if (ctx->mgmt) {
+        rpc_clnt_unref(ctx->mgmt);
+        ctx->mgmt = NULL;
+    }
+
+    /* log infra has to be brought down before destroying
+     * timer registry, as logging uses timer infra
+     */
+    if (gf_log_fini(ctx) != 0)
+        ret = -1;
+
+    /* Join the timer thread */
+    if (fs_init != 0) {
+        gf_timer_registry_destroy(ctx);
+    }
+
+    /* Destroy the context and the global pools */
+    if (glusterfs_ctx_destroy(ctx) != 0)
+        ret = -1;
+
+free_fs:
+    glfs_free_from_ctx(fs);
+
+    /*
+     * Do this as late as possible in case anything else has (or
+     * grows) a dependency on mem-pool allocations.
+     */
+    mem_pools_fini();
+
+fail:
+    if (!ret)
+        ret = err;
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_get_volfile, 3.6.0)
+ssize_t
+pub_glfs_get_volfile(struct glfs *fs, void *buf, size_t len)
+{
+    ssize_t res = -1;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    glfs_lock(fs, _gf_true);
+    if (len >= fs->oldvollen) {
+        gf_msg_trace("glfs", 0, "copying %zu to %p", len, buf);
+        memcpy(buf, fs->oldvolfile, len);
+        res = len;
+    } else {
+        res = len - fs->oldvollen;
+        gf_msg_trace("glfs", 0, "buffer is %zd too short", -res);
+    }
+    glfs_unlock(fs);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return res;
+}
 
+GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_ipc, 3.12.0)
 int
-glfs_init_common (struct glfs *fs)
+priv_glfs_ipc(struct glfs *fs, int opcode, void *xd_in, void **xd_out)
 {
-	int  ret = -1;
+    xlator_t *subvol = NULL;
+    int ret = -1;
 
-	ret = create_master (fs);
-	if (ret)
-		return ret;
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
 
-	ret = gf_thread_create (&fs->poller, NULL, glfs_poller, fs);
-	if (ret)
-		return ret;
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
 
-	ret = glfs_volumes_init (fs);
-	if (ret)
-		return ret;
+    ret = syncop_ipc(subvol, opcode, (dict_t *)xd_in, (dict_t **)xd_out);
+    DECODE_SYNCOP_ERR(ret);
 
-	fs->dev_id = gf_dm_hashfn (fs->volname, strlen (fs->volname));
-	return ret;
-}
+out:
+    glfs_subvol_done(fs, subvol);
+    __GLFS_EXIT_FS;
 
+invalid_fs:
+    return ret;
+}
 
+GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_setfspid, 6.1)
 int
-glfs_init_async (struct glfs *fs, glfs_init_cbk cbk)
+priv_glfs_setfspid(struct glfs *fs, pid_t pid)
 {
-	int  ret = -1;
+    cmd_args_t *cmd_args = NULL;
+    int ret = 0;
 
-	if (!fs || !fs->ctx) {
-		gf_log ("glfs", GF_LOG_ERROR,
-			"fs is not properly initialized.");
-		errno = EINVAL;
-		return ret;
-	}
+    cmd_args = &fs->ctx->cmd_args;
+    cmd_args->client_pid = pid;
+    cmd_args->client_pid_set = 1;
+    ret = syncopctx_setfspid(&pid);
 
-	fs->init_cbk = cbk;
-
-	ret = glfs_init_common (fs);
+    return ret;
+}
 
-	return ret;
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_free, 3.7.16)
+void
+pub_glfs_free(void *ptr)
+{
+    GLFS_FREE(ptr);
 }
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_get_fs, 3.7.16)
+struct glfs *
+pub_glfs_upcall_get_fs(struct glfs_upcall *arg)
+{
+    return arg->fs;
+}
 
-int
-pub_glfs_init (struct glfs *fs)
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_get_reason, 3.7.16)
+enum glfs_upcall_reason
+pub_glfs_upcall_get_reason(struct glfs_upcall *arg)
 {
-	int  ret = -1;
+    return arg->reason;
+}
 
-	if (!fs || !fs->ctx) {
-		gf_log ("glfs", GF_LOG_ERROR,
-			"fs is not properly initialized.");
-		errno = EINVAL;
-		return ret;
-	}
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_get_event, 3.7.16)
+void *
+pub_glfs_upcall_get_event(struct glfs_upcall *arg)
+{
+    return arg->event;
+}
 
-	ret = glfs_init_common (fs);
-	if (ret)
-		return ret;
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_inode_get_object, 3.7.16)
+struct glfs_object *
+pub_glfs_upcall_inode_get_object(struct glfs_upcall_inode *arg)
+{
+    return arg->object;
+}
 
-	ret = glfs_init_wait (fs);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_inode_get_flags, 3.7.16)
+uint64_t
+pub_glfs_upcall_inode_get_flags(struct glfs_upcall_inode *arg)
+{
+    return arg->flags;
+}
 
-	return ret;
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_inode_get_stat, 3.7.16)
+struct stat *
+pub_glfs_upcall_inode_get_stat(struct glfs_upcall_inode *arg)
+{
+    return &arg->buf;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_init, 3.4.0);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_inode_get_expire, 3.7.16)
+uint64_t
+pub_glfs_upcall_inode_get_expire(struct glfs_upcall_inode *arg)
+{
+    return arg->expire_time_attr;
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_inode_get_pobject, 3.7.16)
+struct glfs_object *
+pub_glfs_upcall_inode_get_pobject(struct glfs_upcall_inode *arg)
+{
+    return arg->p_object;
+}
 
-extern xlator_t *
-priv_glfs_active_subvol (struct glfs *);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_inode_get_pstat, 3.7.16)
+struct stat *
+pub_glfs_upcall_inode_get_pstat(struct glfs_upcall_inode *arg)
+{
+    return &arg->p_buf;
+}
 
-int
-pub_glfs_fini (struct glfs *fs)
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_inode_get_oldpobject, 3.7.16)
+struct glfs_object *
+pub_glfs_upcall_inode_get_oldpobject(struct glfs_upcall_inode *arg)
 {
-        int             ret = -1;
-        int             countdown = 100;
-        xlator_t        *subvol = NULL;
-        glusterfs_ctx_t *ctx = NULL;
-        call_pool_t     *call_pool = NULL;
-        int             fs_init = 0;
+    return arg->oldp_object;
+}
 
-        ctx = fs->ctx;
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_inode_get_oldpstat, 3.7.16)
+struct stat *
+pub_glfs_upcall_inode_get_oldpstat(struct glfs_upcall_inode *arg)
+{
+    return &arg->oldp_buf;
+}
 
-        if (ctx->mgmt) {
-                rpc_clnt_disable (ctx->mgmt);
-                ctx->mgmt = NULL;
-        }
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_lease_get_object, 4.1.6)
+struct glfs_object *
+pub_glfs_upcall_lease_get_object(struct glfs_upcall_lease *arg)
+{
+    return arg->object;
+}
 
-        __glfs_entry_fs (fs);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_lease_get_lease_type, 4.1.6)
+uint32_t
+pub_glfs_upcall_lease_get_lease_type(struct glfs_upcall_lease *arg)
+{
+    return arg->lease_type;
+}
 
-        call_pool = fs->ctx->pool;
+/* definitions of the GLFS_SYSRQ_* chars are in glfs.h */
+static struct glfs_sysrq_help {
+    char sysrq;
+    char *msg;
+} glfs_sysrq_help[] = {{GLFS_SYSRQ_HELP, "(H)elp"},
+                       {GLFS_SYSRQ_STATEDUMP, "(S)tatedump"},
+                       {0, NULL}};
 
-        while (countdown--) {
-                /* give some time for background frames to finish */
-                if (!call_pool->cnt)
-                        break;
-                usleep (100000);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_sysrq, 3.10.0)
+int
+pub_glfs_sysrq(struct glfs *fs, char sysrq)
+{
+    glusterfs_ctx_t *ctx = NULL;
+    int ret = 0;
+    int msg_len;
+    char msg[1024] = {
+        0,
+    }; /* should not exceed 1024 chars */
+
+    if (!fs || !fs->ctx) {
+        ret = -1;
+        errno = EINVAL;
+        goto out;
+    }
+
+    ctx = fs->ctx;
+
+    switch (sysrq) {
+        case GLFS_SYSRQ_HELP: {
+            struct glfs_sysrq_help *usage = NULL;
+
+            for (usage = glfs_sysrq_help; usage->sysrq; usage++) {
+                msg_len = strlen(msg);
+                snprintf(msg + msg_len, /* append to msg */
+                         sizeof(msg) - msg_len - 2,
+                         /* - 2 for the " " + terminating \0 */
+                         " %s", usage->msg);
+            }
+
+            /* not really an 'error', but make sure it gets logged */
+            gf_log("glfs", GF_LOG_ERROR, "available events: %s", msg);
+
+            break;
         }
-        /* leaked frames may exist, we ignore */
-
-        /*We deem glfs_fini as successful if there are no pending frames in the call
-         *pool*/
-        ret = (call_pool->cnt == 0)? 0: -1;
+        case GLFS_SYSRQ_STATEDUMP:
+            gf_proc_dump_info(SIGUSR1, ctx);
+            break;
+        default:
+            gf_smsg("glfs", GF_LOG_ERROR, ENOTSUP, API_MSG_INVALID_SYSRQ,
+                    "sysrq=%c", sysrq, NULL);
+            errno = ENOTSUP;
+            ret = -1;
+    }
+out:
+    return ret;
+}
 
-        pthread_mutex_lock (&fs->mutex);
-        {
-                fs_init = fs->init;
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_register, 3.13.0)
+int
+pub_glfs_upcall_register(struct glfs *fs, uint32_t event_list,
+                         glfs_upcall_cbk cbk, void *data)
+{
+    int ret = 0;
+
+    /* list of supported upcall events */
+    uint32_t up_events = (GLFS_EVENT_INODE_INVALIDATE |
+                          GLFS_EVENT_RECALL_LEASE);
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    GF_VALIDATE_OR_GOTO(THIS->name, cbk, out);
+
+    /* Event list should be either GLFS_EVENT_ANY
+     * or list of supported individual events (up_events)
+     */
+    if ((event_list != GLFS_EVENT_ANY) && (event_list & ~up_events)) {
+        errno = EINVAL;
+        ret = -1;
+        gf_smsg(THIS->name, GF_LOG_ERROR, errno, API_MSG_INVALID_ARG,
+                "event_list=(0x%08x)", event_list, NULL);
+        goto out;
+    }
+
+    /* in case other thread does unregister */
+    pthread_mutex_lock(&fs->mutex);
+    {
+        if (event_list & GLFS_EVENT_INODE_INVALIDATE) {
+            /* @todo: Check if features.cache-invalidation is
+             * enabled.
+             */
+            fs->upcall_events |= GF_UPCALL_CACHE_INVALIDATION;
+            ret |= GLFS_EVENT_INODE_INVALIDATE;
         }
-        pthread_mutex_unlock (&fs->mutex);
-
-        if (fs_init != 0) {
-                subvol = priv_glfs_active_subvol (fs);
-                if (subvol) {
-                        /* PARENT_DOWN within priv_glfs_subvol_done() is issued only
-                           on graph switch (new graph should activiate and
-                           decrement the extra @winds count taken in glfs_graph_setup()
-
-                           Since we are explicitly destroying, PARENT_DOWN is necessary
-                        */
-                        xlator_notify (subvol, GF_EVENT_PARENT_DOWN, subvol, 0);
-                        /* TBD: wait for CHILD_DOWN before exiting, in case of
-                           asynchronous cleanup like graceful socket
-                           disconnection in the future.
-                        */
-                }
-                priv_glfs_subvol_done (fs, subvol);
+        if (event_list & GLFS_EVENT_RECALL_LEASE) {
+            /* @todo: Check if features.leases is enabled */
+            fs->upcall_events |= GF_UPCALL_RECALL_LEASE;
+            ret |= GLFS_EVENT_RECALL_LEASE;
         }
+        /* Override cbk function if existing */
+        fs->up_cbk = cbk;
+        fs->up_data = data;
+        fs->cache_upcalls = _gf_true;
+    }
+    pthread_mutex_unlock(&fs->mutex);
 
-        if (gf_log_fini(ctx) != 0)
-                ret = -1;
+out:
+    __GLFS_EXIT_FS;
 
-        return ret;
+invalid_fs:
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fini, 3.4.0);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_unregister, 3.13.0)
+int
+pub_glfs_upcall_unregister(struct glfs *fs, uint32_t event_list)
+{
+    int ret = 0;
+    /* list of supported upcall events */
+    uint32_t up_events = (GLFS_EVENT_INODE_INVALIDATE |
+                          GLFS_EVENT_RECALL_LEASE);
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* Event list should be either GLFS_EVENT_ANY
+     * or list of supported individual events (up_events)
+     */
+    if ((event_list != GLFS_EVENT_ANY) && (event_list & ~up_events)) {
+        errno = EINVAL;
+        ret = -1;
+        gf_smsg(THIS->name, GF_LOG_ERROR, errno, API_MSG_INVALID_ARG,
+                "event_list=(0x%08x)", event_list, NULL);
+        goto out;
+    }
+
+    pthread_mutex_lock(&fs->mutex);
+    {
+        /* We already checked if event_list contains list of supported
+         * upcall events. No other specific checks needed as of now for
+         * unregister */
+        fs->upcall_events &= ~(event_list);
+        ret |= ((event_list == GLFS_EVENT_ANY) ? up_events : event_list);
+
+        /* If there are no upcall events registered, reset cbk */
+        if (fs->upcall_events == 0) {
+            fs->up_cbk = NULL;
+            fs->up_data = NULL;
+            fs->cache_upcalls = _gf_false;
+        }
+    }
+    pthread_mutex_unlock(&fs->mutex);
 
+out:
+    __GLFS_EXIT_FS;
 
-ssize_t
-pub_glfs_get_volfile (struct glfs *fs, void *buf, size_t len)
-{
-        ssize_t         res;
+invalid_fs:
+    return ret;
+}
 
-        glfs_lock(fs);
-        if (len >= fs->oldvollen) {
-                gf_log ("glfs", GF_LOG_TRACE, "copying %lu to %p", len, buf);
-                memcpy(buf,fs->oldvolfile,len);
-                res = len;
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_set_statedump_path, 7.0)
+int
+pub_glfs_set_statedump_path(struct glfs *fs, const char *path)
+{
+    struct stat st;
+    int ret;
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    if (!path) {
+        gf_log("glfs", GF_LOG_ERROR, "path is NULL");
+        errno = EINVAL;
+        goto err;
+    }
+
+    /* If path is not present OR, if it is directory AND has enough permission
+     * to create files, then proceed */
+    ret = sys_stat(path, &st);
+    if (ret && errno != ENOENT) {
+        gf_log("glfs", GF_LOG_ERROR, "%s: not a valid path (%s)", path,
+               strerror(errno));
+        errno = EINVAL;
+        goto err;
+    }
+
+    if (!ret) {
+        /* file is present, now check other things */
+        if (!S_ISDIR(st.st_mode)) {
+            gf_log("glfs", GF_LOG_ERROR, "%s: path is not directory", path);
+            errno = EINVAL;
+            goto err;
         }
-        else {
-                res = len - fs->oldvollen;
-                gf_log ("glfs", GF_LOG_TRACE, "buffer is %ld too short", -res);
+        if (sys_access(path, W_OK | X_OK) < 0) {
+            gf_log("glfs", GF_LOG_ERROR,
+                   "%s: path doesn't have write permission", path);
+            errno = EPERM;
+            goto err;
         }
-        glfs_unlock(fs);
+    }
 
-        return res;
-}
+    /* If set, it needs to be freed, so we don't have leak */
+    GF_FREE(fs->ctx->statedump_path);
+
+    fs->ctx->statedump_path = gf_strdup(path);
+    if (!fs->ctx->statedump_path) {
+        gf_log("glfs", GF_LOG_ERROR,
+               "%s: failed to set statedump path, no memory", path);
+        errno = ENOMEM;
+        goto err;
+    }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_get_volfile, 3.6.0);
+    __GLFS_EXIT_FS;
 
+    return 0;
+err:
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return -1;
+}
diff --git a/api/src/glfs.h b/api/src/glfs.h
index 6d896f0ea13..279d11d58ee 100644
--- a/api/src/glfs.h
+++ b/api/src/glfs.h
@@ -1,5 +1,5 @@
 /*
-  Copyright (c) 2012 Red Hat, Inc. <http://www.redhat.com>
+  Copyright (c) 2012-2018 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
   This file is licensed to you under your choice of the GNU Lesser
@@ -8,7 +8,6 @@
   cases as published by the Free Software Foundation.
 */
 
-
 #ifndef _GLFS_H
 #define _GLFS_H
 
@@ -21,6 +20,17 @@
   both the library and the application.
 */
 
+/* Values for valid flags to be used when using XXXsetattr, to set multiple
+ attribute values passed via the related stat structure.
+ */
+
+#define GFAPI_SET_ATTR_MODE 0x1
+#define GFAPI_SET_ATTR_UID 0x2
+#define GFAPI_SET_ATTR_GID 0x4
+#define GFAPI_SET_ATTR_SIZE 0x8
+#define GFAPI_SET_ATTR_ATIME 0x10
+#define GFAPI_SET_ATTR_MTIME 0x20
+
 #ifndef _FILE_OFFSET_BITS
 #define _FILE_OFFSET_BITS 64
 #endif
@@ -41,14 +51,64 @@
 #include <sys/cdefs.h>
 #include <dirent.h>
 #include <sys/statvfs.h>
+#include <stdint.h>
+#include <sys/time.h>
+
+/*
+ * For off64_t to be defined, we need both
+ * __USE_LARGEFILE64 to be true and __off64_t_defnined to be
+ * false. But, making __USE_LARGEFILE64 true causes other issues
+ * such as redinition of stat and fstat to stat64 and fstat64
+ * respectively which again causes compilation issues.
+ * Without off64_t being defined, this will not compile as
+ * copy_file_range uses off64_t. Hence define it here. First
+ * check whether __off64_t_defined is true or not. <unistd.h>
+ * sets that flag when it defines off64_t. If __off64_t_defined
+ * is false and __USE_FILE_OFFSET64 is true, then go on to define
+ * off64_t using __off64_t.
+ */
+#ifndef GF_BSD_HOST_OS
+#if defined(__USE_FILE_OFFSET64) && !defined(__off64_t_defined)
+typedef __off64_t off64_t;
+#endif /* defined(__USE_FILE_OFFSET64) && !defined(__off64_t_defined) */
+#else
+#include <stdio.h>
+#ifndef _OFF64_T_DECLARED
+/*
+ * Including <stdio.h> (done above) should actually define
+ * _OFF64_T_DECLARED with off64_t data type being available
+ * for consumption. But, off64_t data type is not recognizable
+ * for FreeBSD versions less than 11. Hence, int64_t is typedefed
+ * to off64_t.
+ */
+#define _OFF64_T_DECLARED
+typedef int64_t off64_t;
+#endif /* _OFF64_T_DECLARED */
+#endif /* GF_BSD_HOST_OS */
+
+#if defined(HAVE_SYS_ACL_H) || (defined(USE_POSIX_ACLS) && USE_POSIX_ACLS)
+#include <sys/acl.h>
+#else
+typedef void *acl_t;
+typedef int acl_type_t;
+#endif
 
 /* Portability non glibc c++ build systems */
 #ifndef __THROW
-# if defined __cplusplus
-#  define __THROW       throw ()
-# else
-#  define __THROW
-# endif
+#if defined __cplusplus
+#define __THROW throw()
+#else
+#define __THROW
+#endif
+#endif
+
+#ifndef GF_DARWIN_HOST_OS
+#define GFAPI_PUBLIC(sym, ver)  /**/
+#define GFAPI_PRIVATE(sym, ver) /**/
+#else
+#define GFAPI_PUBLIC(sym, ver) __asm("_" __STRING(sym) "$GFAPI_" __STRING(ver))
+#define GFAPI_PRIVATE(sym, ver)                                                \
+    __asm("_" __STRING(sym) "$GFAPI_PRIVATE_" __STRING(ver))
 #endif
 
 __BEGIN_DECLS
@@ -57,7 +117,6 @@ __BEGIN_DECLS
 struct glfs;
 typedef struct glfs glfs_t;
 
-
 /*
   SYNOPSIS
 
@@ -78,8 +137,8 @@ typedef struct glfs glfs_t;
 
   @volname: Name of the volume. This identifies the server-side volume and
             the fetched volfile (equivalent of --volfile-id command line
-	    parameter to glusterfsd). When used with glfs_set_volfile() the
-	    @volname has no effect (except for appearing in log messages).
+            parameter to glusterfsd). When used with glfs_set_volfile() the
+            @volname has no effect (except for appearing in log messages).
 
   RETURN VALUES
 
@@ -88,8 +147,8 @@ typedef struct glfs glfs_t;
 
 */
 
-glfs_t *glfs_new (const char *volname) __THROW;
-
+glfs_t *
+glfs_new(const char *volname) __THROW GFAPI_PUBLIC(glfs_new, 3.4.0);
 
 /*
   SYNOPSIS
@@ -118,8 +177,9 @@ glfs_t *glfs_new (const char *volname) __THROW;
 
 */
 
-int glfs_set_volfile (glfs_t *fs, const char *volfile);
-
+int
+glfs_set_volfile(glfs_t *fs, const char *volfile) __THROW
+    GFAPI_PUBLIC(glfs_set_volfile, 3.4.0);
 
 /*
   SYNOPSIS
@@ -144,15 +204,16 @@ int glfs_set_volfile (glfs_t *fs, const char *volfile);
        specification file.
 
   @transport: String specifying the transport used to connect to the
-              management daemon. Specifying NULL will result in the usage
-              of the default (tcp) transport type. Permitted values
-              are those what you specify as transport-type in a volume
-              specification file (e.g "tcp", "rdma" etc.)
+              management daemon. Specifying NULL will result in the
+              usage of the default (tcp) transport type. Permitted
+              values are "tcp" or "unix".
 
   @host:      String specifying the address where to find the management daemon.
+              Socket path, while using Unix domain socket as transport type.
               This would either be
-              - FQDN (e.g: "storage01.company.com") or
-              - ASCII (e.g: "192.168.22.1")
+              - FQDN (e.g : "storage01.company.com") or
+              - ASCII (e.g : "192.168.22.1") or
+              - Socket path (e.g : "/var/run/glusterd.socket")
 
   NOTE: This API is special, multiple calls to this function with different
         volfile servers, port or transport-type would create a list of volfile
@@ -169,10 +230,15 @@ int glfs_set_volfile (glfs_t *fs, const char *volfile);
 
 */
 
-int glfs_set_volfile_server (glfs_t *fs, const char *transport,
-                             const char *host, int port) __THROW;
-int glfs_unset_volfile_server (glfs_t *fs, const char *transport,
-                               const char *host, int port) __THROW;
+int
+glfs_set_volfile_server(glfs_t *fs, const char *transport, const char *host,
+                        int port) __THROW
+    GFAPI_PUBLIC(glfs_set_volfile_server, 3.4.0);
+
+int
+glfs_unset_volfile_server(glfs_t *fs, const char *transport, const char *host,
+                          int port) __THROW
+    GFAPI_PUBLIC(glfs_unset_volfile_server, 3.5.1);
 /*
   SYNOPSIS
 
@@ -202,8 +268,9 @@ int glfs_unset_volfile_server (glfs_t *fs, const char *transport,
 
 */
 
-int glfs_set_logging (glfs_t *fs, const char *logfile, int loglevel) __THROW;
-
+int
+glfs_set_logging(glfs_t *fs, const char *logfile, int loglevel) __THROW
+    GFAPI_PUBLIC(glfs_set_logging, 3.4.0);
 
 /*
   SYNOPSIS
@@ -229,8 +296,8 @@ int glfs_set_logging (glfs_t *fs, const char *logfile, int loglevel) __THROW;
 
 */
 
-int glfs_init (glfs_t *fs) __THROW;
-
+int
+glfs_init(glfs_t *fs) __THROW GFAPI_PUBLIC(glfs_init, 3.4.0);
 
 /*
   SYNOPSIS
@@ -262,7 +329,8 @@ int glfs_init (glfs_t *fs) __THROW;
    0 : Success.
 */
 
-int glfs_fini (glfs_t *fs) __THROW;
+int
+glfs_fini(glfs_t *fs) __THROW GFAPI_PUBLIC(glfs_fini, 3.4.0);
 
 /*
   SYNOPSIS
@@ -292,8 +360,9 @@ int glfs_fini (glfs_t *fs) __THROW;
       <0: volfile length exceeds @len by N bytes (@buf unchanged)
 */
 
-ssize_t glfs_get_volfile (glfs_t *fs, void *buf, size_t len) __THROW;
-
+ssize_t
+glfs_get_volfile(glfs_t *fs, void *buf, size_t len) __THROW
+    GFAPI_PUBLIC(glfs_get_volfile, 3.6.0);
 
 /*
   SYNOPSIS
@@ -306,9 +375,9 @@ ssize_t glfs_get_volfile (glfs_t *fs, void *buf, size_t len) __THROW;
        the management server (glusterd) to fetch volume uuid and stores it
        in the glusterfs_context linked to the glfs object fs which can be used
        in the subsequent calls. Later it parses that UUID to convert it from
-       cannonical string format into an opaque byte array and copy it into
-       the volid array. Incase if either of the input parameters, volid or size,
-       is NULL, number of bytes required to copy the volume UUID is returned.
+       canonical string format into an opaque byte array and copy it into
+       the volid array. In case if either of the input parameters, volid or
+  size, is NULL, number of bytes required to copy the volume UUID is returned.
 
   PARAMETERS
 
@@ -323,8 +392,9 @@ ssize_t glfs_get_volfile (glfs_t *fs, void *buf, size_t len) __THROW;
         Others : length of the volume UUID stored.
 */
 
-int glfs_get_volumeid (struct glfs *fs, char *volid, size_t size);
-
+int
+glfs_get_volumeid(glfs_t *fs, char *volid, size_t size) __THROW
+    GFAPI_PUBLIC(glfs_get_volumeid, 3.5.0);
 
 /*
  * FILE OPERATION
@@ -346,6 +416,119 @@ struct glfs_fd;
 typedef struct glfs_fd glfs_fd_t;
 
 /*
+ * Mask for request/result items in the struct glfs_stat.
+ *
+ * Query request/result mask for glfs_stat() (family of functions) and
+ * struct glfs_stat::glfs_st_mask.
+ *
+ * These bits should be set in the mask argument of glfs_stat() (family of
+ * functions) to request particular items when calling glfs_stat().
+ *
+ * NOTE: Lower order 32 bits are used to reflect statx(2) bits. For Gluster
+ * specific attrs/extensions, use higher order 32 bits.
+ *
+ */
+#define GLFS_STAT_TYPE 0x0000000000000001U   /* Want/got stx_mode & S_IFMT */
+#define GLFS_STAT_MODE 0x0000000000000002U   /* Want/got stx_mode & ~S_IFMT */
+#define GLFS_STAT_NLINK 0x0000000000000004U  /* Want/got stx_nlink */
+#define GLFS_STAT_UID 0x0000000000000008U    /* Want/got stx_uid */
+#define GLFS_STAT_GID 0x0000000000000010U    /* Want/got stx_gid */
+#define GLFS_STAT_ATIME 0x0000000000000020U  /* Want/got stx_atime */
+#define GLFS_STAT_MTIME 0x0000000000000040U  /* Want/got stx_mtime */
+#define GLFS_STAT_CTIME 0x0000000000000080U  /* Want/got stx_ctime */
+#define GLFS_STAT_INO 0x0000000000000100U    /* Want/got stx_ino */
+#define GLFS_STAT_SIZE 0x0000000000000200U   /* Want/got stx_size */
+#define GLFS_STAT_BLOCKS 0x0000000000000400U /* Want/got stx_blocks */
+#define GLFS_STAT_BASIC_STATS                                                  \
+    0x00000000000007ffU /* Items in the normal stat struct */
+#define GLFS_STAT_BTIME 0x0000000000000800U /* Want/got stx_btime */
+#define GLFS_STAT_ALL 0x0000000000000fffU   /* All currently supported flags */
+#define GLFS_STAT_RESERVED                                                     \
+    0x8000000000000000U /* Reserved to denote future expansion */
+
+/* Macros for checking validity of struct glfs_stat members.*/
+#define GLFS_STAT_TYPE_VALID(stmask) (stmask & GLFS_STAT_TYPE)
+#define GLFS_STAT_MODE_VALID(stmask) (stmask & GLFS_STAT_MODE)
+#define GLFS_STAT_NLINK_VALID(stmask) (stmask & GLFS_STAT_NLINK)
+#define GLFS_STAT_UID_VALID(stmask) (stmask & GLFS_STAT_UID)
+#define GLFS_STAT_GID_VALID(stmask) (stmask & GLFS_STAT_GID)
+#define GLFS_STAT_ATIME_VALID(stmask) (stmask & GLFS_STAT_ATIME)
+#define GLFS_STAT_MTIME_VALID(stmask) (stmask & GLFS_STAT_MTIME)
+#define GLFS_STAT_CTIME_VALID(stmask) (stmask & GLFS_STAT_CTIME)
+#define GLFS_STAT_INO_VALID(stmask) (stmask & GLFS_STAT_INO)
+#define GLFS_STAT_SIZE_VALID(stmask) (stmask & GLFS_STAT_SIZE)
+#define GLFS_STAT_BLOCKS_VALID(stmask) (stmask & GLFS_STAT_BLOCKS)
+#define GLFS_STAT_BTIME_VALID(stmask) (stmask & GLFS_STAT_BTIME)
+#define GLFS_STAT_GFID_VALID(stmask) (stmask & GLFS_STAT_GFID)
+
+/*
+ * Attributes to be found in glfs_st_attributes and masked in
+ * glfs_st_attributes_mask.
+ *
+ * These give information about the features or the state of a file that might
+ * be of use to programs.
+ *
+ * NOTE: Lower order 32 bits are used to reflect statx(2) attribute bits. For
+ * Gluster specific attrs, use higher order 32 bits.
+ *
+ * NOTE: We do not support any file attributes or state as yet!
+ */
+#define GLFS_STAT_ATTR_RESERVED                                                \
+    0x8000000000000000U /* Reserved to denote future expansion */
+
+/* Extended file attribute structure.
+ *
+ * The caller passes a mask of what they're specifically interested in as a
+ * parameter to glfs_stat().  What glfs_stat() actually got will be indicated
+ * in glfs_st_mask upon return.
+ *
+ * For each bit in the mask argument:
+ *
+ * - if the datum is not supported:
+ *
+ *   - the bit will be cleared, and
+ *
+ *   - the datum value is undefined
+ *
+ * - otherwise, if explicitly requested:
+ *
+ *   - the field will be filled in and the bit will be set;
+ *
+ * - otherwise, if not requested, but available in, it will be filled in
+ * anyway, and the bit will be set upon return;
+ *
+ * - otherwise the field and the bit will be cleared before returning.
+ *
+ */
+
+struct glfs_stat {
+    uint64_t glfs_st_mask;       /* What results were written [uncond] */
+    uint64_t glfs_st_attributes; /* Flags conveying information about the file
+                                    [uncond] */
+    uint64_t glfs_st_attributes_mask; /* Mask to show what's supported in
+                                         st_attributes [ucond] */
+    struct timespec glfs_st_atime;    /* Last access time */
+    struct timespec glfs_st_btime;    /* File creation time */
+    struct timespec glfs_st_ctime;    /* Last attribute change time */
+    struct timespec glfs_st_mtime;    /* Last data modification time */
+    ino_t glfs_st_ino;                /* Inode number */
+    off_t glfs_st_size;               /* File size */
+    blkcnt_t glfs_st_blocks;          /* Number of 512-byte blocks allocated */
+    uint32_t glfs_st_rdev_major; /* Device ID of special file [if bdev/cdev] */
+    uint32_t glfs_st_rdev_minor;
+    uint32_t glfs_st_dev_major; /* ID of device containing file [uncond] */
+    uint32_t glfs_st_dev_minor;
+    blksize_t glfs_st_blksize; /* Preferred general I/O size [uncond] */
+    nlink_t glfs_st_nlink;     /* Number of hard links */
+    uid_t glfs_st_uid;         /* User ID of owner */
+    gid_t glfs_st_gid;         /* Group ID of owner */
+    mode_t glfs_st_mode;       /* File mode */
+};
+
+#define GLFS_LEASE_ID_SIZE 16 /* 128bits */
+typedef char glfs_leaseid_t[GLFS_LEASE_ID_SIZE];
+
+/*
  * PER THREAD IDENTITY MODIFIERS
  *
  * The following operations enable to set a per thread identity context
@@ -362,15 +545,27 @@ typedef struct glfs_fd glfs_fd_t;
  *    caller
  *  - The groups once set, need to be unset by setting the size to 0 (in which
  *    case the list argument is a do not care)
+ *  - In case of leases feature enables, setfsleaseid is used to set and reset
+ *    leaseid before and after every I/O operation.
  *  - Once a process for a thread of operation choses to set the IDs, all glfs
  *    calls made from that thread would default to the IDs set for the thread.
  *    As a result use these APIs with care and ensure that the set IDs are
  *    reverted to global process defaults as required.
  *
  */
-int glfs_setfsuid (uid_t fsuid) __THROW;
-int glfs_setfsgid (gid_t fsgid) __THROW;
-int glfs_setfsgroups (size_t size, const gid_t *list) __THROW;
+int
+glfs_setfsuid(uid_t fsuid) __THROW GFAPI_PUBLIC(glfs_setfsuid, 3.4.2);
+
+int
+glfs_setfsgid(gid_t fsgid) __THROW GFAPI_PUBLIC(glfs_setfsgid, 3.4.2);
+
+int
+glfs_setfsgroups(size_t size, const gid_t *list) __THROW
+    GFAPI_PUBLIC(glfs_setfsgroups, 3.4.2);
+
+int
+glfs_setfsleaseid(glfs_leaseid_t leaseid) __THROW
+    GFAPI_PUBLIC(glfs_setfsleaseid, 4.0.0);
 
 /*
   SYNOPSIS
@@ -397,8 +592,9 @@ int glfs_setfsgroups (size_t size, const gid_t *list) __THROW;
 
  */
 
-glfs_fd_t *glfs_open (glfs_t *fs, const char *path, int flags) __THROW;
-
+glfs_fd_t *
+glfs_open(glfs_t *fs, const char *path, int flags) __THROW
+    GFAPI_PUBLIC(glfs_open, 3.4.0);
 
 /*
   SYNOPSIS
@@ -426,15 +622,20 @@ glfs_fd_t *glfs_open (glfs_t *fs, const char *path, int flags) __THROW;
 
  */
 
-glfs_fd_t *glfs_creat (glfs_t *fs, const char *path, int flags,
-		       mode_t mode) __THROW;
+glfs_fd_t *
+glfs_creat(glfs_t *fs, const char *path, int flags, mode_t mode) __THROW
+    GFAPI_PUBLIC(glfs_creat, 3.4.0);
 
-int glfs_close (glfs_fd_t *fd) __THROW;
+int
+glfs_close(glfs_fd_t *fd) __THROW GFAPI_PUBLIC(glfs_close, 3.4.0);
 
-glfs_t *glfs_from_glfd (glfs_fd_t *fd) __THROW;
+glfs_t *
+glfs_from_glfd(glfs_fd_t *fd) __THROW GFAPI_PUBLIC(glfs_from_glfd, 3.4.0);
 
-int glfs_set_xlator_option (glfs_t *fs, const char *xlator, const char *key,
-			    const char *value) __THROW;
+int
+glfs_set_xlator_option(glfs_t *fs, const char *xlator, const char *key,
+                       const char *value) __THROW
+    GFAPI_PUBLIC(glfs_set_xlator_option, 3.4.0);
 
 /*
 
@@ -457,208 +658,828 @@ int glfs_set_xlator_option (glfs_t *fs, const char *xlator, const char *key,
   time of issuing the async IO call. This can be used by the
   caller to differentiate different instances of the async requests
   in a common callback function.
+
+  @prestat and @poststat are allocated on the stack, that are auto destroyed
+  post the callback function returns.
 */
 
-typedef void (*glfs_io_cbk) (glfs_fd_t *fd, ssize_t ret, void *data);
+typedef void (*glfs_io_cbk)(glfs_fd_t *fd, ssize_t ret,
+                            struct glfs_stat *prestat,
+                            struct glfs_stat *poststat, void *data);
 
 // glfs_{read,write}[_async]
 
-ssize_t glfs_read (glfs_fd_t *fd, void *buf,
-                   size_t count, int flags) __THROW;
-ssize_t glfs_write (glfs_fd_t *fd, const void *buf,
-                    size_t count, int flags) __THROW;
-int glfs_read_async (glfs_fd_t *fd, void *buf, size_t count, int flags,
-		     glfs_io_cbk fn, void *data) __THROW;
-int glfs_write_async (glfs_fd_t *fd, const void *buf, size_t count, int flags,
-		      glfs_io_cbk fn, void *data) __THROW;
+ssize_t
+glfs_read(glfs_fd_t *fd, void *buf, size_t count, int flags) __THROW
+    GFAPI_PUBLIC(glfs_read, 3.4.0);
+
+ssize_t
+glfs_write(glfs_fd_t *fd, const void *buf, size_t count, int flags) __THROW
+    GFAPI_PUBLIC(glfs_write, 3.4.0);
+
+int
+glfs_read_async(glfs_fd_t *fd, void *buf, size_t count, int flags,
+                glfs_io_cbk fn, void *data) __THROW
+    GFAPI_PUBLIC(glfs_read_async, 6.0);
+
+int
+glfs_write_async(glfs_fd_t *fd, const void *buf, size_t count, int flags,
+                 glfs_io_cbk fn, void *data) __THROW
+    GFAPI_PUBLIC(glfs_write_async, 6.0);
 
 // glfs_{read,write}v[_async]
 
-ssize_t glfs_readv (glfs_fd_t *fd, const struct iovec *iov, int iovcnt,
-		    int flags) __THROW;
-ssize_t glfs_writev (glfs_fd_t *fd, const struct iovec *iov, int iovcnt,
-		     int flags) __THROW;
-int glfs_readv_async (glfs_fd_t *fd, const struct iovec *iov, int count,
-		      int flags, glfs_io_cbk fn, void *data) __THROW;
-int glfs_writev_async (glfs_fd_t *fd, const struct iovec *iov, int count,
-		       int flags, glfs_io_cbk fn, void *data) __THROW;
+ssize_t
+glfs_readv(glfs_fd_t *fd, const struct iovec *iov, int iovcnt,
+           int flags) __THROW GFAPI_PUBLIC(glfs_readv, 3.4.0);
+
+ssize_t
+glfs_writev(glfs_fd_t *fd, const struct iovec *iov, int iovcnt,
+            int flags) __THROW GFAPI_PUBLIC(glfs_writev, 3.4.0);
+
+int
+glfs_readv_async(glfs_fd_t *fd, const struct iovec *iov, int count, int flags,
+                 glfs_io_cbk fn, void *data) __THROW
+    GFAPI_PUBLIC(glfs_readv_async, 6.0);
+
+int
+glfs_writev_async(glfs_fd_t *fd, const struct iovec *iov, int count, int flags,
+                  glfs_io_cbk fn, void *data) __THROW
+    GFAPI_PUBLIC(glfs_writev_async, 6.0);
 
 // glfs_p{read,write}[_async]
 
-ssize_t glfs_pread (glfs_fd_t *fd, void *buf, size_t count, off_t offset,
-		    int flags) __THROW;
-ssize_t glfs_pwrite (glfs_fd_t *fd, const void *buf, size_t count,
-		     off_t offset, int flags) __THROW;
-int glfs_pread_async (glfs_fd_t *fd, void *buf, size_t count, off_t offset,
-		      int flags, glfs_io_cbk fn, void *data) __THROW;
-int glfs_pwrite_async (glfs_fd_t *fd, const void *buf, int count, off_t offset,
-		       int flags, glfs_io_cbk fn, void *data) __THROW;
+ssize_t
+glfs_pread(glfs_fd_t *fd, void *buf, size_t count, off_t offset, int flags,
+           struct glfs_stat *poststat) __THROW GFAPI_PUBLIC(glfs_pread, 6.0);
+
+ssize_t
+glfs_pwrite(glfs_fd_t *fd, const void *buf, size_t count, off_t offset,
+            int flags, struct glfs_stat *prestat,
+            struct glfs_stat *poststat) __THROW GFAPI_PUBLIC(glfs_pwrite, 6.0);
+
+int
+glfs_pread_async(glfs_fd_t *fd, void *buf, size_t count, off_t offset,
+                 int flags, glfs_io_cbk fn, void *data) __THROW
+    GFAPI_PUBLIC(glfs_pread_async, 6.0);
+
+int
+glfs_pwrite_async(glfs_fd_t *fd, const void *buf, int count, off_t offset,
+                  int flags, glfs_io_cbk fn, void *data) __THROW
+    GFAPI_PUBLIC(glfs_pwrite_async, 6.0);
 
 // glfs_p{read,write}v[_async]
 
-ssize_t glfs_preadv (glfs_fd_t *fd, const struct iovec *iov, int iovcnt,
-		     off_t offset, int flags) __THROW;
-ssize_t glfs_pwritev (glfs_fd_t *fd, const struct iovec *iov, int iovcnt,
-		      off_t offset, int flags) __THROW;
-int glfs_preadv_async (glfs_fd_t *fd, const struct iovec *iov,
-                       int count, off_t offset, int flags,
-                       glfs_io_cbk fn, void *data) __THROW;
-int glfs_pwritev_async (glfs_fd_t *fd, const struct iovec *iov,
-                        int count, off_t offset, int flags,
-                        glfs_io_cbk fn, void *data) __THROW;
+ssize_t
+glfs_preadv(glfs_fd_t *fd, const struct iovec *iov, int iovcnt, off_t offset,
+            int flags) __THROW GFAPI_PUBLIC(glfs_preadv, 3.4.0);
+
+ssize_t
+glfs_pwritev(glfs_fd_t *fd, const struct iovec *iov, int iovcnt, off_t offset,
+             int flags) __THROW GFAPI_PUBLIC(glfs_pwritev, 3.4.0);
+
+int
+glfs_preadv_async(glfs_fd_t *fd, const struct iovec *iov, int count,
+                  off_t offset, int flags, glfs_io_cbk fn, void *data) __THROW
+    GFAPI_PUBLIC(glfs_preadv_async, 6.0);
+
+int
+glfs_pwritev_async(glfs_fd_t *fd, const struct iovec *iov, int count,
+                   off_t offset, int flags, glfs_io_cbk fn, void *data) __THROW
+    GFAPI_PUBLIC(glfs_pwritev_async, 6.0);
+
+off_t
+glfs_lseek(glfs_fd_t *fd, off_t offset, int whence) __THROW
+    GFAPI_PUBLIC(glfs_lseek, 3.4.0);
+
+ssize_t
+glfs_copy_file_range(struct glfs_fd *glfd_in, off64_t *off_in,
+                     struct glfs_fd *glfd_out, off64_t *off_out, size_t len,
+                     unsigned int flags, struct glfs_stat *statbuf,
+                     struct glfs_stat *prestat,
+                     struct glfs_stat *poststat) __THROW
+    GFAPI_PUBLIC(glfs_copy_file_range, 6.0);
+
+int
+glfs_truncate(glfs_t *fs, const char *path, off_t length) __THROW
+    GFAPI_PUBLIC(glfs_truncate, 3.7.15);
+
+int
+glfs_ftruncate(glfs_fd_t *fd, off_t length, struct glfs_stat *prestat,
+               struct glfs_stat *poststat) __THROW
+    GFAPI_PUBLIC(glfs_ftruncate, 6.0);
+
+int
+glfs_ftruncate_async(glfs_fd_t *fd, off_t length, glfs_io_cbk fn,
+                     void *data) __THROW
+    GFAPI_PUBLIC(glfs_ftruncate_async, 6.0);
+
+int
+glfs_lstat(glfs_t *fs, const char *path, struct stat *buf) __THROW
+    GFAPI_PUBLIC(glfs_lstat, 3.4.0);
+
+int
+glfs_stat(glfs_t *fs, const char *path, struct stat *buf) __THROW
+    GFAPI_PUBLIC(glfs_stat, 3.4.0);
+
+int
+glfs_fstat(glfs_fd_t *fd, struct stat *buf) __THROW
+    GFAPI_PUBLIC(glfs_fstat, 3.4.0);
+
+int
+glfs_fsync(glfs_fd_t *fd, struct glfs_stat *prestat,
+           struct glfs_stat *poststat) __THROW GFAPI_PUBLIC(glfs_fsync, 6.0);
+
+int
+glfs_fsync_async(glfs_fd_t *fd, glfs_io_cbk fn, void *data) __THROW
+    GFAPI_PUBLIC(glfs_fsync_async, 6.0);
+
+int
+glfs_fdatasync(glfs_fd_t *fd, struct glfs_stat *prestat,
+               struct glfs_stat *poststat) __THROW
+    GFAPI_PUBLIC(glfs_fdatasync, 6.0);
+
+int
+glfs_fdatasync_async(glfs_fd_t *fd, glfs_io_cbk fn, void *data) __THROW
+    GFAPI_PUBLIC(glfs_fdatasync_async, 6.0);
+
+int
+glfs_access(glfs_t *fs, const char *path, int mode) __THROW
+    GFAPI_PUBLIC(glfs_access, 3.4.0);
+
+int
+glfs_symlink(glfs_t *fs, const char *oldpath, const char *newpath) __THROW
+    GFAPI_PUBLIC(glfs_symlink, 3.4.0);
+
+int
+glfs_readlink(glfs_t *fs, const char *path, char *buf, size_t bufsiz) __THROW
+    GFAPI_PUBLIC(glfs_readlink, 3.4.0);
+
+int
+glfs_mknod(glfs_t *fs, const char *path, mode_t mode, dev_t dev) __THROW
+    GFAPI_PUBLIC(glfs_mknod, 3.4.0);
+
+int
+glfs_mkdir(glfs_t *fs, const char *path, mode_t mode) __THROW
+    GFAPI_PUBLIC(glfs_mkdir, 3.4.0);
+
+int
+glfs_unlink(glfs_t *fs, const char *path) __THROW
+    GFAPI_PUBLIC(glfs_unlink, 3.4.0);
+
+int
+glfs_rmdir(glfs_t *fs, const char *path) __THROW
+    GFAPI_PUBLIC(glfs_rmdir, 3.4.0);
+
+int
+glfs_rename(glfs_t *fs, const char *oldpath, const char *newpath) __THROW
+    GFAPI_PUBLIC(glfs_rename, 3.4.0);
+
+int
+glfs_link(glfs_t *fs, const char *oldpath, const char *newpath) __THROW
+    GFAPI_PUBLIC(glfs_link, 3.4.0);
+
+glfs_fd_t *
+glfs_opendir(glfs_t *fs, const char *path) __THROW
+    GFAPI_PUBLIC(glfs_opendir, 3.4.0);
+
+/*
+ * @glfs_readdir_r and @glfs_readdirplus_r ARE thread safe AND re-entrant,
+ * but the interface has ambiguity about the size of @dirent to be allocated
+ * before calling the APIs. 512 byte buffer (for @dirent) is sufficient for
+ * all known systems which are tested againt glusterfs/gfapi, but may be
+ * insufficient in the future.
+ */
+
+int
+glfs_readdir_r(glfs_fd_t *fd, struct dirent *dirent,
+               struct dirent **result) __THROW
+    GFAPI_PUBLIC(glfs_readdir_r, 3.4.0);
+
+int
+glfs_readdirplus_r(glfs_fd_t *fd, struct stat *stat, struct dirent *dirent,
+                   struct dirent **result) __THROW
+    GFAPI_PUBLIC(glfs_readdirplus_r, 3.4.0);
+
+/*
+ * @glfs_readdir and @glfs_readdirplus are NEITHER thread safe NOR re-entrant
+ * when called on the same directory handle. However they ARE thread safe
+ * AND re-entrant when called on different directory handles (which may be
+ * referring to the same directory too.)
+ */
+
+struct dirent *
+glfs_readdir(glfs_fd_t *fd) __THROW GFAPI_PUBLIC(glfs_readdir, 3.5.0);
+
+struct dirent *
+glfs_readdirplus(glfs_fd_t *fd, struct stat *stat) __THROW
+    GFAPI_PUBLIC(glfs_readdirplus, 3.5.0);
+
+long
+glfs_telldir(glfs_fd_t *fd) __THROW GFAPI_PUBLIC(glfs_telldir, 3.4.0);
+
+void
+glfs_seekdir(glfs_fd_t *fd, long offset) __THROW
+    GFAPI_PUBLIC(glfs_seekdir, 3.4.0);
+
+int
+glfs_closedir(glfs_fd_t *fd) __THROW GFAPI_PUBLIC(glfs_closedir, 3.4.0);
+
+int
+glfs_statvfs(glfs_t *fs, const char *path, struct statvfs *buf) __THROW
+    GFAPI_PUBLIC(glfs_statvfs, 3.4.0);
+
+int
+glfs_chmod(glfs_t *fs, const char *path, mode_t mode) __THROW
+    GFAPI_PUBLIC(glfs_chmod, 3.4.0);
+
+int
+glfs_fchmod(glfs_fd_t *fd, mode_t mode) __THROW
+    GFAPI_PUBLIC(glfs_fchmod, 3.4.0);
+
+int
+glfs_chown(glfs_t *fs, const char *path, uid_t uid, gid_t gid) __THROW
+    GFAPI_PUBLIC(glfs_chown, 3.4.0);
 
+int
+glfs_lchown(glfs_t *fs, const char *path, uid_t uid, gid_t gid) __THROW
+    GFAPI_PUBLIC(glfs_lchown, 3.4.0);
 
-off_t glfs_lseek (glfs_fd_t *fd, off_t offset, int whence) __THROW;
+int
+glfs_fchown(glfs_fd_t *fd, uid_t uid, gid_t gid) __THROW
+    GFAPI_PUBLIC(glfs_fchown, 3.4.0);
 
-int glfs_truncate (glfs_t *fs, const char *path, off_t length) __THROW;
+int
+glfs_utimens(glfs_t *fs, const char *path,
+             const struct timespec times[2]) __THROW
+    GFAPI_PUBLIC(glfs_utimens, 3.4.0);
 
-int glfs_ftruncate (glfs_fd_t *fd, off_t length) __THROW;
-int glfs_ftruncate_async (glfs_fd_t *fd, off_t length, glfs_io_cbk fn,
-			  void *data) __THROW;
+int
+glfs_lutimens(glfs_t *fs, const char *path,
+              const struct timespec times[2]) __THROW
+    GFAPI_PUBLIC(glfs_lutimens, 3.4.0);
 
-int glfs_lstat (glfs_t *fs, const char *path, struct stat *buf) __THROW;
-int glfs_stat (glfs_t *fs, const char *path, struct stat *buf) __THROW;
-int glfs_fstat (glfs_fd_t *fd, struct stat *buf) __THROW;
+int
+glfs_futimens(glfs_fd_t *fd, const struct timespec times[2]) __THROW
+    GFAPI_PUBLIC(glfs_futimens, 3.4.0);
 
-int glfs_fsync (glfs_fd_t *fd) __THROW;
-int glfs_fsync_async (glfs_fd_t *fd, glfs_io_cbk fn, void *data) __THROW;
+ssize_t
+glfs_getxattr(glfs_t *fs, const char *path, const char *name, void *value,
+              size_t size) __THROW GFAPI_PUBLIC(glfs_getxattr, 3.4.0);
 
-int glfs_fdatasync (glfs_fd_t *fd) __THROW;
-int glfs_fdatasync_async (glfs_fd_t *fd, glfs_io_cbk fn, void *data) __THROW;
+ssize_t
+glfs_lgetxattr(glfs_t *fs, const char *path, const char *name, void *value,
+               size_t size) __THROW GFAPI_PUBLIC(glfs_lgetxattr, 3.4.0);
 
-int glfs_access (glfs_t *fs, const char *path, int mode) __THROW;
+ssize_t
+glfs_fgetxattr(glfs_fd_t *fd, const char *name, void *value,
+               size_t size) __THROW GFAPI_PUBLIC(glfs_fgetxattr, 3.4.0);
 
-int glfs_symlink (glfs_t *fs, const char *oldpath, const char *newpath) __THROW;
+ssize_t
+glfs_listxattr(glfs_t *fs, const char *path, void *value, size_t size) __THROW
+    GFAPI_PUBLIC(glfs_listxattr, 3.4.0);
 
-int glfs_readlink (glfs_t *fs, const char *path,
-                   char *buf, size_t bufsiz) __THROW;
+ssize_t
+glfs_llistxattr(glfs_t *fs, const char *path, void *value, size_t size) __THROW
+    GFAPI_PUBLIC(glfs_llistxattr, 3.4.0);
 
-int glfs_mknod (glfs_t *fs, const char *path, mode_t mode, dev_t dev) __THROW;
+ssize_t
+glfs_flistxattr(glfs_fd_t *fd, void *value, size_t size) __THROW
+    GFAPI_PUBLIC(glfs_flistxattr, 3.4.0);
 
-int glfs_mkdir (glfs_t *fs, const char *path, mode_t mode) __THROW;
+int
+glfs_setxattr(glfs_t *fs, const char *path, const char *name, const void *value,
+              size_t size, int flags) __THROW
+    GFAPI_PUBLIC(glfs_setxattr, 3.4.0);
 
-int glfs_unlink (glfs_t *fs, const char *path) __THROW;
+int
+glfs_lsetxattr(glfs_t *fs, const char *path, const char *name,
+               const void *value, size_t size, int flags) __THROW
+    GFAPI_PUBLIC(glfs_lsetxattr, 3.4.0);
 
-int glfs_rmdir (glfs_t *fs, const char *path) __THROW;
+int
+glfs_fsetxattr(glfs_fd_t *fd, const char *name, const void *value, size_t size,
+               int flags) __THROW GFAPI_PUBLIC(glfs_fsetxattr, 3.4.0);
 
-int glfs_rename (glfs_t *fs, const char *oldpath, const char *newpath) __THROW;
+int
+glfs_removexattr(glfs_t *fs, const char *path, const char *name) __THROW
+    GFAPI_PUBLIC(glfs_removexattr, 3.4.0);
 
-int glfs_link (glfs_t *fs, const char *oldpath, const char *newpath) __THROW;
+int
+glfs_lremovexattr(glfs_t *fs, const char *path, const char *name) __THROW
+    GFAPI_PUBLIC(glfs_lremovexattr, 3.4.0);
 
-glfs_fd_t *glfs_opendir (glfs_t *fs, const char *path) __THROW;
+int
+glfs_fremovexattr(glfs_fd_t *fd, const char *name) __THROW
+    GFAPI_PUBLIC(glfs_fremovexattr, 3.4.0);
+
+int
+glfs_fallocate(glfs_fd_t *fd, int keep_size, off_t offset, size_t len) __THROW
+    GFAPI_PUBLIC(glfs_fallocate, 3.5.0);
+
+int
+glfs_discard(glfs_fd_t *fd, off_t offset, size_t len) __THROW
+    GFAPI_PUBLIC(glfs_discard, 3.5.0);
+
+int
+glfs_discard_async(glfs_fd_t *fd, off_t length, size_t lent, glfs_io_cbk fn,
+                   void *data) __THROW GFAPI_PUBLIC(glfs_discard_async, 6.0);
+
+int
+glfs_zerofill(glfs_fd_t *fd, off_t offset, off_t len) __THROW
+    GFAPI_PUBLIC(glfs_zerofill, 3.5.0);
+
+int
+glfs_zerofill_async(glfs_fd_t *fd, off_t length, off_t len, glfs_io_cbk fn,
+                    void *data) __THROW GFAPI_PUBLIC(glfs_zerofill_async, 6.0);
+
+char *
+glfs_getcwd(glfs_t *fs, char *buf, size_t size) __THROW
+    GFAPI_PUBLIC(glfs_getcwd, 3.4.0);
+
+int
+glfs_chdir(glfs_t *fs, const char *path) __THROW
+    GFAPI_PUBLIC(glfs_chdir, 3.4.0);
+
+int
+glfs_fchdir(glfs_fd_t *fd) __THROW GFAPI_PUBLIC(glfs_fchdir, 3.4.0);
+
+char *
+glfs_realpath(glfs_t *fs, const char *path, char *resolved_path) __THROW
+    GFAPI_PUBLIC(glfs_realpath, 3.7.17);
 
 /*
- * @glfs_readdir_r and @glfs_readdirplus_r ARE thread safe AND re-entrant,
- * but the interface has ambiguity about the size of @dirent to be allocated
- * before calling the APIs. 512 byte buffer (for @dirent) is sufficient for
- * all known systems which are tested againt glusterfs/gfapi, but may be
- * insufficient in the future.
+ * @cmd and @flock are as specified in man fcntl(2).
+ */
+int
+glfs_posix_lock(glfs_fd_t *fd, int cmd, struct flock *flock) __THROW
+    GFAPI_PUBLIC(glfs_posix_lock, 3.4.0);
+
+/*
+  SYNOPSIS
+
+  glfs_file_lock: Request extended byte range lock on a file
+
+  DESCRIPTION
+
+  This function is capable of requesting either advisory or mandatory type
+  byte range locks on a file.
+
+  Note: To set a unique owner key for locks based on a particular file
+  descriptor, make use of glfs_fd_set_lkowner() api to do so before
+  requesting lock via this api. This owner key will be further consumed
+  by other incoming data modifying file operations via the same file
+  descriptor.
+
+  PARAMETERS
+
+  @fd: File descriptor
+
+  @cmd: As specified in man fcntl(2).
+
+  @flock: As specified in man fcntl(2).
+
+  @lk_mode: Required lock type from options available with the
+            enum glfs_lock_mode_t defined below.
+
+  RETURN VALUES
+
+  0   : Success. Lock has been granted.
+  -1  : Failure. @errno will be set indicating the type of failure.
+
  */
 
-int glfs_readdir_r (glfs_fd_t *fd, struct dirent *dirent,
-		    struct dirent **result) __THROW;
+/* Lock modes used by glfs_file_lock() */
+enum glfs_lock_mode { GLFS_LK_ADVISORY = 0, GLFS_LK_MANDATORY };
+typedef enum glfs_lock_mode glfs_lock_mode_t;
+
+int
+glfs_file_lock(glfs_fd_t *fd, int cmd, struct flock *flock,
+               glfs_lock_mode_t lk_mode) __THROW
+    GFAPI_PUBLIC(glfs_file_lock, 3.13.0);
 
-int glfs_readdirplus_r (glfs_fd_t *fd, struct stat *stat, struct dirent *dirent,
-			struct dirent **result) __THROW;
+glfs_fd_t *
+glfs_dup(glfs_fd_t *fd) __THROW GFAPI_PUBLIC(glfs_dup, 3.4.0);
+
+void
+glfs_free(void *ptr) __THROW GFAPI_PUBLIC(glfs_free, 3.7.16);
 
 /*
- * @glfs_readdir and @glfs_readdirplus are NEITHER thread safe NOR re-entrant
- * when called on the same directory handle. However they ARE thread safe
- * AND re-entrant when called on different directory handles (which may be
- * referring to the same directory too.)
+ * glfs_sysrq: send a system request to the @fs instance
+ *
+ * Different commands for @sysrq are possible, the defines for these are listed
+ * below the function definition.
+ *
+ * This function always returns success if the @sysrq is recognized. The return
+ * value does not way anythin about the result of the @sysrq execution. Not all
+ * @sysrq command will be able to return a success/failure status.
+ */
+int
+glfs_sysrq(glfs_t *fs, char sysrq) __THROW GFAPI_PUBLIC(glfs_sysrq, 3.10.0);
+
+#define GLFS_SYSRQ_HELP 'h' /* log a message with supported sysrq commands */
+#define GLFS_SYSRQ_STATEDUMP 's' /* create a statedump */
+
+/*
+ * Structure returned as part of xreaddirplus
+ */
+struct glfs_xreaddirp_stat;
+typedef struct glfs_xreaddirp_stat glfs_xreaddirp_stat_t;
+
+/* Request flags to be used in XREADDIRP operation */
+#define GFAPI_XREADDIRP_NULL                                                   \
+    0x00000000 /* by default, no stat will be fetched */
+#define GFAPI_XREADDIRP_STAT 0x00000001   /* Get stat */
+#define GFAPI_XREADDIRP_HANDLE 0x00000002 /* Get object handle */
+
+/*
+ * This stat structure returned gets freed as part of glfs_free(xstat)
+ */
+struct stat *
+glfs_xreaddirplus_get_stat(glfs_xreaddirp_stat_t *xstat) __THROW
+    GFAPI_PUBLIC(glfs_xreaddirplus_get_stat, 3.11.0);
+
+/*
+ * SYNOPSIS
+ *
+ * glfs_xreaddirplus_r: Extended Readirplus operation
+ *
+ * DESCRIPTION
+ *
+ * This API does readdirplus operation, but along with stat it can fetch other
+ * extra information like object handles etc for each of the dirents returned
+ * based on requested flags. On success it returns the set of flags successfully
+ * processed.
+ *
+ * Note that there are chances that some of the requested information may not be
+ * available or returned (for example if reached EOD). Ensure to validate the
+ * returned value to determine what flags have been successfully processed
+ * & set.
+ *
+ * PARAMETERS
+ *
+ * INPUT:
+ * @glfd: GFAPI file descriptor of the directory
+ * @flags: Flags determining xreaddirp_stat requested
+ *         Current available values are:
+ *              GFAPI_XREADDIRP_NULL
+ *              GFAPI_XREADDIRP_STAT
+ *              GFAPI_XREADDIRP_HANDLE
+ * @ext: Dirent struture to copy the values to
+ *       (though optional recommended to be allocated by application
+ *        esp., in multi-threaded environment)
+ *
+ * OUTPUT:
+ * @res: to store the next dirent value. If NULL and return value is '0',
+ *       it means it reached end of the directory.
+ * @xstat_p: Pointer to contain all the requested data returned
+ *           for that dirent. Application should make use of glfs_free() API
+ *           to free this pointer and the variables returned by
+ *           glfs_xreaddirplus_get_*() APIs.
+ *
+ * RETURN VALUE:
+ * >=0: SUCCESS (value contains the flags successfully processed)
+ *  -1: FAILURE
  */
+int
+glfs_xreaddirplus_r(glfs_fd_t *glfd, uint32_t flags,
+                    glfs_xreaddirp_stat_t **xstat_p, struct dirent *ext,
+                    struct dirent **res) __THROW
+    GFAPI_PUBLIC(glfs_xreaddirplus_r, 3.11.0);
+
+#define GFAPI_MAX_LOCK_OWNER_LEN 255
 
-struct dirent *glfs_readdir (glfs_fd_t *fd) __THROW;
+/*
+ *
+ * DESCRIPTION
+ *
+ * This API allows application to set lk_owner on a fd.
+ * A glfd can be associated with only single lk_owner. In case if there
+ * is need to set another lk_owner, applications can make use of
+ * 'glfs_dup' to get duplicate glfd and set new lk_owner on that second
+ * glfd.
+ *
+ * Also its not recommended to override or clear lk_owner value as the
+ * same shall be used to flush any outstanding locks while closing the fd.
+ *
+ * PARAMETERS
+ *
+ * INPUT:
+ * @glfd: GFAPI file descriptor
+ * @len: Size of lk_owner buffer. Max value can be GFAPI_MAX_LOCK_OWNER_LEN
+ * @data: lk_owner data buffer.
+ *
+ * OUTPUT:
+ * 0: SUCCESS
+ * -1: FAILURE
+ */
+int
+glfs_fd_set_lkowner(glfs_fd_t *glfd, void *data, int len) __THROW
+    GFAPI_PUBLIC(glfs_fd_set_lkowner, 3.10.7);
+
+/*
+ * Applications (currently NFS-Ganesha) can make use of this
+ * structure to read upcall notifications sent by server either
+ * by polling or registering a callback function.
+ *
+ * On success, applications need to check for 'reason' to decide
+ * if any upcall event is received.
+ *
+ * Currently supported upcall_events -
+ *      GLFS_UPCALL_INODE_INVALIDATE -
+ *              'event_arg' - glfs_upcall_inode
+ *
+ * After processing the event, applications need to free 'event_arg' with
+ * glfs_free().
+ *
+ * Also similar to I/Os, the application should ideally stop polling
+ * or unregister upcall_cbk function before calling glfs_fini(..).
+ * Hence making an assumption that 'fs' & ctx structures cannot be
+ * freed while in this routine.
+ */
+struct glfs_upcall;
+typedef struct glfs_upcall glfs_upcall_t;
+
+glfs_t *
+glfs_upcall_get_fs(glfs_upcall_t *arg) __THROW
+    GFAPI_PUBLIC(glfs_upcall_get_fs, 3.7.16);
+
+enum glfs_upcall_reason {
+    GLFS_UPCALL_EVENT_NULL = 0,
+    GLFS_UPCALL_INODE_INVALIDATE, /* invalidate cache entry */
+    GLFS_UPCALL_RECALL_LEASE,     /* recall lease */
+};
+typedef enum glfs_upcall_reason glfs_upcall_reason_t;
+
+glfs_upcall_reason_t
+glfs_upcall_get_reason(glfs_upcall_t *arg) __THROW
+    GFAPI_PUBLIC(glfs_upcall_get_reason, 3.7.16);
+
+/*
+ * Applications first need to make use of above API i.e,
+ * "glfs_upcall_get_reason" to determine which upcall event it has
+ * received. Post that below API - "glfs_upcall_get_event" should
+ * be used to get corresponding upcall event object.
+ *
+ * Below are the upcall_reason and corresponding upcall_event objects:
+ *      ==========================================================
+ *      glfs_upcall_reason           -    event_object
+ *      ==========================================================
+ *      GLFS_UPCALL_EVENT_NULL       -    NULL
+ *      GLFS_UPCALL_INODE_INVALIDATE -    struct glfs_upcall_inode
+ *      GLFS_UPCALL_RECALL_LEASE     -    struct glfs_upcall_lease
+ *
+ * After processing upcall event, glfs_free() should be called on the
+ * glfs_upcall.
+ */
+void *
+glfs_upcall_get_event(glfs_upcall_t *arg) __THROW
+    GFAPI_PUBLIC(glfs_upcall_get_event, 3.7.16);
+
+/*
+ * SYNOPSIS
+ *
+ * glfs_upcall_cbk: Upcall callback definition
+ *
+ * This is function type definition of the callback function pointer
+ * which has to be provided by the caller while registering for any
+ * upcall events.
+ *
+ * This function is called whenever any upcall which the application
+ * has registered for is received from the server.
+ *
+ * @up_arg: Upcall structure whose contents need to be interpreted by
+ * making use of glfs_upcall_* helper routines.
+ *
+ * @data: The same context pointer provided by the caller at the time of
+ * registering of upcall events. This may be used by the caller for any
+ * of its internal use while processing upcalls.
+ */
+typedef void (*glfs_upcall_cbk)(glfs_upcall_t *up_arg, void *data);
 
-struct dirent *glfs_readdirplus (glfs_fd_t *fd, struct stat *stat) __THROW;
+/*
+ * List of upcall events supported by gluster/gfapi
+ */
+#define GLFS_EVENT_INODE_INVALIDATE 0x00000001 /* invalidate cache entry */
+#define GLFS_EVENT_RECALL_LEASE 0x00000002     /* Recall lease */
+#define GLFS_EVENT_ANY 0xffffffff              /* for all the above events */
 
-long glfs_telldir (glfs_fd_t *fd) __THROW;
+/*
+ * SYNOPSIS
+ *
+ * glfs_upcall_register: Register for upcall events
+ *
+ * DESCRIPTION
+ *
+ * This function is used to register for various upcall events application
+ * is interested in and the callback function to be invoked when such
+ * events are triggered.
+ *
+ * Multiple calls of this routine shall override cbk function. That means
+ * only one cbk function can be used for all the upcall events registered
+ * and that shall be the one last updated.
+ *
+ * PARAMETERS:
+ *
+ * INPUT:
+ * @fs: The 'virtual mount' object
+ *
+ * @event_list: List of upcall events to be registered.
+ *              Current available values are:
+ *               - GLFS_EVENT_INODE_INVALIDATE
+ *               - GLFS_EVENT_RECALL_LEASE
+ *
+ * @cbk: The cbk routine to be invoked in case of any upcall received
+ * @data: Any opaque pointer provided by caller which shall be using while
+ * making cbk calls. This pointer may be used by caller for any of its
+ * internal use while processing upcalls. Can be NULL.
+ *
+ * RETURN VALUE:
+ * >0: SUCCESS (value contains the events successfully registered)
+ * -1: FAILURE
+ */
+int
+glfs_upcall_register(glfs_t *fs, uint32_t event_list, glfs_upcall_cbk cbk,
+                     void *data) __THROW
+    GFAPI_PUBLIC(glfs_upcall_register, 3.13.0);
 
-void glfs_seekdir (glfs_fd_t *fd, long offset) __THROW;
+/*
+ * SYNOPSIS
+ *
+ * glfs_upcall_unregister: Unregister for upcall events
+ *
+ * DESCRIPTION
+ *
+ * This function is used to unregister the upcall events application
+ * is not interested in. In case if the caller unregisters all the events
+ * it has registered for, it shall no more receive any upcall event.
+ *
+ * PARAMETERS:
+ *
+ * INPUT:
+ * @fs: The 'virtual mount' object
+ *
+ * @event_list: List of upcall events to be unregistered.
+ *              Current available values are:
+ *               - GLFS_EVENT_INODE_INVALIDATE
+ *               - GLFS_EVENT_RECALL_LEASE
+ * RETURN VALUE:
+ * >0: SUCCESS (value contains the events successfully unregistered)
+ * -1: FAILURE
+ */
+int
+glfs_upcall_unregister(glfs_t *fs, uint32_t event_list) __THROW
+    GFAPI_PUBLIC(glfs_upcall_unregister, 3.13.0);
+
+/* Lease Types */
+enum glfs_lease_types {
+    GLFS_LEASE_NONE = 0,
+    GLFS_RD_LEASE = 1,
+    GLFS_RW_LEASE = 2,
+};
+typedef enum glfs_lease_types glfs_lease_types_t;
+
+/* Lease cmds */
+enum glfs_lease_cmds {
+    GLFS_GET_LEASE = 1,
+    GLFS_SET_LEASE = 2,
+    GLFS_UNLK_LEASE = 3,
+};
+typedef enum glfs_lease_cmds glfs_lease_cmds_t;
+
+struct glfs_lease {
+    glfs_lease_cmds_t cmd;
+    glfs_lease_types_t lease_type;
+    glfs_leaseid_t lease_id;
+    unsigned int lease_flags;
+};
+typedef struct glfs_lease glfs_lease_t;
+
+typedef void (*glfs_recall_cbk)(glfs_lease_t lease, void *data);
 
-int glfs_closedir (glfs_fd_t *fd) __THROW;
+/*
+  SYNOPSIS
 
-int glfs_statvfs (glfs_t *fs, const char *path, struct statvfs *buf) __THROW;
+  glfs_lease: Takes a lease on a file.
 
-int glfs_chmod (glfs_t *fs, const char *path, mode_t mode) __THROW;
+  DESCRIPTION
 
-int glfs_fchmod (glfs_fd_t *fd, mode_t mode) __THROW;
+  This function takes lease on an open file.
 
-int glfs_chown (glfs_t *fs, const char *path, uid_t uid, gid_t gid) __THROW;
+  PARAMETERS
 
-int glfs_lchown (glfs_t *fs, const char *path, uid_t uid, gid_t gid) __THROW;
+  @glfd: The fd of the file on which lease should be taken,
+   this fd is returned by glfs_open/glfs_create.
 
-int glfs_fchown (glfs_fd_t *fd, uid_t uid, gid_t gid) __THROW;
+  @lease: Struct that defines the lease operation to be performed
+   on the file.
+      @lease.cmd - Can be one of the following values
+         GF_GET_LEASE:  Get the lease type currently present on the file,
+                        lease.lease_type will contain GF_RD_LEASE
+                        or GF_RW_LEASE or 0 if no leases.
+         GF_SET_LEASE:  Set the lease of given lease.lease_type on the file.
+         GF_UNLK_LEASE: Unlock the lease present on the given fd.
+                        Note that the every lease request should have
+                        a corresponding unlk_lease.
 
-int glfs_utimens (glfs_t *fs, const char *path,
-                  struct timespec times[2]) __THROW;
+      @lease.lease_type - Can be one of the following values
+         GF_RD_LEASE:   Read lease on a file, shared lease.
+         GF_RW_LEASE:   Read-Write lease on a file, exclusive lease.
 
-int glfs_lutimens (glfs_t *fs, const char *path,
-                   struct timespec times[2]) __THROW;
+      @lease.lease_id - A unique identification of lease, 128bits.
 
-int glfs_futimens (glfs_fd_t *fd, struct timespec times[2]) __THROW;
+  @fn: This is the function that is invoked when the lease has to be recalled
+  @data: It is a cookie, this pointer is returned as a part of recall
 
-ssize_t glfs_getxattr (glfs_t *fs, const char *path, const char *name,
-		       void *value, size_t size) __THROW;
+  fn and data field are stored as a part of glfs_fd, hence if there are multiple
+  glfs_lease calls, each of them updates the fn and data fields. glfs_recall_cbk
+  will be invoked with the last updated fn and data
 
-ssize_t glfs_lgetxattr (glfs_t *fs, const char *path, const char *name,
-			void *value, size_t size) __THROW;
+  RETURN VALUES
+  0:  Successful completion
+  <0: Failure. @errno will be set with the type of failure
+*/
 
-ssize_t glfs_fgetxattr (glfs_fd_t *fd, const char *name,
-			void *value, size_t size) __THROW;
+int
+glfs_lease(glfs_fd_t *glfd, glfs_lease_t *lease, glfs_recall_cbk fn,
+           void *data) __THROW GFAPI_PUBLIC(glfs_lease, 4.0.0);
 
-ssize_t glfs_listxattr (glfs_t *fs, const char *path,
-                        void *value, size_t size) __THROW;
+/*
+  SYNOPSIS
 
-ssize_t glfs_llistxattr (glfs_t *fs, const char *path, void *value,
-			 size_t size) __THROW;
+  glfs_fsetattr: Function to set attributes.
+  glfs_setattr: Function to set attributes
 
-ssize_t glfs_flistxattr (glfs_fd_t *fd, void *value, size_t size) __THROW;
+  DESCRIPTION
 
-int glfs_setxattr (glfs_t *fs, const char *path, const char *name,
-		   const void *value, size_t size, int flags) __THROW;
+  The functions are used to set attributes on the file.
 
-int glfs_lsetxattr (glfs_t *fs, const char *path, const char *name,
-		    const void *value, size_t size, int flags) __THROW;
+  PARAMETERS
 
-int glfs_fsetxattr (glfs_fd_t *fd, const char *name,
-		    const void *value, size_t size, int flags) __THROW;
+  @glfs_fsetattr
 
-int glfs_removexattr (glfs_t *fs, const char *path, const char *name) __THROW;
+     @glfd: The fd of the file for which the attributes are to be set,
+            this fd is returned by glfs_open/glfs_create.
 
-int glfs_lremovexattr (glfs_t *fs, const char *path, const char *name) __THROW;
+  @glfs_setattr
 
-int glfs_fremovexattr (glfs_fd_t *fd, const char *name) __THROW;
+         @fs: File object.
 
-int glfs_fallocate(glfs_fd_t *fd, int keep_size,
-                   off_t offset, size_t len) __THROW;
+         @path: The path of the file that is being operated on.
 
-int glfs_discard(glfs_fd_t *fd, off_t offset, size_t len) __THROW;
+         @follow: Flag used to resolve symlink.
 
 
-int glfs_discard_async (glfs_fd_t *fd, off_t length, size_t lent,
-			glfs_io_cbk fn, void *data) __THROW;
+  @stat: Struct that has information about the file.
 
-int glfs_zerofill(glfs_fd_t *fd, off_t offset, off_t len) __THROW;
+  @valid: This is the mask bit, that accepts GFAPI_SET_ATTR* masks.
+          Refer glfs.h to see the mask definitions.
 
-int glfs_zerofill_async (glfs_fd_t *fd, off_t length, off_t len,
-                        glfs_io_cbk fn, void *data) __THROW;
+  Both functions are similar in functionality, just that the
+  func setattr() uses file path whereas the func fsetattr()
+  uses the fd.
 
-char *glfs_getcwd (glfs_t *fs, char *buf, size_t size) __THROW;
+  RETURN VALUES
+  0:  Successful completion
+  <0: Failure. @errno will be set with the type of failure
 
-int glfs_chdir (glfs_t *fs, const char *path) __THROW;
+ */
 
-int glfs_fchdir (glfs_fd_t *fd) __THROW;
+int
+glfs_fsetattr(struct glfs_fd *glfd, struct glfs_stat *stat) __THROW
+    GFAPI_PUBLIC(glfs_fsetattr, 6.0);
 
-char *glfs_realpath (glfs_t *fs, const char *path, char *resolved_path) __THROW;
+int
+glfs_setattr(struct glfs *fs, const char *path, struct glfs_stat *stat,
+             int follow) __THROW GFAPI_PUBLIC(glfs_setattr, 6.0);
 
 /*
- * @cmd and @flock are as specified in man fcntl(2).
+  SYNOPSIS
+
+  glfs_set_statedump_path: Function to set statedump path.
+
+  DESCRIPTION
+
+  This function is used to set statedump directory
+
+  PARAMETERS
+
+  @fs: The 'virtual mount' object to be configured with the volume
+       specification file.
+
+  @path: statedump path. Should be a directory. But the API won't fail if the
+  directory doesn't exist yet, as one may create it later.
+
+  RETURN VALUES
+
+   0 : Success.
+  -1 : Failure. @errno will be set with the type of failure.
+
  */
-int glfs_posix_lock (glfs_fd_t *fd, int cmd, struct flock *flock) __THROW;
 
-glfs_fd_t *glfs_dup (glfs_fd_t *fd) __THROW;
+int
+glfs_set_statedump_path(struct glfs *fs, const char *path) __THROW
+    GFAPI_PUBLIC(glfs_set_statedump_path, 7.0);
 
 __END_DECLS
-
 #endif /* !_GLFS_H */
diff --git a/autogen.sh b/autogen.sh
index fef2f037426..c8cdc3f89fa 100755
--- a/autogen.sh
+++ b/autogen.sh
@@ -85,11 +85,7 @@ $TOOL --automake --copy --force
 echo Running ${AUTOCONF}...
 $AUTOCONF
 echo Running ${AUTOMAKE}...
-$AUTOMAKE --add-missing --copy --foreign
-
-# Run autogen in the argp-standalone sub-directory
-echo "Running autogen.sh in argp-standalone ..."
-( cd contrib/argp-standalone;./autogen.sh )
+$AUTOMAKE --add-missing --force-missing --copy
 
 # Instruct user on next steps
 echo
diff --git a/build-aux/checkpatch.pl b/build-aux/checkpatch.pl
new file mode 100755
index 00000000000..17ae4e4d579
--- /dev/null
+++ b/build-aux/checkpatch.pl
@@ -0,0 +1,4326 @@
+#!/usr/bin/perl -w
+# (c) 2001, Dave Jones. (the file handling bit)
+# (c) 2005, Joel Schopp <jschopp@austin.ibm.com> (the ugly bit)
+# (c) 2007,2008, Andy Whitcroft <apw@uk.ibm.com> (new conditions, test suite)
+# (c) 2008-2010 Andy Whitcroft <apw@canonical.com>
+# (c) 2014 Gluster Community <gluster-devel@gluster.org>
+# Licensed under the terms of the GNU GPL License version 2
+
+use strict;
+use POSIX;
+
+my $P = $0;
+$P =~ s@.*/@@g;
+
+my $V = '0.32.1';
+
+use Getopt::Long qw(:config no_auto_abbrev);
+
+my $quiet = 0;
+my $tree = 1;
+my $chk_signoff = 1;
+my $chk_patch = 1;
+my $tst_only;
+my $emacs = 0;
+my $terse = 0;
+my $file = 0;
+my $check = 0;
+my $check_orig = 0;
+my $summary = 1;
+my $mailback = 0;
+my $summary_file = 0;
+my $show_types = 0;
+my $fix = 0;
+my $fix_inplace = 0;
+my $root;
+my %debug;
+my %camelcase = ();
+my %use_type = ();
+my @use = ();
+my %ignore_type = ();
+my @ignore = ();
+my $help = 0;
+my $configuration_file = ".checkpatch.conf";
+my $max_line_length = 80;
+my $ignore_perl_version = 0;
+my $minimum_perl_version = 5.10.0;
+my $gerrit_url = $ENV{GERRIT_URL};
+
+sub help {
+    my ($exitcode) = @_;
+
+    print << "EOM";
+Usage: $P [OPTION]... [FILE]...
+Version: $V
+
+Options:
+  -q, --quiet                quiet
+  --patch                    treat FILE as patchfile (default)
+  --emacs                    emacs compile window format
+  --gerrit-url=STRING        URL the patch was reviewed at
+  --terse                    one line per report
+  -f, --file                 treat FILE as regular source file
+  --subjective, --strict     enable more subjective tests
+  --types TYPE(,TYPE2...)    show only these comma separated message types
+  --ignore TYPE(,TYPE2...)   ignore various comma separated message types
+  --max-line-length=n        set the maximum line length, if exceeded, warn
+  --show-types               show the message "types" in the output
+  --root=PATH                PATH to the glusterfs tree root
+  --no-summary               suppress the per-file summary
+  --mailback                 only produce a report in case of warnings/errors
+  --summary-file             include the filename in summary
+  --debug KEY=[0|1]          turn on/off debugging of KEY, where KEY is one of
+                             'values', 'possible', 'type', and 'attr' (default
+                             is all off)
+  --test-only=WORD           report only warnings/errors containing WORD literally
+  --fix                      EXPERIMENTAL - may create horrible results
+                             If correctable single-line errors exist, create
+                             "<inputfile>.EXPERIMENTAL-checkpatch-fixes"
+                             with potential errors corrected to the preferred
+                             checkpatch style
+  --fix-inplace              EXPERIMENTAL - may create horrible results
+                             Is the same as --fix, but overwrites the input
+                             file.  It's your fault if there's no backup or git
+  --ignore-perl-version      override checking of perl version.  expect
+                             runtime errors.
+  -h, --help, --version      display this help and exit
+
+When FILE is - read standard input.
+EOM
+
+exit($exitcode);
+}
+
+my $conf = which_conf($configuration_file);
+if (-f $conf) {
+    my @conf_args;
+    open(my $conffile, '<', "$conf")
+        or warn "$P: Can't find a readable $configuration_file file $!\n";
+
+    while (<$conffile>) {
+        my $line = $_;
+
+        $line =~ s/\s*\n?$//g;
+        $line =~ s/^\s*//g;
+        $line =~ s/\s+/ /g;
+
+        next if ($line =~ m/^\s*#/);
+        next if ($line =~ m/^\s*$/);
+
+        my @words = split(" ", $line);
+        foreach my $word (@words) {
+            last if ($word =~ m/^#/);
+            push (@conf_args, $word);
+        }
+    }
+    close($conffile);
+    unshift(@ARGV, @conf_args) if @conf_args;
+}
+
+GetOptions(
+    'q|quiet+'  => \$quiet,
+    'patch!'    => \$chk_patch,
+    'emacs!'    => \$emacs,
+    'gerrit-url=s' => \$gerrit_url,
+    'terse!'    => \$terse,
+    'f|file!'   => \$file,
+    'subjective!'       => \$check,
+    'strict!'   => \$check,
+    'ignore=s'  => \@ignore,
+    'types=s'   => \@use,
+    'show-types!'       => \$show_types,
+    'max-line-length=i' => \$max_line_length,
+    'root=s'    => \$root,
+    'summary!'  => \$summary,
+    'mailback!' => \$mailback,
+    'summary-file!'     => \$summary_file,
+    'fix!'              => \$fix,
+    'fix-inplace!'      => \$fix_inplace,
+    'ignore-perl-version!' => \$ignore_perl_version,
+    'debug=s'   => \%debug,
+    'test-only=s'       => \$tst_only,
+    'h|help'    => \$help,
+    'version'   => \$help
+) or help(1);
+
+help(0) if ($help);
+
+$fix = 1 if ($fix_inplace);
+$check_orig = $check;
+
+my $exit = 0;
+
+if ($^V && $^V lt $minimum_perl_version) {
+    printf "$P: requires at least perl version %vd\n", $minimum_perl_version;
+    if (!$ignore_perl_version) {
+        exit(1);
+    }
+}
+
+if ($#ARGV < 0) {
+    print "$P: no input files\n";
+    exit(1);
+}
+
+sub hash_save_array_words {
+    my ($hashRef, $arrayRef) = @_;
+
+    my @array = split(/,/, join(',', @$arrayRef));
+    foreach my $word (@array) {
+        $word =~ s/\s*\n?$//g;
+        $word =~ s/^\s*//g;
+        $word =~ s/\s+/ /g;
+        $word =~ tr/[a-z]/[A-Z]/;
+
+        next if ($word =~ m/^\s*#/);
+        next if ($word =~ m/^\s*$/);
+
+        $hashRef->{$word}++;
+    }
+}
+
+sub hash_show_words {
+    my ($hashRef, $prefix) = @_;
+
+    if ($quiet == 0 && keys %$hashRef) {
+        print "NOTE: $prefix message types:";
+        foreach my $word (sort keys %$hashRef) {
+            print " $word";
+        }
+        print "\n\n";
+    }
+}
+
+hash_save_array_words(\%ignore_type, \@ignore);
+hash_save_array_words(\%use_type, \@use);
+
+my $dbg_values = 0;
+my $dbg_possible = 0;
+my $dbg_type = 0;
+my $dbg_attr = 0;
+for my $key (keys %debug) {
+    ## no critic
+    eval "\${dbg_$key} = '$debug{$key}';";
+    die "$@" if ($@);
+}
+
+my $rpt_cleaners = 0;
+
+if ($terse) {
+    $emacs = 1;
+    $quiet++;
+}
+
+if ($tree) {
+    if (defined $root) {
+        if (!top_of_glusterfs_tree($root)) {
+            die "$P: $root: --root does not point at a valid tree\n";
+        }
+    } else {
+        if (top_of_glusterfs_tree('.')) {
+            $root = '.';
+        } elsif ($0 =~ m@(.*)/extras/[^/]*$@ &&
+                 top_of_glusterfs_tree($1)) {
+            $root = $1;
+        }
+    }
+
+    if (!defined $root) {
+        print "Must be run from the top-level dir. of a GlusterFS tree\n";
+        exit(2);
+    }
+}
+
+my $emitted_corrupt = 0;
+
+our $Ident      = qr{
+                        [A-Za-z_][A-Za-z\d_]*
+                        (?:\s*\#\#\s*[A-Za-z_][A-Za-z\d_]*)*
+                }x;
+our $Storage    = qr{extern|static|asmlinkage};
+our $Sparse     = qr{
+                        __user|
+                        __kernel|
+                        __force|
+                        __iomem|
+                        __must_check|
+                        __init_refok|
+                        __kprobes|
+                        __ref|
+                        __rcu
+                }x;
+our $InitAttributePrefix = qr{__(?:mem|cpu|dev|net_|)};
+our $InitAttributeData = qr{$InitAttributePrefix(?:initdata\b)};
+our $InitAttributeConst = qr{$InitAttributePrefix(?:initconst\b)};
+our $InitAttributeInit = qr{$InitAttributePrefix(?:init\b)};
+our $InitAttribute = qr{$InitAttributeData|$InitAttributeConst|$InitAttributeInit};
+
+# Notes to $Attribute:
+# We need \b after 'init' otherwise 'initconst' will cause a false positive in a check
+our $Attribute  = qr{
+                        const|
+                        __percpu|
+                        __nocast|
+                        __safe|
+                        __bitwise__|
+                        __packed__|
+                        __packed2__|
+                        __naked|
+                        __maybe_unused|
+                        __always_unused|
+                        __noreturn|
+                        __used|
+                        __cold|
+                        __noclone|
+                        __deprecated|
+                        __read_mostly|
+                        __kprobes|
+                        $InitAttribute|
+                        ____cacheline_aligned|
+                        ____cacheline_aligned_in_smp|
+                        ____cacheline_internodealigned_in_smp|
+                        __weak
+                  }x;
+our $Modifier;
+our $Inline     = qr{inline|__always_inline|noinline|__inline|__inline__};
+our $Member     = qr{->$Ident|\.$Ident|\[[^]]*\]};
+our $Lval       = qr{$Ident(?:$Member)*};
+
+our $Int_type   = qr{(?i)llu|ull|ll|lu|ul|l|u};
+our $Binary     = qr{(?i)0b[01]+$Int_type?};
+our $Hex        = qr{(?i)0x[0-9a-f]+$Int_type?};
+our $Int        = qr{[0-9]+$Int_type?};
+our $Octal      = qr{0[0-7]+$Int_type?};
+our $Float_hex  = qr{(?i)0x[0-9a-f]+p-?[0-9]+[fl]?};
+our $Float_dec  = qr{(?i)(?:[0-9]+\.[0-9]*|[0-9]*\.[0-9]+)(?:e-?[0-9]+)?[fl]?};
+our $Float_int  = qr{(?i)[0-9]+e-?[0-9]+[fl]?};
+our $Float      = qr{$Float_hex|$Float_dec|$Float_int};
+our $Constant   = qr{$Float|$Binary|$Octal|$Hex|$Int};
+our $Assignment = qr{\*\=|/=|%=|\+=|-=|<<=|>>=|&=|\^=|\|=|=};
+our $Compare    = qr{<=|>=|==|!=|<|(?<!-)>};
+our $Arithmetic = qr{\+|-|\*|\/|%};
+our $Operators  = qr{
+                        <=|>=|==|!=|
+                        =>|->|<<|>>|<|>|!|~|
+                        &&|\|\||,|\^|\+\+|--|&|\||$Arithmetic
+                  }x;
+
+our $c90_Keywords = qr{do|for|while|if|else|return|goto|continue|switch|default|case|break}x;
+
+our $NonptrType;
+our $NonptrTypeWithAttr;
+our $Type;
+our $Declare;
+
+our $NON_ASCII_UTF8     = qr{
+        [\xC2-\xDF][\x80-\xBF]               # non-overlong 2-byte
+        |  \xE0[\xA0-\xBF][\x80-\xBF]        # excluding overlongs
+        | [\xE1-\xEC\xEE\xEF][\x80-\xBF]{2}  # straight 3-byte
+        |  \xED[\x80-\x9F][\x80-\xBF]        # excluding surrogates
+        |  \xF0[\x90-\xBF][\x80-\xBF]{2}     # planes 1-3
+        | [\xF1-\xF3][\x80-\xBF]{3}          # planes 4-15
+        |  \xF4[\x80-\x8F][\x80-\xBF]{2}     # plane 16
+}x;
+
+our $UTF8       = qr{
+        [\x09\x0A\x0D\x20-\x7E]              # ASCII
+        | $NON_ASCII_UTF8
+}x;
+
+our $typeTypedefs = qr{(?x:
+        (?:__)?(?:u|s|be|le)(?:8|16|32|64)|
+        atomic_t
+)};
+
+our $logFunctions = qr{(?x:
+        printk(?:_ratelimited|_once|)|
+        (?:[a-z0-9]+_){1,2}(?:printk|emerg|alert|crit|err|warning|warn|notice|info|debug|dbg|vdbg|devel|cont|WARN)(?:_ratelimited|_once|)|
+        WARN(?:_RATELIMIT|_ONCE|)|
+        panic|
+        MODULE_[A-Z_]+|
+        seq_vprintf|seq_printf|seq_puts
+)};
+
+our $signature_tags = qr{(?xi:
+        Signed-off-by:|
+        Acked-by:|
+        Tested-by:|
+        Reviewed-by:|
+        Reviewed-on:|
+        Reported-by:|
+        Original-author:|
+        Original-Author:|
+        Original-Authors:|
+        Suggested-by:|
+        To:|
+        Cc:
+)};
+
+our $url_tags = qr{http:|https:};
+
+our @typeList = (
+        qr{void},
+        qr{(?:unsigned\s+)?char},
+        qr{(?:unsigned\s+)?short},
+        qr{(?:unsigned\s+)?int},
+        qr{(?:unsigned\s+)?long},
+        qr{(?:unsigned\s+)?long\s+int},
+        qr{(?:unsigned\s+)?long\s+long},
+        qr{(?:unsigned\s+)?long\s+long\s+int},
+        qr{unsigned},
+        qr{float},
+        qr{double},
+        qr{bool},
+        qr{struct\s+$Ident},
+        qr{union\s+$Ident},
+        qr{enum\s+$Ident},
+        qr{${Ident}_t},
+        qr{${Ident}_handler},
+        qr{${Ident}_handler_fn},
+);
+our @typeListWithAttr = (
+        @typeList,
+        qr{struct\s+$InitAttribute\s+$Ident},
+        qr{union\s+$InitAttribute\s+$Ident},
+);
+
+our @modifierList = (
+        qr{fastcall},
+);
+
+our @mode_permission_funcs = (
+        ["module_param", 3],
+        ["module_param_(?:array|named|string)", 4],
+        ["module_param_array_named", 5],
+        ["debugfs_create_(?:file|u8|u16|u32|u64|x8|x16|x32|x64|size_t|atomic_t|bool|blob|regset32|u32_array)", 2],
+        ["proc_create(?:_data|)", 2],
+        ["(?:CLASS|DEVICE|SENSOR)_ATTR", 2],
+);
+
+#Create a search pattern for all these functions to speed up a loop below
+our $mode_perms_search = "";
+foreach my $entry (@mode_permission_funcs) {
+        $mode_perms_search .= '|' if ($mode_perms_search ne "");
+        $mode_perms_search .= $entry->[0];
+}
+
+our $declaration_macros = qr{(?x:
+        (?:$Storage\s+)?(?:DECLARE|DEFINE)_[A-Z]+\s*\(|
+        (?:$Storage\s+)?LIST_HEAD\s*\(
+)};
+
+our $allowed_asm_includes = qr{(?x:
+        irq|
+        memory
+)};
+# memory.h: ARM has a custom one
+
+sub build_types {
+    my $mods = "(?x:\n" . join("|\n  ", @modifierList) . "\n)";
+    my $all = "(?x:\n" . join("|\n  ", @typeList) . "\n)";
+    my $allWithAttr = "(?x:\n" . join("|\n  ", @typeListWithAttr) . "\n)";
+    $Modifier   = qr{(?:$Attribute|$Sparse|$mods)};
+    $NonptrType = qr{
+                        (?:$Modifier\s+|const\s+)*
+                        (?:
+                                (?:typeof|__typeof__)\s*\([^\)]*\)|
+                                (?:$typeTypedefs\b)|
+                                (?:${all}\b)
+                        )
+                        (?:\s+$Modifier|\s+const)*
+                  }x;
+        $NonptrTypeWithAttr     = qr{
+                        (?:$Modifier\s+|const\s+)*
+                        (?:
+                                (?:typeof|__typeof__)\s*\([^\)]*\)|
+                                (?:$typeTypedefs\b)|
+                                (?:${allWithAttr}\b)
+                        )
+                        (?:\s+$Modifier|\s+const)*
+                  }x;
+        $Type   = qr{
+                        $NonptrType
+                        (?:(?:\s|\*|\[\])+\s*const|(?:\s|\*|\[\])+|(?:\s*\[\s*\])+)?
+                        (?:\s+$Inline|\s+$Modifier)*
+                  }x;
+        $Declare        = qr{(?:$Storage\s+(?:$Inline\s+)?)?$Type};
+}
+build_types();
+
+our $Typecast   = qr{\s*(\(\s*$NonptrType\s*\)){0,1}\s*};
+
+# Using $balanced_parens, $LvalOrFunc, or $FuncArg
+# requires at least perl version v5.10.0
+# Any use must be runtime checked with $^V
+
+our $balanced_parens = qr/(\((?:[^\(\)]++|(?-1))*\))/;
+our $LvalOrFunc = qr{((?:[\&\*]\s*)?$Lval)\s*($balanced_parens{0,1})\s*};
+our $FuncArg = qr{$Typecast{0,1}($LvalOrFunc|$Constant)};
+
+sub deparenthesize {
+    my ($string) = @_;
+    return "" if (!defined($string));
+
+    while ($string =~ /^\s*\(.*\)\s*$/) {
+        $string =~ s@^\s*\(\s*@@;
+        $string =~ s@\s*\)\s*$@@;
+    }
+
+    $string =~ s@\s+@ @g;
+
+    return $string;
+}
+
+sub seed_camelcase_file {
+    my ($file) = @_;
+
+    return if (!(-f $file));
+
+    local $/;
+
+    open(my $include_file, '<', "$file")
+        or warn "$P: Can't read '$file' $!\n";
+    my $text = <$include_file>;
+    close($include_file);
+
+    my @lines = split('\n', $text);
+
+    foreach my $line (@lines) {
+        next if ($line !~ /(?:[A-Z][a-z]|[a-z][A-Z])/);
+        if ($line =~ /^[ \t]*(?:#[ \t]*define|typedef\s+$Type)\s+(\w*(?:[A-Z][a-z]|[a-z][A-Z])\w*)/) {
+            $camelcase{$1} = 1;
+        } elsif ($line =~ /^\s*$Declare\s+(\w*(?:[A-Z][a-z]|[a-z][A-Z])\w*)\s*[\(\[,;]/) {
+            $camelcase{$1} = 1;
+        } elsif ($line =~ /^\s*(?:union|struct|enum)\s+(\w*(?:[A-Z][a-z]|[a-z][A-Z])\w*)\s*[;\{]/) {
+            $camelcase{$1} = 1;
+        }
+    }
+}
+
+my $camelcase_seeded = 0;
+sub seed_camelcase_includes {
+    return if ($camelcase_seeded);
+
+    my $files;
+    my $camelcase_cache = "";
+    my @include_files = ();
+
+    $camelcase_seeded = 1;
+
+    if (-e ".git") {
+        my $git_last_include_commit = `git log --no-merges --pretty=format:"%h%n" -1 -- include`;
+        chomp $git_last_include_commit;
+        $camelcase_cache = ".checkpatch-camelcase.git.$git_last_include_commit";
+    } else {
+        my $last_mod_date = 0;
+        $files = `find $root/include -name "*.h"`;
+        @include_files = split('\n', $files);
+        foreach my $file (@include_files) {
+            my $date = POSIX::strftime("%Y%m%d%H%M",
+                                       localtime((stat $file)[9]));
+            $last_mod_date = $date if ($last_mod_date < $date);
+        }
+        $camelcase_cache = ".checkpatch-camelcase.date.$last_mod_date";
+    }
+
+    if ($camelcase_cache ne "" && -f $camelcase_cache) {
+        open(my $camelcase_file, '<', "$camelcase_cache")
+            or warn "$P: Can't read '$camelcase_cache' $!\n";
+        while (<$camelcase_file>) {
+            chomp;
+            $camelcase{$_} = 1;
+        }
+        close($camelcase_file);
+        return;
+    }
+
+    if (-e ".git") {
+        $files = `git ls-files "include/*.h"`;
+        @include_files = split('\n', $files);
+    }
+
+    foreach my $file (@include_files) {
+        seed_camelcase_file($file);
+    }
+
+    if ($camelcase_cache ne "") {
+        unlink glob ".checkpatch-camelcase.*";
+        open(my $camelcase_file, '>', "$camelcase_cache")
+            or warn "$P: Can't write '$camelcase_cache' $!\n";
+        foreach (sort { lc($a) cmp lc($b) } keys(%camelcase)) {
+            print $camelcase_file ("$_\n");
+        }
+        close($camelcase_file);
+    }
+}
+
+$chk_signoff = 0 if ($file);
+
+my @rawlines = ();
+my @lines = ();
+my @fixed = ();
+my $vname;
+for my $filename (@ARGV) {
+    my $FILE;
+    if ($file) {
+        open($FILE, '-|', "diff -u /dev/null $filename") ||
+            die "$P: $filename: diff failed - $!\n";
+    } elsif ($filename eq '-') {
+        open($FILE, '<&STDIN');
+    } else {
+        open($FILE, '<', "$filename") ||
+            die "$P: $filename: open failed - $!\n";
+    }
+    if ($filename eq '-') {
+        $vname = 'Your patch';
+    } else {
+        $vname = $filename;
+    }
+    while (<$FILE>) {
+        chomp;
+        push(@rawlines, $_);
+    }
+    close($FILE);
+    if (!process($filename)) {
+        $exit = 1;
+    }
+    @rawlines = ();
+    @lines = ();
+    @fixed = ();
+}
+
+exit($exit);
+
+sub top_of_glusterfs_tree {
+    my ($root) = @_;
+
+    # Add here if the tree changes
+    my @tree_check = (
+        "api",
+        "AUTHORS",
+        "autogen.sh",
+        "build-aux",
+        "ChangeLog",
+        "cli",
+        "configure.ac",
+        "contrib",
+        "CONTRIBUTING",
+        "COPYING-GPLV2",
+        "COPYING-LGPLV3",
+        "doc",
+        "extras",
+        "geo-replication",
+        "glusterfs-api.pc.in",
+        "glusterfsd",
+        "glusterfs.spec.in",
+        "heal",
+        "INSTALL",
+        "libgfchangelog.pc.in",
+        "libglusterfs",
+        "MAINTAINERS",
+        "Makefile.am",
+        "NEWS",
+        "README.md",
+        "rfc.sh",
+        "rpc",
+        "run-tests.sh",
+        "tests",
+        "THANKS",
+        "xlators",
+        );
+
+    foreach my $check (@tree_check) {
+        if (! -e $root . '/' . $check) {
+            return 0;
+        }
+    }
+    return 1;
+}
+
+sub parse_email {
+    my ($formatted_email) = @_;
+
+    my $name = "";
+    my $address = "";
+    my $comment = "";
+
+    if ($formatted_email =~ /^(.*)<(\S+\@\S+)>(.*)$/) {
+        $name = $1;
+        $address = $2;
+        $comment = $3 if defined $3;
+    } elsif ($formatted_email =~ /^\s*<(\S+\@\S+)>(.*)$/) {
+        $address = $1;
+        $comment = $2 if defined $2;
+    } elsif ($formatted_email =~ /(\S+\@\S+)(.*)$/) {
+        $address = $1;
+        $comment = $2 if defined $2;
+        $formatted_email =~ s/$address.*$//;
+        $name = $formatted_email;
+        $name = trim($name);
+        $name =~ s/^\"|\"$//g;
+        # If there's a name left after stripping spaces and
+        # leading quotes, and the address doesn't have both
+        # leading and trailing angle brackets, the address
+        # is invalid. ie:
+        #   "joe smith joe@smith.com" bad
+        #   "joe smith <joe@smith.com" bad
+        if ($name ne "" && $address !~ /^<[^>]+>$/) {
+            $name = "";
+            $address = "";
+            $comment = "";
+        }
+    }
+
+    $name = trim($name);
+    $name =~ s/^\"|\"$//g;
+    $address = trim($address);
+    $address =~ s/^\<|\>$//g;
+
+    if ($name =~ /[^\w \-]/i) { ##has "must quote" chars
+        $name =~ s/(?<!\\)"/\\"/g; ##escape quotes
+        $name = "\"$name\"";
+    }
+
+    return ($name, $address, $comment);
+}
+
+sub format_email {
+    my ($name, $address) = @_;
+
+    my $formatted_email;
+
+    $name = trim($name);
+    $name =~ s/^\"|\"$//g;
+    $address = trim($address);
+
+    if ($name =~ /[^\w \-]/i) { ##has "must quote" chars
+        $name =~ s/(?<!\\)"/\\"/g; ##escape quotes
+        $name = "\"$name\"";
+    }
+
+    if ("$name" eq "") {
+        $formatted_email = "$address";
+    } else {
+        $formatted_email = "$name <$address>";
+    }
+
+    return $formatted_email;
+}
+
+sub which_conf {
+    my ($conf) = @_;
+
+    foreach my $path (split(/:/, ".:$ENV{HOME}:.scripts")) {
+        if (-e "$path/$conf") {
+            return "$path/$conf";
+        }
+    }
+
+    return "";
+}
+
+sub expand_tabs {
+    my ($str) = @_;
+
+    my $res = '';
+    my $n = 0;
+    for my $c (split(//, $str)) {
+        if ($c eq "\t") {
+            $res .= ' ';
+            $n++;
+            for (; ($n % 8) != 0; $n++) {
+                $res .= ' ';
+            }
+            next;
+        }
+        $res .= $c;
+        $n++;
+    }
+    return $res;
+}
+sub copy_spacing {
+    (my $res = shift) =~ tr/\t/ /c;
+    return $res;
+}
+
+sub line_stats {
+    my ($line) = @_;
+
+    # Drop the diff line leader and expand tabs
+    $line =~ s/^.//;
+    $line = expand_tabs($line);
+
+    # Pick the indent from the front of the line.
+    my ($white) = ($line =~ /^(\s*)/);
+
+    return (length($line), length($white));
+}
+
+my $sanitise_quote = '';
+
+sub sanitise_line_reset {
+    my ($in_comment) = @_;
+
+    if ($in_comment) {
+        $sanitise_quote = '*/';
+    } else {
+        $sanitise_quote = '';
+    }
+}
+sub sanitise_line {
+    my ($line) = @_;
+
+    my $res = '';
+    my $l = '';
+
+    my $qlen = 0;
+    my $off = 0;
+    my $c;
+
+    # Always copy over the diff marker.
+    $res = substr($line, 0, 1);
+
+    for ($off = 1; $off < length($line); $off++) {
+        $c = substr($line, $off, 1);
+
+        # Comments we are wacking completly including the begin
+        # and end, all to $;.
+        if ($sanitise_quote eq '' && substr($line, $off, 2) eq '/*') {
+            $sanitise_quote = '*/';
+
+            substr($res, $off, 2, "$;$;");
+            $off++;
+            next;
+        }
+        if ($sanitise_quote eq '*/' && substr($line, $off, 2) eq '*/') {
+            $sanitise_quote = '';
+            substr($res, $off, 2, "$;$;");
+            $off++;
+            next;
+        }
+        if ($sanitise_quote eq '' && substr($line, $off, 2) eq '//') {
+            $sanitise_quote = '//';
+
+            substr($res, $off, 2, $sanitise_quote);
+            $off++;
+            next;
+        }
+
+        # A \ in a string means ignore the next character.
+        if (($sanitise_quote eq "'" || $sanitise_quote eq '"') &&
+            $c eq "\\") {
+            substr($res, $off, 2, 'XX');
+            $off++;
+            next;
+        }
+        # Regular quotes.
+        if ($c eq "'" || $c eq '"') {
+            if ($sanitise_quote eq '') {
+                $sanitise_quote = $c;
+
+                substr($res, $off, 1, $c);
+                next;
+            } elsif ($sanitise_quote eq $c) {
+                $sanitise_quote = '';
+            }
+        }
+
+        #print "c<$c> SQ<$sanitise_quote>\n";
+        if ($off != 0 && $sanitise_quote eq '*/' && $c ne "\t") {
+            substr($res, $off, 1, $;);
+        } elsif ($off != 0 && $sanitise_quote eq '//' && $c ne "\t") {
+            substr($res, $off, 1, $;);
+        } elsif ($off != 0 && $sanitise_quote && $c ne "\t") {
+            substr($res, $off, 1, 'X');
+        } else {
+            substr($res, $off, 1, $c);
+        }
+    }
+
+    if ($sanitise_quote eq '//') {
+        $sanitise_quote = '';
+    }
+
+    # The pathname on a #include may be surrounded by '<' and '>'.
+    if ($res =~ /^.\s*\#\s*include\s+\<(.*)\>/) {
+        my $clean = 'X' x length($1);
+        $res =~ s@\<.*\>@<$clean>@;
+
+    # The whole of a #error is a string.
+    } elsif ($res =~ /^.\s*\#\s*(?:error|warning)\s+(.*)\b/) {
+        my $clean = 'X' x length($1);
+        $res =~ s@(\#\s*(?:error|warning)\s+).*@$1$clean@;
+    }
+
+    return $res;
+}
+
+sub get_quoted_string {
+    my ($line, $rawline) = @_;
+
+    return "" if ($line !~ m/(\"[X]+\")/g);
+    return substr($rawline, $-[0], $+[0] - $-[0]);
+}
+
+sub ctx_statement_block {
+    my ($linenr, $remain, $off) = @_;
+    my $line = $linenr - 1;
+    my $blk = '';
+    my $soff = $off;
+    my $coff = $off - 1;
+    my $coff_set = 0;
+
+    my $loff = 0;
+
+    my $type = '';
+    my $level = 0;
+    my @stack = ();
+    my $p;
+    my $c;
+    my $len = 0;
+
+    my $remainder;
+    while (1) {
+        @stack = (['', 0]) if ($#stack == -1);
+
+        #warn "CSB: blk<$blk> remain<$remain>\n";
+        # If we are about to drop off the end, pull in more
+        # context.
+        if ($off >= $len) {
+            for (; $remain > 0; $line++) {
+                last if (!defined $lines[$line]);
+                next if ($lines[$line] =~ /^-/);
+                $remain--;
+                $loff = $len;
+                $blk .= $lines[$line] . "\n";
+                $len = length($blk);
+                $line++;
+                last;
+            }
+            # Bail if there is no further context.
+            #warn "CSB: blk<$blk> off<$off> len<$len>\n";
+            if ($off >= $len) {
+                last;
+            }
+            if ($level == 0 && substr($blk, $off) =~ /^.\s*#\s*define/) {
+                $level++;
+                $type = '#';
+            }
+        }
+        $p = $c;
+        $c = substr($blk, $off, 1);
+        $remainder = substr($blk, $off);
+
+        #warn "CSB: c<$c> type<$type> level<$level> remainder<$remainder> coff_set<$coff_set>\n";
+
+        # Handle nested #if/#else.
+        if ($remainder =~ /^#\s*(?:ifndef|ifdef|if)\s/) {
+            push(@stack, [ $type, $level ]);
+        } elsif ($remainder =~ /^#\s*(?:else|elif)\b/) {
+            ($type, $level) = @{$stack[$#stack - 1]};
+        } elsif ($remainder =~ /^#\s*endif\b/) {
+            ($type, $level) = @{pop(@stack)};
+        }
+
+        # Statement ends at the ';' or a close '}' at the
+        # outermost level.
+        if ($level == 0 && $c eq ';') {
+            last;
+        }
+
+        # An else is really a conditional as long as its not else if
+        if ($level == 0 && $coff_set == 0 &&
+            (!defined($p) || $p =~ /(?:\s|\}|\+)/) &&
+            $remainder =~ /^(else)(?:\s|{)/ &&
+            $remainder !~ /^else\s+if\b/) {
+            $coff = $off + length($1) - 1;
+            $coff_set = 1;
+            #warn "CSB: mark coff<$coff> soff<$soff> 1<$1>\n";
+            #warn "[" . substr($blk, $soff, $coff - $soff + 1) . "]\n";
+        }
+
+        if (($type eq '' || $type eq '(') && $c eq '(') {
+            $level++;
+            $type = '(';
+        }
+        if ($type eq '(' && $c eq ')') {
+            $level--;
+            $type = ($level != 0)? '(' : '';
+
+            if ($level == 0 && $coff < $soff) {
+                $coff = $off;
+                $coff_set = 1;
+                #warn "CSB: mark coff<$coff>\n";
+            }
+        }
+        if (($type eq '' || $type eq '{') && $c eq '{') {
+            $level++;
+            $type = '{';
+        }
+        if ($type eq '{' && $c eq '}') {
+            $level--;
+            $type = ($level != 0)? '{' : '';
+
+            if ($level == 0) {
+                if (substr($blk, $off + 1, 1) eq ';') {
+                    $off++;
+                }
+                last;
+            }
+        }
+        # Preprocessor commands end at the newline unless escaped.
+        if ($type eq '#' && $c eq "\n" && $p ne "\\") {
+            $level--;
+            $type = '';
+            $off++;
+            last;
+        }
+        $off++;
+    }
+    # We are truly at the end, so shuffle to the next line.
+    if ($off == $len) {
+        $loff = $len + 1;
+        $line++;
+        $remain--;
+    }
+
+    my $statement = substr($blk, $soff, $off - $soff + 1);
+    my $condition = substr($blk, $soff, $coff - $soff + 1);
+
+    #warn "STATEMENT<$statement>\n";
+    #warn "CONDITION<$condition>\n";
+
+    #print "coff<$coff> soff<$off> loff<$loff>\n";
+
+    return ($statement, $condition,
+            $line, $remain + 1, $off - $loff + 1, $level);
+}
+
+sub statement_lines {
+    my ($stmt) = @_;
+
+    # Strip the diff line prefixes and rip blank lines at start and end.
+    $stmt =~ s/(^|\n)./$1/g;
+    $stmt =~ s/^\s*//;
+    $stmt =~ s/\s*$//;
+
+    my @stmt_lines = ($stmt =~ /\n/g);
+
+    return $#stmt_lines + 2;
+}
+
+sub statement_rawlines {
+    my ($stmt) = @_;
+
+    my @stmt_lines = ($stmt =~ /\n/g);
+
+    return $#stmt_lines + 2;
+}
+
+sub statement_block_size {
+    my ($stmt) = @_;
+
+    $stmt =~ s/(^|\n)./$1/g;
+    $stmt =~ s/^\s*{//;
+    $stmt =~ s/}\s*$//;
+    $stmt =~ s/^\s*//;
+    $stmt =~ s/\s*$//;
+
+    my @stmt_lines = ($stmt =~ /\n/g);
+    my @stmt_statements = ($stmt =~ /;/g);
+
+    my $stmt_lines = $#stmt_lines + 2;
+    my $stmt_statements = $#stmt_statements + 1;
+
+    if ($stmt_lines > $stmt_statements) {
+        return $stmt_lines;
+    } else {
+        return $stmt_statements;
+    }
+}
+
+sub ctx_statement_full {
+    my ($linenr, $remain, $off) = @_;
+    my ($statement, $condition, $level);
+
+    my (@chunks);
+
+    # Grab the first conditional/block pair.
+    ($statement, $condition, $linenr, $remain, $off, $level) =
+        ctx_statement_block($linenr, $remain, $off);
+    #print "F: c<$condition> s<$statement> remain<$remain>\n";
+    push(@chunks, [ $condition, $statement ]);
+    if (!($remain > 0 && $condition =~ /^\s*(?:\n[+-])?\s*(?:if|else|do)\b/s)) {
+        return ($level, $linenr, @chunks);
+    }
+
+    # Pull in the following conditional/block pairs and see if they
+    # could continue the statement.
+    for (;;) {
+        ($statement, $condition, $linenr, $remain, $off, $level) =
+            ctx_statement_block($linenr, $remain, $off);
+        #print "C: c<$condition> s<$statement> remain<$remain>\n";
+        last if (!($remain > 0 && $condition =~ /^(?:\s*\n[+-])*\s*(?:else|do)\b/s));
+        #print "C: push\n";
+        push(@chunks, [ $condition, $statement ]);
+    }
+
+    return ($level, $linenr, @chunks);
+}
+
+sub ctx_block_get {
+    my ($linenr, $remain, $outer, $open, $close, $off) = @_;
+    my $line;
+    my $start = $linenr - 1;
+    my $blk = '';
+    my @o;
+    my @c;
+    my @res = ();
+
+    my $level = 0;
+    my @stack = ($level);
+    for ($line = $start; $remain > 0; $line++) {
+        next if ($rawlines[$line] =~ /^-/);
+        $remain--;
+
+        $blk .= $rawlines[$line];
+
+        # Handle nested #if/#else.
+        if ($lines[$line] =~ /^.\s*#\s*(?:ifndef|ifdef|if)\s/) {
+            push(@stack, $level);
+        } elsif ($lines[$line] =~ /^.\s*#\s*(?:else|elif)\b/) {
+            $level = $stack[$#stack - 1];
+        } elsif ($lines[$line] =~ /^.\s*#\s*endif\b/) {
+            $level = pop(@stack);
+        }
+
+        foreach my $c (split(//, $lines[$line])) {
+            ##print "C<$c>L<$level><$open$close>O<$off>\n";
+            if ($off > 0) {
+                $off--;
+                next;
+            }
+
+            if ($c eq $close && $level > 0) {
+                $level--;
+                last if ($level == 0);
+            } elsif ($c eq $open) {
+                $level++;
+            }
+        }
+
+        if (!$outer || $level <= 1) {
+            push(@res, $rawlines[$line]);
+        }
+
+        last if ($level == 0);
+    }
+
+    return ($level, @res);
+}
+sub ctx_block_outer {
+    my ($linenr, $remain) = @_;
+
+    my ($level, @r) = ctx_block_get($linenr, $remain, 1, '{', '}', 0);
+    return @r;
+}
+sub ctx_block {
+    my ($linenr, $remain) = @_;
+
+    my ($level, @r) = ctx_block_get($linenr, $remain, 0, '{', '}', 0);
+    return @r;
+}
+sub ctx_statement {
+    my ($linenr, $remain, $off) = @_;
+
+    my ($level, @r) = ctx_block_get($linenr, $remain, 0, '(', ')', $off);
+    return @r;
+}
+sub ctx_block_level {
+    my ($linenr, $remain) = @_;
+
+    return ctx_block_get($linenr, $remain, 0, '{', '}', 0);
+}
+sub ctx_statement_level {
+    my ($linenr, $remain, $off) = @_;
+
+    return ctx_block_get($linenr, $remain, 0, '(', ')', $off);
+}
+
+sub ctx_locate_comment {
+    my ($first_line, $end_line) = @_;
+
+    # Catch a comment on the end of the line itself.
+    my ($current_comment) = ($rawlines[$end_line - 1] =~ m@.*(/\*.*\*/)\s*(?:\\\s*)?$@);
+    return $current_comment if (defined $current_comment);
+
+    # Look through the context and try and figure out if there is a
+    # comment.
+    my $in_comment = 0;
+    $current_comment = '';
+    for (my $linenr = $first_line; $linenr < $end_line; $linenr++) {
+        my $line = $rawlines[$linenr - 1];
+        #warn "           $line\n";
+        if ($linenr == $first_line and $line =~ m@^.\s*\*@) {
+            $in_comment = 1;
+        }
+        if ($line =~ m@/\*@) {
+            $in_comment = 1;
+        }
+        if (!$in_comment && $current_comment ne '') {
+            $current_comment = '';
+        }
+        $current_comment .= $line . "\n" if ($in_comment);
+        if ($line =~ m@\*/@) {
+            $in_comment = 0;
+        }
+    }
+
+    chomp($current_comment);
+    return($current_comment);
+}
+sub ctx_has_comment {
+    my ($first_line, $end_line) = @_;
+    my $cmt = ctx_locate_comment($first_line, $end_line);
+
+    ##print "LINE: $rawlines[$end_line - 1 ]\n";
+    ##print "CMMT: $cmt\n";
+
+    return ($cmt ne '');
+}
+
+sub raw_line {
+    my ($linenr, $cnt) = @_;
+
+    my $offset = $linenr - 1;
+    $cnt++;
+
+    my $line;
+    while ($cnt) {
+        $line = $rawlines[$offset++];
+        next if (defined($line) && $line =~ /^-/);
+        $cnt--;
+    }
+    return $line;
+}
+
+sub cat_vet {
+    my ($vet) = @_;
+    my ($res, $coded);
+
+    $res = '';
+    while ($vet =~ /([^[:cntrl:]]*)([[:cntrl:]]|$)/g) {
+        $res .= $1;
+        if ($2 ne '') {
+            $coded = sprintf("^%c", unpack('C', $2) + 64);
+            $res .= $coded;
+        }
+    }
+    $res =~ s/$/\$/;
+    return $res;
+}
+
+my $av_preprocessor = 0;
+my $av_pending;
+my @av_paren_type;
+my $av_pend_colon;
+
+sub annotate_reset {
+    $av_preprocessor = 0;
+    $av_pending = '_';
+    @av_paren_type = ('E');
+    $av_pend_colon = 'O';
+}
+
+sub annotate_values {
+    my ($stream, $type) = @_;
+
+    my $res;
+    my $var = '_' x length($stream);
+    my $cur = $stream;
+
+    print "$stream\n" if ($dbg_values > 1);
+
+    while (length($cur)) {
+        @av_paren_type = ('E') if ($#av_paren_type < 0);
+        print " <" . join('', @av_paren_type) .
+            "> <$type> <$av_pending>" if ($dbg_values > 1);
+        if ($cur =~ /^(\s+)/o) {
+            print "WS($1)\n" if ($dbg_values > 1);
+            if ($1 =~ /\n/ && $av_preprocessor) {
+                $type = pop(@av_paren_type);
+                $av_preprocessor = 0;
+            }
+
+        } elsif ($cur =~ /^(\(\s*$Type\s*)\)/ && $av_pending eq '_') {
+            print "CAST($1)\n" if ($dbg_values > 1);
+            push(@av_paren_type, $type);
+            $type = 'c';
+
+        } elsif ($cur =~ /^($Type)\s*(?:$Ident|,|\)|\(|\s*$)/) {
+            print "DECLARE($1)\n" if ($dbg_values > 1);
+            $type = 'T';
+
+        } elsif ($cur =~ /^($Modifier)\s*/) {
+            print "MODIFIER($1)\n" if ($dbg_values > 1);
+            $type = 'T';
+
+        } elsif ($cur =~ /^(\#\s*define\s*$Ident)(\(?)/o) {
+            print "DEFINE($1,$2)\n" if ($dbg_values > 1);
+            $av_preprocessor = 1;
+            push(@av_paren_type, $type);
+            if ($2 ne '') {
+                $av_pending = 'N';
+            }
+            $type = 'E';
+
+        } elsif ($cur =~ /^(\#\s*(?:undef\s*$Ident|include\b))/o) {
+            print "UNDEF($1)\n" if ($dbg_values > 1);
+            $av_preprocessor = 1;
+            push(@av_paren_type, $type);
+
+        } elsif ($cur =~ /^(\#\s*(?:ifdef|ifndef|if))/o) {
+            print "PRE_START($1)\n" if ($dbg_values > 1);
+            $av_preprocessor = 1;
+
+            push(@av_paren_type, $type);
+            push(@av_paren_type, $type);
+            $type = 'E';
+
+        } elsif ($cur =~ /^(\#\s*(?:else|elif))/o) {
+            print "PRE_RESTART($1)\n" if ($dbg_values > 1);
+            $av_preprocessor = 1;
+
+            push(@av_paren_type, $av_paren_type[$#av_paren_type]);
+
+            $type = 'E';
+
+        } elsif ($cur =~ /^(\#\s*(?:endif))/o) {
+            print "PRE_END($1)\n" if ($dbg_values > 1);
+
+            $av_preprocessor = 1;
+
+            # Assume all arms of the conditional end as this
+            # one does, and continue as if the #endif was not here.
+            pop(@av_paren_type);
+            push(@av_paren_type, $type);
+            $type = 'E';
+
+        } elsif ($cur =~ /^(\\\n)/o) {
+            print "PRECONT($1)\n" if ($dbg_values > 1);
+
+        } elsif ($cur =~ /^(__attribute__)\s*\(?/o) {
+            print "ATTR($1)\n" if ($dbg_values > 1);
+            $av_pending = $type;
+            $type = 'N';
+
+        } elsif ($cur =~ /^(sizeof)\s*(\()?/o) {
+            print "SIZEOF($1)\n" if ($dbg_values > 1);
+            if (defined $2) {
+                $av_pending = 'V';
+            }
+            $type = 'N';
+
+        } elsif ($cur =~ /^(if|while|for)\b/o) {
+            print "COND($1)\n" if ($dbg_values > 1);
+            $av_pending = 'E';
+            $type = 'N';
+
+        } elsif ($cur =~/^(case)/o) {
+            print "CASE($1)\n" if ($dbg_values > 1);
+            $av_pend_colon = 'C';
+            $type = 'N';
+
+        } elsif ($cur =~/^(return|else|goto|typeof|__typeof__)\b/o) {
+            print "KEYWORD($1)\n" if ($dbg_values > 1);
+            $type = 'N';
+
+        } elsif ($cur =~ /^(\()/o) {
+            print "PAREN('$1')\n" if ($dbg_values > 1);
+            push(@av_paren_type, $av_pending);
+            $av_pending = '_';
+            $type = 'N';
+
+        } elsif ($cur =~ /^(\))/o) {
+            my $new_type = pop(@av_paren_type);
+            if ($new_type ne '_') {
+                $type = $new_type;
+                print "PAREN('$1') -> $type\n"
+                    if ($dbg_values > 1);
+            } else {
+                print "PAREN('$1')\n" if ($dbg_values > 1);
+            }
+
+        } elsif ($cur =~ /^($Ident)\s*\(/o) {
+            print "FUNC($1)\n" if ($dbg_values > 1);
+            $type = 'V';
+            $av_pending = 'V';
+
+        } elsif ($cur =~ /^($Ident\s*):(?:\s*\d+\s*(,|=|;))?/) {
+            if (defined $2 && $type eq 'C' || $type eq 'T') {
+                $av_pend_colon = 'B';
+            } elsif ($type eq 'E') {
+                $av_pend_colon = 'L';
+            }
+            print "IDENT_COLON($1,$type>$av_pend_colon)\n" if ($dbg_values > 1);
+            $type = 'V';
+
+        } elsif ($cur =~ /^($Ident|$Constant)/o) {
+            print "IDENT($1)\n" if ($dbg_values > 1);
+            $type = 'V';
+
+        } elsif ($cur =~ /^($Assignment)/o) {
+            print "ASSIGN($1)\n" if ($dbg_values > 1);
+            $type = 'N';
+
+        } elsif ($cur =~/^(;|{|})/) {
+            print "END($1)\n" if ($dbg_values > 1);
+            $type = 'E';
+            $av_pend_colon = 'O';
+
+        } elsif ($cur =~/^(,)/) {
+            print "COMMA($1)\n" if ($dbg_values > 1);
+            $type = 'C';
+
+        } elsif ($cur =~ /^(\?)/o) {
+            print "QUESTION($1)\n" if ($dbg_values > 1);
+            $type = 'N';
+
+        } elsif ($cur =~ /^(:)/o) {
+            print "COLON($1,$av_pend_colon)\n" if ($dbg_values > 1);
+
+            substr($var, length($res), 1, $av_pend_colon);
+            if ($av_pend_colon eq 'C' || $av_pend_colon eq 'L') {
+                $type = 'E';
+            } else {
+                $type = 'N';
+            }
+            $av_pend_colon = 'O';
+
+        } elsif ($cur =~ /^(\[)/o) {
+            print "CLOSE($1)\n" if ($dbg_values > 1);
+            $type = 'N';
+
+        } elsif ($cur =~ /^(-(?![->])|\+(?!\+)|\*|\&\&|\&)/o) {
+            my $variant;
+
+            print "OPV($1)\n" if ($dbg_values > 1);
+            if ($type eq 'V') {
+                $variant = 'B';
+            } else {
+                $variant = 'U';
+            }
+
+            substr($var, length($res), 1, $variant);
+            $type = 'N';
+
+        } elsif ($cur =~ /^($Operators)/o) {
+            print "OP($1)\n" if ($dbg_values > 1);
+            if ($1 ne '++' && $1 ne '--') {
+                $type = 'N';
+            }
+
+        } elsif ($cur =~ /(^.)/o) {
+            print "C($1)\n" if ($dbg_values > 1);
+        }
+        if (defined $1) {
+            $cur = substr($cur, length($1));
+            $res .= $type x length($1);
+        }
+    }
+
+    return ($res, $var);
+}
+
+sub possible {
+    my ($possible, $line) = @_;
+    my $notPermitted = qr{(?:
+                ^(?:
+                        $Modifier|
+                        $Storage|
+                        $Type|
+                        DEFINE_\S+
+                )$|
+                ^(?:
+                        goto|
+                        return|
+                        case|
+                        else|
+                        asm|__asm__|
+                        do|
+                        \#|
+                        \#\#|
+                )(?:\s|$)|
+                ^(?:typedef|struct|enum)\b
+            )}x;
+    warn "CHECK<$possible> ($line)\n" if ($dbg_possible > 2);
+    if ($possible !~ $notPermitted) {
+        # Check for modifiers.
+        $possible =~ s/\s*$Storage\s*//g;
+        $possible =~ s/\s*$Sparse\s*//g;
+        if ($possible =~ /^\s*$/) {
+
+        } elsif ($possible =~ /\s/) {
+            $possible =~ s/\s*$Type\s*//g;
+            for my $modifier (split(' ', $possible)) {
+                if ($modifier !~ $notPermitted) {
+                    warn "MODIFIER: $modifier ($possible) ($line)\n" if ($dbg_possible);
+                    push(@modifierList, $modifier);
+                }
+            }
+
+        } else {
+            warn "POSSIBLE: $possible ($line)\n" if ($dbg_possible);
+            push(@typeList, $possible);
+        }
+        build_types();
+    } else {
+        warn "NOTPOSS: $possible ($line)\n" if ($dbg_possible > 1);
+    }
+}
+
+my $prefix = '';
+
+sub show_type {
+    my ($type) = @_;
+
+    return defined $use_type{$type} if (scalar keys %use_type > 0);
+
+    return !defined $ignore_type{$type};
+}
+
+sub report {
+    my ($level, $type, $msg) = @_;
+
+    if (!show_type($type) ||
+        (defined $tst_only && $msg !~ /\Q$tst_only\E/)) {
+        return 0;
+    }
+    my $line;
+    if ($show_types) {
+        $line = "$prefix$level:$type: $msg\n";
+    } else {
+        $line = "$prefix$level: $msg\n";
+    }
+    $line = (split('\n', $line))[0] . "\n" if ($terse);
+
+    if ($quiet == 0) {
+        push(our @report, $line);
+    }
+    return 1;
+}
+
+sub report_dump {
+    our @report;
+}
+
+sub ERROR {
+    my ($type, $msg) = @_;
+
+    if (report("ERROR", $type, $msg)) {
+        our $clean = 0;
+        our $cnt_error++;
+        return 1;
+    }
+    return 0;
+}
+sub WARN {
+    my ($type, $msg) = @_;
+
+    if (report("WARNING", $type, $msg)) {
+        ## Warning is okay to submit
+        our $clean = 0;
+        our $cnt_warn++;
+        return 1;
+    }
+    return 0;
+}
+sub CHK {
+    my ($type, $msg) = @_;
+
+    if ($check && report("CHECK", $type, $msg)) {
+        our $clean = 0;
+        our $cnt_chk++;
+        return 1;
+    }
+    return 0;
+}
+
+sub check_absolute_file {
+    my ($absolute, $herecurr) = @_;
+    my $file = $absolute;
+
+    ##print "absolute<$absolute>\n";
+
+    # See if any suffix of this path is a path within the tree.
+    while ($file =~ s@^[^/]*/@@) {
+        if (-f "$root/$file") {
+            ##print "file<$file>\n";
+            last;
+        }
+    }
+    if (! -f _)  {
+        return 0;
+    }
+
+    # It is, so see if the prefix is acceptable.
+    my $prefix = $absolute;
+    substr($prefix, -length($file)) = '';
+
+    ##print "prefix<$prefix>\n";
+    if ($prefix ne ".../") {
+        WARN("USE_RELATIVE_PATH",
+             "use relative pathname instead of absolute in changelog text\n" . $herecurr);
+    }
+}
+
+sub trim {
+    my ($string) = @_;
+
+    $string =~ s/^\s+|\s+$//g;
+
+    return $string;
+}
+
+sub ltrim {
+    my ($string) = @_;
+
+    $string =~ s/^\s+//;
+
+    return $string;
+}
+
+sub rtrim {
+    my ($string) = @_;
+
+    $string =~ s/\s+$//;
+
+    return $string;
+}
+
+sub string_find_replace {
+    my ($string, $find, $replace) = @_;
+
+    $string =~ s/$find/$replace/g;
+
+    return $string;
+}
+
+sub tabify {
+    my ($leading) = @_;
+
+    my $source_indent = 8;
+    my $max_spaces_before_tab = $source_indent - 1;
+    my $spaces_to_tab = " " x $source_indent;
+
+    #convert leading spaces to tabs
+    1 while $leading =~ s@^([\t]*)$spaces_to_tab@$1\t@g;
+    #Remove spaces before a tab
+    1 while $leading =~ s@^([\t]*)( {1,$max_spaces_before_tab})\t@$1\t@g;
+    return "$leading";
+}
+
+sub pos_last_openparen {
+    my ($line) = @_;
+
+    my $pos = 0;
+
+    my $opens = $line =~ tr/\(/\(/;
+    my $closes = $line =~ tr/\)/\)/;
+
+    my $last_openparen = 0;
+
+    if (($opens == 0) || ($closes >= $opens)) {
+        return -1;
+    }
+
+    my $len = length($line);
+
+    for ($pos = 0; $pos < $len; $pos++) {
+        my $string = substr($line, $pos);
+        if ($string =~ /^($FuncArg|$balanced_parens)/) {
+            $pos += length($1) - 1;
+        } elsif (substr($line, $pos, 1) eq '(') {
+            $last_openparen = $pos;
+        } elsif (index($string, '(') == -1) {
+            last;
+        }
+    }
+
+    return length(expand_tabs(substr($line, 0, $last_openparen))) + 1;
+}
+
+sub process {
+    my $filename = shift;
+
+    my $linenr=0;
+    my $prevline="";
+    my $prevrawline="";
+    my $stashline="";
+    my $stashrawline="";
+
+    my $length;
+    my $indent;
+    my $previndent=0;
+    my $stashindent=0;
+
+    our $clean = 1;
+    my $signoff = 0;
+    my $subject_trailing_dot = 0;
+    my $is_patch = 0;
+
+    my $in_header_lines = 1;
+    my $in_commit_log = 0; #Scanning lines before patch
+
+    my $non_utf8_charset = 0;
+
+    our @report = ();
+    our $cnt_lines = 0;
+    our $cnt_error = 0;
+    our $cnt_warn = 0;
+    our $cnt_chk = 0;
+
+    # Trace the real file/line as we go.
+    my $realfile = '';
+    my $realline = 0;
+    my $realcnt = 0;
+    my $here = '';
+    my $in_comment = 0;
+    my $comment_edge = 0;
+    my $first_line = 0;
+    my $p1_prefix = '';
+
+    my $prev_values = 'E';
+
+    # suppression flags
+    my %suppress_ifbraces;
+    my %suppress_whiletrailers;
+    my %suppress_export;
+    my $suppress_statement = 0;
+
+    my %signatures = ();
+
+    # Pre-scan the patch sanitizing the lines.
+    # Pre-scan the patch looking for any __setup documentation.
+    #
+    my @setup_docs = ();
+    my $setup_docs = 0;
+
+    my $camelcase_file_seeded = 0;
+
+    sanitise_line_reset();
+    my $line;
+    foreach my $rawline (@rawlines) {
+        $linenr++;
+        $line = $rawline;
+
+        push(@fixed, $rawline) if ($fix);
+
+        if ($rawline=~/^\+\+\+\s+(\S+)/) {
+            $setup_docs = 0;
+            if ($1 =~ m@Documentation/kernel-parameters.txt$@) {
+                $setup_docs = 1;
+            }
+            #next;
+        }
+        if ($rawline=~/^\@\@ -\d+(?:,\d+)? \+(\d+)(,(\d+))? \@\@/) {
+            $realline=$1-1;
+            if (defined $2) {
+                $realcnt=$3+1;
+            } else {
+                $realcnt=1+1;
+            }
+            $in_comment = 0;
+
+            # Guestimate if this is a continuing comment.  Run
+            # the context looking for a comment "edge".  If this
+            # edge is a close comment then we must be in a comment
+            # at context start.
+            my $edge;
+            my $cnt = $realcnt;
+            for (my $ln = $linenr + 1; $cnt > 0; $ln++) {
+                next if (defined $rawlines[$ln - 1] &&
+                         $rawlines[$ln - 1] =~ /^-/);
+                $cnt--;
+                #print "RAW<$rawlines[$ln - 1]>\n";
+                last if (!defined $rawlines[$ln - 1]);
+                if ($rawlines[$ln - 1] =~ m@(/\*|\*/)@ &&
+                    $rawlines[$ln - 1] !~ m@"[^"]*(?:/\*|\*/)[^"]*"@) {
+                    ($edge) = $1;
+                    last;
+                }
+            }
+            if (defined $edge && $edge eq '*/') {
+                $in_comment = 1;
+            }
+
+            # Guestimate if this is a continuing comment.  If this
+            # is the start of a diff block and this line starts
+            # ' *' then it is very likely a comment.
+            if (!defined $edge &&
+                $rawlines[$linenr] =~ m@^.\s*(?:\*\*+| \*)(?:\s|$)@)
+            {
+                $in_comment = 1;
+            }
+
+            ##print "COMMENT:$in_comment edge<$edge> $rawline\n";
+            sanitise_line_reset($in_comment);
+
+        } elsif ($realcnt && $rawline =~ /^(?:\+| |$)/) {
+            # Standardise the strings and chars within the input to
+            # simplify matching -- only bother with positive lines.
+            $line = sanitise_line($rawline);
+        }
+        push(@lines, $line);
+
+        if ($realcnt > 1) {
+            $realcnt-- if ($line =~ /^(?:\+| |$)/);
+        } else {
+            $realcnt = 0;
+        }
+
+        #print "==>$rawline\n";
+        #print "-->$line\n";
+
+        if ($setup_docs && $line =~ /^\+/) {
+            push(@setup_docs, $line);
+        }
+    }
+
+    $prefix = '';
+
+    $realcnt = 0;
+    $linenr = 0;
+    foreach my $line (@lines) {
+        $linenr++;
+        my $sline = $line;      #copy of $line
+        $sline =~ s/$;/ /g;     #with comments as spaces
+
+        my $rawline = $rawlines[$linenr - 1];
+
+#extract the line range in the file after the patch is applied
+        if ($line=~/^\@\@ -\d+(?:,\d+)? \+(\d+)(,(\d+))? \@\@/) {
+            $is_patch = 1;
+            $first_line = $linenr + 1;
+            $realline=$1-1;
+            if (defined $2) {
+                $realcnt=$3+1;
+            } else {
+                $realcnt=1+1;
+            }
+            annotate_reset();
+            $prev_values = 'E';
+
+            %suppress_ifbraces = ();
+            %suppress_whiletrailers = ();
+            %suppress_export = ();
+            $suppress_statement = 0;
+            next;
+
+# track the line number as we move through the hunk, note that
+# new versions of GNU diff omit the leading space on completely
+# blank context lines so we need to count that too.
+        } elsif ($line =~ /^( |\+|$)/) {
+            $realline++;
+            $realcnt-- if ($realcnt != 0);
+
+            # Measure the line length and indent.
+            ($length, $indent) = line_stats($rawline);
+
+            # Track the previous line.
+            ($prevline, $stashline) = ($stashline, $line);
+            ($previndent, $stashindent) = ($stashindent, $indent);
+            ($prevrawline, $stashrawline) = ($stashrawline, $rawline);
+
+            #warn "line<$line>\n";
+
+        } elsif ($realcnt == 1) {
+            $realcnt--;
+        }
+
+        my $hunk_line = ($realcnt != 0);
+
+#make up the handle for any error we report on this line
+        $prefix = "$filename:$realline: " if ($emacs && $file);
+        $prefix = "$filename:$linenr: " if ($emacs && !$file);
+
+        $here = "#$linenr: " if (!$file);
+        $here = "#$realline: " if ($file);
+
+        my $found_file = 0;
+        # extract the filename as it passes
+        if ($line =~ /^diff --git.*?(\S+)$/) {
+            $realfile = $1;
+            $realfile =~ s@^([^/]*)/@@ if (!$file);
+            $in_commit_log = 0;
+            $found_file = 1;
+        } elsif ($line =~ /^\+\+\+\s+(\S+)/) {
+            $realfile = $1;
+            $realfile =~ s@^([^/]*)/@@ if (!$file);
+            $in_commit_log = 0;
+
+            $p1_prefix = $1;
+            if (!$file && $tree && $p1_prefix ne '' &&
+                -e "$root/$p1_prefix") {
+                WARN("PATCH_PREFIX",
+                     "patch prefix '$p1_prefix' exists, appears to be a -p0 patch\n");
+            }
+
+            $found_file = 1;
+        }
+
+        if ($found_file) {
+            if ($realfile =~ m@^(drivers/net/|net/)@) {
+                $check = 1;
+            } else {
+                $check = $check_orig;
+            }
+            next;
+        }
+
+        $here .= "FILE: $realfile:$realline:" if ($realcnt != 0);
+
+        my $hereline = "$here\n$rawline\n";
+        my $herecurr = "$here\n$rawline\n";
+        my $hereprev = "$here\n$prevrawline\n$rawline\n";
+
+        $cnt_lines++ if ($realcnt != 0);
+
+# Check for incorrect file permissions
+        if ($line =~ /^new (file )?mode.*[7531]\d{0,2}$/) {
+            my $permhere = $here . "FILE: $realfile\n";
+            if ($realfile !~ m@scripts/@ &&
+                $realfile !~ /\.(py|pl|awk|sh|t)$/) {
+                ERROR("EXECUTE_PERMISSIONS",
+                      "do not set execute permissions for source files\n" . $permhere);
+            }
+        }
+
+        next if ($realfile =~ /(checkpatch.pl)/);
+        next if ($realfile =~ /\.(md|txt|doc|8|pdf|tex)$/);
+
+# Check that the subject does not have a trailing dot
+        if ($in_header_lines &&
+            $line =~ /^Subject: \[PATCH\] (.+)\.(\s*)$/) {
+                $subject_trailing_dot++;
+        }
+
+# Check the patch for a signoff:
+        if ($line =~ /^\s*signed-off-by:/i) {
+            $signoff++;
+            $in_commit_log = 0;
+        }
+
+# Check signature styles
+        if (!$in_header_lines &&
+            $line =~ /^(\s*)([a-z0-9_-]+by:|$signature_tags)(\s*)(.*)/i) {
+            my $space_before = $1;
+            my $sign_off = $2;
+            my $space_after = $3;
+            my $email = $4;
+            my $ucfirst_sign_off = ucfirst(lc($sign_off));
+
+            if ($sign_off !~ /$signature_tags/) {
+                WARN("BAD_SIGN_OFF",
+                     "Non-standard signature: $sign_off\n" . $herecurr);
+            }
+            if (defined $space_before && $space_before ne "") {
+                if (WARN("BAD_SIGN_OFF",
+                         "Do not use whitespace before $ucfirst_sign_off\n" . $herecurr) &&
+                    $fix) {
+                    $fixed[$linenr - 1] =
+                        "$ucfirst_sign_off $email";
+                }
+            }
+            if ($sign_off =~ /-by:$/i && $sign_off ne $ucfirst_sign_off) {
+                if (WARN("BAD_SIGN_OFF",
+                         "'$ucfirst_sign_off' is the preferred signature form\n" . $herecurr) &&
+                    $fix) {
+                    $fixed[$linenr - 1] =
+                        "$ucfirst_sign_off $email";
+                }
+
+            }
+            if (!defined $space_after || $space_after ne " ") {
+                if (WARN("BAD_SIGN_OFF",
+                         "Use a single space after $ucfirst_sign_off\n" . $herecurr) &&
+                    $fix) {
+                    $fixed[$linenr - 1] =
+                        "$ucfirst_sign_off $email";
+                }
+            }
+
+            # Check if email is really Gerrit URL
+            if ($email =~ /^($url_tags)(.*)/) {
+                my $uri = $1;
+                my $url = $2;
+                if ($uri && $url !~ /$gerrit_url/) {
+                    ERROR("BAD_URL",
+                          "Unrecognized url address: '$email'\n" . $herecurr);
+                }
+            } else {
+                my ($email_name, $email_address, $comment) = parse_email($email);
+                my $suggested_email = format_email(($email_name, $email_address));
+                if ($suggested_email eq "") {
+                    ERROR("BAD_SIGN_OFF",
+                          "Unrecognized email address: '$email'\n" . $herecurr);
+                } else {
+                    my $dequoted = $suggested_email;
+                    $dequoted =~ s/^"//;
+                    $dequoted =~ s/" </ </;
+                    # Don't force email to have quotes
+                    # Allow just an angle bracketed address
+                    if ("$dequoted$comment" ne $email &&
+                        "<$email_address>$comment" ne $email &&
+                        "$suggested_email$comment" ne $email) {
+                        WARN("BAD_SIGN_OFF",
+                             "email address '$email' might be better as '$suggested_email$comment'\n" . $herecurr);
+                    }
+                }
+            }
+
+# Check for duplicate signatures
+            my $sig_nospace = $line;
+            $sig_nospace =~ s/\s//g;
+            $sig_nospace = lc($sig_nospace);
+            if (defined $signatures{$sig_nospace}) {
+                WARN("BAD_SIGN_OFF",
+                     "Duplicate signature\n" . $herecurr);
+            } else {
+                $signatures{$sig_nospace} = 1;
+            }
+        }
+
+# Check for wrappage within a valid hunk of the file
+        if ($realcnt != 0 && $line !~ m{^(?:\+|-| |\\ No newline|$)}) {
+            ERROR("CORRUPTED_PATCH",
+                  "patch seems to be corrupt (line wrapped?)\n" .
+                  $herecurr) if (!$emitted_corrupt++);
+        }
+
+# Check for absolute kernel paths.
+        if ($tree) {
+            while ($line =~ m{(?:^|\s)(/\S*)}g) {
+                my $file = $1;
+
+                if ($file =~ m{^(.*?)(?::\d+)+:?$} &&
+                    check_absolute_file($1, $herecurr)) {
+                    #
+                } else {
+                    check_absolute_file($file, $herecurr);
+                }
+            }
+        }
+
+# UTF-8 regex found at http://www.w3.org/International/questions/qa-forms-utf-8.en.php
+        if (($realfile =~ /^$/ || $line =~ /^\+/) &&
+            $rawline !~ m/^$UTF8*$/) {
+            my ($utf8_prefix) = ($rawline =~ /^($UTF8*)/);
+
+            my $blank = copy_spacing($rawline);
+            my $ptr = substr($blank, 0, length($utf8_prefix)) . "^";
+            my $hereptr = "$hereline$ptr\n";
+
+            CHK("INVALID_UTF8",
+                "Invalid UTF-8, patch and commit message should be encoded in UTF-8\n" . $hereptr);
+        }
+
+# Check if it's the start of a commit log
+# (not a header line and we haven't seen the patch filename)
+        if ($in_header_lines && $realfile =~ /^$/ &&
+            $rawline !~ /^(commit\b|from\b|[\w-]+:).+$/i) {
+            $in_header_lines = 0;
+            $in_commit_log = 1;
+        }
+
+# Check if there is UTF-8 in a commit log when a mail header has explicitly
+# declined it, i.e defined some charset where it is missing.
+        if ($in_header_lines &&
+            $rawline =~ /^Content-Type:.+charset="(.+)".*$/ &&
+            $1 !~ /utf-8/i) {
+            $non_utf8_charset = 1;
+        }
+
+        if ($in_commit_log && $non_utf8_charset && $realfile =~ /^$/ &&
+            $rawline =~ /$NON_ASCII_UTF8/) {
+            WARN("UTF8_BEFORE_PATCH",
+                 "8-bit UTF-8 used in possible commit log\n" . $herecurr);
+        }
+
+# ignore non-hunk lines and lines being removed
+        next if (!$hunk_line || $line =~ /^-/);
+
+#trailing whitespace
+        if ($line =~ /^\+.*\015/) {
+            my $herevet = "$here\n" . cat_vet($rawline) . "\n";
+            if (ERROR("DOS_LINE_ENDINGS",
+                      "DOS line endings\n" . $herevet) &&
+                $fix) {
+                $fixed[$linenr - 1] =~ s/[\s\015]+$//;
+            }
+        } elsif ($rawline =~ /^\+.*\S\s+$/ || $rawline =~ /^\+\s+$/) {
+            my $herevet = "$here\n" . cat_vet($rawline) . "\n";
+            if (ERROR("TRAILING_WHITESPACE",
+                      "trailing whitespace\n" . $herevet) &&
+                $fix) {
+                $fixed[$linenr - 1] =~ s/\s+$//;
+            }
+
+            $rpt_cleaners = 1;
+        }
+
+        if (($realfile =~ /Makefile.*/) &&
+            ($line =~ /\+(EXTRA_[A-Z]+FLAGS).*/)) {
+            my $flag = $1;
+            my $replacement = {
+                            'EXTRA_AFLAGS' =>   'asflags-y',
+                            'EXTRA_CFLAGS' =>   'ccflags-y',
+                            'EXTRA_CPPFLAGS' => 'cppflags-y',
+                            'EXTRA_LDFLAGS' =>  'ldflags-y',
+            };
+
+            WARN("DEPRECATED_VARIABLE",
+                 "Use of $flag is deprecated, please use \`$replacement->{$flag} instead.\n" . $herecurr) if ($replacement->{$flag});
+        }
+
+# check we are in .spec file, then ignore this hunk
+        next if ($realfile eq "glusterfs.spec.in");
+
+# check we are in a valid source file if not then ignore this hunk
+        next if ($realfile !~ /\.(h|c|pl|py|l|y|sh|in)$/);
+
+#line length limit
+        if ($line =~ /^\+/ && $prevrawline !~ /\/\*\*/ &&
+            $rawline !~ /^.\s*\*\s*\@$Ident\s/ &&
+            !($line =~ /^\+\s*$logFunctions\s*\(\s*(?:(KERN_\S+\s*|[^"]*))?"[X\t]*"\s*(?:|,|\)\s*;)\s*$/ ||
+              $line =~ /^\+\s*"[^"]*"\s*(?:\s*|,|\)\s*;)\s*$/) &&
+            $length > $max_line_length)
+        {
+            WARN("LONG_LINE",
+                 "line over $max_line_length characters\n" . $herecurr);
+        }
+
+# check for spaces before a quoted newline
+        if ($rawline =~ /^.*\".*\s\\n/) {
+            if (WARN("QUOTED_WHITESPACE_BEFORE_NEWLINE",
+                     "unnecessary whitespace before a quoted newline\n" . $herecurr) &&
+                $fix) {
+                $fixed[$linenr - 1] =~ s/^(\+.*\".*)\s+\\n/$1\\n/;
+            }
+
+        }
+
+# check for adding lines without a newline.
+        if ($line =~ /^\+/ && defined $lines[$linenr] && $lines[$linenr] =~ /^\\ No newline at end of file/) {
+            WARN("MISSING_EOF_NEWLINE",
+                 "adding a line without newline at end of file\n" . $herecurr);
+        }
+
+# check we are in a valid source file C or perl if not then ignore this hunk
+        next if ($realfile !~ /\.(h|c|pl)$/);
+
+# check for space before tabs.
+        if ($rawline =~ /^\+/ && $rawline =~ / \t/) {
+            my $herevet = "$here\n" . cat_vet($rawline) . "\n";
+            if (WARN("SPACE_BEFORE_TAB",
+                     "please, no space before tabs\n" . $herevet) &&
+                $fix) {
+                while ($fixed[$linenr - 1] =~
+                       s/(^\+.*) {8,8}\t/$1\t\t/) {}
+                while ($fixed[$linenr - 1] =~
+                       s/(^\+.*) +\t/$1\t/) {}
+            }
+        }
+
+# check for && or || at the start of a line
+        if ($rawline =~ /^\+\s*(&&|\|\|)/) {
+            CHK("LOGICAL_CONTINUATIONS",
+                "Logical continuations should be on the previous line\n" . $hereprev);
+        }
+
+# check multi-line statement indentation matches previous line
+        if ($^V && $^V ge 5.10.0 &&
+            $prevline =~ /^\+([ \t]*)((?:$c90_Keywords(?:\s+if)\s*)|(?:$Declare\s*)?(?:$Ident|\(\s*\*\s*$Ident\s*\))\s*|$Ident\s*=\s*$Ident\s*)\(.*(\&\&|\|\||,)\s*$/) {
+            $prevline =~ /^\+(\t*)(.*)$/;
+            my $oldindent = $1;
+            my $rest = $2;
+
+            my $pos = pos_last_openparen($rest);
+            if ($pos >= 0) {
+                $line =~ /^(\+| )([ \t]*)/;
+                my $newindent = $2;
+
+                my $goodtabindent = $oldindent .
+                    "\t" x ($pos / 8) .
+                    " "  x ($pos % 8);
+                my $goodspaceindent = $oldindent . " "  x $pos;
+
+                if ($newindent ne $goodtabindent &&
+                    $newindent ne $goodspaceindent) {
+
+                    if (CHK("PARENTHESIS_ALIGNMENT",
+                            "Alignment should match open parenthesis\n" . $hereprev) &&
+                        $fix && $line =~ /^\+/) {
+                        $fixed[$linenr - 1] =~
+                            s/^\+[ \t]*/\+$goodtabindent/;
+                    }
+                }
+            }
+        }
+
+        if ($line =~ /^\+.*\*[ \t]*\)[ \t]+(?!$Assignment|$Arithmetic)/) {
+            if (CHK("SPACING",
+                    "No space is necessary after a cast\n" . $hereprev) &&
+                $fix) {
+                $fixed[$linenr - 1] =~
+                    s/^(\+.*\*[ \t]*\))[ \t]+/$1/;
+            }
+        }
+
+
+# check for missing blank lines after declarations
+        if ($sline =~ /^\+\s+\S/ &&  #Not at char 1
+            # actual declarations
+            ($prevline =~ /^\+\s+$Declare\s*$Ident\s*[=,;:\[]/ ||
+             # foo bar; where foo is some local typedef or #define
+             $prevline =~ /^\+\s+$Ident(?:\s+|\s*\*\s*)$Ident\s*[=,;\[]/ ||
+             # known declaration macros
+             $prevline =~ /^\+\s+$declaration_macros/) &&
+            # for "else if" which can look like "$Ident $Ident"
+            !($prevline =~ /^\+\s+$c90_Keywords\b/ ||
+              # other possible extensions of declaration lines
+              $prevline =~ /(?:$Compare|$Assignment|$Operators)\s*$/ ||
+              # not starting a section or a macro "\" extended line
+              $prevline =~ /(?:\{\s*|\\)$/) &&
+            # looks like a declaration
+            !($sline =~ /^\+\s+$Declare\s*$Ident\s*[=,;:\[]/ ||
+              # foo bar; where foo is some local typedef or #define
+              $sline =~ /^\+\s+$Ident(?:\s+|\s*\*\s*)$Ident\s*[=,;\[]/ ||
+              # known declaration macros
+              $sline =~ /^\+\s+$declaration_macros/ ||
+              # start of struct or union or enum
+              $sline =~ /^\+\s+(?:union|struct|enum|typedef)\b/ ||
+              # start or end of block or continuation of declaration
+              $sline =~ /^\+\s+(?:$|[\{\}\.\#\"\?\:\(\[])/ ||
+              # bitfield continuation
+              $sline =~ /^\+\s+$Ident\s*:\s*\d+\s*[,;]/ ||
+              # other possible extensions of declaration lines
+              $sline =~ /^\+\s+\(?\s*(?:$Compare|$Assignment|$Operators)/) &&
+            # indentation of previous and current line are the same
+            (($prevline =~ /\+(\s+)\S/) && $sline =~ /^\+$1\S/)) {
+            WARN("SPACING",
+                 "Missing a blank line after declarations\n" . $hereprev);
+        }
+
+# check we are in a valid C source file if not then ignore this hunk
+        next if ($realfile !~ /\.(h|c)$/);
+
+# check for RCS/CVS revision markers
+        if ($rawline =~ /^\+.*\$(Revision|Log|Id)(?:\$|)/) {
+            WARN("CVS_KEYWORD",
+                 "CVS style keyword markers, these will _not_ be updated\n". $herecurr);
+        }
+
+# Check for potential 'bare' types
+        my ($stat, $cond, $line_nr_next, $remain_next, $off_next,
+            $realline_next);
+#print "LINE<$line>\n";
+        if ($linenr >= $suppress_statement &&
+            $realcnt && $sline =~ /.\s*\S/) {
+            ($stat, $cond, $line_nr_next, $remain_next, $off_next) =
+                ctx_statement_block($linenr, $realcnt, 0);
+            $stat =~ s/\n./\n /g;
+            $cond =~ s/\n./\n /g;
+
+#print "linenr<$linenr> <$stat>\n";
+            # If this statement has no statement boundaries within
+            # it there is no point in retrying a statement scan
+            # until we hit end of it.
+            my $frag = $stat; $frag =~ s/;+\s*$//;
+            if ($frag !~ /(?:{|;)/) {
+#print "skip<$line_nr_next>\n";
+                $suppress_statement = $line_nr_next;
+            }
+
+            # Find the real next line.
+            $realline_next = $line_nr_next;
+            if (defined $realline_next &&
+                (!defined $lines[$realline_next - 1] ||
+                 substr($lines[$realline_next - 1], $off_next) =~ /^\s*$/)) {
+                $realline_next++;
+            }
+
+            my $s = $stat;
+            $s =~ s/{.*$//s;
+
+            # Ignore goto labels.
+            if ($s =~ /$Ident:\*$/s) {
+
+                # Ignore functions being called
+            } elsif ($s =~ /^.\s*$Ident\s*\(/s) {
+
+            } elsif ($s =~ /^.\s*else\b/s) {
+
+                # declarations always start with types
+            } elsif ($prev_values eq 'E' && $s =~ /^.\s*(?:$Storage\s+)?(?:$Inline\s+)?(?:const\s+)?((?:\s*$Ident)+?)\b(?:\s+$Sparse)?\s*\**\s*(?:$Ident|\(\*[^\)]*\))(?:\s*$Modifier)?\s*(?:;|=|,|\()/s) {
+                my $type = $1;
+                $type =~ s/\s+/ /g;
+                possible($type, "A:" . $s);
+
+                # definitions in global scope can only start with types
+            } elsif ($s =~ /^.(?:$Storage\s+)?(?:$Inline\s+)?(?:const\s+)?($Ident)\b\s*(?!:)/s) {
+                possible($1, "B:" . $s);
+            }
+
+            # any (foo ... *) is a pointer cast, and foo is a type
+            while ($s =~ /\(($Ident)(?:\s+$Sparse)*[\s\*]+\s*\)/sg) {
+                possible($1, "C:" . $s);
+            }
+
+            # Check for any sort of function declaration.
+            # int foo(something bar, other baz);
+            # void (*store_gdt)(x86_descr_ptr *);
+            if ($prev_values eq 'E' && $s =~ /^(.(?:typedef\s*)?(?:(?:$Storage|$Inline)\s*)*\s*$Type\s*(?:\b$Ident|\(\*\s*$Ident\))\s*)\(/s) {
+                my ($name_len) = length($1);
+
+                my $ctx = $s;
+                substr($ctx, 0, $name_len + 1, '');
+                $ctx =~ s/\)[^\)]*$//;
+
+                for my $arg (split(/\s*,\s*/, $ctx)) {
+                    if ($arg =~ /^(?:const\s+)?($Ident)(?:\s+$Sparse)*\s*\**\s*(:?\b$Ident)?$/s || $arg =~ /^($Ident)$/s) {
+
+                        possible($1, "D:" . $s);
+                    }
+                }
+            }
+
+        }
+
+#
+# Checks which may be anchored in the context.
+#
+
+# Check for switch () and associated case and default
+# statements should be at the same indent.
+        if ($line=~/\bswitch\s*\(.*\)/) {
+            my $err = '';
+            my $sep = '';
+            my @ctx = ctx_block_outer($linenr, $realcnt);
+            shift(@ctx);
+            for my $ctx (@ctx) {
+                my ($clen, $cindent) = line_stats($ctx);
+                if ($ctx =~ /^\+\s*(case\s+|default:)/ &&
+                    $indent != $cindent) {
+                    $err .= "$sep$ctx\n";
+                    $sep = '';
+                } else {
+                    $sep = "[...]\n";
+                }
+            }
+            if ($err ne '') {
+                ERROR("SWITCH_CASE_INDENT_LEVEL",
+                      "switch and case should be at the same indent\n$hereline$err");
+            }
+        }
+
+# if/while/etc brace do not go on next line, unless defining a do while loop,
+# or if that brace on the next line is for something else
+        if ($line =~ /(.*)\b((?:if|while|for|switch)\s*\(|do\b|else\b)/ && $line !~ /^.\s*\#/) {
+            my $pre_ctx = "$1$2";
+
+            my ($level, @ctx) = ctx_statement_level($linenr, $realcnt, 0);
+
+            if ($line =~ /^\+\t{6,}/) {
+                WARN("DEEP_INDENTATION",
+                     "Too many leading tabs - consider code refactoring\n" . $herecurr);
+            }
+
+            my $ctx_cnt = $realcnt - $#ctx - 1;
+            my $ctx = join("\n", @ctx);
+
+            my $ctx_ln = $linenr;
+            my $ctx_skip = $realcnt;
+
+            while ($ctx_skip > $ctx_cnt || ($ctx_skip == $ctx_cnt &&
+                                            defined $lines[$ctx_ln - 1] &&
+                                            $lines[$ctx_ln - 1] =~ /^-/)) {
+                ##print "SKIP<$ctx_skip> CNT<$ctx_cnt>\n";
+                $ctx_skip-- if (!defined $lines[$ctx_ln - 1] || $lines[$ctx_ln - 1] !~ /^-/);
+                $ctx_ln++;
+            }
+
+            #print "realcnt<$realcnt> ctx_cnt<$ctx_cnt>\n";
+            #print "pre<$pre_ctx>\nline<$line>\nctx<$ctx>\nnext<$lines[$ctx_ln - 1]>\n";
+
+            if ($ctx !~ /{\s*/ && defined($lines[$ctx_ln -1]) && $lines[$ctx_ln - 1] =~ /^\+\s*{/) {
+                ERROR("OPEN_BRACE",
+                      "that open brace { should be on the previous line\n" .
+                      "$here\n$ctx\n$rawlines[$ctx_ln - 1]\n");
+            }
+            if ($level == 0 && $pre_ctx !~ /}\s*while\s*\($/ &&
+                $ctx =~ /\)\s*\;\s*$/ &&
+                defined $lines[$ctx_ln - 1])
+            {
+                my ($nlength, $nindent) = line_stats($lines[$ctx_ln - 1]);
+                if ($nindent > $indent) {
+                    WARN("TRAILING_SEMICOLON",
+                         "trailing semicolon indicates no statements, indent implies otherwise\n" .
+                         "$here\n$ctx\n$rawlines[$ctx_ln - 1]\n");
+                }
+            }
+        }
+
+# Check relative indent for conditionals and blocks.
+        if ($line =~ /\b(?:(?:if|while|for)\s*\(|do\b)/ && $line !~ /^.\s*#/ && $line !~ /\}\s*while\s*/) {
+            ($stat, $cond, $line_nr_next, $remain_next, $off_next) =
+                ctx_statement_block($linenr, $realcnt, 0)
+                if (!defined $stat);
+            my ($s, $c) = ($stat, $cond);
+
+            substr($s, 0, length($c), '');
+
+            # Make sure we remove the line prefixes as we have
+            # none on the first line, and are going to readd them
+            # where necessary.
+            $s =~ s/\n./\n/gs;
+
+            # Find out how long the conditional actually is.
+            my @newlines = ($c =~ /\n/gs);
+            my $cond_lines = 1 + $#newlines;
+
+            # We want to check the first line inside the block
+            # starting at the end of the conditional, so remove:
+            #  1) any blank line termination
+            #  2) any opening brace { on end of the line
+            #  3) any do (...) {
+            my $continuation = 0;
+            my $check = 0;
+            $s =~ s/^.*\bdo\b//;
+            $s =~ s/^\s*{//;
+            if ($s =~ s/^\s*\\//) {
+                $continuation = 1;
+            }
+            if ($s =~ s/^\s*?\n//) {
+                $check = 1;
+                $cond_lines++;
+            }
+
+            # Also ignore a loop construct at the end of a
+            # preprocessor statement.
+            if (($prevline =~ /^.\s*#\s*define\s/ ||
+                 $prevline =~ /\\\s*$/) && $continuation == 0) {
+                $check = 0;
+            }
+
+            my $cond_ptr = -1;
+            $continuation = 0;
+            while ($cond_ptr != $cond_lines) {
+                $cond_ptr = $cond_lines;
+
+                # If we see an #else/#elif then the code
+                # is not linear.
+                if ($s =~ /^\s*\#\s*(?:else|elif)/) {
+                    $check = 0;
+                }
+
+                # Ignore:
+                #  1) blank lines, they should be at 0,
+                #  2) preprocessor lines, and
+                #  3) labels.
+                if ($continuation ||
+                    $s =~ /^\s*?\n/ ||
+                    $s =~ /^\s*#\s*?/ ||
+                    $s =~ /^\s*$Ident\s*:/) {
+                    $continuation = ($s =~ /^.*?\\\n/) ? 1 : 0;
+                    if ($s =~ s/^.*?\n//) {
+                        $cond_lines++;
+                    }
+                }
+            }
+
+            my (undef, $sindent) = line_stats("+" . $s);
+            my $stat_real = raw_line($linenr, $cond_lines);
+
+            # Check if either of these lines are modified, else
+            # this is not this patch's fault.
+            if (!defined($stat_real) ||
+                $stat !~ /^\+/ && $stat_real !~ /^\+/) {
+                $check = 0;
+            }
+            if (defined($stat_real) && $cond_lines > 1) {
+                $stat_real = "[...]\n$stat_real";
+            }
+
+            #print "line<$line> prevline<$prevline> indent<$indent> sindent<$sindent> check<$check> continuation<$continuation> s<$s> cond_lines<$cond_lines> stat_real<$stat_real> stat<$stat>\n";
+
+            if ($check && (($sindent % 8) != 0 ||
+                           ($sindent <= $indent && $s ne ''))) {
+                WARN("SUSPECT_CODE_INDENT",
+                     "suspect code indent for conditional statements ($indent, $sindent)\n" . $herecurr . "$stat_real\n");
+            }
+        }
+
+        # Track the 'values' across context and added lines.
+        my $opline = $line; $opline =~ s/^./ /;
+        my ($curr_values, $curr_vars) =
+            annotate_values($opline . "\n", $prev_values);
+        $curr_values = $prev_values . $curr_values;
+        if ($dbg_values) {
+            my $outline = $opline; $outline =~ s/\t/ /g;
+            print "$linenr > .$outline\n";
+            print "$linenr > $curr_values\n";
+            print "$linenr >  $curr_vars\n";
+        }
+        $prev_values = substr($curr_values, -1);
+
+#ignore lines not being added
+        next if ($line =~ /^[^\+]/);
+
+# TEST: allow direct testing of the type matcher.
+        if ($dbg_type) {
+            if ($line =~ /^.\s*$Declare\s*$/) {
+                ERROR("TEST_TYPE",
+                      "TEST: is type\n" . $herecurr);
+            } elsif ($dbg_type > 1 && $line =~ /^.+($Declare)/) {
+                ERROR("TEST_NOT_TYPE",
+                      "TEST: is not type ($1 is)\n". $herecurr);
+            }
+            next;
+        }
+# TEST: allow direct testing of the attribute matcher.
+        if ($dbg_attr) {
+            if ($line =~ /^.\s*$Modifier\s*$/) {
+                ERROR("TEST_ATTR",
+                      "TEST: is attr\n" . $herecurr);
+            } elsif ($dbg_attr > 1 && $line =~ /^.+($Modifier)/) {
+                ERROR("TEST_NOT_ATTR",
+                      "TEST: is not attr ($1 is)\n". $herecurr);
+            }
+            next;
+        }
+
+# check for initialisation to aggregates open brace on the next line
+        if ($line =~ /^.\s*{/ &&
+            $prevline =~ /(?:^|[^=])=\s*$/) {
+            ERROR("OPEN_BRACE",
+                  "that open brace { should be on the previous line\n" . $hereprev);
+        }
+
+#
+# Checks which are anchored on the added line.
+#
+
+# check for malformed paths in #include statements (uses RAW line)
+        if ($rawline =~ m{^.\s*\#\s*include\s+[<"](.*)[">]}) {
+            my $path = $1;
+            if ($path =~ m{//}) {
+                ERROR("MALFORMED_INCLUDE",
+                      "malformed #include filename\n" . $herecurr);
+            }
+            if ($path =~ "^uapi/" && $realfile =~ m@\binclude/uapi/@) {
+                ERROR("UAPI_INCLUDE",
+                      "No #include in ...include/uapi/... should use a uapi/ path prefix\n" . $herecurr);
+            }
+        }
+
+# no C99 // comments
+        if ($line =~ m{//}) {
+            if (ERROR("C99_COMMENTS",
+                      "do not use C99 // comments\n" . $herecurr) &&
+                $fix) {
+                my $line = $fixed[$linenr - 1];
+                if ($line =~ /\/\/(.*)$/) {
+                    my $comment = trim($1);
+                    $fixed[$linenr - 1] =~ s@\/\/(.*)$@/\* $comment \*/@;
+                }
+            }
+        }
+        # Remove C99 comments.
+        $line =~ s@//.*@@;
+        $opline =~ s@//.*@@;
+
+# EXPORT_SYMBOL should immediately follow the thing it is exporting, consider
+# the whole statement.
+#print "APW <$lines[$realline_next - 1]>\n";
+        if (defined $realline_next &&
+            exists $lines[$realline_next - 1] &&
+            !defined $suppress_export{$realline_next} &&
+            ($lines[$realline_next - 1] =~ /EXPORT_SYMBOL.*\((.*)\)/ ||
+             $lines[$realline_next - 1] =~ /EXPORT_UNUSED_SYMBOL.*\((.*)\)/)) {
+            # Handle definitions which produce identifiers with
+            # a prefix:
+            #   XXX(foo);
+            #   EXPORT_SYMBOL(something_foo);
+            my $name = $1;
+            if ($stat =~ /^(?:.\s*}\s*\n)?.([A-Z_]+)\s*\(\s*($Ident)/ &&
+                $name =~ /^${Ident}_$2/) {
+#print "FOO C name<$name>\n";
+                $suppress_export{$realline_next} = 1;
+
+            } elsif ($stat !~ /(?:
+                                \n.}\s*$|
+                                ^.DEFINE_$Ident\(\Q$name\E\)|
+                                ^.DECLARE_$Ident\(\Q$name\E\)|
+                                ^.LIST_HEAD\(\Q$name\E\)|
+                                ^.(?:$Storage\s+)?$Type\s*\(\s*\*\s*\Q$name\E\s*\)\s*\(|
+                                \b\Q$name\E(?:\s+$Attribute)*\s*(?:;|=|\[|\()
+                            )/x) {
+#print "FOO A<$lines[$realline_next - 1]> stat<$stat> name<$name>\n";
+                $suppress_export{$realline_next} = 2;
+            } else {
+                $suppress_export{$realline_next} = 1;
+            }
+        }
+        if (!defined $suppress_export{$linenr} &&
+            $prevline =~ /^.\s*$/ &&
+            ($line =~ /EXPORT_SYMBOL.*\((.*)\)/ ||
+             $line =~ /EXPORT_UNUSED_SYMBOL.*\((.*)\)/)) {
+#print "FOO B <$lines[$linenr - 1]>\n";
+            $suppress_export{$linenr} = 2;
+        }
+        if (defined $suppress_export{$linenr} &&
+            $suppress_export{$linenr} == 2) {
+            WARN("EXPORT_SYMBOL",
+                 "EXPORT_SYMBOL(foo); should immediately follow its function/variable\n" . $herecurr);
+        }
+
+# check for global initialisers.
+        if ($line =~ /^\+(\s*$Type\s*$Ident\s*(?:\s+$Modifier))*\s*=\s*(0|NULL|false)\s*;/) {
+            if (ERROR("GLOBAL_INITIALISERS",
+                      "do not initialise globals to 0 or NULL\n" .
+                      $herecurr) &&
+                $fix) {
+                $fixed[$linenr - 1] =~ s/($Type\s*$Ident\s*(?:\s+$Modifier))*\s*=\s*(0|NULL|false)\s*;/$1;/;
+            }
+        }
+# check for static initialisers.
+        if ($line =~ /^\+.*\bstatic\s.*=\s*(0|NULL|false)\s*;/) {
+            if (ERROR("INITIALISED_STATIC",
+                      "do not initialise statics to 0 or NULL\n" .
+                      $herecurr) &&
+                $fix) {
+                $fixed[$linenr - 1] =~ s/(\bstatic\s.*?)\s*=\s*(0|NULL|false)\s*;/$1;/;
+            }
+        }
+
+# check for static const char * arrays.
+        if ($line =~ /\bstatic\s+const\s+char\s*\*\s*(\w+)\s*\[\s*\]\s*=\s*/) {
+            WARN("STATIC_CONST_CHAR_ARRAY",
+                 "static const char * array should probably be static const char * const\n" .
+                 $herecurr);
+        }
+
+# check for static char foo[] = "bar" declarations.
+        if ($line =~ /\bstatic\s+char\s+(\w+)\s*\[\s*\]\s*=\s*"/) {
+            WARN("STATIC_CONST_CHAR_ARRAY",
+                 "static char array declaration should probably be static const char\n" .
+                 $herecurr);
+        }
+
+# check for non-global char *foo[] = {"bar", ...} declarations.
+        if ($line =~ /^.\s+(?:static\s+|const\s+)?char\s+\*\s*\w+\s*\[\s*\]\s*=\s*\{/) {
+            WARN("STATIC_CONST_CHAR_ARRAY",
+                 "char * array declaration might be better as static const\n" .
+                 $herecurr);
+        }
+
+# check for function declarations without arguments like "int foo()"
+        if ($line =~ /(\b$Type\s+$Ident)\s*\(\s*\)/) {
+            if (ERROR("FUNCTION_WITHOUT_ARGS",
+                      "Bad function definition - $1() should probably be $1(void)\n" . $herecurr) &&
+                $fix) {
+                $fixed[$linenr - 1] =~ s/(\b($Type)\s+($Ident))\s*\(\s*\)/$2 $3(void)/;
+            }
+        }
+
+# * goes on variable not on type
+        # (char*[ const])
+        while ($line =~ m{(\($NonptrType(\s*(?:$Modifier\b\s*|\*\s*)+)\))}g) {
+            #print "AA<$1>\n";
+            my ($ident, $from, $to) = ($1, $2, $2);
+
+            # Should start with a space.
+            $to =~ s/^(\S)/ $1/;
+            # Should not end with a space.
+            $to =~ s/\s+$//;
+            # '*'s should not have spaces between.
+            while ($to =~ s/\*\s+\*/\*\*/) {
+            }
+
+##   print "1: from<$from> to<$to> ident<$ident>\n";
+            if ($from ne $to) {
+                if (ERROR("POINTER_LOCATION",
+                          "\"(foo$from)\" should be \"(foo$to)\"\n" .  $herecurr) &&
+                    $fix) {
+                    my $sub_from = $ident;
+                    my $sub_to = $ident;
+                    $sub_to =~ s/\Q$from\E/$to/;
+                    $fixed[$linenr - 1] =~
+                        s@\Q$sub_from\E@$sub_to@;
+                }
+            }
+        }
+        while ($line =~ m{(\b$NonptrType(\s*(?:$Modifier\b\s*|\*\s*)+)($Ident))}g) {
+            #print "BB<$1>\n";
+            my ($match, $from, $to, $ident) = ($1, $2, $2, $3);
+
+            # Should start with a space.
+            $to =~ s/^(\S)/ $1/;
+            # Should not end with a space.
+            $to =~ s/\s+$//;
+            # '*'s should not have spaces between.
+            while ($to =~ s/\*\s+\*/\*\*/) {
+            }
+            # Modifiers should have spaces.
+            $to =~ s/(\b$Modifier$)/$1 /;
+
+##  print "2: from<$from> to<$to> ident<$ident>\n";
+            if ($from ne $to && $ident !~ /^$Modifier$/) {
+                if (ERROR("POINTER_LOCATION",
+                          "\"foo${from}bar\" should be \"foo${to}bar\"\n" .  $herecurr) &&
+                    $fix) {
+
+                    my $sub_from = $match;
+                    my $sub_to = $match;
+                    $sub_to =~ s/\Q$from\E/$to/;
+                    $fixed[$linenr - 1] =~
+                        s@\Q$sub_from\E@$sub_to@;
+                }
+            }
+        }
+
+# function brace can't be on same line, except for #defines of do while,
+# or if closed on same line
+        if (($line=~/$Type\s*$Ident\(.*\).*\s\{/) and
+            !($line=~/\#\s*define.*do\s\{/) and !($line=~/}/)) {
+            ERROR("OPEN_BRACE",
+                  "open brace '{' following function declarations go on the next line\n" . $herecurr);
+        }
+
+# open braces for enum, union and struct go on the same line.
+        if ($line =~ /^.\s*{/ &&
+            $prevline =~ /^.\s*(?:typedef\s+)?(enum|union|struct)(?:\s+$Ident)?\s*$/) {
+            ERROR("OPEN_BRACE",
+                  "open brace '{' following $1 go on the same line\n" . $hereprev);
+        }
+
+# missing space after union, struct or enum definition
+        if ($line =~ /^.\s*(?:typedef\s+)?(enum|union|struct)(?:\s+$Ident){1,2}[=\{]/) {
+            if (WARN("SPACING",
+                     "missing space after $1 definition\n" . $herecurr) &&
+                $fix) {
+                $fixed[$linenr - 1] =~
+                    s/^(.\s*(?:typedef\s+)?(?:enum|union|struct)(?:\s+$Ident){1,2})([=\{])/$1 $2/;
+            }
+        }
+
+# Function pointer declarations
+# check spacing between type, funcptr, and args
+# canonical declaration is "type (*funcptr)(args...)"
+        if ($line =~ /^.\s*($Declare)\((\s*)\*(\s*)($Ident)(\s*)\)(\s*)\(/) {
+            my $declare = $1;
+            my $pre_pointer_space = $2;
+            my $post_pointer_space = $3;
+            my $funcname = $4;
+            my $post_funcname_space = $5;
+            my $pre_args_space = $6;
+
+# the $Declare variable will capture all spaces after the type
+# so check it for a missing trailing missing space but pointer return types
+# don't need a space so don't warn for those.
+            my $post_declare_space = "";
+            if ($declare =~ /(\s+)$/) {
+                $post_declare_space = $1;
+                $declare = rtrim($declare);
+            }
+            if ($declare !~ /\*$/ && $post_declare_space =~ /^$/) {
+                WARN("SPACING",
+                     "missing space after return type\n" . $herecurr);
+                $post_declare_space = " ";
+            }
+
+# unnecessary space "type ( *funcptr)(args...)"
+            if (defined $pre_pointer_space &&
+                $pre_pointer_space =~ /^\s/) {
+                WARN("SPACING",
+                     "Unnecessary space after function pointer open parenthesis\n" . $herecurr);
+            }
+
+# unnecessary space "type (* funcptr)(args...)"
+            if (defined $post_pointer_space &&
+                $post_pointer_space =~ /^\s/) {
+                WARN("SPACING",
+                     "Unnecessary space before function pointer name\n" . $herecurr);
+            }
+
+# unnecessary space "type (*funcptr )(args...)"
+            if (defined $post_funcname_space &&
+                $post_funcname_space =~ /^\s/) {
+                WARN("SPACING",
+                     "Unnecessary space after function pointer name\n" . $herecurr);
+            }
+
+# unnecessary space "type (*funcptr) (args...)"
+            if (defined $pre_args_space &&
+                $pre_args_space =~ /^\s/) {
+                WARN("SPACING",
+                     "Unnecessary space before function pointer arguments\n" . $herecurr);
+            }
+
+            if (show_type("SPACING") && $fix) {
+                $fixed[$linenr - 1] =~
+                    s/^(.\s*)$Declare\s*\(\s*\*\s*$Ident\s*\)\s*\(/$1 . $declare . $post_declare_space . '(*' . $funcname . ')('/ex;
+            }
+        }
+
+# check for spacing round square brackets; allowed:
+#  1. with a type on the left -- int [] a;
+#  2. at the beginning of a line for slice initialisers -- [0...10] = 5,
+#  3. inside a curly brace -- = { [0...10] = 5 }
+        while ($line =~ /(.*?\s)\[/g) {
+            my ($where, $prefix) = ($-[1], $1);
+            if ($prefix !~ /$Type\s+$/ &&
+                ($where != 0 || $prefix !~ /^.\s+$/) &&
+                $prefix !~ /[{,]\s+$/) {
+                if (ERROR("BRACKET_SPACE",
+                          "space prohibited before open square bracket '['\n" . $herecurr) &&
+                    $fix) {
+                    $fixed[$linenr - 1] =~
+                        s/^(\+.*?)\s+\[/$1\[/;
+                }
+            }
+        }
+
+# Check operator spacing.
+        if (!($line=~/\#\s*include/)) {
+            my $fixed_line = "";
+            my $line_fixed = 0;
+
+            my $ops = qr{
+                                <<=|>>=|<=|>=|==|!=|
+                                \+=|-=|\*=|\/=|%=|\^=|\|=|&=|
+                                =>|->|<<|>>|<|>|=|!|~|
+                                &&|\|\||,|\^|\+\+|--|&|\||\+|-|\*|\/|%|
+                                \?:|\?|:
+                        }x;
+            my @elements = split(/($ops|;)/, $opline);
+
+##                      print("element count: <" . $#elements . ">\n");
+##                      foreach my $el (@elements) {
+##                              print("el: <$el>\n");
+##                      }
+
+            my @fix_elements = ();
+            my $off = 0;
+
+            foreach my $el (@elements) {
+                push(@fix_elements, substr($rawline, $off, length($el)));
+                $off += length($el);
+            }
+
+            $off = 0;
+
+            my $blank = copy_spacing($opline);
+            my $last_after = -1;
+
+            for (my $n = 0; $n < $#elements; $n += 2) {
+
+                my $good = $fix_elements[$n] . $fix_elements[$n + 1];
+
+##                              print("n: <$n> good: <$good>\n");
+
+                $off += length($elements[$n]);
+
+                # Pick up the preceding and succeeding characters.
+                my $ca = substr($opline, 0, $off);
+                my $cc = '';
+                if (length($opline) >= ($off + length($elements[$n + 1]))) {
+                    $cc = substr($opline, $off + length($elements[$n + 1]));
+                }
+                my $cb = "$ca$;$cc";
+
+                my $a = '';
+                $a = 'V' if ($elements[$n] ne '');
+                $a = 'W' if ($elements[$n] =~ /\s$/);
+                $a = 'C' if ($elements[$n] =~ /$;$/);
+                $a = 'B' if ($elements[$n] =~ /(\[|\()$/);
+                $a = 'O' if ($elements[$n] eq '');
+                $a = 'E' if ($ca =~ /^\s*$/);
+
+                my $op = $elements[$n + 1];
+
+                my $c = '';
+                if (defined $elements[$n + 2]) {
+                    $c = 'V' if ($elements[$n + 2] ne '');
+                    $c = 'W' if ($elements[$n + 2] =~ /^\s/);
+                    $c = 'C' if ($elements[$n + 2] =~ /^$;/);
+                    $c = 'B' if ($elements[$n + 2] =~ /^(\)|\]|;)/);
+                    $c = 'O' if ($elements[$n + 2] eq '');
+                    $c = 'E' if ($elements[$n + 2] =~ /^\s*\\$/);
+                } else {
+                    $c = 'E';
+                }
+
+                my $ctx = "${a}x${c}";
+
+                my $at = "(ctx:$ctx)";
+
+                my $ptr = substr($blank, 0, $off) . "^";
+                my $hereptr = "$hereline$ptr\n";
+
+                # Pull out the value of this operator.
+                my $op_type = substr($curr_values, $off + 1, 1);
+
+                # Get the full operator variant.
+                my $opv = $op . substr($curr_vars, $off, 1);
+
+                # Ignore operators passed as parameters.
+                if ($op_type ne 'V' &&
+                    $ca =~ /\s$/ && $cc =~ /^\s*,/) {
+
+#                               # Ignore comments
+#                               } elsif ($op =~ /^$;+$/) {
+
+                    # ; should have either the end of line or a space or \ after it
+                } elsif ($op eq ';') {
+                    if ($ctx !~ /.x[WEBC]/ &&
+                        $cc !~ /^\\/ && $cc !~ /^;/) {
+                        if (ERROR("SPACING",
+                                  "space required after that '$op' $at\n" . $hereptr)) {
+                            $good = $fix_elements[$n] . trim($fix_elements[$n + 1]) . " ";
+                            $line_fixed = 1;
+                        }
+                    }
+
+                    # // is a comment
+                } elsif ($op eq '//') {
+
+                    #   :   when part of a bitfield
+                } elsif ($opv eq ':B') {
+                    # skip the bitfield test for now
+
+                    # No spaces for:
+                    #   ->
+                } elsif ($op eq '->') {
+                    if ($ctx =~ /Wx.|.xW/) {
+                        if (ERROR("SPACING",
+                                  "spaces prohibited around that '$op' $at\n" . $hereptr)) {
+                            $good = rtrim($fix_elements[$n]) . trim($fix_elements[$n + 1]);
+                            if (defined $fix_elements[$n + 2]) {
+                                $fix_elements[$n + 2] =~ s/^\s+//;
+                            }
+                            $line_fixed = 1;
+                        }
+                    }
+
+                    # , must have a space on the right.
+                } elsif ($op eq ',') {
+                    if ($ctx !~ /.x[WEC]/ && $cc !~ /^}/) {
+                        if (ERROR("SPACING",
+                                  "space required after that '$op' $at\n" . $hereptr)) {
+                            $good = $fix_elements[$n] . trim($fix_elements[$n + 1]) . " ";
+                            $line_fixed = 1;
+                            $last_after = $n;
+                        }
+                    }
+
+                    # '*' as part of a type definition -- reported already.
+                } elsif ($opv eq '*_') {
+                    #warn "'*' is part of type\n";
+
+                    # unary operators should have a space before and
+                    # none after.  May be left adjacent to another
+                    # unary operator, or a cast
+                } elsif ($op eq '!' || $op eq '~' ||
+                         $opv eq '*U' || $opv eq '-U' ||
+                         $opv eq '&U' || $opv eq '&&U') {
+                    if ($ctx !~ /[WEBC]x./ && $ca !~ /(?:\)|!|~|\*|-|\&|\||\+\+|\-\-|\{)$/) {
+                        if (ERROR("SPACING",
+                                  "space required before that '$op' $at\n" . $hereptr)) {
+                            if ($n != $last_after + 2) {
+                                $good = $fix_elements[$n] . " " . ltrim($fix_elements[$n + 1]);
+                                $line_fixed = 1;
+                            }
+                        }
+                    }
+                    if ($op eq '*' && $cc =~/\s*$Modifier\b/) {
+                        # A unary '*' may be const
+
+                    } elsif ($ctx =~ /.xW/) {
+                        if (ERROR("SPACING",
+                                  "space prohibited after that '$op' $at\n" . $hereptr)) {
+                            $good = $fix_elements[$n] . rtrim($fix_elements[$n + 1]);
+                            if (defined $fix_elements[$n + 2]) {
+                                $fix_elements[$n + 2] =~ s/^\s+//;
+                            }
+                            $line_fixed = 1;
+                        }
+                    }
+
+                    # unary ++ and unary -- are allowed no space on one side.
+                } elsif ($op eq '++' or $op eq '--') {
+                    if ($ctx !~ /[WEOBC]x[^W]/ && $ctx !~ /[^W]x[WOBEC]/) {
+                        if (ERROR("SPACING",
+                                  "space required one side of that '$op' $at\n" . $hereptr)) {
+                            $good = $fix_elements[$n] . trim($fix_elements[$n + 1]) . " ";
+                            $line_fixed = 1;
+                        }
+                    }
+                    if ($ctx =~ /Wx[BE]/ ||
+                        ($ctx =~ /Wx./ && $cc =~ /^;/)) {
+                        if (ERROR("SPACING",
+                                  "space prohibited before that '$op' $at\n" . $hereptr)) {
+                            $good = rtrim($fix_elements[$n]) . trim($fix_elements[$n + 1]);
+                            $line_fixed = 1;
+                        }
+                    }
+                    if ($ctx =~ /ExW/) {
+                        if (ERROR("SPACING",
+                                  "space prohibited after that '$op' $at\n" . $hereptr)) {
+                            $good = $fix_elements[$n] . trim($fix_elements[$n + 1]);
+                            if (defined $fix_elements[$n + 2]) {
+                                $fix_elements[$n + 2] =~ s/^\s+//;
+                            }
+                            $line_fixed = 1;
+                        }
+                    }
+
+                    # << and >> may either have or not have spaces both sides
+                } elsif ($op eq '<<' or $op eq '>>' or
+                         $op eq '&' or $op eq '^' or $op eq '|' or
+                         $op eq '+' or $op eq '-' or
+                         $op eq '*' or $op eq '/' or
+                         $op eq '%')
+                {
+                    if ($ctx =~ /Wx[^WCE]|[^WCE]xW/) {
+                        if (ERROR("SPACING",
+                                  "need consistent spacing around '$op' $at\n" . $hereptr)) {
+                            $good = rtrim($fix_elements[$n]) . " " . trim($fix_elements[$n + 1]) . " ";
+                            if (defined $fix_elements[$n + 2]) {
+                                $fix_elements[$n + 2] =~ s/^\s+//;
+                            }
+                            $line_fixed = 1;
+                        }
+                    }
+
+                    # A colon needs no spaces before when it is
+                    # terminating a case value or a label.
+                } elsif ($opv eq ':C' || $opv eq ':L') {
+                    if ($ctx =~ /Wx./) {
+                        if (ERROR("SPACING",
+                                  "space prohibited before that '$op' $at\n" . $hereptr)) {
+                            $good = rtrim($fix_elements[$n]) . trim($fix_elements[$n + 1]);
+                            $line_fixed = 1;
+                        }
+                    }
+
+                    # All the others need spaces both sides.
+                } elsif ($ctx !~ /[EWC]x[CWE]/) {
+                    my $ok = 0;
+
+                    # Ignore email addresses <foo@bar>
+                    if (($op eq '<' &&
+                         $cc =~ /^\S+\@\S+>/) ||
+                        ($op eq '>' &&
+                         $ca =~ /<\S+\@\S+$/))
+                    {
+                        $ok = 1;
+                    }
+
+                    # messages are ERROR, but ?: are CHK
+                    if ($ok == 0) {
+                        my $msg_type = \&ERROR;
+                        $msg_type = \&CHK if (($op eq '?:' || $op eq '?' || $op eq ':') && $ctx =~ /VxV/);
+
+                        if (&{$msg_type}("SPACING",
+                                         "spaces required around that '$op' $at\n" . $hereptr)) {
+                            $good = rtrim($fix_elements[$n]) . " " . trim($fix_elements[$n + 1]) . " ";
+                            if (defined $fix_elements[$n + 2]) {
+                                $fix_elements[$n + 2] =~ s/^\s+//;
+                            }
+                            $line_fixed = 1;
+                        }
+                    }
+                }
+                $off += length($elements[$n + 1]);
+
+##                              print("n: <$n> GOOD: <$good>\n");
+
+                $fixed_line = $fixed_line . $good;
+            }
+
+            if (($#elements % 2) == 0) {
+                $fixed_line = $fixed_line . $fix_elements[$#elements];
+            }
+
+            if ($fix && $line_fixed && $fixed_line ne $fixed[$linenr - 1]) {
+                $fixed[$linenr - 1] = $fixed_line;
+            }
+
+
+        }
+
+# check for whitespace before a non-naked semicolon
+        if ($line =~ /^\+.*\S\s+;\s*$/) {
+            if (WARN("SPACING",
+                     "space prohibited before semicolon\n" . $herecurr) &&
+                $fix) {
+                1 while $fixed[$linenr - 1] =~
+                    s/^(\+.*\S)\s+;/$1;/;
+            }
+        }
+
+# check for multiple assignments
+        if ($line =~ /^.\s*$Lval\s*=\s*$Lval\s*=(?!=)/) {
+            CHK("MULTIPLE_ASSIGNMENTS",
+                "multiple assignments should be avoided\n" . $herecurr);
+        }
+
+## # check for multiple declarations, allowing for a function declaration
+## # continuation.
+##              if ($line =~ /^.\s*$Type\s+$Ident(?:\s*=[^,{]*)?\s*,\s*$Ident.*/ &&
+##                  $line !~ /^.\s*$Type\s+$Ident(?:\s*=[^,{]*)?\s*,\s*$Type\s*$Ident.*/) {
+##
+##                      # Remove any bracketed sections to ensure we do not
+##                      # falsly report the parameters of functions.
+##                      my $ln = $line;
+##                      while ($ln =~ s/\([^\(\)]*\)//g) {
+##                      }
+##                      if ($ln =~ /,/) {
+##                              WARN("MULTIPLE_DECLARATION",
+##                                   "declaring multiple variables together should be avoided\n" . $herecurr);
+##                      }
+##              }
+
+#need space before brace following if, while, etc
+        if (($line =~ /\(.*\)\{/ && $line !~ /\($Type\)\{/) ||
+            $line =~ /do\{/) {
+            if (ERROR("SPACING",
+                      "space required before the open brace '{'\n" . $herecurr) &&
+                $fix) {
+                $fixed[$linenr - 1] =~ s/^(\+.*(?:do|\)))\{/$1 {/;
+            }
+        }
+
+## # check for blank lines before declarations
+##              if ($line =~ /^.\t+$Type\s+$Ident(?:\s*=.*)?;/ &&
+##                  $prevrawline =~ /^.\s*$/) {
+##                      WARN("SPACING",
+##                           "No blank lines before declarations\n" . $hereprev);
+##              }
+##
+
+# closing brace should have a space following it when it has anything
+# on the line
+        if ($line =~ /}(?!(?:,|;|\)))\S/) {
+            if (ERROR("SPACING",
+                      "space required after that close brace '}'\n" . $herecurr) &&
+                $fix) {
+                $fixed[$linenr - 1] =~
+                    s/}((?!(?:,|;|\)))\S)/} $1/;
+            }
+        }
+
+# check spacing on square brackets
+        if ($line =~ /\[\s/ && $line !~ /\[\s*$/) {
+            if (ERROR("SPACING",
+                      "space prohibited after that open square bracket '['\n" . $herecurr) &&
+                $fix) {
+                $fixed[$linenr - 1] =~
+                    s/\[\s+/\[/;
+            }
+        }
+        if ($line =~ /\s\]/) {
+            if (ERROR("SPACING",
+                      "space prohibited before that close square bracket ']'\n" . $herecurr) &&
+                $fix) {
+                $fixed[$linenr - 1] =~
+                    s/\s+\]/\]/;
+            }
+        }
+
+# check spacing on parentheses
+        if ($line =~ /\(\s/ && $line !~ /\(\s*(?:\\)?$/ &&
+            $line !~ /for\s*\(\s+;/) {
+            if (ERROR("SPACING",
+                      "space prohibited after that open parenthesis '('\n" . $herecurr) &&
+                $fix) {
+                $fixed[$linenr - 1] =~
+                    s/\(\s+/\(/;
+            }
+        }
+        if ($line =~ /(\s+)\)/ && $line !~ /^.\s*\)/ &&
+            $line !~ /for\s*\(.*;\s+\)/ &&
+            $line !~ /:\s+\)/) {
+            if (ERROR("SPACING",
+                      "space prohibited before that close parenthesis ')'\n" . $herecurr) &&
+                $fix) {
+                $fixed[$linenr - 1] =~
+                    s/\s+\)/\)/;
+            }
+        }
+
+#goto labels aren't indented, allow a single space however
+        if ($line=~/^.\s+[A-Za-z\d_]+:(?![0-9]+)/ and
+            !($line=~/^. [A-Za-z\d_]+:/) and !($line=~/^.\s+default:/)) {
+            if (WARN("INDENTED_LABEL",
+                     "labels should not be indented\n" . $herecurr) &&
+                $fix) {
+                $fixed[$linenr - 1] =~
+                    s/^(.)\s+/$1/;
+            }
+        }
+
+# return is not a function
+        if (defined($stat) && $stat =~ /^.\s*return(\s*)\(/s) {
+            my $spacing = $1;
+            if ($^V && $^V ge 5.10.0 &&
+                $stat =~ /^.\s*return\s*($balanced_parens)\s*;\s*$/) {
+                my $value = $1;
+                $value = deparenthesize($value);
+                if ($value =~ m/^\s*$FuncArg\s*(?:\?|$)/) {
+                    ERROR("RETURN_PARENTHESES",
+                          "return is not a function, parentheses are not required\n" . $herecurr);
+                }
+            } elsif ($spacing !~ /\s+/) {
+                ERROR("SPACING",
+                      "space required before the open parenthesis '('\n" . $herecurr);
+            }
+        }
+
+# unnecessary return in a void function
+# at end-of-function, with the previous line a single leading tab, then return;
+# and the line before that not a goto label target like "out:"
+        if ($sline =~ /^[ \+]}\s*$/ &&
+            $prevline =~ /^\+\treturn\s*;\s*$/ &&
+            $linenr >= 3 &&
+            $lines[$linenr - 3] =~ /^[ +]/ &&
+            $lines[$linenr - 3] !~ /^[ +]\s*$Ident\s*:/) {
+            WARN("RETURN_VOID",
+                 "void function return statements are not generally useful\n" . $hereprev);
+        }
+
+# if statements using unnecessary parentheses - ie: if ((foo == bar))
+        if ($^V && $^V ge 5.10.0 &&
+            $line =~ /\bif\s*((?:\(\s*){2,})/) {
+            my $openparens = $1;
+            my $count = $openparens =~ tr@\(@\(@;
+            my $msg = "";
+            if ($line =~ /\bif\s*(?:\(\s*){$count,$count}$LvalOrFunc\s*($Compare)\s*$LvalOrFunc(?:\s*\)){$count,$count}/) {
+                my $comp = $4;  #Not $1 because of $LvalOrFunc
+                $msg = " - maybe == should be = ?" if ($comp eq "==");
+                WARN("UNNECESSARY_PARENTHESES",
+                     "Unnecessary parentheses$msg\n" . $herecurr);
+            }
+        }
+
+# Return of what appears to be an errno should normally be -'ve
+        if ($line =~ /^.\s*return\s*(E[A-Z]*)\s*;/) {
+            my $name = $1;
+            if ($name ne 'EOF' && $name ne 'ERROR') {
+                WARN("USE_NEGATIVE_ERRNO",
+                     "return of an errno should typically be -ve (return -$1)\n" . $herecurr);
+            }
+        }
+
+# Need a space before open parenthesis after if, while etc
+        if ($line =~ /\b(if|while|for|switch)\(/) {
+            if (ERROR("SPACING",
+                      "space required before the open parenthesis '('\n" . $herecurr) &&
+                $fix) {
+                $fixed[$linenr - 1] =~
+                    s/\b(if|while|for|switch)\(/$1 \(/;
+            }
+        }
+
+# Check for illegal assignment in if conditional -- and check for trailing
+# statements after the conditional.
+        if ($line =~ /do\s*(?!{)/) {
+            ($stat, $cond, $line_nr_next, $remain_next, $off_next) =
+                ctx_statement_block($linenr, $realcnt, 0)
+                if (!defined $stat);
+            my ($stat_next) = ctx_statement_block($line_nr_next,
+                                                  $remain_next, $off_next);
+            $stat_next =~ s/\n./\n /g;
+            ##print "stat<$stat> stat_next<$stat_next>\n";
+
+            if ($stat_next =~ /^\s*while\b/) {
+                # If the statement carries leading newlines,
+                # then count those as offsets.
+                my ($whitespace) =
+                    ($stat_next =~ /^((?:\s*\n[+-])*\s*)/s);
+                my $offset =
+                    statement_rawlines($whitespace) - 1;
+
+                $suppress_whiletrailers{$line_nr_next +
+                                            $offset} = 1;
+            }
+        }
+        if (!defined $suppress_whiletrailers{$linenr} &&
+            defined($stat) && defined($cond) &&
+            $line =~ /\b(?:if|while|for)\s*\(/ && $line !~ /^.\s*#/) {
+            my ($s, $c) = ($stat, $cond);
+
+            if ($c =~ /\bif\s*\(.*[^<>!=]=[^=].*/s) {
+                ERROR("ASSIGN_IN_IF",
+                      "do not use assignment in if condition\n" . $herecurr);
+            }
+
+            # Find out what is on the end of the line after the
+            # conditional.
+            substr($s, 0, length($c), '');
+            $s =~ s/\n.*//g;
+            $s =~ s/$;//g;      # Remove any comments
+            if (length($c) && $s !~ /^\s*{?\s*\\*\s*$/ &&
+                $c !~ /}\s*while\s*/)
+            {
+                # Find out how long the conditional actually is.
+                my @newlines = ($c =~ /\n/gs);
+                my $cond_lines = 1 + $#newlines;
+                my $stat_real = '';
+
+                $stat_real = raw_line($linenr, $cond_lines)
+                    . "\n" if ($cond_lines);
+                if (defined($stat_real) && $cond_lines > 1) {
+                    $stat_real = "[...]\n$stat_real";
+                }
+
+                ERROR("TRAILING_STATEMENTS",
+                      "trailing statements should be on next line\n" . $herecurr . $stat_real);
+            }
+        }
+
+# Check for bitwise tests written as boolean
+        if ($line =~ /
+                        (?:
+                                (?:\[|\(|\&\&|\|\|)
+                                \s*0[xX][0-9]+\s*
+                                (?:\&\&|\|\|)
+                        |
+                                (?:\&\&|\|\|)
+                                \s*0[xX][0-9]+\s*
+                                (?:\&\&|\|\||\)|\])
+                        )/x)
+        {
+            WARN("HEXADECIMAL_BOOLEAN_TEST",
+                 "boolean test with hexadecimal, perhaps just 1 \& or \|?\n" . $herecurr);
+        }
+
+# if and else should not have general statements after it
+        if ($line =~ /^.\s*(?:}\s*)?else\b(.*)/) {
+            my $s = $1;
+            $s =~ s/$;//g;      # Remove any comments
+            if ($s !~ /^\s*(?:\sif|(?:{|)\s*\\?\s*$)/) {
+                ERROR("TRAILING_STATEMENTS",
+                      "trailing statements should be on next line\n" . $herecurr);
+            }
+        }
+# if should not continue a brace
+        if ($line =~ /}\s*if\b/) {
+            ERROR("TRAILING_STATEMENTS",
+                  "trailing statements should be on next line\n" .
+                  $herecurr);
+        }
+# case and default should not have general statements after them
+        if ($line =~ /^.\s*(?:case\s*.*|default\s*):/g &&
+            $line !~ /\G(?:
+                        (?:\s*$;*)(?:\s*{)?(?:\s*$;*)(?:\s*\\)?\s*$|
+                        \s*return\s+
+                    )/xg)
+        {
+            ERROR("TRAILING_STATEMENTS",
+                  "trailing statements should be on next line\n" . $herecurr);
+        }
+
+        # Check for }<nl>else {, these must be at the same
+        # indent level to be relevant to each other.
+        if ($prevline=~/}\s*$/ and $line=~/^.\s*else\s*/ and
+            $previndent == $indent) {
+            ERROR("ELSE_AFTER_BRACE",
+                  "else should follow close brace '}'\n" . $hereprev);
+        }
+
+        if ($prevline=~/}\s*$/ and $line=~/^.\s*while\s*/ and
+            $previndent == $indent) {
+            my ($s, $c) = ctx_statement_block($linenr, $realcnt, 0);
+
+            # Find out what is on the end of the line after the
+            # conditional.
+            substr($s, 0, length($c), '');
+            $s =~ s/\n.*//g;
+
+            if ($s =~ /^\s*;/) {
+                ERROR("WHILE_AFTER_BRACE",
+                      "while should follow close brace '}'\n" . $hereprev);
+            }
+        }
+
+#Specific variable tests
+        while ($line =~ m{($Constant|$Lval)}g) {
+            my $var = $1;
+
+#gcc binary extension
+            if ($var =~ /^$Binary$/) {
+                if (WARN("GCC_BINARY_CONSTANT",
+                         "Avoid gcc v4.3+ binary constant extension: <$var>\n" . $herecurr) &&
+                    $fix) {
+                    my $hexval = sprintf("0x%x", oct($var));
+                    $fixed[$linenr - 1] =~
+                        s/\b$var\b/$hexval/;
+                }
+            }
+
+#CamelCase
+            if ($var !~ /^$Constant$/ &&
+                $var =~ /[A-Z][a-z]|[a-z][A-Z]/ &&
+#Ignore Page<foo> variants
+                $var !~ /^(?:Clear|Set|TestClear|TestSet|)Page[A-Z]/ &&
+#Ignore SI style variants like nS, mV and dB (ie: max_uV, regulator_min_uA_show)
+                $var !~ /^(?:[a-z_]*?)_?[a-z][A-Z](?:_[a-z_]+)?$/) {
+                while ($var =~ m{($Ident)}g) {
+                    my $word = $1;
+                    next if ($word !~ /[A-Z][a-z]|[a-z][A-Z]/);
+                    if ($check) {
+                        seed_camelcase_includes();
+                        if (!$file && !$camelcase_file_seeded) {
+                            seed_camelcase_file($realfile);
+                            $camelcase_file_seeded = 1;
+                        }
+                    }
+                    if (!defined $camelcase{$word}) {
+                        $camelcase{$word} = 1;
+                        CHK("CAMELCASE",
+                            "Avoid CamelCase: <$word>\n" . $herecurr);
+                    }
+                }
+            }
+        }
+
+#no spaces allowed after \ in define
+        if ($line =~ /\#\s*define.*\\\s+$/) {
+            if (WARN("WHITESPACE_AFTER_LINE_CONTINUATION",
+                     "Whitespace after \\ makes next lines useless\n" . $herecurr) &&
+                $fix) {
+                $fixed[$linenr - 1] =~ s/\s+$//;
+            }
+        }
+
+#warn if <asm/foo.h> is #included and <linux/foo.h> is available (uses RAW line)
+        if ($tree && $rawline =~ m{^.\s*\#\s*include\s*\<asm\/(.*)\.h\>}) {
+            my $file = "$1.h";
+            my $checkfile = "include/linux/$file";
+            if (-f "$root/$checkfile" &&
+                $realfile ne $checkfile &&
+                $1 !~ /$allowed_asm_includes/)
+            {
+                if ($realfile =~ m{^arch/}) {
+                    CHK("ARCH_INCLUDE_LINUX",
+                        "Consider using #include <linux/$file> instead of <asm/$file>\n" . $herecurr);
+                } else {
+                    WARN("INCLUDE_LINUX",
+                         "Use #include <linux/$file> instead of <asm/$file>\n" . $herecurr);
+                }
+            }
+        }
+
+# multi-statement macros should be enclosed in a do while loop, grab the
+# first statement and ensure its the whole macro if its not enclosed
+# in a known good container
+        if ($realfile !~ m@/vmlinux.lds.h$@ &&
+            $line =~ /^.\s*\#\s*define\s*$Ident(\()?/) {
+            my $ln = $linenr;
+            my $cnt = $realcnt;
+            my ($off, $dstat, $dcond, $rest);
+            my $ctx = '';
+            ($dstat, $dcond, $ln, $cnt, $off) =
+                ctx_statement_block($linenr, $realcnt, 0);
+            $ctx = $dstat;
+            #print "dstat<$dstat> dcond<$dcond> cnt<$cnt> off<$off>\n";
+            #print "LINE<$lines[$ln-1]> len<" . length($lines[$ln-1]) . "\n";
+
+            $dstat =~ s/^.\s*\#\s*define\s+$Ident(?:\([^\)]*\))?\s*//;
+            $dstat =~ s/$;//g;
+            $dstat =~ s/\\\n.//g;
+            $dstat =~ s/^\s*//s;
+            $dstat =~ s/\s*$//s;
+
+            # Flatten any parentheses and braces
+            while ($dstat =~ s/\([^\(\)]*\)/1/ ||
+                   $dstat =~ s/\{[^\{\}]*\}/1/ ||
+                   $dstat =~ s/\[[^\[\]]*\]/1/)
+            {
+            }
+
+            # Flatten any obvious string concatentation.
+            while ($dstat =~ s/("X*")\s*$Ident/$1/ ||
+                   $dstat =~ s/$Ident\s*("X*")/$1/)
+            {
+            }
+
+            my $exceptions = qr{
+                                $Declare|
+                                module_param_named|
+                                MODULE_PARM_DESC|
+                                DECLARE_PER_CPU|
+                                DEFINE_PER_CPU|
+                                __typeof__\(|
+                                union|
+                                struct|
+                                \.$Ident\s*=\s*|
+                                ^\"|\"$
+                        }x;
+            #print "REST<$rest> dstat<$dstat> ctx<$ctx>\n";
+            if ($dstat ne '' &&
+                $dstat !~ /^(?:$Ident|-?$Constant),$/ &&                        # 10, // foo(),
+                $dstat !~ /^(?:$Ident|-?$Constant);$/ &&                        # foo();
+                $dstat !~ /^[!~-]?(?:$Lval|$Constant)$/ &&              # 10 // foo() // !foo // ~foo // -foo // foo->bar // foo.bar->baz
+                $dstat !~ /^'X'$/ &&                                    # character constants
+                $dstat !~ /$exceptions/ &&
+                $dstat !~ /^\.$Ident\s*=/ &&                            # .foo =
+                $dstat !~ /^(?:\#\s*$Ident|\#\s*$Constant)\s*$/ &&              # stringification #foo
+                $dstat !~ /^do\s*$Constant\s*while\s*$Constant;?$/ &&   # do {...} while (...); // do {...} while (...)
+                $dstat !~ /^for\s*$Constant$/ &&                                # for (...)
+                $dstat !~ /^for\s*$Constant\s+(?:$Ident|-?$Constant)$/ &&       # for (...) bar()
+                $dstat !~ /^do\s*{/ &&                                  # do {...
+                $dstat !~ /^\(\{/ &&                                             # ({...
+                $ctx !~ /^.\s*#\s*define\s+TRACE_(?:SYSTEM|INCLUDE_FILE|INCLUDE_PATH)\b/)
+            {
+                $ctx =~ s/\n*$//;
+                my $herectx = $here . "\n";
+                my $cnt = statement_rawlines($ctx);
+
+                for (my $n = 0; $n < $cnt; $n++) {
+                    $herectx .= raw_line($linenr, $n) . "\n";
+                }
+
+                if ($dstat =~ /;/) {
+                    ERROR("MULTISTATEMENT_MACRO_USE_DO_WHILE",
+                          "Macros with multiple statements should be enclosed in a do - while loop\n" . "$herectx");
+                } else {
+                    ERROR("COMPLEX_MACRO",
+                          "Macros with complex values should be enclosed in parenthesis\n" . "$herectx");
+                }
+            }
+
+# check for line continuations outside of #defines, preprocessor #, and asm
+
+        } else {
+            if ($prevline !~ /^..*\\$/ &&
+                $line !~ /^\+\s*\#.*\\$/ &&             # preprocessor
+                $line !~ /^\+.*\b(__asm__|asm)\b.*\\$/ &&       # asm
+                $line =~ /^\+.*\\$/) {
+                WARN("LINE_CONTINUATIONS",
+                     "Avoid unnecessary line continuations\n" . $herecurr);
+            }
+        }
+
+# do {} while (0) macro tests:
+# single-statement macros do not need to be enclosed in do while (0) loop,
+# macro should not end with a semicolon
+        if ($^V && $^V ge 5.10.0 &&
+            $realfile !~ m@/vmlinux.lds.h$@ &&
+            $line =~ /^.\s*\#\s*define\s+$Ident(\()?/) {
+            my $ln = $linenr;
+            my $cnt = $realcnt;
+            my ($off, $dstat, $dcond, $rest);
+            my $ctx = '';
+            ($dstat, $dcond, $ln, $cnt, $off) =
+                ctx_statement_block($linenr, $realcnt, 0);
+            $ctx = $dstat;
+
+            $dstat =~ s/\\\n.//g;
+
+            if ($dstat =~ /^\+\s*#\s*define\s+$Ident\s*${balanced_parens}\s*do\s*{(.*)\s*}\s*while\s*\(\s*0\s*\)\s*([;\s]*)\s*$/) {
+                my $stmts = $2;
+                my $semis = $3;
+
+                $ctx =~ s/\n*$//;
+                my $cnt = statement_rawlines($ctx);
+                my $herectx = $here . "\n";
+
+                for (my $n = 0; $n < $cnt; $n++) {
+                    $herectx .= raw_line($linenr, $n) . "\n";
+                }
+
+                if (($stmts =~ tr/;/;/) == 1 &&
+                    $stmts !~ /^\s*(if|while|for|switch)\b/) {
+                    WARN("SINGLE_STATEMENT_DO_WHILE_MACRO",
+                         "Single statement macros should not use a do {} while (0) loop\n" . "$herectx");
+                }
+                if (defined $semis && $semis ne "") {
+                    WARN("DO_WHILE_MACRO_WITH_TRAILING_SEMICOLON",
+                         "do {} while (0) macros should not be semicolon terminated\n" . "$herectx");
+                }
+            } elsif ($dstat =~ /^\+\s*#\s*define\s+$Ident.*;\s*$/) {
+                $ctx =~ s/\n*$//;
+                my $cnt = statement_rawlines($ctx);
+                my $herectx = $here . "\n";
+
+                for (my $n = 0; $n < $cnt; $n++) {
+                    $herectx .= raw_line($linenr, $n) . "\n";
+                }
+
+                WARN("TRAILING_SEMICOLON",
+                     "macros should not use a trailing semicolon\n" . "$herectx");
+            }
+        }
+
+# make sure symbols are always wrapped with VMLINUX_SYMBOL() ...
+# all assignments may have only one of the following with an assignment:
+#       .
+#       ALIGN(...)
+#       VMLINUX_SYMBOL(...)
+        if ($realfile eq 'vmlinux.lds.h' && $line =~ /(?:(?:^|\s)$Ident\s*=|=\s*$Ident(?:\s|$))/) {
+            WARN("MISSING_VMLINUX_SYMBOL",
+                 "vmlinux.lds.h needs VMLINUX_SYMBOL() around C-visible symbols\n" . $herecurr);
+        }
+
+# check for redundant bracing round if etc
+        if ($line =~ /(^.*)\bif\b/ && $1 !~ /else\s*$/) {
+            my ($level, $endln, @chunks) =
+                ctx_statement_full($linenr, $realcnt, 1);
+            #print "chunks<$#chunks> linenr<$linenr> endln<$endln> level<$level>\n";
+            #print "APW: <<$chunks[1][0]>><<$chunks[1][1]>>\n";
+            if ($#chunks > 0 && $level == 0) {
+                my @allowed = ();
+                my $allow = 0;
+                my $seen = 0;
+                my $herectx = $here . "\n";
+                my $ln = $linenr - 1;
+                for my $chunk (@chunks) {
+                    my ($cond, $block) = @{$chunk};
+
+                    # If the condition carries leading newlines, then count those as offsets.
+                    my ($whitespace) = ($cond =~ /^((?:\s*\n[+-])*\s*)/s);
+                    my $offset = statement_rawlines($whitespace) - 1;
+
+                    $allowed[$allow] = 0;
+                    #print "COND<$cond> whitespace<$whitespace> offset<$offset>\n";
+
+                    # We have looked at and allowed this specific line.
+                    $suppress_ifbraces{$ln + $offset} = 1;
+
+                    $herectx .= "$rawlines[$ln + $offset]\n[...]\n";
+                    $ln += statement_rawlines($block) - 1;
+
+                    substr($block, 0, length($cond), '');
+
+                    $seen++ if ($block =~ /^\s*{/);
+
+                    #print "cond<$cond> block<$block> allowed<$allowed[$allow]>\n";
+                    if (statement_lines($cond) > 1) {
+                        #print "APW: ALLOWED: cond<$cond>\n";
+                        $allowed[$allow] = 1;
+                    }
+                    if ($block =~/\b(?:if|for|while)\b/) {
+                        #print "APW: ALLOWED: block<$block>\n";
+                        $allowed[$allow] = 1;
+                    }
+                    if (statement_block_size($block) > 1) {
+                        #print "APW: ALLOWED: lines block<$block>\n";
+                        $allowed[$allow] = 1;
+                    }
+                    $allow++;
+                }
+                if ($seen) {
+                    my $sum_allowed = 0;
+                    foreach (@allowed) {
+                        $sum_allowed += $_;
+                    }
+                    if ($sum_allowed != 0 && $sum_allowed != $allow
+		    			  && $seen != $allow) {
+                        CHK("BRACES",
+                            "braces {} should be used on all arms of this statement\n" . $herectx);
+                    }
+                }
+            }
+        }
+        if (!defined $suppress_ifbraces{$linenr - 1} &&
+            $line =~ /\b(if|while|for|else)\b/) {
+            my $allowed = 0;
+
+            # Check the pre-context.
+            if (substr($line, 0, $-[0]) =~ /(\}\s*)$/) {
+                #print "APW: ALLOWED: pre<$1>\n";
+                $allowed = 1;
+            }
+
+            my ($level, $endln, @chunks) =
+                ctx_statement_full($linenr, $realcnt, $-[0]);
+
+            # Check the condition.
+            my ($cond, $block) = @{$chunks[0]};
+            #print "CHECKING<$linenr> cond<$cond> block<$block>\n";
+            if (defined $cond) {
+                substr($block, 0, length($cond), '');
+            }
+            if (statement_lines($cond) > 1) {
+                #print "APW: ALLOWED: cond<$cond>\n";
+                $allowed = 1;
+            }
+            if ($block =~/\b(?:if|for|while)\b/) {
+                #print "APW: ALLOWED: block<$block>\n";
+                $allowed = 1;
+            }
+            if (statement_block_size($block) > 1) {
+                #print "APW: ALLOWED: lines block<$block>\n";
+                $allowed = 1;
+            }
+            # Check the post-context.
+            if (defined $chunks[1]) {
+                my ($cond, $block) = @{$chunks[1]};
+                if (defined $cond) {
+                    substr($block, 0, length($cond), '');
+                }
+                if ($block =~ /^\s*\{/) {
+                    #print "APW: ALLOWED: chunk-1 block<$block>\n";
+                    $allowed = 1;
+                }
+            }
+        }
+
+# check for unnecessary blank lines around braces
+        if (($line =~ /^.\s*}\s*$/ && $prevrawline =~ /^.\s*$/)) {
+            CHK("BRACES",
+                "Blank lines aren't necessary before a close brace '}'\n" . $hereprev);
+        }
+        if (($rawline =~ /^.\s*$/ && $prevline =~ /^..*{\s*$/)) {
+            CHK("BRACES",
+                "Blank lines aren't necessary after an open brace '{'\n" . $hereprev);
+        }
+
+# no volatiles please
+        my $asm_volatile = qr{\b(__asm__|asm)\s+(__volatile__|volatile)\b};
+        if ($line =~ /\bvolatile\b/ && $line !~ /$asm_volatile/) {
+            WARN("VOLATILE",
+                 "Use of volatile is usually wrong: see Documentation/volatile-considered-harmful.txt\n" . $herecurr);
+        }
+
+# warn about #if 0
+        if ($line =~ /^.\s*\#\s*if\s+0\b/) {
+            CHK("REDUNDANT_CODE",
+                "if this code is redundant consider removing it\n" .
+                $herecurr);
+        }
+
+# check for needless "if (<foo>) fn(<foo>)" uses
+        if ($prevline =~ /\bif\s*\(\s*($Lval)\s*\)/) {
+            my $expr = '\s*\(\s*' . quotemeta($1) . '\s*\)\s*;';
+            if ($line =~ /\b(kfree|usb_free_urb|debugfs_remove(?:_recursive)?)$expr/) {
+                WARN('NEEDLESS_IF',
+                     "$1(NULL) is safe this check is probably not required\n" . $hereprev);
+            }
+        }
+
+# check for bad placement of section $InitAttribute (e.g.: __initdata)
+        if ($line =~ /(\b$InitAttribute\b)/) {
+            my $attr = $1;
+            if ($line =~ /^\+\s*static\s+(?:const\s+)?(?:$attr\s+)?($NonptrTypeWithAttr)\s+(?:$attr\s+)?($Ident(?:\[[^]]*\])?)\s*[=;]/) {
+                my $ptr = $1;
+                my $var = $2;
+                if ((($ptr =~ /\b(union|struct)\s+$attr\b/ &&
+                      ERROR("MISPLACED_INIT",
+                            "$attr should be placed after $var\n" . $herecurr)) ||
+                     ($ptr !~ /\b(union|struct)\s+$attr\b/ &&
+                      WARN("MISPLACED_INIT",
+                           "$attr should be placed after $var\n" . $herecurr))) &&
+                    $fix) {
+                    $fixed[$linenr - 1] =~ s/(\bstatic\s+(?:const\s+)?)(?:$attr\s+)?($NonptrTypeWithAttr)\s+(?:$attr\s+)?($Ident(?:\[[^]]*\])?)\s*([=;])\s*/"$1" . trim(string_find_replace($2, "\\s*$attr\\s*", " ")) . " " . trim(string_find_replace($3, "\\s*$attr\\s*", "")) . " $attr" . ("$4" eq ";" ? ";" : " = ")/e;
+                }
+            }
+        }
+
+# check for $InitAttributeData (ie: __initdata) with const
+        if ($line =~ /\bconst\b/ && $line =~ /($InitAttributeData)/) {
+            my $attr = $1;
+            $attr =~ /($InitAttributePrefix)(.*)/;
+            my $attr_prefix = $1;
+            my $attr_type = $2;
+            if (ERROR("INIT_ATTRIBUTE",
+                      "Use of const init definition must use ${attr_prefix}initconst\n" . $herecurr) &&
+                $fix) {
+                $fixed[$linenr - 1] =~
+                    s/$InitAttributeData/${attr_prefix}initconst/;
+            }
+        }
+
+# check for $InitAttributeConst (ie: __initconst) without const
+        if ($line !~ /\bconst\b/ && $line =~ /($InitAttributeConst)/) {
+            my $attr = $1;
+            if (ERROR("INIT_ATTRIBUTE",
+                      "Use of $attr requires a separate use of const\n" . $herecurr) &&
+                $fix) {
+                my $lead = $fixed[$linenr - 1] =~
+                    /(^\+\s*(?:static\s+))/;
+                $lead = rtrim($1);
+                $lead = "$lead " if ($lead !~ /^\+$/);
+                $lead = "${lead}const ";
+                $fixed[$linenr - 1] =~ s/(^\+\s*(?:static\s+))/$lead/;
+            }
+        }
+
+# don't use __constant_<foo> functions outside of include/uapi/
+        if ($realfile !~ m@^include/uapi/@ &&
+            $line =~ /(__constant_(?:htons|ntohs|[bl]e(?:16|32|64)_to_cpu|cpu_to_[bl]e(?:16|32|64)))\s*\(/) {
+            my $constant_func = $1;
+            my $func = $constant_func;
+            $func =~ s/^__constant_//;
+            if (WARN("CONSTANT_CONVERSION",
+                     "$constant_func should be $func\n" . $herecurr) &&
+                $fix) {
+                $fixed[$linenr - 1] =~ s/\b$constant_func\b/$func/g;
+            }
+        }
+
+# prefer usleep_range over udelay
+        if ($line =~ /\budelay\s*\(\s*(\d+)\s*\)/) {
+            my $delay = $1;
+            # ignore udelay's < 10, however
+            if (! ($delay < 10) ) {
+                CHK("USLEEP_RANGE",
+                    "usleep_range is preferred over udelay; see Documentation/timers/timers-howto.txt\n" . $herecurr);
+            }
+            if ($delay > 2000) {
+                WARN("LONG_UDELAY",
+                     "long udelay - prefer mdelay; see arch/arm/include/asm/delay.h\n" . $herecurr);
+            }
+        }
+
+# warn about unexpectedly long msleep's
+        if ($line =~ /\bmsleep\s*\((\d+)\);/) {
+            if ($1 < 20) {
+                WARN("MSLEEP",
+                     "msleep < 20ms can sleep for up to 20ms; see Documentation/timers/timers-howto.txt\n" . $herecurr);
+            }
+        }
+
+# check for comparisons of jiffies
+        if ($line =~ /\bjiffies\s*$Compare|$Compare\s*jiffies\b/) {
+            WARN("JIFFIES_COMPARISON",
+                 "Comparing jiffies is almost always wrong; prefer time_after, time_before and friends\n" . $herecurr);
+        }
+
+# check for comparisons of get_jiffies_64()
+        if ($line =~ /\bget_jiffies_64\s*\(\s*\)\s*$Compare|$Compare\s*get_jiffies_64\s*\(\s*\)/) {
+            WARN("JIFFIES_COMPARISON",
+                 "Comparing get_jiffies_64() is almost always wrong; prefer time_after64, time_before64 and friends\n" . $herecurr);
+        }
+
+# warn about spacing in #ifdefs
+        if ($line =~ /^.\s*\#\s*(ifdef|ifndef|elif)\s\s+/) {
+            if (ERROR("SPACING",
+                      "exactly one space required after that #$1\n" . $herecurr) &&
+                $fix) {
+                $fixed[$linenr - 1] =~
+                    s/^(.\s*\#\s*(ifdef|ifndef|elif))\s{2,}/$1 /;
+            }
+
+        }
+
+# check for spinlock_t definitions without a comment.
+        if ($line =~ /^.\s*(struct\s+mutex|spinlock_t)\s+\S+;/ ||
+            $line =~ /^.\s*(DEFINE_MUTEX)\s*\(/) {
+            my $which = $1;
+            if (!ctx_has_comment($first_line, $linenr)) {
+                CHK("UNCOMMENTED_DEFINITION",
+                    "$1 definition without comment\n" . $herecurr);
+            }
+        }
+# check for memory barriers without a comment.
+        if ($line =~ /\b(mb|rmb|wmb|read_barrier_depends|smp_mb|smp_rmb|smp_wmb|smp_read_barrier_depends)\(/) {
+            if (!ctx_has_comment($first_line, $linenr)) {
+                WARN("MEMORY_BARRIER",
+                     "memory barrier without comment\n" . $herecurr);
+            }
+        }
+# check of hardware specific defines
+        if ($line =~ m@^.\s*\#\s*if.*\b(__i386__|__powerpc64__|__sun__|__s390x__)\b@ && $realfile !~ m@include/asm-@) {
+            CHK("ARCH_DEFINES",
+                "architecture specific defines should be avoided\n" .  $herecurr);
+        }
+
+# Check that the storage class is at the beginning of a declaration
+        if ($line =~ /\b$Storage\b/ && $line !~ /^.\s*$Storage\b/) {
+            WARN("STORAGE_CLASS",
+                 "storage class should be at the beginning of the declaration\n" . $herecurr)
+        }
+
+# check the location of the inline attribute, that it is between
+# storage class and type.
+        if ($line =~ /\b$Type\s+$Inline\b/ ||
+            $line =~ /\b$Inline\s+$Storage\b/) {
+            ERROR("INLINE_LOCATION",
+                  "inline keyword should sit between storage class and type\n" . $herecurr);
+        }
+
+# Check for __inline__ and __inline, prefer inline
+        if ($realfile !~ m@\binclude/uapi/@ &&
+            $line =~ /\b(__inline__|__inline)\b/) {
+            if (WARN("INLINE",
+                     "plain inline is preferred over $1\n" . $herecurr) &&
+                $fix) {
+                $fixed[$linenr - 1] =~ s/\b(__inline__|__inline)\b/inline/;
+
+            }
+        }
+
+# Check for __attribute__ packed, prefer __packed
+        if ($realfile !~ m@\binclude/uapi/@ &&
+            $line =~ /\b__attribute__\s*\(\s*\(.*\bpacked\b/) {
+            WARN("PREFER_PACKED",
+                 "__packed is preferred over __attribute__((packed))\n" . $herecurr);
+        }
+
+# Check for __attribute__ aligned, prefer __aligned
+        if ($realfile !~ m@\binclude/uapi/@ &&
+            $line =~ /\b__attribute__\s*\(\s*\(.*aligned/) {
+            WARN("PREFER_ALIGNED",
+                 "__aligned(size) is preferred over __attribute__((aligned(size)))\n" . $herecurr);
+        }
+
+# Check for __attribute__ format(printf, prefer __printf
+        if ($realfile !~ m@\binclude/uapi/@ &&
+            $line =~ /\b__attribute__\s*\(\s*\(\s*format\s*\(\s*printf/) {
+            if (WARN("PREFER_PRINTF",
+                     "__printf(string-index, first-to-check) is preferred over __attribute__((format(printf, string-index, first-to-check)))\n" . $herecurr) &&
+                $fix) {
+                $fixed[$linenr - 1] =~ s/\b__attribute__\s*\(\s*\(\s*format\s*\(\s*printf\s*,\s*(.*)\)\s*\)\s*\)/"__printf(" . trim($1) . ")"/ex;
+
+            }
+        }
+
+# Check for __attribute__ format(scanf, prefer __scanf
+        if ($realfile !~ m@\binclude/uapi/@ &&
+            $line =~ /\b__attribute__\s*\(\s*\(\s*format\s*\(\s*scanf\b/) {
+            if (WARN("PREFER_SCANF",
+                     "__scanf(string-index, first-to-check) is preferred over __attribute__((format(scanf, string-index, first-to-check)))\n" . $herecurr) &&
+                $fix) {
+                $fixed[$linenr - 1] =~ s/\b__attribute__\s*\(\s*\(\s*format\s*\(\s*scanf\s*,\s*(.*)\)\s*\)\s*\)/"__scanf(" . trim($1) . ")"/ex;
+            }
+        }
+
+# check for sizeof(&)
+        if ($line =~ /\bsizeof\s*\(\s*\&/) {
+            WARN("SIZEOF_ADDRESS",
+                 "sizeof(& should be avoided\n" . $herecurr);
+        }
+
+# check for sizeof without parenthesis
+        if ($line =~ /\bsizeof\s+((?:\*\s*|)$Lval|$Type(?:\s+$Lval|))/) {
+            if (WARN("SIZEOF_PARENTHESIS",
+                     "sizeof $1 should be sizeof($1)\n" . $herecurr) &&
+                $fix) {
+                $fixed[$linenr - 1] =~ s/\bsizeof\s+((?:\*\s*|)$Lval|$Type(?:\s+$Lval|))/"sizeof(" . trim($1) . ")"/ex;
+            }
+        }
+
+# check for line continuations in quoted strings with odd counts of "
+        if ($rawline =~ /\\$/ && $rawline =~ tr/"/"/ % 2) {
+            WARN("LINE_CONTINUATIONS",
+                 "Avoid line continuations in quoted strings\n" . $herecurr);
+        }
+
+# check for struct spinlock declarations
+        if ($line =~ /^.\s*\bstruct\s+spinlock\s+\w+\s*;/) {
+            WARN("USE_SPINLOCK_T",
+                 "struct spinlock should be spinlock_t\n" . $herecurr);
+        }
+
+# check for seq_printf uses that could be seq_puts
+        if ($sline =~ /\bseq_printf\s*\(.*"\s*\)\s*;\s*$/) {
+            my $fmt = get_quoted_string($line, $rawline);
+            if ($fmt ne "" && $fmt !~ /[^\\]\%/) {
+                if (WARN("PREFER_SEQ_PUTS",
+                         "Prefer seq_puts to seq_printf\n" . $herecurr) &&
+                    $fix) {
+                    $fixed[$linenr - 1] =~ s/\bseq_printf\b/seq_puts/;
+                }
+            }
+        }
+
+# Check for misused memsets
+        if ($^V && $^V ge 5.10.0 &&
+            defined $stat &&
+            $stat =~ /^\+(?:.*?)\bmemset\s*\(\s*$FuncArg\s*,\s*$FuncArg\s*\,\s*$FuncArg\s*\)/s) {
+
+            my $ms_addr = $2;
+            my $ms_val = $7;
+            my $ms_size = $12;
+
+            if ($ms_size =~ /^(0x|)0$/i) {
+                ERROR("MEMSET",
+                      "memset to 0's uses 0 as the 2nd argument, not the 3rd\n" . "$here\n$stat\n");
+            } elsif ($ms_size =~ /^(0x|)1$/i) {
+                WARN("MEMSET",
+                     "single byte memset is suspicious. Swapped 2nd/3rd argument?\n" . "$here\n$stat\n");
+            }
+        }
+
+# typecasts on min/max could be min_t/max_t
+        if ($^V && $^V ge 5.10.0 &&
+            defined $stat &&
+            $stat =~ /^\+(?:.*?)\b(min|max)\s*\(\s*$FuncArg\s*,\s*$FuncArg\s*\)/) {
+            if (defined $2 || defined $7) {
+                my $call = $1;
+                my $cast1 = deparenthesize($2);
+                my $arg1 = $3;
+                my $cast2 = deparenthesize($7);
+                my $arg2 = $8;
+                my $cast;
+
+                if ($cast1 ne "" && $cast2 ne "" && $cast1 ne $cast2) {
+                    $cast = "$cast1 or $cast2";
+                } elsif ($cast1 ne "") {
+                    $cast = $cast1;
+                } else {
+                    $cast = $cast2;
+                }
+                WARN("MINMAX",
+                     "$call() should probably be ${call}_t($cast, $arg1, $arg2)\n" . "$here\n$stat\n");
+            }
+        }
+
+# check usleep_range arguments
+        if ($^V && $^V ge 5.10.0 &&
+            defined $stat &&
+            $stat =~ /^\+(?:.*?)\busleep_range\s*\(\s*($FuncArg)\s*,\s*($FuncArg)\s*\)/) {
+            my $min = $1;
+            my $max = $7;
+            if ($min eq $max) {
+                WARN("USLEEP_RANGE",
+                     "usleep_range should not use min == max args; see Documentation/timers/timers-howto.txt\n" . "$here\n$stat\n");
+            } elsif ($min =~ /^\d+$/ && $max =~ /^\d+$/ &&
+                     $min > $max) {
+                WARN("USLEEP_RANGE",
+                     "usleep_range args reversed, use min then max; see Documentation/timers/timers-howto.txt\n" . "$here\n$stat\n");
+            }
+        }
+
+# check for naked sscanf
+        if ($^V && $^V ge 5.10.0 &&
+            defined $stat &&
+            $line =~ /\bsscanf\b/ &&
+            ($stat !~ /$Ident\s*=\s*sscanf\s*$balanced_parens/ &&
+             $stat !~ /\bsscanf\s*$balanced_parens\s*(?:$Compare)/ &&
+             $stat !~ /(?:$Compare)\s*\bsscanf\s*$balanced_parens/)) {
+            my $lc = $stat =~ tr@\n@@;
+            $lc = $lc + $linenr;
+            my $stat_real = raw_line($linenr, 0);
+            for (my $count = $linenr + 1; $count <= $lc; $count++) {
+                $stat_real = $stat_real . "\n" . raw_line($count, 0);
+            }
+            WARN("NAKED_SSCANF",
+                 "unchecked sscanf return value\n" . "$here\n$stat_real\n");
+        }
+
+# check for simple sscanf that should be kstrto<foo>
+        if ($^V && $^V ge 5.10.0 &&
+            defined $stat &&
+            $line =~ /\bsscanf\b/) {
+            my $lc = $stat =~ tr@\n@@;
+            $lc = $lc + $linenr;
+            my $stat_real = raw_line($linenr, 0);
+            for (my $count = $linenr + 1; $count <= $lc; $count++) {
+                $stat_real = $stat_real . "\n" . raw_line($count, 0);
+            }
+            if ($stat_real =~ /\bsscanf\b\s*\(\s*$FuncArg\s*,\s*("[^"]+")/) {
+                my $format = $6;
+                my $count = $format =~ tr@%@%@;
+                if ($count == 1 &&
+                    $format =~ /^"\%(?i:ll[udxi]|[udxi]ll|ll|[hl]h?[udxi]|[udxi][hl]h?|[hl]h?|[udxi])"$/) {
+                    WARN("SSCANF_TO_KSTRTO",
+                         "Prefer kstrto<type> to single variable sscanf\n" . "$here\n$stat_real\n");
+                }
+            }
+        }
+
+# check for new externs in .h files.
+        if ($realfile =~ /\.h$/ &&
+            $line =~ /^\+\s*(extern\s+)$Type\s*$Ident\s*\(/s) {
+            if (CHK("AVOID_EXTERNS",
+                    "extern prototypes should be avoided in .h files\n" . $herecurr) &&
+                $fix) {
+                $fixed[$linenr - 1] =~ s/(.*)\bextern\b\s*(.*)/$1$2/;
+            }
+        }
+
+# check for new externs in .c files.
+        if ($realfile =~ /\.c$/ && defined $stat &&
+            $stat =~ /^.\s*(?:extern\s+)?$Type\s+($Ident)(\s*)\(/s)
+        {
+            my $function_name = $1;
+            my $paren_space = $2;
+
+            my $s = $stat;
+            if (defined $cond) {
+                substr($s, 0, length($cond), '');
+            }
+            if ($s =~ /^\s*;/ &&
+                $function_name ne 'uninitialized_var')
+            {
+                WARN("AVOID_EXTERNS",
+                     "externs should be avoided in .c files\n" .  $herecurr);
+            }
+
+            if ($paren_space =~ /\n/) {
+                WARN("FUNCTION_ARGUMENTS",
+                     "arguments for function declarations should follow identifier\n" . $herecurr);
+            }
+
+        } elsif ($realfile =~ /\.c$/ && defined $stat &&
+                 $stat =~ /^.\s*extern\s+/)
+        {
+            WARN("AVOID_EXTERNS",
+                 "externs should be avoided in .c files\n" .  $herecurr);
+        }
+
+# check for pointless casting of kmalloc return
+        if ($line =~ /\*\s*\)\s*[kv][czm]alloc(_node){0,1}\b/) {
+            WARN("UNNECESSARY_CASTS",
+                 "unnecessary cast may hide bugs, see http://c-faq.com/malloc/mallocnocast.html\n" . $herecurr);
+        }
+
+# alloc style
+# p = alloc(sizeof(struct foo), ...) should be p = alloc(sizeof(*p), ...)
+        if ($^V && $^V ge 5.10.0 &&
+            $line =~ /\b($Lval)\s*\=\s*(?:$balanced_parens)?\s*([kv][mz]alloc(?:_node)?)\s*\(\s*(sizeof\s*\(\s*struct\s+$Lval\s*\))/) {
+            CHK("ALLOC_SIZEOF_STRUCT",
+                "Prefer $3(sizeof(*$1)...) over $3($4...)\n" . $herecurr);
+        }
+
+# check for multiple semicolons
+        if ($line =~ /;\s*;\s*$/) {
+            if (WARN("ONE_SEMICOLON",
+                     "Statements terminations use 1 semicolon\n" . $herecurr) &&
+                $fix) {
+                $fixed[$linenr - 1] =~ s/(\s*;\s*){2,}$/;/g;
+            }
+        }
+
+# check for case / default statements not preceeded by break/fallthrough/switch
+        if ($line =~ /^.\s*(?:case\s+(?:$Ident|$Constant)\s*|default):/) {
+            my $has_break = 0;
+            my $has_statement = 0;
+            my $count = 0;
+            my $prevline = $linenr;
+            while ($prevline > 1 && $count < 3 && !$has_break) {
+                $prevline--;
+                my $rline = $rawlines[$prevline - 1];
+                my $fline = $lines[$prevline - 1];
+                last if ($fline =~ /^\@\@/);
+                next if ($fline =~ /^\-/);
+                next if ($fline =~ /^.(?:\s*(?:case\s+(?:$Ident|$Constant)[\s$;]*|default):[\s$;]*)*$/);
+                $has_break = 1 if ($rline =~ /fall[\s_-]*(through|thru)/i);
+                next if ($fline =~ /^.[\s$;]*$/);
+                $has_statement = 1;
+                $count++;
+                $has_break = 1 if ($fline =~ /\bswitch\b|\b(?:break\s*;[\s$;]*$|return\b|goto\b|continue\b)/);
+            }
+            if (!$has_break && $has_statement) {
+                WARN("MISSING_BREAK",
+                     "Possible switch case/default not preceeded by break or fallthrough comment\n" . $herecurr);
+            }
+        }
+
+# check for switch/default statements without a break;
+        if ($^V && $^V ge 5.10.0 &&
+            defined $stat &&
+            $stat =~ /^\+[$;\s]*(?:case[$;\s]+\w+[$;\s]*:[$;\s]*|)*[$;\s]*\bdefault[$;\s]*:[$;\s]*;/g) {
+            my $ctx = '';
+            my $herectx = $here . "\n";
+            my $cnt = statement_rawlines($stat);
+            for (my $n = 0; $n < $cnt; $n++) {
+                $herectx .= raw_line($linenr, $n) . "\n";
+            }
+            WARN("DEFAULT_NO_BREAK",
+                 "switch default: should use break\n" . $herectx);
+        }
+
+# check for comparisons against true and false
+        if ($line =~ /\+\s*(.*?)\b(true|false|$Lval)\s*(==|\!=)\s*(true|false|$Lval)\b(.*)$/i) {
+            my $lead = $1;
+            my $arg = $2;
+            my $test = $3;
+            my $otype = $4;
+            my $trail = $5;
+            my $op = "!";
+
+            ($arg, $otype) = ($otype, $arg) if ($arg =~ /^(?:true|false)$/i);
+
+            my $type = lc($otype);
+            if ($type =~ /^(?:true|false)$/) {
+                if (("$test" eq "==" && "$type" eq "true") ||
+                    ("$test" eq "!=" && "$type" eq "false")) {
+                    $op = "";
+                }
+
+                CHK("BOOL_COMPARISON",
+                    "Using comparison to $otype is error prone\n" . $herecurr);
+            }
+        }
+
+# check for semaphores initialized locked
+        if ($line =~ /^.\s*sema_init.+,\W?0\W?\)/) {
+            WARN("CONSIDER_COMPLETION",
+                 "consider using a completion\n" . $herecurr);
+        }
+
+# check for %L{u,d,i} in strings
+        my $string;
+        while ($line =~ /(?:^|")([X\t]*)(?:"|$)/g) {
+            $string = substr($rawline, $-[1], $+[1] - $-[1]);
+            $string =~ s/%%/__/g;
+            if ($string =~ /(?<!%)%L[udi]/) {
+                WARN("PRINTF_L",
+                     "\%Ld/%Lu are not-standard C, use %lld/%llu\n" . $herecurr);
+                last;
+            }
+        }
+
+
+# Mode permission misuses where it seems decimal should be octal
+# This uses a shortcut match to avoid unnecessary uses of a slow foreach loop
+        if ($^V && $^V ge 5.10.0 &&
+            $line =~ /$mode_perms_search/) {
+            foreach my $entry (@mode_permission_funcs) {
+                my $func = $entry->[0];
+                my $arg_pos = $entry->[1];
+
+                my $skip_args = "";
+                if ($arg_pos > 1) {
+                    $arg_pos--;
+                    $skip_args = "(?:\\s*$FuncArg\\s*,\\s*){$arg_pos,$arg_pos}";
+                }
+                my $test = "\\b$func\\s*\\(${skip_args}([\\d]+)\\s*[,\\)]";
+                if ($line =~ /$test/) {
+                    my $val = $1;
+                    $val = $6 if ($skip_args ne "");
+
+                    if ($val !~ /^0$/ &&
+                        (($val =~ /^$Int$/ && $val !~ /^$Octal$/) ||
+                         length($val) ne 4)) {
+                        ERROR("NON_OCTAL_PERMISSIONS",
+                              "Use 4 digit octal (0777) not decimal permissions\n" . $herecurr);
+                    }
+                }
+            }
+        }
+    }
+
+    # If we have no input at all, then there is nothing to report on
+    # so just keep quiet.
+    if ($#rawlines == -1) {
+        exit(0);
+    }
+
+    # In mailback mode only produce a report in the negative, for
+    # things that appear to be patches.
+    if ($mailback && ($clean == 1 || !$is_patch)) {
+        exit(0);
+    }
+
+    # This is not a patch, and we are are in 'no-patch' mode so
+    # just keep quiet.
+    if (!$chk_patch && !$is_patch) {
+        exit(0);
+    }
+
+    if (!$is_patch) {
+        ERROR("NOT_UNIFIED_DIFF",
+              "Does not appear to be a unified-diff format patch\n");
+    }
+    if ($is_patch && $subject_trailing_dot != 0) {
+        ERROR("SUBJECT_TRAILING_DOT",
+              "The subject of the patch should not end with a dot.\n");
+    }
+    if ($is_patch && $chk_signoff && $signoff == 0) {
+        ERROR("MISSING_SIGN_OFF",
+              "Missing Signed-off-by: line(s)\n");
+    }
+
+    print report_dump();
+    if ($summary && !($clean == 1 && $quiet == 1)) {
+        print "$filename " if ($summary_file);
+        if ($cnt_error > 0) {
+            print "Patch not according to coding guidelines! please fix.\n";
+            print "total: $cnt_error errors, $cnt_warn warnings, " .
+                (($check)? "$cnt_chk checks, " : "") .
+                "$cnt_lines lines checked\n"; exit 1;
+        } else {
+            print "total: $cnt_warn warnings, " .
+                (($check)? "$cnt_chk checks, " : "") .
+                "$cnt_lines lines checked\n";
+            print "Patch found to have warnings, please fix if necessary.\n" if ($cnt_warn > 0);
+            exit 2;
+        }
+        print "\n" if ($quiet == 0);
+    }
+
+    if ($quiet == 0) {
+
+        if ($^V lt 5.10.0) {
+            print("NOTE: perl $^V is not modern enough to detect all possible issues.\n");
+            print("An upgrade to at least perl v5.10.0 is suggested.\n\n");
+        }
+
+        # If there were whitespace errors which cleanpatch can fix
+        # then suggest that.
+        if ($rpt_cleaners) {
+            print "NOTE: whitespace errors detected, you may wish to use scripts/cleanpatch or\n";
+            print "      scripts/cleanfile\n\n";
+            $rpt_cleaners = 0;
+        }
+    }
+
+    hash_show_words(\%use_type, "Used");
+    hash_show_words(\%ignore_type, "Ignored");
+
+    if ($clean == 0 && $fix && "@rawlines" ne "@fixed") {
+        my $newfile = $filename;
+        $newfile .= ".EXPERIMENTAL-checkpatch-fixes" if (!$fix_inplace);
+        my $linecount = 0;
+        my $f;
+
+        open($f, '>', $newfile)
+            or die "$P: Can't open $newfile for write\n";
+        foreach my $fixed_line (@fixed) {
+            $linecount++;
+            if ($file) {
+                if ($linecount > 3) {
+                    $fixed_line =~ s/^\+//;
+                    print $f $fixed_line. "\n";
+                }
+            } else {
+                print $f $fixed_line . "\n";
+            }
+        }
+        close($f);
+
+        if (!$quiet) {
+            print << "EOM";
+Wrote EXPERIMENTAL --fix correction(s) to '$newfile'
+
+Do _NOT_ trust the results written to this file.
+Do _NOT_ submit these changes without inspecting them for correctness.
+
+This EXPERIMENTAL file is simply a convenience to help rewrite patches.
+No warranties, expressed or implied...
+EOM
+        }
+    }
+
+    if ($clean == 1 && $quiet == 0) {
+        print "$vname has no obvious style problems and is ready for submission.\n"
+    }
+    if ($clean == 0 && $quiet == 0) {
+        print << "EOM";
+$vname has style problems, please review.
+
+If any of these errors are false positives, please report
+them to the maintainer, see MAINTAINERS
+EOM
+    }
+
+    return $clean;
+}
diff --git a/build-aux/config.guess.dist b/build-aux/config.guess.dist
new file mode 100755
index 00000000000..881ba7a0438
--- /dev/null
+++ b/build-aux/config.guess.dist
@@ -0,0 +1,14 @@
+#!/bin/sh
+#
+# This script is intentionally left empty. Distributions that package GlusterFS
+# may want to to replace it with an updated copy from the automake project.
+#
+
+cat << EOM
+It is not expected to execute this script. When you are building from a
+released tarball (generated with 'make dist'), you are expected to pass
+--build=... and --host=... to ./configure or replace this config.guess script
+in the sources with an updated version.
+EOM
+
+exit 0
diff --git a/build-aux/config.sub.dist b/build-aux/config.sub.dist
new file mode 100755
index 00000000000..c5a0dbad282
--- /dev/null
+++ b/build-aux/config.sub.dist
@@ -0,0 +1,14 @@
+#!/bin/sh
+#
+# This script is intentionally left empty. Distributions that package GlusterFS
+# may want to to replace it with an updated copy from the automake project.
+#
+
+cat << EOM
+It is not expected to execute this script. When you are building from a
+released tarball (generated with 'make dist'), you are expected to pass
+--build=... and --host=... to ./configure or replace this config.sub script in
+the sources with an updated version.
+EOM
+
+exit 0
diff --git a/build-aux/pkg-version b/build-aux/pkg-version
index 83d4a5f9136..17ceab70c03 100755
--- a/build-aux/pkg-version
+++ b/build-aux/pkg-version
@@ -1,9 +1,13 @@
-#!/bin/sh
+#!/bin/bash
 
 # To override version/release from git,
 # create VERSION file containing text with version/release
 # eg. v3.4.0-1
-PKG_VERSION=`cat VERSION 2> /dev/null || git describe --tags --match "v[0-9]*"`
+
+# One thing to note, If one does 'git clone --depth N glusterfs.git',
+# the git describe command doesn't work. Hence you notice below that
+# we have added timestamp as version (YYYY.MM.DD) and release (HH.mmss)
+PKG_VERSION=`cat VERSION 2> /dev/null || git describe --tags --match "v[0-9]*" 2>/dev/null`
 
 get_version()
 {
@@ -18,7 +22,11 @@ get_version()
       sub(/^v/,"") ; print $1
     }'
 
-    echo $PKG_VERSION | awk "$AWK_VERSION" | tr -cd '[:alnum:].'
+    version=$(echo $PKG_VERSION | awk "$AWK_VERSION" | tr -cd '[:alnum:].')
+    if [ "x${version}" == "x" ] ; then
+       version=$(date +%Y.%m.%d | tr -d '\n')
+    fi
+    echo $version | tr -d '\n'
 }
 
 get_release()
@@ -37,7 +45,11 @@ get_release()
       else if (NF == 4) print $2, $3, "git" substr($4, 2)
     }'
 
-    echo $PKG_VERSION | awk "$AWK_RELEASE" | tr -cd '[:alnum:].'
+    release=$(echo $PKG_VERSION | awk "$AWK_RELEASE" | tr -cd '[:alnum:].')
+    if [ "x${release}" == "x" ] ; then
+       release=$(date +%H.%M%S | tr -d '\n')
+    fi
+    echo $release | tr -d '\n'
 }
 
 if test "x$1" = "x--full"; then
diff --git a/build-aux/xdrgen b/build-aux/xdrgen
deleted file mode 100755
index 0cefc9b4890..00000000000
--- a/build-aux/xdrgen
+++ /dev/null
@@ -1,104 +0,0 @@
-#!/bin/sh
-
-append_licence_header ()
-{
-    local src_file=$1;
-    local dst_file=$2;
-
-    cat >$dst_file <<EOF
-/*
-  Copyright (c) 2007-2014 Red Hat, Inc. <http://www.redhat.com>
-  This file is part of GlusterFS.
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-*/
-
-#include "compat.h"
-#include "xdr-common.h"
-#include "xdr-nfs3.h"
-
-#if defined(__GNUC__)
-#if __GNUC__ >= 4
-#if !defined(__clang__)
-#if !defined(__NetBSD__)
-#pragma GCC diagnostic ignored "-Wunused-but-set-variable"
-#pragma GCC diagnostic ignored "-Wunused-variable"
-#endif
-#else
-#pragma clang diagnostic ignored "-Wunused-variable"
-#pragma clang diagnostic ignored "-Wunused-value"
-#endif
-#endif
-#endif
-
-EOF
-
-    cat $src_file >> $dst_file;
-
-}
-
-gen_sources ()
-{
-    local xfile="$1";
-
-    local cfile="${1%.x}.c";
-    local hfile="${1%.x}.h";
-
-    local tmp_cfile="$1.c.tmp";
-    local tmp_hfile="$1.h.tmp";
-
-    rm -f $cfile;
-    rpcgen -c -o $cfile $xfile;
-    append_licence_header $cfile $tmp_cfile;
-    mv $tmp_cfile $cfile;
-    # remove unwanted temporary files (if any)
-    rm -f $tmp_cfile;
-    echo "Generated $cfile"
-    return
-}
-
-gen_headers ()
-{
-    local xfile="$1";
-
-    local cfile="${1%.x}.c";
-    local hfile="${1%.x}.h";
-
-    local tmp_cfile="$1.c.tmp";
-    local tmp_hfile="$1.h.tmp";
-
-    # no need for a temporary file here as there are no changes from glusterfs
-    rm -f $hfile;
-    rpcgen -h -o $hfile $xfile;
-    # the '#ifdef' part of file should be fixed
-    sed -e 's/-/_/g' $hfile > ${hfile}.new && mv ${hfile}.new $hfile;
-    # Gen header to temp file and append generated file
-    append_licence_header $hfile $tmp_hfile;
-    # now move the destination file to actual original file
-    mv $tmp_hfile $hfile;
-    rm -f $tmp_hfile;
-    echo "Generated $hfile";
-    return
-}
-
-main ()
-{
-    if [ $# -ne 2 ]; then
-        echo "wrong number of arguments given"
-        echo " $0 [header|source] <XDR-definition-file>.x"
-        exit 1;
-    fi
-
-    if [ $1 = "header" ]; then
-        gen_headers $2
-    fi
-
-    if [ $1 = "source" ]; then
-        gen_sources $2
-    fi
-}
-
-main "$@";
diff --git a/cli/src/Makefile.am b/cli/src/Makefile.am
index 434122b2a78..16063f27c7f 100644
--- a/cli/src/Makefile.am
+++ b/cli/src/Makefile.am
@@ -1,10 +1,11 @@
 sbin_PROGRAMS = gluster
 
-gluster_SOURCES = cli.c registry.c input.c cli-cmd.c cli-rl.c \
+gluster_SOURCES = cli.c registry.c input.c cli-cmd.c cli-rl.c cli-cmd-global.c \
 	 cli-cmd-volume.c cli-cmd-peer.c cli-rpc-ops.c cli-cmd-parser.c\
 	 cli-cmd-system.c cli-cmd-misc.c cli-xml-output.c cli-quotad-client.c cli-cmd-snapshot.c
 
 gluster_LDADD = $(top_builddir)/libglusterfs/src/libglusterfs.la $(GF_LDADD) \
+		$(top_builddir)/libglusterd/src/libglusterd.la \
 		$(RLLIBS) $(top_builddir)/rpc/xdr/src/libgfxdr.la \
 		$(top_builddir)/rpc/rpc-lib/src/libgfrpc.la \
 		$(XML_LIBS)
@@ -13,20 +14,24 @@ gluster_LDFLAGS = $(GF_LDFLAGS)
 noinst_HEADERS = cli.h cli-mem-types.h cli-cmd.h cli-quotad-client.h
 
 AM_CPPFLAGS = $(GF_CPPFLAGS) \
-	-I$(top_srcdir)/libglusterfs/src -I$(top_srcdir)/rpc/rpc-lib/src\
-	-I$(top_srcdir)/rpc/xdr/src\
+	-I$(top_srcdir)/libglusterfs/src -I$(top_srcdir)/rpc/rpc-lib/src \
+	-I$(top_srcdir)/rpc/xdr/src -I$(top_srcdir)/libglusterd/src \
+	-I$(top_builddir)/rpc/xdr/src \
 	-DDATADIR=\"$(localstatedir)\" \
 	-DCONFDIR=\"$(sysconfdir)/glusterfs\" \
-	-DGSYNCD_PREFIX=\"$(libexecdir)/glusterfs\"\
-	-DSYNCDAEMON_COMPILE=$(SYNCDAEMON_COMPILE) -DSBIN_DIR=\"$(sbindir)\"\
-	$(XML_CPPFLAGS)
+	-DGSYNCD_PREFIX=\"$(GLUSTERFS_LIBEXECDIR)\"\
+	-DGLFSHEAL_PREFIX=\"$(GLUSTERFS_LIBEXECDIR)\"\
+	-DSYNCDAEMON_COMPILE=$(SYNCDAEMON_COMPILE)
 
-AM_CFLAGS = -Wall $(GF_CFLAGS)
+AM_CFLAGS = -Wall $(GF_CFLAGS) $(XML_CFLAGS)
 
 CLEANFILES =
 
 $(top_builddir)/libglusterfs/src/libglusterfs.la:
 	$(MAKE) -C $(top_builddir)/libglusterfs/src/ all
 
+$(top_builddir)/libglusterd/src/libglusterd.la:
+	$(MAKE) -C $(top_builddir)/libglusterd/src/ all
+
 install-data-hook:
 	$(mkdir_p) $(DESTDIR)$(localstatedir)/run/gluster
diff --git a/cli/src/cli-cmd-global.c b/cli/src/cli-cmd-global.c
new file mode 100644
index 00000000000..2c9a5f01bb1
--- /dev/null
+++ b/cli/src/cli-cmd-global.c
@@ -0,0 +1,195 @@
+/*
+   Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#include <stdio.h>
+#include <string.h>
+#include <stdlib.h>
+#include <stdint.h>
+#include <pthread.h>
+
+#include <sys/socket.h>
+#include <netdb.h>
+#include <sys/types.h>
+#include <sys/wait.h>
+#include <netinet/in.h>
+
+#include "cli.h"
+#include "cli-cmd.h"
+#include "cli-mem-types.h"
+#include "cli1-xdr.h"
+#include <glusterfs/run.h>
+#include <glusterfs/syscall.h>
+#include <glusterfs/common-utils.h>
+
+int
+cli_cmd_global_help_cbk(struct cli_state *state, struct cli_cmd_word *in_word,
+                        const char **words, int wordcount);
+int
+cli_cmd_get_state_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                      const char **words, int wordcount);
+
+int
+cli_cmd_ganesha_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                    const char **words, int wordcount);
+
+struct cli_cmd global_cmds[] = {
+    {
+        "global help",
+        cli_cmd_global_help_cbk,
+        "list global commands",
+    },
+    {
+        "get-state [<daemon>] [[odir </path/to/output/dir/>] "
+        "[file <filename>]] [detail|volumeoptions]",
+        cli_cmd_get_state_cbk,
+        "Get local state representation of mentioned daemon",
+    },
+    {
+        "nfs-ganesha {enable| disable} ",
+        cli_cmd_ganesha_cbk,
+        "Enable/disable NFS-Ganesha support",
+    },
+    {NULL, NULL, NULL}};
+
+int
+cli_cmd_global_help_cbk(struct cli_state *state, struct cli_cmd_word *in_word,
+                        const char **words, int wordcount)
+{
+    struct cli_cmd *cmd = NULL;
+    struct cli_cmd *global_cmd = NULL;
+    int count = 0;
+
+    cmd = GF_MALLOC(sizeof(global_cmds), cli_mt_cli_cmd);
+    memcpy(cmd, global_cmds, sizeof(global_cmds));
+    count = (sizeof(global_cmds) / sizeof(struct cli_cmd));
+    cli_cmd_sort(cmd, count);
+
+    cli_out("\ngluster global commands");
+    cli_out("========================\n");
+    for (global_cmd = cmd; global_cmd->pattern; global_cmd++)
+        if (_gf_false == global_cmd->disable)
+            cli_out("%s - %s", global_cmd->pattern, global_cmd->desc);
+
+    cli_out("\n");
+    GF_FREE(cmd);
+    return 0;
+}
+
+int
+cli_cmd_global_register(struct cli_state *state)
+{
+    int ret = 0;
+    struct cli_cmd *cmd = NULL;
+    for (cmd = global_cmds; cmd->pattern; cmd++) {
+        ret = cli_cmd_register(&state->tree, cmd);
+        if (ret)
+            goto out;
+    }
+out:
+    return ret;
+}
+
+int
+cli_cmd_ganesha_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                    const char **words, int wordcount)
+
+{
+    int sent = 0;
+    int parse_error = 0;
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *options = NULL;
+    cli_local_t *local = NULL;
+    char *op_errstr = NULL;
+
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_GANESHA];
+
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame)
+        goto out;
+
+    ret = cli_cmd_ganesha_parse(state, words, wordcount, &options, &op_errstr);
+    if (ret) {
+        if (op_errstr) {
+            cli_err("%s", op_errstr);
+            GF_FREE(op_errstr);
+        } else
+            cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
+
+    CLI_LOCAL_INIT(local, words, frame, options);
+
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, options);
+    }
+
+out:
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("Setting global option failed");
+    }
+
+    CLI_STACK_DESTROY(frame);
+    return ret;
+}
+
+int
+cli_cmd_get_state_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                      const char **words, int wordcount)
+{
+    int sent = 0;
+    int parse_error = 0;
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *options = NULL;
+    cli_local_t *local = NULL;
+    char *op_errstr = NULL;
+
+    ret = cli_cmd_get_state_parse(state, words, wordcount, &options,
+                                  &op_errstr);
+
+    if (ret) {
+        if (op_errstr) {
+            cli_err("%s", op_errstr);
+            cli_usage_out(word->pattern);
+            GF_FREE(op_errstr);
+        } else
+            cli_usage_out(word->pattern);
+
+        parse_error = 1;
+        goto out;
+    }
+
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
+
+    CLI_LOCAL_INIT(local, words, frame, options);
+
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_GET_STATE];
+    if (proc->fn)
+        ret = proc->fn(frame, THIS, options);
+out:
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("Getting daemon state failed");
+    }
+
+    CLI_STACK_DESTROY(frame);
+
+    return ret;
+}
diff --git a/cli/src/cli-cmd-misc.c b/cli/src/cli-cmd-misc.c
index 566d7c978d9..e961d88da86 100644
--- a/cli/src/cli-cmd-misc.c
+++ b/cli/src/cli-cmd-misc.c
@@ -13,86 +13,105 @@
 #include <stdint.h>
 #include <pthread.h>
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 #include "cli.h"
 #include "cli-cmd.h"
 #include "cli-mem-types.h"
 #include "protocol-common.h"
 
-extern struct rpc_clnt *global_rpc;
-
-extern rpc_clnt_prog_t *cli_rpc_prog;
-
 extern struct cli_cmd volume_cmds[];
+extern struct cli_cmd bitrot_cmds[];
+extern struct cli_cmd quota_cmds[];
 extern struct cli_cmd cli_probe_cmds[];
 extern struct cli_cmd cli_log_cmds[];
 extern struct cli_cmd cli_system_cmds[];
 extern struct cli_cmd cli_bd_cmds[];
 extern struct cli_cmd snapshot_cmds[];
+extern struct cli_cmd global_cmds[];
 struct cli_cmd cli_misc_cmds[];
 
 int
-cli_cmd_quit_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                   const char **words, int wordcount)
+cli_cmd_quit_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                 const char **words, int wordcount)
 {
-        exit (0);
+    exit(0);
 }
 
-int
-cli_cmd_display_help (struct cli_state *state, struct cli_cmd_word *in_word,
-                      const char **words, int wordcount)
+static gf_boolean_t
+cli_is_help_command(const char *pattern)
 {
-        struct cli_cmd        *cmd[] = {volume_cmds, cli_probe_cmds,
-                                       cli_misc_cmds, snapshot_cmds,
-                                       NULL};
-        struct cli_cmd        *cmd_ind = NULL;
-        int                   i = 0;
-
-         /* cli_system_cmds commands for internal usage
-           they are not exposed
-         */
-        for (i=0; cmd[i]!=NULL; i++)
-                for (cmd_ind = cmd[i]; cmd_ind->pattern; cmd_ind++)
-                        if (_gf_false == cmd_ind->disable)
-                                cli_out ("%s - %s", cmd_ind->pattern,
-                                         cmd_ind->desc);
-
-        return 0;
+    /* FixFixFix
+     * This is not the best way to determine whether
+     * this is a help command
+     */
+    if (strstr(pattern, "help"))
+        return _gf_true;
+
+    return _gf_false;
 }
 
-struct cli_cmd cli_misc_cmds[] = {
-        { "quit",
-          cli_cmd_quit_cbk,
-          "quit"},
+int
+cli_cmd_display_help(struct cli_state *state, struct cli_cmd_word *in_word,
+                     const char **words, int wordcount)
+{
+    static struct cli_cmd *cmd[] = {
+        cli_misc_cmds, cli_probe_cmds, volume_cmds, bitrot_cmds,
+        quota_cmds,    snapshot_cmds,  global_cmds, NULL};
+    struct cli_cmd *cmd_ind = NULL;
+    int i = 0;
+    gf_boolean_t list_all = _gf_false;
+
+    /* cli_system_cmds commands for internal usage
+      they are not exposed
+    */
+
+    /* If "help all" */
+    if (wordcount == 2)
+        list_all = _gf_true;
+
+    for (i = 0; cmd[i] != NULL; i++) {
+        for (cmd_ind = cmd[i]; cmd_ind->pattern; cmd_ind++) {
+            if ((_gf_false == cmd_ind->disable) &&
+                cli_is_help_command(cmd_ind->pattern)) {
+                if (list_all && (cmd_ind->cbk)) {
+                    cmd_ind->cbk(state, in_word, words, wordcount);
+                } else {
+                    cli_out(" %-25s- %s", cmd_ind->pattern, cmd_ind->desc);
+                }
+            }
+        }
+    }
+
+    cli_out("\n");
+    return 0;
+}
 
-        { "help",
-           cli_cmd_display_help,
-           "display command options"},
+struct cli_cmd cli_help_cmds[] = {
+    {"help [all]", cli_cmd_display_help, "display help for command classes"},
 
-        { "exit",
-          cli_cmd_quit_cbk,
-          "exit"},
+    {NULL, NULL, NULL}};
 
-        { NULL, NULL, NULL }
-};
+struct cli_cmd cli_misc_cmds[] = {{"quit", cli_cmd_quit_cbk, "quit"},
+                                  {"exit", cli_cmd_quit_cbk, "exit"},
 
+                                  {NULL, NULL, NULL}};
 
 int
-cli_cmd_misc_register (struct cli_state *state)
+cli_cmd_misc_register(struct cli_state *state)
 {
-        int  ret = 0;
-        struct cli_cmd *cmd = NULL;
-
-        for (cmd = cli_misc_cmds; cmd->pattern; cmd++) {
-
-                ret = cli_cmd_register (&state->tree, cmd);
-                if (ret)
-                        goto out;
-        }
+    int ret = 0;
+    struct cli_cmd *cmd = NULL;
+
+    for (cmd = cli_misc_cmds; cmd->pattern; cmd++) {
+        ret = cli_cmd_register(&state->tree, cmd);
+        if (ret)
+            goto out;
+    }
+
+    for (cmd = cli_help_cmds; cmd->pattern; cmd++) {
+        ret = cli_cmd_register(&state->tree, cmd);
+        if (ret)
+            goto out;
+    }
 out:
-        return ret;
+    return ret;
 }
diff --git a/cli/src/cli-cmd-parser.c b/cli/src/cli-cmd-parser.c
index 28888ba656d..34620b4a31b 100644
--- a/cli/src/cli-cmd-parser.c
+++ b/cli/src/cli-cmd-parser.c
@@ -12,3193 +12,4306 @@
 #include <stdlib.h>
 #include <stdint.h>
 #include <pthread.h>
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
+#include <fnmatch.h>
+#include <time.h>
 
 #include "cli.h"
 #include "cli-cmd.h"
 #include "cli-mem-types.h"
-#include "dict.h"
+#include <glusterfs/dict.h>
+#include <glusterfs/list.h>
 
 #include "protocol-common.h"
 #include "cli1-xdr.h"
 
 #define MAX_SNAP_DESCRIPTION_LEN 1024
 
-struct snap_config_opt_vals_ snap_confopt_vals[] = {
-        {.op_name        = "snap-max-hard-limit",
-         .question       = "Changing snapshot-max-hard-limit "
-                           "will lead to deletion of snapshots "
-                           "if they exceed the new limit.\n"
-                           "Do you want to continue?"
-        },
-        {.op_name        = "snap-max-soft-limit",
-         .question       = "Changing snapshot-max-soft-limit "
-                           "will lead to deletion of snapshots "
-                           "if they exceed the new limit.\n"
-                           "Do you want to continue?"
-        },
-        {.op_name        = "both",
-        .question        = "Changing snapshot-max-hard-limit & "
-                           "snapshot-max-soft-limit will lead to "
-                           "deletion of snapshots if they exceed "
-                           "the new limit.\nDo you want to continue?"
-        },
-        {.op_name        = NULL,
-        }
-};
+static struct snap_config_opt_vals_ snap_confopt_vals[] = {
+    {.op_name = "snap-max-hard-limit",
+     .question = "Changing snapshot-max-hard-limit "
+                 "will limit the creation of new snapshots "
+                 "if they exceed the new limit.\n"
+                 "Do you want to continue?"},
+    {.op_name = "snap-max-soft-limit",
+     .question = "If Auto-delete is enabled, snap-max-soft-limit will"
+                 " trigger deletion of oldest snapshot, on the "
+                 "creation of new snapshot, when the "
+                 "snap-max-soft-limit is reached.\n"
+                 "Do you want to change the snap-max-soft-limit?"},
+    {.op_name = "both",
+     .question = "Changing snapshot-max-hard-limit "
+                 "will limit the creation of new snapshots "
+                 "if they exceed the new snapshot-max-hard-limit.\n"
+                 "If Auto-delete is enabled, snap-max-soft-limit will"
+                 " trigger deletion of oldest snapshot, on the "
+                 "creation of new snapshot, when the "
+                 "snap-max-soft-limit is reached.\n"
+                 "Do you want to continue?"},
+    {
+        .op_name = NULL,
+    }};
 
 enum cli_snap_config_set_types {
-        GF_SNAP_CONFIG_SET_HARD = 0,
-        GF_SNAP_CONFIG_SET_SOFT = 1,
-        GF_SNAP_CONFIG_SET_BOTH = 2,
+    GF_SNAP_CONFIG_SET_HARD = 0,
+    GF_SNAP_CONFIG_SET_SOFT = 1,
+    GF_SNAP_CONFIG_SET_BOTH = 2,
 };
 typedef enum cli_snap_config_set_types cli_snap_config_set_types;
 
+typedef struct _cli_brick {
+    struct list_head list;
+    const char *name;
+    int32_t len;
+} cli_brick_t;
+
+int
+cli_cmd_validate_volume(char *volname);
+
 static const char *
-id_sel (void *wcon)
+id_sel(void *wcon)
 {
-        return (const char *)wcon;
+    return (const char *)wcon;
 }
 
 static char *
-str_getunamb (const char *tok, char **opwords)
+str_getunamb(const char *tok, char **opwords)
 {
-        return (char *)cli_getunamb (tok, (void **)opwords, id_sel);
+    return (char *)cli_getunamb(tok, (void **)opwords, id_sel);
 }
 
 int32_t
-cli_cmd_bricks_parse (const char **words, int wordcount, int brick_index,
-                      char **bricks, int *brick_count)
+cli_cmd_ta_brick_parse(const char **words, int wordcount, char **ta_brick)
 {
-        int                    ret = 0;
-        char             *tmp_list = NULL;
-        char    brick_list[120000] = {0,};
-        char                *space = " ";
-        char            *delimiter = NULL;
-        char            *host_name = NULL;
-        char        *free_list_ptr = NULL;
-        char               *tmpptr = NULL;
-        int                      j = 0;
-        int         brick_list_len = 0;
-        char             *tmp_host = NULL;
-
-        GF_ASSERT (words);
-        GF_ASSERT (wordcount);
-        GF_ASSERT (bricks);
-        GF_ASSERT (brick_index > 0);
-        GF_ASSERT (brick_index < wordcount);
-
-        strncpy (brick_list, space, strlen (space));
-        brick_list_len++;
-        while (brick_index < wordcount) {
-                if (validate_brick_name ((char *)words[brick_index])) {
-                        cli_err ("Wrong brick type: %s, use <HOSTNAME>:"
-                                 "<export-dir-abs-path>", words[brick_index]);
-                        ret = -1;
-                        goto out;
-                } else {
-                        delimiter = strrchr (words[brick_index], ':');
-                        ret = gf_canonicalize_path (delimiter + 1);
-                        if (ret)
-                                goto out;
-                }
-
-                if ((brick_list_len + strlen (words[brick_index]) + 1) > sizeof (brick_list)) {
-                        cli_err ("Total brick list is larger than a request. "
-                                 "Can take (brick_count %d)", *brick_count);
-                        ret = -1;
-                        goto out;
-                }
+    char *host_name = NULL;
+    char *tmp_host = NULL;
+    char *delimiter = NULL;
+    cli_brick_t *brick = NULL;
+    int ret = 0;
+
+    GF_ASSERT(words);
+    GF_ASSERT(wordcount);
+
+    if (validate_brick_name((char *)words[wordcount - 1])) {
+        cli_err(
+            "Wrong brick type: %s, use <HOSTNAME>:"
+            "<export-dir-abs-path>",
+            words[wordcount - 1]);
+        ret = -1;
+        goto out;
+    } else {
+        delimiter = strrchr(words[wordcount - 1], ':');
+        ret = gf_canonicalize_path(delimiter + 1);
+        if (ret)
+            goto out;
+    }
 
-                tmp_host = gf_strdup ((char *)words[brick_index]);
-                if (!tmp_host) {
-                        gf_log ("cli", GF_LOG_ERROR, "Out of memory");
-                        ret = -1;
-                        goto out;
-                }
-                get_host_name (tmp_host, &host_name);
-                if (!host_name) {
-                        ret = -1;
-                        gf_log("cli",GF_LOG_ERROR, "Unable to allocate "
-                               "memory");
-                        goto out;
-                }
+    tmp_host = gf_strdup((char *)words[wordcount - 1]);
+    if (!tmp_host) {
+        gf_log("cli", GF_LOG_ERROR, "Out of memory");
+        ret = -1;
+        goto out;
+    }
+    get_host_name(tmp_host, &host_name);
+    if (!host_name) {
+        ret = -1;
+        gf_log("cli", GF_LOG_ERROR,
+               "Unable to retrieve "
+               "hostname");
+        goto out;
+    }
+
+    if (!(strcmp(host_name, "localhost") && strcmp(host_name, "127.0.0.1") &&
+          strncmp(host_name, "0.", 2))) {
+        cli_err(
+            "Please provide a valid hostname/ip other "
+            "than localhost, 127.0.0.1 or loopback "
+            "address (0.0.0.0 to 0.255.255.255).");
+        ret = -1;
+        goto out;
+    }
+    if (!valid_internet_address(host_name, _gf_false, _gf_false)) {
+        cli_err(
+            "internet address '%s' does not conform to "
+            "standards",
+            host_name);
+    }
+
+    brick = GF_MALLOC(sizeof(cli_brick_t), gf_common_list_node);
+    if (brick == NULL) {
+        ret = -1;
+        gf_log("cli", GF_LOG_ERROR, "Out of memory");
+        goto out;
+    }
 
-                if (!(strcmp (host_name, "localhost") &&
-                      strcmp (host_name, "127.0.0.1") &&
-                      strncmp (host_name, "0.", 2))) {
-                        cli_err ("Please provide a valid hostname/ip other "
-                                 "than localhost, 127.0.0.1 or loopback "
-                                 "address (0.0.0.0 to 0.255.255.255).");
-                        ret = -1;
-                        GF_FREE (tmp_host);
-                        goto out;
-                }
-                if (!valid_internet_address (host_name, _gf_false)) {
-                        cli_err ("internet address '%s' does not conform to "
-                                 "standards", host_name);
-                }
-                GF_FREE (tmp_host);
-                tmp_list = gf_strdup (brick_list + 1);
-                if (free_list_ptr) {
-                        GF_FREE (free_list_ptr);
-                        free_list_ptr = NULL;
-                }
-                free_list_ptr = tmp_list;
-                j = 0;
-                while(j < *brick_count) {
-                        strtok_r (tmp_list, " ", &tmpptr);
-                        if (!(strcmp (tmp_list, words[brick_index]))) {
-                                ret = -1;
-                                cli_err ("Found duplicate"
-                                         " exports %s",words[brick_index]);
-                                goto out;
-                       }
-                       tmp_list = tmpptr;
-                       j++;
-                }
-                strcat (brick_list, words[brick_index]);
-                strcat (brick_list, " ");
-                brick_list_len += (strlen (words[brick_index]) + 1);
-                ++(*brick_count);
-                ++brick_index;
-        }
+    brick->name = words[wordcount - 1];
+    brick->len = strlen(words[wordcount - 1]);
+    *ta_brick = GF_MALLOC(brick->len + 3, gf_common_mt_char);
+    if (*ta_brick == NULL) {
+        ret = -1;
+        gf_log("cli", GF_LOG_ERROR, "Out of memory");
+        goto out;
+    }
 
-        *bricks = gf_strdup (brick_list);
-        if (!*bricks)
-                ret = -1;
+    strcat(*ta_brick, " ");
+    strcat(*ta_brick, brick->name);
+    strcat(*ta_brick, " ");
 out:
-        GF_FREE (free_list_ptr);
-        return ret;
+    if (tmp_host) {
+        GF_FREE(tmp_host);
+        tmp_host = NULL;
+    }
+    if (brick) {
+        GF_FREE(brick);
+        brick = NULL;
+    }
+
+    return ret;
 }
 
 int32_t
-cli_cmd_create_disperse_check(struct cli_state * state, int * disperse,
-                              int * redundancy, int count)
+cli_cmd_bricks_parse(const char **words, int wordcount, int brick_index,
+                     char **bricks, int *brick_count)
 {
-        int i = 0;
-        int tmp = 0;
-        gf_answer_t answer = GF_ANSWER_NO;
-        char question[128];
+    int ret = 0;
+    char *delimiter = NULL;
+    char *host_name = NULL;
+    char *tmp_host = NULL;
+    char *bricks_str = NULL;
+    int len = 0;
+    int brick_list_len = 1; /* For initial space */
+    struct list_head brick_list = {
+        0,
+    };
+    cli_brick_t *brick = NULL;
+
+    GF_ASSERT(words);
+    GF_ASSERT(wordcount);
+    GF_ASSERT(bricks);
+    GF_ASSERT(brick_index > 0);
+    GF_ASSERT(brick_index < wordcount);
+
+    INIT_LIST_HEAD(&brick_list);
+
+    while (brick_index < wordcount) {
+        if (validate_brick_name((char *)words[brick_index])) {
+            cli_err(
+                "Wrong brick type: %s, use <HOSTNAME>:"
+                "<export-dir-abs-path>",
+                words[brick_index]);
+            ret = -1;
+            goto out;
+        } else {
+            delimiter = strrchr(words[brick_index], ':');
+            ret = gf_canonicalize_path(delimiter + 1);
+            if (ret)
+                goto out;
+        }
 
-        const char * question1 = "There isn't an optimal redundancy value "
-                                 "for this configuration. Do you want to "
-                                 "create the volume with redundancy 1 ?";
+        tmp_host = gf_strdup((char *)words[brick_index]);
+        if (!tmp_host) {
+            gf_log("cli", GF_LOG_ERROR, "Out of memory");
+            ret = -1;
+            goto out;
+        }
+        get_host_name(tmp_host, &host_name);
+        if (!host_name) {
+            ret = -1;
+            gf_log("cli", GF_LOG_ERROR,
+                   "Unable to allocate "
+                   "memory");
+            GF_FREE(tmp_host);
+            goto out;
+        }
+
+        if (!(strcmp(host_name, "localhost") &&
+              strcmp(host_name, "127.0.0.1") && strncmp(host_name, "0.", 2))) {
+            cli_err(
+                "Please provide a valid hostname/ip other "
+                "than localhost, 127.0.0.1 or loopback "
+                "address (0.0.0.0 to 0.255.255.255).");
+            ret = -1;
+            GF_FREE(tmp_host);
+            goto out;
+        }
+        if (!valid_internet_address(host_name, _gf_false, _gf_false)) {
+            cli_err(
+                "internet address '%s' does not conform to "
+                "standards",
+                host_name);
+        }
+        GF_FREE(tmp_host);
+        list_for_each_entry(brick, &brick_list, list)
+        {
+            if (strcmp(brick->name, words[brick_index]) == 0) {
+                ret = -1;
+                cli_err("Found duplicate exports %s", words[brick_index]);
+                goto out;
+            }
+        }
 
-        const char * question2 = "The optimal redundancy for this "
-                                 "configuration is %d. Do you want to create "
-                                 "the volume with this value ?";
+        brick = GF_MALLOC(sizeof(cli_brick_t), gf_common_list_node);
+        if (brick == NULL) {
+            ret = -1;
+            gf_log("cli", GF_LOG_ERROR, "Out of memory");
+            goto out;
+        }
+        len = strlen(words[brick_index]);
+        brick->name = words[brick_index];
+        brick->len = len;
+        list_add_tail(&brick->list, &brick_list);
 
-        const char * question3 = "This configuration is not optimal on most "
-                                 "workloads. Do you want to use it ?";
+        brick_list_len += len + 1; /* Brick name + space */
+        ++(*brick_count);
+        ++brick_index;
+    }
 
-        if (*disperse <= 0) {
-                if (count < 3) {
-                        cli_err ("number of bricks must be greater "
-                                 "than 2");
+    /* If brick count is not valid exit here */
+    if (!*brick_count) {
+        cli_err("No bricks specified");
+        ret = -1;
+        goto out;
+    }
 
-                        return -1;
-                }
-                *disperse = count;
-        }
+    brick_list_len++; /* For terminating null char */
 
-        if (*redundancy == 0) {
-                tmp = *disperse - 1;
-                for (i = tmp / 2;
-                     (i > 0) && ((tmp & -tmp) != tmp);
-                     i--, tmp--);
+    bricks_str = GF_MALLOC(brick_list_len, gf_common_mt_char);
+    if (bricks_str == NULL) {
+        ret = -1;
+        gf_log("cli", GF_LOG_ERROR, "Out of memory");
+        goto out;
+    }
+    *bricks = bricks_str;
+    *bricks_str = ' ';
+    bricks_str++;
+    while (!list_empty(&brick_list)) {
+        brick = list_first_entry(&brick_list, cli_brick_t, list);
+        list_del_init(&brick->list);
+        memcpy(bricks_str, brick->name, brick->len);
+        bricks_str[brick->len] = ' ';
+        bricks_str += brick->len + 1;
+        GF_FREE(brick);
+    }
+    *bricks_str = 0;
 
-                if (i == 0) {
-                        answer = cli_cmd_get_confirmation(state, question1);
-                        if (answer == GF_ANSWER_NO)
-                                return -1;
+out:
+    while (!list_empty(&brick_list)) {
+        brick = list_first_entry(&brick_list, cli_brick_t, list);
+        list_del_init(&brick->list);
+        GF_FREE(brick);
+    }
 
-                        *redundancy = 1;
-                }
-                else
-                {
-                        *redundancy = *disperse - tmp;
-                        if (*redundancy > 1) {
-                                sprintf(question, question2, *redundancy);
-                                answer = cli_cmd_get_confirmation(state,
-                                                                  question);
-                                if (answer == GF_ANSWER_NO)
-                                        return -1;
-                        }
-                }
+    return ret;
+}
 
-                tmp = 0;
+int32_t
+cli_cmd_create_disperse_check(struct cli_state *state, int *disperse,
+                              int *redundancy, int *data, int count)
+{
+    int i = 0;
+    int tmp = 0;
+    gf_answer_t answer = GF_ANSWER_NO;
+    char question[128];
+
+    const char *question1 =
+        "There isn't an optimal redundancy value "
+        "for this configuration. Do you want to "
+        "create the volume with redundancy 1 ?";
+
+    const char *question2 =
+        "The optimal redundancy for this "
+        "configuration is %d. Do you want to create "
+        "the volume with this value ?";
+
+    const char *question3 =
+        "This configuration is not optimal on most "
+        "workloads. Do you want to use it ?";
+
+    const char *question4 =
+        "Redundancy for this configuration is %d. "
+        "Do you want to create "
+        "the volume with this value ?";
+
+    if (*data > 0) {
+        if (*disperse > 0 && *redundancy > 0) {
+            if (*disperse != (*data + *redundancy)) {
+                cli_err(
+                    "Disperse count(%d) should be equal "
+                    "to sum of disperse-data count(%d) and "
+                    "redundancy count(%d)",
+                    *disperse, *data, *redundancy);
+                return -1;
+            }
+        } else if (*redundancy > 0) {
+            *disperse = *data + *redundancy;
+        } else if (*disperse > 0) {
+            *redundancy = *disperse - *data;
+        } else {
+            if ((count - *data) >= *data) {
+                cli_err(
+                    "Please provide redundancy count "
+                    "along with disperse-data count");
+                return -1;
+            } else {
+                sprintf(question, question4, count - *data);
+                answer = cli_cmd_get_confirmation(state, question);
+                if (answer == GF_ANSWER_NO)
+                    return -1;
+                *redundancy = count - *data;
+                *disperse = count;
+            }
         }
-        else {
-                tmp = *disperse - *redundancy;
+    }
+
+    if (*disperse <= 0) {
+        if (count < 3) {
+            cli_err(
+                "number of bricks must be greater "
+                "than 2");
+
+            return -1;
         }
+        *disperse = count;
+    }
 
-        if (*redundancy > (*disperse - 1) / 2) {
-                cli_err ("redundancy must be less than %d for a "
-                         "disperse %d volume",
-                         (*disperse + 1) / 2, *disperse);
+    if (*redundancy == -1) {
+        tmp = *disperse - 1;
+        for (i = tmp / 2; (i > 0) && ((tmp & -tmp) != tmp); i--, tmp--)
+            ;
 
+        if (i == 0) {
+            answer = cli_cmd_get_confirmation(state, question1);
+            if (answer == GF_ANSWER_NO)
                 return -1;
-        }
 
-        if ((tmp & -tmp) != tmp) {
-                answer = cli_cmd_get_confirmation(state, question3);
+            *redundancy = 1;
+        } else {
+            *redundancy = *disperse - tmp;
+            if (*redundancy > 1) {
+                sprintf(question, question2, *redundancy);
+                answer = cli_cmd_get_confirmation(state, question);
                 if (answer == GF_ANSWER_NO)
-                        return -1;
+                    return -1;
+            }
         }
 
-        return 0;
-}
-
-int32_t
-cli_cmd_volume_create_parse (struct cli_state *state, const char **words,
-                             int wordcount, dict_t **options)
-{
-        dict_t  *dict = NULL;
-        char    *volname = NULL;
-        int     ret = -1;
-        gf1_cluster_type type = GF_CLUSTER_TYPE_NONE;
-        int     count = 1;
-        int     sub_count = 1;
-        int     brick_index = 0;
-        int     i = 0;
-        char    *trans_type = NULL;
-        int32_t index = 0;
-        char    *bricks = NULL;
-        int32_t brick_count = 0;
-        char    *opwords[] = { "replica", "stripe", "transport", "disperse",
-                               "redundancy", NULL };
-
-        char    *invalid_volnames[] = {"volume", "type", "subvolumes", "option",
-                                       "end-volume", "all", "volume_not_in_ring",
-                                       "description", "force",
-                                       "snap-max-hard-limit",
-                                       "snap-max-soft-limit", "auto-delete",
-                                       "activate-on-create", NULL};
-        char    *w = NULL;
-        char    *ptr = NULL;
-        int      op_count = 0;
-        int32_t  replica_count = 1;
-        int32_t  stripe_count = 1;
-        int32_t  disperse_count = -1;
-        int32_t  redundancy_count = 0;
-        gf_boolean_t is_force = _gf_false;
-        int wc = wordcount;
-
-        GF_ASSERT (words);
-        GF_ASSERT (options);
-
-        dict = dict_new ();
-
-        if (!dict)
-                goto out;
+        tmp = 0;
+    } else {
+        tmp = *disperse - *redundancy;
+    }
 
-        if (wordcount < 3)
-                goto out;
+    if ((*redundancy < 1) || (*redundancy > (*disperse - 1) / 2)) {
+        cli_err(
+            "redundancy must be greater than or equal to 1 and "
+            "less than %d for a disperse %d volume",
+            (*disperse + 1) / 2, *disperse);
 
-        volname = (char *)words[2];
+        return -1;
+    }
 
-        GF_ASSERT (volname);
+    if ((tmp & -tmp) != tmp) {
+        answer = cli_cmd_get_confirmation(state, question3);
+        if (answer == GF_ANSWER_NO)
+            return -1;
+    }
 
-        /* Validate the volume name here itself */
-        {
-                if (volname[0] == '-')
-                        goto out;
-
-                for (i = 0; invalid_volnames[i]; i++) {
-                        if (!strcmp (volname, invalid_volnames[i])) {
-                                cli_err ("\"%s\" cannot be the name of a volume.",
-                                         volname);
-                                goto out;
-                        }
-                }
+    return 0;
+}
 
-                if (strchr (volname, '/'))
-                        goto out;
+static int32_t
+cli_validate_disperse_volume(char *word, gf1_cluster_type type,
+                             const char **words, int32_t wordcount,
+                             int32_t index, int32_t *disperse_count,
+                             int32_t *redundancy_count, int32_t *data_count)
+{
+    int ret = -1;
+
+    switch (type) {
+        case GF_CLUSTER_TYPE_NONE:
+        case GF_CLUSTER_TYPE_DISPERSE:
+            if (strcmp(word, "disperse") == 0) {
+                if (*disperse_count >= 0) {
+                    cli_err("disperse option given twice");
+                    goto out;
+                }
+                if (wordcount < (index + 2)) {
+                    goto out;
+                }
+                ret = gf_string2int(words[index + 1], disperse_count);
+                if (ret == -1 && errno == EINVAL) {
+                    *disperse_count = 0;
+                    ret = 1;
+                } else if (ret == -1) {
+                    goto out;
+                } else {
+                    if (*disperse_count < 3) {
+                        cli_err(
+                            "disperse count must "
+                            "be greater than 2");
+                        goto out;
+                    }
+                    ret = 2;
+                }
+            } else if (strcmp(word, "disperse-data") == 0) {
+                if (*data_count >= 0) {
+                    cli_err("disperse-data option given twice");
+                    goto out;
+                }
+                if (wordcount < (index + 2)) {
+                    goto out;
+                }
+                ret = gf_string2int(words[index + 1], data_count);
+                if (ret == -1 || *data_count < 2) {
+                    cli_err("disperse-data must be greater than 1");
+                    goto out;
+                }
+                ret = 2;
+            } else if (strcmp(word, "redundancy") == 0) {
+                if (*redundancy_count >= 0) {
+                    cli_err("redundancy option given twice");
+                    goto out;
+                }
+                if (wordcount < (index + 2)) {
+                    goto out;
+                }
+                ret = gf_string2int(words[index + 1], redundancy_count);
+                if (ret == -1 || *redundancy_count < 1) {
+                    cli_err("redundancy must be greater than 0");
+                    goto out;
+                }
+                ret = 2;
+            }
+            break;
+        case GF_CLUSTER_TYPE_REPLICATE:
+            cli_err(
+                "replicated-dispersed volume is not "
+                "supported");
+            goto out;
+        default:
+            cli_err("Invalid type given");
+            break;
+    }
+out:
+    return ret;
+}
 
-                if (strlen (volname) > GD_VOLUME_NAME_MAX) {
-                        cli_err("Volume name exceeds %d characters.",
-                                GD_VOLUME_NAME_MAX);
-                        goto out;
-                }
+int32_t
+cli_validate_volname(const char *volname)
+{
+    int32_t ret = -1;
+    int32_t i = -1;
+    int volname_len;
+    static const char *const invalid_volnames[] = {"volume",
+                                                   "type",
+                                                   "subvolumes",
+                                                   "option",
+                                                   "end-volume",
+                                                   "all",
+                                                   "volume_not_in_ring",
+                                                   "description",
+                                                   "force",
+                                                   "snap-max-hard-limit",
+                                                   "snap-max-soft-limit",
+                                                   "auto-delete",
+                                                   "activate-on-create",
+                                                   NULL};
+
+    if (volname[0] == '-')
+        goto out;
 
-                for (i = 0; i < strlen (volname); i++)
-                        if (!isalnum (volname[i]) && (volname[i] != '_') &&
-                           (volname[i] != '-')) {
-                                cli_err ("Volume name should not contain \"%c\""
-                                         " character.\nVolume names can only"
-                                         "contain alphanumeric, '-' and '_' "
-                                         "characters.",volname[i]);
-                                goto out;
-                        }
+    for (i = 0; invalid_volnames[i]; i++) {
+        if (!strcmp(volname, invalid_volnames[i])) {
+            cli_err("\"%s\" cannot be the name of a volume.", volname);
+            goto out;
         }
+    }
 
-        if (wordcount < 4) {
-                ret = -1;
-                goto out;
-        }
+    if (strchr(volname, '/'))
+        goto out;
 
-        type = GF_CLUSTER_TYPE_NONE;
-        index = 3;
+    volname_len = strlen(volname);
+    if (volname_len > GD_VOLUME_NAME_MAX) {
+        cli_err("Volume name exceeds %d characters.", GD_VOLUME_NAME_MAX);
+        goto out;
+    }
+
+    for (i = 0; i < volname_len; i++) {
+        if (!isalnum(volname[i]) && (volname[i] != '_') &&
+            (volname[i] != '-')) {
+            cli_err(
+                "Volume name should not contain \"%c\""
+                " character.\nVolume names can only"
+                "contain alphanumeric, '-' and '_' "
+                "characters.",
+                volname[i]);
+            goto out;
+        }
+    }
+
+    ret = 0;
+out:
+    return ret;
+}
 
-        while (op_count < 3) {
-                ret = -1;
-                w = str_getunamb (words[index], opwords);
-                if (!w) {
-                        break;
-                } else if ((strcmp (w, "replica")) == 0) {
-                        switch (type) {
-                        case GF_CLUSTER_TYPE_STRIPE_REPLICATE:
-                        case GF_CLUSTER_TYPE_REPLICATE:
-                                cli_err ("replica option given twice");
-                                goto out;
-                        case GF_CLUSTER_TYPE_NONE:
-                                type = GF_CLUSTER_TYPE_REPLICATE;
-                                break;
-                        case GF_CLUSTER_TYPE_STRIPE:
-                                type = GF_CLUSTER_TYPE_STRIPE_REPLICATE;
-                                break;
-                        case GF_CLUSTER_TYPE_DISPERSE:
-                                cli_err ("replicated-dispersed volume is not "
-                                         "supported");
-                                goto out;
-                        }
+int32_t
+cli_cmd_volume_create_parse(struct cli_state *state, const char **words,
+                            int wordcount, dict_t **options, char **brick_list)
+{
+    dict_t *dict = NULL;
+    char *volname = NULL;
+    int ret = -1;
+    gf1_cluster_type type = GF_CLUSTER_TYPE_NONE;
+    int sub_count = 1;
+    int brick_index = 0;
+    char *trans_type = NULL;
+    int32_t index = 0;
+    char *bricks = NULL;
+    char *ta_brick = NULL;
+    int32_t brick_count = 0;
+    static char *opwords[] = {"replica",  "stripe",       "transport",
+                              "disperse", "redundancy",   "disperse-data",
+                              "arbiter",  "thin-arbiter", NULL};
+
+    char *w = NULL;
+    int op_count = 0;
+    int32_t replica_count = 1;
+    int32_t arbiter_count = 0;
+    int32_t thin_arbiter_count = 0;
+    int32_t stripe_count = 1;
+    int32_t disperse_count = -1;
+    int32_t redundancy_count = -1;
+    int32_t disperse_data_count = -1;
+    gf_boolean_t is_force = _gf_false;
+    int wc = wordcount;
+    gf_answer_t answer = GF_ANSWER_NO;
+    const char *question = NULL;
+
+    GF_ASSERT(words);
+    GF_ASSERT(options);
+
+    dict = dict_new();
+
+    if (!dict)
+        goto out;
 
-                        if (wordcount < (index+2)) {
-                                ret = -1;
-                                goto out;
-                        }
-                        replica_count = strtol (words[index+1], NULL, 0);
-                        if (replica_count < 2) {
-                                cli_err ("replica count should be greater"
-                                         " than 1");
-                                ret = -1;
-                                goto out;
-                        }
-                        ret = dict_set_int32 (dict, "replica-count", replica_count);
-                        if (ret)
-                                goto out;
-
-                        index += 2;
-
-                } else if ((strcmp (w, "stripe")) == 0) {
-                        switch (type) {
-                        case GF_CLUSTER_TYPE_STRIPE_REPLICATE:
-                        case GF_CLUSTER_TYPE_STRIPE:
-                                cli_err ("stripe option given twice");
-                                goto out;
-                        case GF_CLUSTER_TYPE_NONE:
-                                type = GF_CLUSTER_TYPE_STRIPE;
-                                break;
-                        case GF_CLUSTER_TYPE_REPLICATE:
-                                type = GF_CLUSTER_TYPE_STRIPE_REPLICATE;
-                                break;
-                        case GF_CLUSTER_TYPE_DISPERSE:
-                                cli_err ("striped-dispersed volume is not "
-                                         "supported");
-                                goto out;
-                        }
-                        if (wordcount < (index + 2)) {
-                                ret = -1;
-                                goto out;
-                        }
-                        stripe_count = strtol (words[index+1], NULL, 0);
-                        if (stripe_count < 2) {
-                                cli_err ("stripe count should be greater"
-                                         " than 1");
-                                ret = -1;
-                                goto out;
-                        }
-                        ret = dict_set_int32 (dict, "stripe-count", stripe_count);
-                        if (ret)
-                                goto out;
+    if (wordcount < 3)
+        goto out;
 
-                        index += 2;
+    volname = (char *)words[2];
 
-                } else if ((strcmp (w, "transport")) == 0) {
-                        if (trans_type) {
-                                cli_err ("'transport' option given more"
-                                         " than one time");
-                                goto out;
-                        }
-                        if ((strcasecmp (words[index+1], "tcp") == 0)) {
-                                trans_type = gf_strdup ("tcp");
-                        } else if ((strcasecmp (words[index+1], "rdma") == 0)) {
-                                trans_type = gf_strdup ("rdma");
-                        } else if ((strcasecmp (words[index+1], "tcp,rdma") == 0) ||
-                                   (strcasecmp (words[index+1], "rdma,tcp") == 0)) {
-                                trans_type = gf_strdup ("tcp,rdma");
-                        } else {
-                                gf_log ("", GF_LOG_ERROR, "incorrect transport"
-                                        " protocol specified");
-                                ret = -1;
-                                goto out;
-                        }
-                        index += 2;
-
-                } else if ((strcmp (w, "disperse")) == 0) {
-                        switch (type) {
-                        case GF_CLUSTER_TYPE_DISPERSE:
-                                if (disperse_count >= 0) {
-                                        cli_err ("disperse option given "
-                                                 "twice");
-                                        goto out;
-                                }
-                                break;
-                        case GF_CLUSTER_TYPE_NONE:
-                                type = GF_CLUSTER_TYPE_DISPERSE;
-                                break;
-                        case GF_CLUSTER_TYPE_STRIPE_REPLICATE:
-                                cli_err ("striped-replicated-dispersed volume "
-                                         "is not supported");
-                                goto out;
-                        case GF_CLUSTER_TYPE_STRIPE:
-                                cli_err ("striped-dispersed volume is not "
-                                         "supported");
-                                goto out;
-                        case GF_CLUSTER_TYPE_REPLICATE:
-                                cli_err ("replicated-dispersed volume is not "
-                                         "supported");
-                                goto out;
-                        }
+    GF_ASSERT(volname);
 
-                        if (wordcount >= (index+2)) {
-                                disperse_count = strtol (words[index + 1],
-                                                         &ptr, 0);
-                                if (*ptr != 0)
-                                        disperse_count = 0;
-                                else {
-                                        if (disperse_count < 3) {
-                                                cli_err ("disperse count must "
-                                                         "be greater than 2");
-                                                ret = -1;
-                                                goto out;
-                                        }
-                                        index++;
-                                }
-                        }
+    /* Validate the volume name here itself */
+    if (cli_validate_volname(volname) < 0)
+        goto out;
 
-                        index++;
-
-                } else if ((strcmp (w, "redundancy")) == 0) {
-                        switch (type) {
-                        case GF_CLUSTER_TYPE_NONE:
-                                type = GF_CLUSTER_TYPE_DISPERSE;
-                                break;
-                        case GF_CLUSTER_TYPE_DISPERSE:
-                                if (redundancy_count > 0) {
-                                        cli_err ("redundancy option given "
-                                                 "twice");
-                                        goto out;
-                                }
-                                break;
-                        case GF_CLUSTER_TYPE_STRIPE_REPLICATE:
-                                cli_err ("striped-replicated-dispersed volume "
-                                         "is not supported");
-                                goto out;
-                        case GF_CLUSTER_TYPE_STRIPE:
-                                cli_err ("striped-dispersed volume is not "
-                                         "supported");
-                                goto out;
-                        case GF_CLUSTER_TYPE_REPLICATE:
-                                cli_err ("replicated-dispersed volume is not "
-                                         "supported");
-                                goto out;
-                        }
+    if (wordcount < 4) {
+        ret = -1;
+        goto out;
+    }
 
-                        if (wordcount < (index+2)) {
-                                ret = -1;
-                                goto out;
-                        }
-                        redundancy_count = strtol (words[index+1], NULL, 0);
-                        if (redundancy_count < 1) {
-                                cli_err ("redundancy must be greater than 0");
-                                ret = -1;
-                                goto out;
-                        }
+    type = GF_CLUSTER_TYPE_NONE;
+    index = 3;
 
-                        index += 2;
+    while (op_count < 3) {
+        ret = -1;
+        w = str_getunamb(words[index], opwords);
+        if (!w) {
+            break;
+        } else if ((strcmp(w, "replica")) == 0) {
+            switch (type) {
+                case GF_CLUSTER_TYPE_STRIPE_REPLICATE:
+                case GF_CLUSTER_TYPE_REPLICATE:
+                    cli_err("replica option given twice");
+                    goto out;
+                case GF_CLUSTER_TYPE_NONE:
+                    type = GF_CLUSTER_TYPE_REPLICATE;
+                    break;
+                case GF_CLUSTER_TYPE_STRIPE:
+                    cli_err("stripe option not supported");
+                    goto out;
+                case GF_CLUSTER_TYPE_DISPERSE:
+                    cli_err(
+                        "replicated-dispersed volume is not "
+                        "supported");
+                    goto out;
+                default:
+                    cli_err("Invalid type given");
+                    goto out;
+            }
+
+            if (wordcount < (index + 2)) {
+                ret = -1;
+                goto out;
+            }
 
-                }              else {
-                        GF_ASSERT (!"opword mismatch");
+            replica_count = strtol(words[index + 1], NULL, 0);
+            if (replica_count < 2) {
+                cli_err(
+                    "replica count should be greater"
+                    " than 1");
+                ret = -1;
+                goto out;
+            }
+
+            index += 2;
+            if (words[index]) {
+                if (!strcmp(words[index], "arbiter")) {
+                    ret = gf_string2int(words[index + 1], &arbiter_count);
+                    if ((ret == -1) || (arbiter_count != 1)) {
+                        cli_err(
+                            "For arbiter "
+                            "configuration, "
+                            "replica count must be"
+                            " 2 and arbiter count "
+                            "must be 1. The 3rd "
+                            "brick of the replica "
+                            "will be the arbiter");
                         ret = -1;
                         goto out;
-                }
-                op_count++;
-        }
-
-        if (!trans_type)
-                trans_type = gf_strdup ("tcp");
-
-        /* reset the count value now */
-        count = 1;
-
-        if (index >= wordcount) {
+                    }
+                    ret = dict_set_int32(dict, "arbiter-count", arbiter_count);
+                    if (ret)
+                        goto out;
+                    index += 2;
+                } else if (!strcmp(words[index], "thin-arbiter")) {
+                    ret = gf_string2int(words[index + 1], &thin_arbiter_count);
+                    if ((ret == -1) || (thin_arbiter_count != 1) ||
+                        (replica_count != 2)) {
+                        cli_err(
+                            "For thin-arbiter "
+                            "configuration, "
+                            "replica count must be"
+                            " 2 and thin-arbiter count "
+                            "must be 1. The 3rd "
+                            "brick of the replica "
+                            "will be the thin-arbiter brick");
+                        ret = -1;
+                        goto out;
+                    }
+                    ret = dict_set_int32(dict, "thin-arbiter-count",
+                                         thin_arbiter_count);
+                    if (ret)
+                        goto out;
+                    index += 2;
+                }
+            }
+
+            /* Do this to keep glusterd happy with sending
+               "replica 3 arbiter 1" options to server */
+            if ((arbiter_count == 1) && (replica_count == 2))
+                replica_count += arbiter_count;
+
+            if (replica_count == 2 && thin_arbiter_count == 0) {
+                if (strcmp(words[wordcount - 1], "force")) {
+                    question =
+                        "Replica 2 volumes are prone"
+                        " to split-brain. Use "
+                        "Arbiter or Replica 3 to "
+                        "avoid this. See: "
+                        "http://docs.gluster.org/en/latest/"
+                        "Administrator%20Guide/"
+                        "Split%20brain%20and%20ways%20to%20deal%20with%20it/."
+                        "\nDo you still want to "
+                        "continue?\n";
+                    answer = cli_cmd_get_confirmation(state, question);
+                    if (GF_ANSWER_NO == answer) {
+                        gf_log("cli", GF_LOG_ERROR,
+                               "Volume create "
+                               "cancelled, exiting");
+                        ret = -1;
+                        goto out;
+                    }
+                }
+            }
+            ret = dict_set_int32(dict, "replica-count", replica_count);
+            if (ret)
+                goto out;
+
+        } else if ((strcmp(w, "stripe")) == 0) {
+            cli_err("stripe option not supported");
+            goto out;
+        } else if ((strcmp(w, "transport")) == 0) {
+            if (trans_type) {
+                cli_err(
+                    "'transport' option given more"
+                    " than one time");
+                goto out;
+            }
+            if ((strcasecmp(words[index + 1], "tcp") == 0)) {
+                trans_type = gf_strdup("tcp");
+            } else if ((strcasecmp(words[index + 1], "rdma") == 0)) {
+                trans_type = gf_strdup("rdma");
+            } else if ((strcasecmp(words[index + 1], "tcp,rdma") == 0) ||
+                       (strcasecmp(words[index + 1], "rdma,tcp") == 0)) {
+                trans_type = gf_strdup("tcp,rdma");
+            } else {
+                gf_log("", GF_LOG_ERROR,
+                       "incorrect transport"
+                       " protocol specified");
                 ret = -1;
                 goto out;
+            }
+            index += 2;
+
+        } else if ((strcmp(w, "disperse") == 0) ||
+                   (strcmp(w, "redundancy") == 0) ||
+                   (strcmp(w, "disperse-data") == 0)) {
+            ret = cli_validate_disperse_volume(
+                w, type, words, wordcount, index, &disperse_count,
+                &redundancy_count, &disperse_data_count);
+            if (ret < 0)
+                goto out;
+            index += ret;
+            type = GF_CLUSTER_TYPE_DISPERSE;
+        } else if ((strcmp(w, "arbiter") == 0)) {
+            cli_err(
+                "arbiter option must be preceded by replica "
+                "option.");
+            ret = -1;
+            goto out;
+        } else if ((strcmp(w, "thin-arbiter") == 0)) {
+            cli_err(
+                "thin-arbiter option must be preceded by replica "
+                "option.");
+            ret = -1;
+            goto out;
+        } else {
+            GF_ASSERT(!"opword mismatch");
+            ret = -1;
+            goto out;
         }
+        op_count++;
+    }
 
-        brick_index = index;
+    if (!trans_type)
+        trans_type = gf_strdup("tcp");
 
-        if (strcmp (words[wordcount - 1], "force") == 0) {
-                is_force = _gf_true;
-                wc = wordcount - 1;
-        }
+    if (index >= wordcount) {
+        ret = -1;
+        goto out;
+    }
 
-        ret = cli_cmd_bricks_parse (words, wc, brick_index, &bricks,
-                                    &brick_count);
-        if (ret)
-                goto out;
+    brick_index = index;
 
-        /* If brick-count is not valid when replica or stripe is
-           given, exit here */
-        if (!brick_count) {
-                cli_err ("No bricks specified");
-                ret = -1;
-                goto out;
-        }
+    if (strcmp(words[wordcount - 1], "force") == 0) {
+        is_force = _gf_true;
+        wc = wordcount - 1;
+    }
 
-        if (type == GF_CLUSTER_TYPE_DISPERSE) {
-                ret = cli_cmd_create_disperse_check(state, &disperse_count,
-                                                    &redundancy_count,
-                                                    brick_count);
-                if (!ret)
-                        ret = dict_set_int32 (dict, "disperse-count",
-                                              disperse_count);
-                if (!ret)
-                        ret = dict_set_int32 (dict, "redundancy-count",
-                                              redundancy_count);
-                if (ret)
-                        goto out;
+    // Exclude the thin-arbiter-brick i.e. last brick in the bricks list
+    if (thin_arbiter_count == 1) {
+        ret = cli_cmd_bricks_parse(words, wc - 1, brick_index, &bricks,
+                                   &brick_count);
+        if (ret)
+            goto out;
 
-                sub_count = disperse_count;
-        } else
-                sub_count = stripe_count * replica_count;
-
-        if (brick_count % sub_count) {
-                if (type == GF_CLUSTER_TYPE_STRIPE)
-                        cli_err ("number of bricks is not a multiple of "
-                                 "stripe count");
-                else if (type == GF_CLUSTER_TYPE_REPLICATE)
-                        cli_err ("number of bricks is not a multiple of "
-                                 "replica count");
-                else if (type == GF_CLUSTER_TYPE_DISPERSE)
-                        cli_err ("number of bricks is not a multiple of "
-                                 "disperse count");
-                else
-                        cli_err ("number of bricks given doesn't match "
-                                 "required count");
+        ret = cli_cmd_ta_brick_parse(words, wc, &ta_brick);
 
-                ret = -1;
-                goto out;
-        }
+    } else {
+        ret = cli_cmd_bricks_parse(words, wc, brick_index, &bricks,
+                                   &brick_count);
+    }
 
-        /* Everything is parsed fine. start setting info in dict */
-        ret = dict_set_str (dict, "volname", volname);
-        if (ret)
-                goto out;
+    if (ret)
+        goto out;
 
-        ret = dict_set_int32 (dict, "type", type);
+    if (type == GF_CLUSTER_TYPE_DISPERSE) {
+        ret = cli_cmd_create_disperse_check(state, &disperse_count,
+                                            &redundancy_count,
+                                            &disperse_data_count, brick_count);
+        if (!ret)
+            ret = dict_set_int32(dict, "disperse-count", disperse_count);
+        if (!ret)
+            ret = dict_set_int32(dict, "redundancy-count", redundancy_count);
         if (ret)
-                goto out;
+            goto out;
+
+        sub_count = disperse_count;
+    } else
+        sub_count = stripe_count * replica_count;
+
+    if (brick_count % sub_count) {
+        if (type == GF_CLUSTER_TYPE_STRIPE)
+            cli_err(
+                "number of bricks is not a multiple of "
+                "stripe count");
+        else if (type == GF_CLUSTER_TYPE_REPLICATE)
+            cli_err(
+                "number of bricks is not a multiple of "
+                "replica count");
+        else if (type == GF_CLUSTER_TYPE_DISPERSE)
+            cli_err(
+                "number of bricks is not a multiple of "
+                "disperse count");
+        else
+            cli_err(
+                "number of bricks given doesn't match "
+                "required count");
 
-        ret = dict_set_dynstr (dict, "transport", trans_type);
-        if (ret)
-                goto out;
-        trans_type = NULL;
+        ret = -1;
+        goto out;
+    }
 
-        ret = dict_set_dynstr (dict, "bricks", bricks);
-        if (ret)
-                goto out;
+    /* Everything is parsed fine. start setting info in dict */
+    ret = dict_set_str(dict, "volname", volname);
+    if (ret)
+        goto out;
 
-        ret = dict_set_int32 (dict, "count", brick_count);
-        if (ret)
-                goto out;
+    ret = dict_set_int32(dict, "type", type);
+    if (ret)
+        goto out;
+
+    ret = dict_set_dynstr(dict, "transport", trans_type);
+    if (ret)
+        goto out;
+    trans_type = NULL;
 
-        ret = dict_set_int32 (dict, "force", is_force);
+    ret = dict_set_dynstr(dict, "bricks", bricks);
+    if (ret)
+        goto out;
+
+    if (thin_arbiter_count == 1) {
+        ret = dict_set_dynstr(dict, "ta-brick", ta_brick);
         if (ret)
-                goto out;
+            goto out;
+    }
 
-        *options = dict;
+    ret = dict_set_int32(dict, "count", brick_count);
+    if (ret)
+        goto out;
+
+    ret = dict_set_int32(dict, "force", is_force);
+    if (ret)
+        goto out;
 
+    *options = dict;
+    *brick_list = bricks;
 out:
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Unable to parse create volume CLI");
-                if (dict)
-                        dict_destroy (dict);
-        }
+    if (ret) {
+        GF_FREE(bricks);
+        GF_FREE(ta_brick);
+        gf_log("cli", GF_LOG_ERROR, "Unable to parse create volume CLI");
+        if (dict)
+            dict_unref(dict);
+    }
 
-        GF_FREE (trans_type);
+    GF_FREE(trans_type);
 
-        return ret;
+    return ret;
 }
 
 int32_t
-cli_cmd_volume_reset_parse (const char **words, int wordcount, dict_t **options)
+cli_cmd_volume_reset_parse(const char **words, int wordcount, dict_t **options)
 {
-        dict_t  *dict = NULL;
-        char    *volname = NULL;
-        int     ret = -1;
+    dict_t *dict = NULL;
+    char *volname = NULL;
+    int ret = -1;
 
-        GF_ASSERT (words);
-        GF_ASSERT (options);
+    GF_ASSERT(words);
+    GF_ASSERT(options);
 
-        dict = dict_new ();
+    dict = dict_new();
 
-        if (!dict)
-                goto out;
+    if (!dict)
+        goto out;
 
-        if (wordcount < 3)
-                goto out;
+    if (wordcount < 3)
+        goto out;
 
-        if (wordcount > 5)
-                goto out;
+    if (wordcount > 5)
+        goto out;
 
-        volname = (char *)words[2];
+    volname = (char *)words[2];
 
-        if (!volname) {
-                ret = -1;
-                goto out;
-        }
+    if (!volname) {
+        ret = -1;
+        goto out;
+    }
 
-        ret = dict_set_str (dict, "volname", volname);
+    ret = dict_set_str(dict, "volname", volname);
+    if (ret)
+        goto out;
+
+    if (wordcount == 3) {
+        ret = dict_set_str(dict, "key", "all");
         if (ret)
-                goto out;
+            goto out;
+    }
 
-        if (wordcount == 3) {
-                ret = dict_set_str (dict, "key", "all");
-                if (ret)
-                        goto out;
+    if (wordcount >= 4) {
+        if (!strcmp("force", (char *)words[3])) {
+            ret = dict_set_int32(dict, "force", 1);
+            if (ret)
+                goto out;
+            ret = dict_set_str(dict, "key", "all");
+            if (ret)
+                goto out;
+        } else {
+            ret = dict_set_str(dict, "key", (char *)words[3]);
+            if (ret)
+                goto out;
         }
+    }
 
-        if (wordcount >= 4) {
-                if (!strcmp ("force", (char*)words[3])) {
-                        ret = dict_set_int32 (dict, "force", 1);
-                        if (ret)
-                                goto out;
-                        ret = dict_set_str (dict, "key", "all");
-                        if (ret)
-                                goto out;
-                } else {
-                        ret = dict_set_str (dict, "key", (char *)words[3]);
-                        if (ret)
-                                goto out;
-                }
+    if (wordcount == 5) {
+        if (strcmp("force", (char *)words[4])) {
+            ret = -1;
+            goto out;
+        } else {
+            ret = dict_set_int32(dict, "force", 1);
+            if (ret)
+                goto out;
         }
+    }
+
+    *options = dict;
+
+out:
+    if (ret && dict) {
+        dict_unref(dict);
+    }
+
+    return ret;
+}
 
-        if (wordcount == 5) {
-                if (strcmp ("force", (char*)words[4])) {
+int32_t
+cli_cmd_get_state_parse(struct cli_state *state, const char **words,
+                        int wordcount, dict_t **options, char **op_errstr)
+{
+    dict_t *dict = NULL;
+    int ret = -1;
+    char *odir = NULL;
+    char *filename = NULL;
+    char *daemon_name = NULL;
+    int count = 0;
+    uint32_t cmd = 0;
+
+    GF_VALIDATE_OR_GOTO("cli", options, out);
+    GF_VALIDATE_OR_GOTO("cli", words, out);
+
+    dict = dict_new();
+    if (!dict)
+        goto out;
+
+    if (wordcount < 1 || wordcount > 7) {
+        *op_errstr = gf_strdup(
+            "Problem parsing arguments."
+            " Check usage.");
+        goto out;
+    }
+
+    if (wordcount >= 1) {
+        gf_asprintf(&daemon_name, "%s", "glusterd");
+
+        for (count = 1; count < wordcount; count++) {
+            if (strcmp(words[count], "odir") == 0 ||
+                strcmp(words[count], "file") == 0) {
+                if (strcmp(words[count], "odir") == 0) {
+                    if (++count < wordcount) {
+                        odir = (char *)words[count];
+                        continue;
+                    } else {
+                        ret = -1;
+                        goto out;
+                    }
+                } else if (strcmp(words[count], "file") == 0) {
+                    if (++count < wordcount) {
+                        filename = (char *)words[count];
+                        continue;
+                    } else {
+                        ret = -1;
+                        goto out;
+                    }
+                }
+            } else {
+                if (count > 1) {
+                    if (count == wordcount - 1) {
+                        if (strcmp(words[count], "detail") == 0) {
+                            cmd = GF_CLI_GET_STATE_DETAIL;
+                            continue;
+                        } else if (strcmp(words[count], "volumeoptions") == 0) {
+                            cmd = GF_CLI_GET_STATE_VOLOPTS;
+                            continue;
+                        }
+                    } else {
+                        *op_errstr = gf_strdup(
+                            "Problem"
+                            " parsing arguments. "
+                            "Check usage.");
                         ret = -1;
                         goto out;
+                    }
+                }
+                if (strcmp(words[count], "glusterd") == 0) {
+                    continue;
                 } else {
-                        ret = dict_set_int32 (dict, "force", 1);
-                        if (ret)
-                                goto out;
+                    if (count == wordcount - 1) {
+                        if (strcmp(words[count], "detail") == 0) {
+                            cmd = GF_CLI_GET_STATE_DETAIL;
+                            continue;
+                        } else if (strcmp(words[count], "volumeoptions") == 0) {
+                            cmd = GF_CLI_GET_STATE_VOLOPTS;
+                            continue;
+                        }
+                    }
+
+                    *op_errstr = gf_strdup(
+                        "glusterd is "
+                        "the only supported daemon.");
+                    ret = -1;
+                    goto out;
                 }
+            }
         }
 
-        *options = dict;
+        ret = dict_set_dynstr(dict, "daemon", daemon_name);
+        if (ret) {
+            *op_errstr = gf_strdup(
+                "Command failed. Please check "
+                " log file for more details.");
+            gf_log(THIS->name, GF_LOG_ERROR,
+                   "Setting daemon name to dictionary failed");
+            goto out;
+        }
+        daemon_name = NULL;
+
+        if (odir) {
+            ret = dict_set_str(dict, "odir", odir);
+            if (ret) {
+                *op_errstr = gf_strdup(
+                    "Command failed. Please"
+                    " check log file for"
+                    " more details.");
+                gf_log(THIS->name, GF_LOG_ERROR,
+                       "Setting output directory to"
+                       "dictionary failed");
+                goto out;
+            }
+        }
+
+        if (filename) {
+            ret = dict_set_str(dict, "filename", filename);
+            if (ret) {
+                *op_errstr = gf_strdup(
+                    "Command failed. Please"
+                    " check log file for"
+                    " more  details.");
+                gf_log(THIS->name, GF_LOG_ERROR,
+                       "Setting filename to dictionary failed");
+                goto out;
+            }
+        }
+
+        if (cmd) {
+            ret = dict_set_uint32(dict, "getstate-cmd", cmd);
+            if (ret) {
+                *op_errstr = gf_strdup(
+                    "Command failed. Please"
+                    " check log file for"
+                    " more details.");
+                gf_log(THIS->name, GF_LOG_ERROR,
+                       "Setting "
+                       "get-state command type to dictionary "
+                       "failed");
+                goto out;
+            }
+        }
+    }
 
 out:
-        if (ret && dict) {
-                dict_destroy (dict);
-        }
+    if (dict)
+        *options = dict;
+
+    if (ret && dict)
+        dict_unref(dict);
 
-                return ret;
+    GF_FREE(daemon_name);
+
+    return ret;
 }
 
 int32_t
-cli_cmd_quota_parse (const char **words, int wordcount, dict_t **options)
+cli_cmd_inode_quota_parse(const char **words, int wordcount, dict_t **options)
 {
-        dict_t          *dict    = NULL;
-        char            *volname = NULL;
-        int              ret     = -1;
-        int              i       = 0;
-        char             key[20] = {0, };
-        uint64_t         value   = 0;
-        gf_quota_type    type    = GF_QUOTA_OPTION_TYPE_NONE;
-        char           *opwords[] = { "enable", "disable", "limit-usage",
-                                      "remove", "list", "alert-time",
-                                      "soft-timeout", "hard-timeout",
-                                      "default-soft-limit", NULL};
-        char            *w       = NULL;
-        uint32_t         time    = 0;
-        double           percent = 0;
-
-        GF_ASSERT (words);
-        GF_ASSERT (options);
-
-        dict = dict_new ();
-        if (!dict)
-                goto out;
+    dict_t *dict = NULL;
+    char *volname = NULL;
+    int ret = -1;
 
-        if (wordcount < 4)
-                goto out;
+    GF_ASSERT(words);
+    GF_ASSERT(options);
 
-        volname = (char *)words[2];
-        if (!volname) {
-                ret = -1;
-                goto out;
-        }
+    dict = dict_new();
+    if (!dict) {
+        gf_log("cli", GF_LOG_ERROR, "dict_new failed");
+        goto out;
+    }
 
-        /* Validate the volume name here itself */
-        {
-                if (volname[0] == '-')
-                        goto out;
+    if (wordcount != 4)
+        goto out;
 
-                if (!strcmp (volname, "all")) {
-                        cli_err ("\"all\" cannot be the name of a volume.");
-                        goto out;
-                }
+    volname = (char *)words[2];
+    if (!volname) {
+        ret = -1;
+        goto out;
+    }
 
-                if (strchr (volname, '/'))
-                        goto out;
+    /* Validate the volume name here itself */
+    if (cli_validate_volname(volname) < 0)
+        goto out;
 
-                if (strlen (volname) > GD_VOLUME_NAME_MAX) {
-                        cli_err("Volname can not exceed %d characters.",
-                                GD_VOLUME_NAME_MAX);
-                        goto out;
-                }
+    ret = dict_set_str(dict, "volname", volname);
+    if (ret < 0)
+        goto out;
+
+    if (strcmp(words[3], "enable") != 0) {
+        cli_out("Invalid quota option : %s", words[3]);
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_set_int32(dict, "type", GF_QUOTA_OPTION_TYPE_ENABLE_OBJECTS);
+    if (ret < 0)
+        goto out;
 
-                for (i = 0; i < strlen (volname); i++)
-                       if (!isalnum (volname[i]) && (volname[i] != '_') &&
-                          (volname[i] != '-')) {
-                                cli_err ("Volume name should not contain \"%c\""
-                                         " character.\nVolume names can only"
-                                         "contain alphanumeric, '-' and '_' "
-                                         "characters.",volname[i]);
-                                goto out;
-                       }
+    *options = dict;
+out:
+    if (ret < 0) {
+        if (dict)
+            dict_unref(dict);
+    }
+
+    return ret;
+}
+
+int32_t
+cli_cmd_quota_parse(const char **words, int wordcount, dict_t **options)
+{
+    dict_t *dict = NULL;
+    char *volname = NULL;
+    int ret = -1;
+    int i = -1;
+    char key[20] = {
+        0,
+    };
+    int64_t value = 0;
+    gf_quota_type type = GF_QUOTA_OPTION_TYPE_NONE;
+    static char *opwords[] = {"enable",
+                              "disable",
+                              "limit-usage",
+                              "remove",
+                              "list",
+                              "alert-time",
+                              "soft-timeout",
+                              "hard-timeout",
+                              "default-soft-limit",
+                              "limit-objects",
+                              "list-objects",
+                              "remove-objects",
+                              NULL};
+    char *w = NULL;
+    uint32_t time = 0;
+    double percent = 0;
+    char *end_ptr = NULL;
+    int64_t limit = 0;
+
+    GF_ASSERT(words);
+    GF_ASSERT(options);
+
+    dict = dict_new();
+    if (!dict) {
+        gf_log("cli", GF_LOG_ERROR, "dict_new failed");
+        goto out;
+    }
+
+    if (wordcount < 4) {
+        if ((wordcount == 3) && !(strcmp(words[2], "help"))) {
+            ret = 1;
         }
+        goto out;
+    }
 
-        ret = dict_set_str (dict, "volname", volname);
-        if (ret < 0)
-                goto out;
+    volname = (char *)words[2];
+    if (!volname) {
+        ret = -1;
+        goto out;
+    }
 
-        w = str_getunamb (words[3], opwords);
-        if (!w) {
-                cli_out ("Invalid quota option : %s", words[3]);
-                ret = - 1;
-                goto out;
+    /* Validate the volume name here itself */
+    if (cli_validate_volname(volname) < 0)
+        goto out;
+
+    ret = dict_set_str(dict, "volname", volname);
+    if (ret < 0)
+        goto out;
+
+    w = str_getunamb(words[3], opwords);
+    if (!w) {
+        cli_out("Invalid quota option : %s", words[3]);
+        ret = -1;
+        goto out;
+    }
+
+    if (strcmp(w, "enable") == 0) {
+        if (wordcount == 4) {
+            type = GF_QUOTA_OPTION_TYPE_ENABLE;
+            ret = 0;
+            goto set_type;
+        } else {
+            ret = -1;
+            goto out;
         }
+    }
 
-        if (strcmp (w, "enable") == 0) {
-                if (wordcount == 4) {
-                        type = GF_QUOTA_OPTION_TYPE_ENABLE;
-                        ret = 0;
-                        goto set_type;
-                } else {
-                        ret = -1;
-                        goto out;
-                }
+    if (strcmp(w, "disable") == 0) {
+        if (wordcount == 4) {
+            type = GF_QUOTA_OPTION_TYPE_DISABLE;
+            ret = 0;
+            goto set_type;
+        } else {
+            ret = -1;
+            goto out;
         }
+    }
 
-        if (strcmp (w, "disable") == 0) {
-                if (wordcount == 4) {
-                        type = GF_QUOTA_OPTION_TYPE_DISABLE;
-                        ret = 0;
-                        goto set_type;
-                } else {
-                        ret = -1;
-                        goto out;
-                }
+    if (strcmp(w, "limit-usage") == 0) {
+        type = GF_QUOTA_OPTION_TYPE_LIMIT_USAGE;
+    } else if (strcmp(w, "limit-objects") == 0) {
+        type = GF_QUOTA_OPTION_TYPE_LIMIT_OBJECTS;
+    }
+
+    if (type == GF_QUOTA_OPTION_TYPE_LIMIT_USAGE ||
+        type == GF_QUOTA_OPTION_TYPE_LIMIT_OBJECTS) {
+        if (wordcount < 6 || wordcount > 7) {
+            ret = -1;
+            goto out;
         }
 
-        if (strcmp (w, "limit-usage") == 0) {
+        if (words[4][0] != '/') {
+            cli_err("Please enter absolute path");
+            ret = -1;
+            goto out;
+        }
+        ret = dict_set_str(dict, "path", (char *)words[4]);
+        if (ret)
+            goto out;
+
+        if (!words[5]) {
+            cli_err("Please enter the limit value to be set");
+            ret = -1;
+            goto out;
+        }
+
+        if (type == GF_QUOTA_OPTION_TYPE_LIMIT_USAGE) {
+            ret = gf_string2bytesize_int64(words[5], &value);
+            if (ret != 0 || value <= 0) {
+                if (errno == ERANGE || value <= 0) {
+                    ret = -1;
+                    cli_err(
+                        "Please enter an integer "
+                        "value in the range of "
+                        "(1 - %" PRId64 ")",
+                        INT64_MAX);
+                } else
+                    cli_err(
+                        "Please enter a correct "
+                        "value");
+                goto out;
+            }
+        } else {
+            errno = 0;
+            limit = strtol(words[5], &end_ptr, 10);
+            if (errno == ERANGE || errno == EINVAL || limit <= 0 ||
+                strcmp(end_ptr, "") != 0) {
+                ret = -1;
+                cli_err(
+                    "Please enter an integer value in "
+                    "the range 1 - %" PRId64,
+                    INT64_MAX);
+                goto out;
+            }
+        }
 
-                if (wordcount < 6 || wordcount > 7) {
-                        ret = -1;
-                        goto out;
-                }
+        ret = dict_set_str(dict, "hard-limit", (char *)words[5]);
+        if (ret < 0)
+            goto out;
 
-                type = GF_QUOTA_OPTION_TYPE_LIMIT_USAGE;
+        if (wordcount == 7) {
+            ret = gf_string2percent(words[6], &percent);
+            if (ret != 0 || percent > 100) {
+                ret = -1;
+                cli_err(
+                    "Please enter a correct value "
+                    "in the range of 0 to 100");
+                goto out;
+            }
 
-                if (words[4][0] != '/') {
-                        cli_err ("Please enter absolute path");
-                        ret = -1;
-                        goto out;
-                }
-                ret = dict_set_str (dict, "path", (char *) words[4]);
-                if (ret)
-                        goto out;
+            ret = dict_set_str(dict, "soft-limit", (char *)words[6]);
+            if (ret < 0)
+                goto out;
+        }
 
-                if (!words[5]) {
-                        cli_err ("Please enter the limit value to be set");
-                        ret = -1;
-                        goto out;
-                }
+        goto set_type;
+    }
 
-                ret = gf_string2bytesize_uint64 (words[5], &value);
-                if (ret != 0) {
-                        if (errno == ERANGE)
-                                cli_err ("Value too large: %s", words[5]);
-                        else
-                                cli_err ("Please enter a correct value");
-                        goto out;
-                }
+    if (strcmp(w, "remove") == 0) {
+        if (wordcount != 5) {
+            ret = -1;
+            goto out;
+        }
 
-                ret  = dict_set_str (dict, "hard-limit", (char *) words[5]);
-                if (ret < 0)
-                        goto out;
+        type = GF_QUOTA_OPTION_TYPE_REMOVE;
 
-                if (wordcount == 7) {
+        if (words[4][0] != '/') {
+            cli_err("Please enter absolute path");
+            ret = -1;
+            goto out;
+        }
 
-                        ret = gf_string2percent (words[6], &percent);
-                        if (ret != 0) {
-                                cli_err ("Please enter a correct value");
-                                goto out;
-                        }
+        ret = dict_set_str(dict, "path", (char *)words[4]);
+        if (ret < 0)
+            goto out;
+        goto set_type;
+    }
 
-                        ret = dict_set_str (dict, "soft-limit",
-                                            (char *) words[6]);
-                        if (ret < 0)
-                                goto out;
-                }
+    if (strcmp(w, "remove-objects") == 0) {
+        if (wordcount != 5) {
+            ret = -1;
+            goto out;
+        }
 
-                goto set_type;
+        type = GF_QUOTA_OPTION_TYPE_REMOVE_OBJECTS;
+
+        if (words[4][0] != '/') {
+            cli_err("Please enter absolute path");
+            ret = -1;
+            goto out;
         }
-        if (strcmp (w, "remove") == 0) {
-                if (wordcount != 5) {
-                        ret = -1;
-                        goto out;
-                }
 
-                type = GF_QUOTA_OPTION_TYPE_REMOVE;
+        ret = dict_set_str(dict, "path", (char *)words[4]);
+        if (ret < 0)
+            goto out;
+        goto set_type;
+    }
 
-                if (words[4][0] != '/') {
-                        cli_err ("Please enter absolute path");
-                        ret = -1;
-                        goto out;
-                }
+    if (strcmp(w, "list") == 0) {
+        type = GF_QUOTA_OPTION_TYPE_LIST;
 
-                ret = dict_set_str (dict, "path", (char *) words[4]);
-                if (ret < 0)
-                        goto out;
-                goto set_type;
+        if (words[4] && words[4][0] != '/') {
+            cli_err("Please enter absolute path");
+            ret = -1;
+            goto out;
         }
 
-        if (strcmp (w, "list") == 0) {
-                if (wordcount < 4) {
-                        ret = -1;
-                        goto out;
-                }
+        i = 4;
+        while (i < wordcount) {
+            snprintf(key, 20, "path%d", i - 4);
 
-                type = GF_QUOTA_OPTION_TYPE_LIST;
+            ret = dict_set_str(dict, key, (char *)words[i++]);
+            if (ret < 0)
+                goto out;
+        }
 
-                i = 4;
-                while (i < wordcount) {
-                        snprintf (key, 20, "path%d", i-4);
+        ret = dict_set_int32(dict, "count", i - 4);
+        if (ret < 0)
+            goto out;
 
-                        ret = dict_set_str (dict, key, (char *) words [i++]);
-                        if (ret < 0)
-                                goto out;
-                }
+        goto set_type;
+    }
 
-                ret = dict_set_int32 (dict, "count", i - 4);
-                if (ret < 0)
-                        goto out;
+    if (strcmp(w, "list-objects") == 0) {
+        type = GF_QUOTA_OPTION_TYPE_LIST_OBJECTS;
 
-                goto set_type;
+        i = 4;
+        while (i < wordcount) {
+            snprintf(key, 20, "path%d", i - 4);
+
+            ret = dict_set_str(dict, key, (char *)words[i++]);
+            if (ret < 0) {
+                gf_log("cli", GF_LOG_ERROR,
+                       "Failed to set "
+                       "quota patch in request dictionary");
+                goto out;
+            }
         }
 
+        ret = dict_set_int32(dict, "count", i - 4);
+        if (ret < 0) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Failed to set quota "
+                   "limit count in request dictionary");
+            goto out;
+        }
 
-        if (strcmp (w, "alert-time") == 0) {
-                if (wordcount != 5) {
-                        ret = -1;
-                        goto out;
-                }
-                type = GF_QUOTA_OPTION_TYPE_ALERT_TIME;
+        goto set_type;
+    }
 
-                ret = gf_string2time (words[4], &time);
-                if (ret) {
-                        cli_err ("Invalid argument %s. Please enter a valid "
-                                 "string", words[4]);
-                        goto out;
-                }
+    if (strcmp(w, "alert-time") == 0) {
+        if (wordcount != 5) {
+            ret = -1;
+            goto out;
+        }
+        type = GF_QUOTA_OPTION_TYPE_ALERT_TIME;
 
-                ret = dict_set_str (dict, "value", (char *)words[4]);
-                if (ret < 0)
-                        goto out;
-                goto set_type;
+        ret = gf_string2time(words[4], &time);
+        if (ret) {
+            cli_err(
+                "Invalid argument %s. Please enter a valid "
+                "string",
+                words[4]);
+            goto out;
         }
-        if (strcmp (w, "soft-timeout") == 0) {
-                if (wordcount != 5) {
-                        ret = -1;
-                        goto out;
-                }
-                type = GF_QUOTA_OPTION_TYPE_SOFT_TIMEOUT;
 
-                ret = gf_string2time (words[4], &time);
-                if (ret) {
-                        cli_err ("Invalid argument %s. Please enter a valid "
-                                 "string", words[4]);
-                        goto out;
-                }
+        ret = dict_set_str(dict, "value", (char *)words[4]);
+        if (ret < 0)
+            goto out;
+        goto set_type;
+    }
 
-                ret = dict_set_str (dict, "value", (char *)words[4]);
-                if (ret < 0)
-                        goto out;
-                goto set_type;
+    if (strcmp(w, "soft-timeout") == 0) {
+        if (wordcount != 5) {
+            ret = -1;
+            goto out;
         }
-        if (strcmp (w, "hard-timeout") == 0) {
-                if(wordcount != 5) {
-                        ret = -1;
-                        goto out;
-                }
-                type = GF_QUOTA_OPTION_TYPE_HARD_TIMEOUT;
+        type = GF_QUOTA_OPTION_TYPE_SOFT_TIMEOUT;
 
-                ret = gf_string2time (words[4], &time);
-                if (ret) {
-                        cli_err ("Invalid argument %s. Please enter a valid "
-                                 "string", words[4]);
-                        goto out;
-                }
+        ret = gf_string2time(words[4], &time);
+        if (ret) {
+            cli_err(
+                "Invalid argument %s. Please enter a valid "
+                "string",
+                words[4]);
+            goto out;
+        }
 
-                ret = dict_set_str (dict, "value", (char *)words[4]);
-                if (ret < 0)
-                        goto out;
-                goto set_type;
+        ret = dict_set_str(dict, "value", (char *)words[4]);
+        if (ret < 0)
+            goto out;
+        goto set_type;
+    }
+
+    if (strcmp(w, "hard-timeout") == 0) {
+        if (wordcount != 5) {
+            ret = -1;
+            goto out;
         }
-        if (strcmp (w, "default-soft-limit") == 0) {
-                if(wordcount != 5) {
-                        ret = -1;
-                        goto out;
-                }
-                type = GF_QUOTA_OPTION_TYPE_DEFAULT_SOFT_LIMIT;
+        type = GF_QUOTA_OPTION_TYPE_HARD_TIMEOUT;
 
-                ret = dict_set_str (dict, "value", (char *)words[4]);
-                if (ret < 0)
-                        goto out;
-                goto set_type;
-        } else {
-                GF_ASSERT (!"opword mismatch");
+        ret = gf_string2time(words[4], &time);
+        if (ret) {
+            cli_err(
+                "Invalid argument %s. Please enter a valid "
+                "string",
+                words[4]);
+            goto out;
         }
 
-set_type:
-        ret = dict_set_int32 (dict, "type", type);
+        ret = dict_set_str(dict, "value", (char *)words[4]);
         if (ret < 0)
-                goto out;
+            goto out;
+        goto set_type;
+    }
+    if (strcmp(w, "default-soft-limit") == 0) {
+        if (wordcount != 5) {
+            ret = -1;
+            goto out;
+        }
+        type = GF_QUOTA_OPTION_TYPE_DEFAULT_SOFT_LIMIT;
 
-        *options = dict;
+        ret = dict_set_str(dict, "value", (char *)words[4]);
+        if (ret < 0)
+            goto out;
+        goto set_type;
+    } else {
+        GF_ASSERT(!"opword mismatch");
+    }
+
+set_type:
+    ret = dict_set_int32(dict, "type", type);
+    if (ret < 0)
+        goto out;
+
+    *options = dict;
 out:
-        if (ret < 0) {
-                if (dict)
-                        dict_destroy (dict);
-        }
+    if (ret < 0) {
+        if (dict)
+            dict_unref(dict);
+    }
 
-        return ret;
+    return ret;
 }
 
-static inline gf_boolean_t
-cli_is_key_spl (char *key)
+static gf_boolean_t
+cli_is_key_spl(char *key)
 {
-        return (strcmp (key, "group") == 0);
+    return (strcmp(key, "group") == 0);
 }
 
-static int
-cli_add_key_group (dict_t *dict, char *key, char *value, char **op_errstr)
+static int32_t
+cli_add_key_group_value(dict_t *dict, const char *name, const char *value,
+                        int32_t id, char **op_errstr)
 {
-        int             ret = -1;
-        int             opt_count = 0;
-        char            iter_key[1024] = {0,};
-        char            iter_val[1024] = {0,};
-        char            *saveptr = NULL;
-        char            *tok_key = NULL;
-        char            *tok_val = NULL;
-        char            *dkey = NULL;
-        char            *dval = NULL;
-        char            *tagpath = NULL;
-        char            *buf = NULL;
-        char            line[PATH_MAX + 256] = {0,};
-        char            errstr[2048] = "";
-        FILE            *fp = NULL;
-
-        ret = gf_asprintf (&tagpath, "%s/groups/%s",
-                           GLUSTERD_DEFAULT_WORKDIR, value);
-        if (ret == -1) {
-                tagpath = NULL;
-                goto out;
-        }
+    char *key = NULL;
+    char *data = NULL;
+    int32_t ret = -1;
 
-        fp = fopen (tagpath, "r");
-        if (!fp) {
-                ret = -1;
-                snprintf(errstr, sizeof(errstr), "Unable to open file '%s'."
-                         " Error: %s", tagpath, strerror (errno));
-                if (op_errstr)
-                        *op_errstr = gf_strdup(errstr);
-                goto out;
-        }
+    ret = gf_asprintf(&key, "%s%d", name, id);
+    if (ret < 0) {
+        goto out;
+    }
+    data = gf_strdup(value);
+    if (data == NULL) {
+        gf_log(THIS->name, GF_LOG_ERROR, "Failed to allocate memory for data");
+        ret = -1;
+        goto out;
+    }
 
-        opt_count = 0;
-        buf = line;
-        while (fscanf (fp, "%s", buf) != EOF) {
+    ret = dict_set_dynstr(dict, key, data);
+    if (ret == 0) {
+        data = NULL;
+    }
 
-                opt_count++;
-                tok_key = strtok_r (line, "=", &saveptr);
-                tok_val = strtok_r (NULL, "=", &saveptr);
-                if (!tok_key || !tok_val) {
-                        ret = -1;
-                        snprintf(errstr, sizeof(errstr), "'%s' file format "
-                                 "not valid.", tagpath);
-                        if (op_errstr)
-                                *op_errstr = gf_strdup(errstr);
-                        goto out;
-                }
+out:
+    GF_FREE(key);
+    GF_FREE(data);
 
-                snprintf (iter_key, sizeof (iter_key), "key%d", opt_count);
-                dkey = gf_strdup (tok_key);
-                ret = dict_set_dynstr (dict, iter_key, dkey);
-                if (ret)
-                        goto out;
-                dkey = NULL;
+    if ((ret != 0) && (op_errstr != NULL)) {
+        *op_errstr = gf_strdup("Failed to allocate memory");
+    }
 
-                snprintf (iter_val, sizeof (iter_val), "value%d", opt_count);
-                dval = gf_strdup (tok_val);
-                ret = dict_set_dynstr (dict, iter_val, dval);
-                if (ret)
-                        goto out;
-                dval = NULL;
+    return ret;
+}
 
-        }
+static int
+cli_add_key_group(dict_t *dict, char *key, char *value, char **op_errstr)
+{
+    int ret = -1;
+    int opt_count = 0;
+    char *saveptr = NULL;
+    char *tok_key = NULL;
+    char *tok_val = NULL;
+    char *tagpath = NULL;
+    char line[PATH_MAX + 256] = {
+        0,
+    };
+    FILE *fp = NULL;
+
+    ret = gf_asprintf(&tagpath, "%s/groups/%s", GLUSTERD_DEFAULT_WORKDIR,
+                      value);
+    if (ret == -1) {
+        tagpath = NULL;
+        goto out;
+    }
 
-        if (!opt_count) {
-                ret = -1;
-                snprintf(errstr, sizeof(errstr), "'%s' file format "
-                         "not valid.", tagpath);
-                if (op_errstr)
-                        *op_errstr = gf_strdup(errstr);
-                goto out;
+    fp = fopen(tagpath, "r");
+    if (!fp) {
+        ret = -1;
+        if (op_errstr) {
+            gf_asprintf(op_errstr,
+                        "Unable to open file '%s'. "
+                        "Error: %s",
+                        tagpath, strerror(errno));
+        }
+        goto out;
+    }
+
+    opt_count = 0;
+    while (fgets(line, sizeof(line), fp) != NULL) {
+        if (strlen(line) >= sizeof(line) - 1) {
+            ret = -1;
+            if (op_errstr != NULL) {
+                *op_errstr = gf_strdup("Line too long");
+            }
+            goto out;
+        }
+
+        /* Treat line that start with "#" as comments */
+        if ('#' == line[0])
+            continue;
+
+        opt_count++;
+        tok_key = strtok_r(line, "=", &saveptr);
+        tok_val = strtok_r(NULL, "\r\n", &saveptr);
+        if (!tok_key || !tok_val) {
+            ret = -1;
+            if (op_errstr) {
+                gf_asprintf(op_errstr,
+                            "'%s' file format "
+                            "not valid.",
+                            tagpath);
+            }
+            goto out;
+        }
+
+        ret = cli_add_key_group_value(dict, "key", tok_key, opt_count,
+                                      op_errstr);
+        if (ret != 0) {
+            goto out;
+        }
+        ret = cli_add_key_group_value(dict, "value", tok_val, opt_count,
+                                      op_errstr);
+        if (ret != 0) {
+            goto out;
+        }
+    }
+
+    if (!opt_count) {
+        ret = -1;
+        if (op_errstr) {
+            gf_asprintf(op_errstr, "'%s' file format not valid.", tagpath);
         }
-        ret = dict_set_int32 (dict, "count", opt_count);
+        goto out;
+    }
+    ret = dict_set_int32(dict, "count", opt_count);
 out:
 
-        GF_FREE (tagpath);
+    GF_FREE(tagpath);
 
-        if (ret) {
-                GF_FREE (dkey);
-                GF_FREE (dval);
-        }
-
-        if (fp)
-                fclose (fp);
+    if (fp)
+        fclose(fp);
 
-        return ret;
+    return ret;
 }
 
 int32_t
-cli_cmd_volume_set_parse (const char **words, int wordcount, dict_t **options,
-                          char **op_errstr)
+cli_cmd_volume_set_parse(struct cli_state *state, const char **words,
+                         int wordcount, dict_t **options, char **op_errstr)
 {
-        dict_t                  *dict = NULL;
-        char                    *volname = NULL;
-        int                     ret = -1;
-        int                     count = 0;
-        char                    *key = NULL;
-        char                    *value = NULL;
-        int                     i = 0;
-        char                    str[50] = {0,};
+    dict_t *dict = NULL;
+    char *volname = NULL;
+    int ret = -1;
+    int count = 0;
+    char *key = NULL;
+    char *value = NULL;
+    int i = 0;
+    char str[50] = {
+        0,
+    };
+    const char *question = NULL;
+    gf_answer_t answer = GF_ANSWER_NO;
+
+    GF_ASSERT(words);
+    GF_ASSERT(options);
+
+    dict = dict_new();
+
+    if (!dict)
+        goto out;
 
-        GF_ASSERT (words);
-        GF_ASSERT (options);
+    if (wordcount < 3)
+        goto out;
 
-        dict = dict_new ();
+    volname = (char *)words[2];
 
-        if (!dict)
-                goto out;
+    GF_ASSERT(volname);
 
-        if (wordcount < 3)
-                goto out;
+    ret = dict_set_str(dict, "volname", volname);
 
-        volname = (char *)words[2];
+    if (ret)
+        goto out;
 
-        GF_ASSERT (volname);
+    if (!strcmp(volname, "all")) {
+        ret = dict_set_str(dict, "globalname", "All");
+        if (ret) {
+            gf_log(THIS->name, GF_LOG_ERROR, "dict set on global key failed.");
+            goto out;
+        }
 
-        ret = dict_set_str (dict, "volname", volname);
+        ret = dict_set_int32(dict, "hold_global_locks", _gf_true);
+        if (ret) {
+            gf_log(THIS->name, GF_LOG_ERROR, "dict set on global key failed.");
+            goto out;
+        }
+    }
 
+    if ((!strcmp(volname, "help") || !strcmp(volname, "help-xml")) &&
+        wordcount == 3) {
+        ret = dict_set_str(dict, volname, volname);
         if (ret)
-                goto out;
+            goto out;
 
-        if ((!strcmp (volname, "help") || !strcmp (volname, "help-xml"))
-            && wordcount == 3 ) {
-                ret = dict_set_str (dict, volname, volname);
-                if (ret)
-                        goto out;
-
-        } else if (wordcount < 5) {
-                ret = -1;
-                goto out;
-
-        } else if (wordcount == 5  && cli_is_key_spl ((char *)words[3])) {
-                key = (char *) words[3];
-                value = (char *) words[4];
-                if ( !key || !value) {
-                        ret = -1;
-                        goto out;
-                }
+    } else if (wordcount < 5) {
+        ret = -1;
+        goto out;
 
-                ret = gf_strip_whitespace (value, strlen (value));
-                if (ret == -1)
-                        goto out;
+    } else if (wordcount == 5 && cli_is_key_spl((char *)words[3])) {
+        key = (char *)words[3];
+        value = (char *)words[4];
+        if (!key || !value) {
+            ret = -1;
+            goto out;
+        }
 
-                if (strlen (value) == 0) {
-                        ret = -1;
-                        goto out;
-                }
+        ret = gf_strip_whitespace(value, strlen(value));
+        if (ret == -1)
+            goto out;
 
-                ret = cli_add_key_group (dict, key, value, op_errstr);
-                if (ret == 0)
-                        *options = dict;
-                goto out;
+        if (strlen(value) == 0) {
+            ret = -1;
+            goto out;
         }
 
-        for (i = 3; i < wordcount; i+=2) {
+        ret = cli_add_key_group(dict, key, value, op_errstr);
+        if (ret == 0)
+            *options = dict;
+        goto out;
+    }
 
-                key = (char *) words[i];
-                value = (char *) words[i+1];
+    for (i = 3; i < wordcount; i += 2) {
+        key = (char *)words[i];
+        value = (char *)words[i + 1];
 
-                if ( !key || !value) {
-                        ret = -1;
-                        goto out;
-                }
+        if (!key || !value) {
+            ret = -1;
+            goto out;
+        }
 
-                count++;
+        count++;
 
-                ret = gf_strip_whitespace (value, strlen (value));
-                if (ret == -1)
-                        goto out;
+        if (fnmatch("user.*", key, FNM_NOESCAPE) != 0) {
+            ret = gf_strip_whitespace(value, strlen(value));
+            if (ret == -1)
+                goto out;
+        }
 
-                if (strlen (value) == 0) {
-                        ret = -1;
-                        goto out;
-                }
+        if (strlen(value) == 0) {
+            ret = -1;
+            goto out;
+        }
 
-                if (cli_is_key_spl (key)) {
-                        ret = -1;
-                        goto out;
-                }
+        if (cli_is_key_spl(key)) {
+            ret = -1;
+            goto out;
+        }
 
-                sprintf (str, "key%d", count);
-                ret = dict_set_str (dict, str, key);
-                if (ret)
-                        goto out;
+        sprintf(str, "key%d", count);
+        ret = dict_set_str(dict, str, key);
+        if (ret)
+            goto out;
 
-                sprintf (str, "value%d", count);
-                ret = dict_set_str (dict, str, value);
+        sprintf(str, "value%d", count);
+        ret = dict_set_str(dict, str, value);
 
-                if (ret)
-                        goto out;
+        if (ret)
+            goto out;
+
+        if ((!strcmp(key, "cluster.enable-shared-storage")) &&
+            (!strcmp(value, "disable"))) {
+            question =
+                "Disabling cluster.enable-shared-storage "
+                "will delete the shared storage volume"
+                "(gluster_shared_storage), which is used "
+                "by snapshot scheduler, geo-replication "
+                "and NFS-Ganesha. Do you still want to "
+                "continue?";
+            answer = cli_cmd_get_confirmation(state, question);
+            if (GF_ANSWER_NO == answer) {
+                gf_log("cli", GF_LOG_ERROR,
+                       "Operation "
+                       "cancelled, exiting");
+                *op_errstr = gf_strdup("Aborted by user.");
+                ret = -1;
+                goto out;
+            }
+        }
+        if ((!strcmp(key, "nfs.disable")) && (!strcmp(value, "off"))) {
+            question =
+                "Gluster NFS is being deprecated in favor "
+                "of NFS-Ganesha Enter \"yes\" to continue "
+                "using Gluster NFS";
+            answer = cli_cmd_get_confirmation(state, question);
+            if (GF_ANSWER_NO == answer) {
+                gf_log("cli", GF_LOG_ERROR,
+                       "Operation "
+                       "cancelled, exiting");
+                *op_errstr = gf_strdup("Aborted by user.");
+                ret = -1;
+                goto out;
+            }
         }
+    }
 
-        ret = dict_set_int32 (dict, "count", wordcount-3);
+    ret = dict_set_int32(dict, "count", wordcount - 3);
 
-        if (ret)
-                goto out;
+    if (ret)
+        goto out;
 
-        *options = dict;
+    *options = dict;
 
 out:
-        if (ret)
-                dict_destroy (dict);
+    if (ret && dict)
+        dict_unref(dict);
 
-        return ret;
+    return ret;
 }
 
 int32_t
-cli_cmd_volume_add_brick_parse (const char **words, int wordcount,
-                                dict_t **options)
+cli_cmd_volume_add_brick_parse(struct cli_state *state, const char **words,
+                               int wordcount, dict_t **options, int *ret_type)
 {
-        dict_t  *dict = NULL;
-        char    *volname = NULL;
-        int     ret = -1;
-        int     brick_count = 0, brick_index = 0;
-        char    *bricks = NULL;
-        char    *opwords_cl[] = { "replica", "stripe", NULL };
-        gf1_cluster_type type = GF_CLUSTER_TYPE_NONE;
-        int     count = 1;
-        char    *w = NULL;
-        int     index;
-        gf_boolean_t is_force = _gf_false;
-        int wc = wordcount;
-
-        GF_ASSERT (words);
-        GF_ASSERT (options);
-
-        dict = dict_new ();
-
-        if (!dict)
-                goto out;
+    dict_t *dict = NULL;
+    char *volname = NULL;
+    int ret = -1;
+    int brick_count = 0, brick_index = 0;
+    char *bricks = NULL;
+    static char *opwords_cl[] = {"replica", "stripe", NULL};
+    gf1_cluster_type type = GF_CLUSTER_TYPE_NONE;
+    int count = 1;
+    int arbiter_count = 0;
+    char *w = NULL;
+    int index;
+    gf_boolean_t is_force = _gf_false;
+    int wc = wordcount;
+    gf_answer_t answer = GF_ANSWER_NO;
+    const char *question = NULL;
+
+    GF_ASSERT(words);
+    GF_ASSERT(options);
+
+    dict = dict_new();
+
+    if (!dict)
+        goto out;
 
-        if (wordcount < 3)
-                goto out;
+    if (wordcount < 3)
+        goto out;
 
-        volname = (char *)words[2];
+    volname = (char *)words[2];
 
-        GF_ASSERT (volname);
+    GF_ASSERT(volname);
 
-        ret = dict_set_str (dict, "volname", volname);
+    ret = dict_set_str(dict, "volname", volname);
 
-        if (ret)
-                goto out;
+    if (ret)
+        goto out;
 
-        if (wordcount < 4) {
+    if (wordcount < 4) {
+        ret = -1;
+        goto out;
+    }
+    if (wordcount < 6) {
+        /* seems no options are given, go directly to the parse_brick */
+        brick_index = 3;
+        type = GF_CLUSTER_TYPE_NONE;
+        goto parse_bricks;
+    }
+
+    w = str_getunamb(words[3], opwords_cl);
+    if (!w) {
+        type = GF_CLUSTER_TYPE_NONE;
+        index = 3;
+    } else if ((strcmp(w, "replica")) == 0) {
+        type = GF_CLUSTER_TYPE_REPLICATE;
+        count = strtol(words[4], NULL, 0);
+        if (!count || (count < 2)) {
+            cli_err("replica count should be greater than 1");
+            ret = -1;
+            goto out;
+        }
+        ret = dict_set_int32(dict, "replica-count", count);
+        if (ret)
+            goto out;
+        index = 5;
+        if (words[index] && !strcmp(words[index], "arbiter")) {
+            arbiter_count = strtol(words[6], NULL, 0);
+            if (arbiter_count != 1 || count != 3) {
+                cli_err(
+                    "For arbiter configuration, replica "
+                    "count must be 3 and arbiter count "
+                    "must be 1. The 3rd brick of the "
+                    "replica will be the arbiter");
                 ret = -1;
                 goto out;
-        }
-        if (wordcount < 6) {
-                /* seems no options are given, go directly to the parse_brick */
-                brick_index = 3;
-                type = GF_CLUSTER_TYPE_NONE;
-                goto parse_bricks;
+            }
+            ret = dict_set_int32(dict, "arbiter-count", arbiter_count);
+            if (ret)
+                goto out;
+            index = 7;
         }
 
-        w = str_getunamb (words[3], opwords_cl);
-        if (!w) {
-                type = GF_CLUSTER_TYPE_NONE;
-                index = 3;
-        } else if ((strcmp (w, "replica")) == 0) {
-                type = GF_CLUSTER_TYPE_REPLICATE;
-                if (wordcount < 5) {
-                        ret = -1;
-                        goto out;
-                }
-                count = strtol (words[4], NULL, 0);
-                if (!count || (count < 2)) {
-                        cli_err ("replica count should be greater than 1");
-                        ret = -1;
-                        goto out;
-                }
-                ret = dict_set_int32 (dict, "replica-count", count);
-                if (ret)
-                        goto out;
-                index = 5;
-        } else if ((strcmp (w, "stripe")) == 0) {
-                type = GF_CLUSTER_TYPE_STRIPE;
-                if (wordcount < 5) {
-                        ret = -1;
-                        goto out;
-                }
-                count = strtol (words[4], NULL, 0);
-                if (!count || (count < 2)) {
-                        cli_err ("stripe count should be greater than 1");
-                        ret = -1;
-                        goto out;
+        if (count == 2) {
+            if (strcmp(words[wordcount - 1], "force")) {
+                question =
+                    "Replica 2 volumes are prone to "
+                    "split-brain. Use Arbiter or "
+                    "Replica 3 to avoid this. See: "
+                    "http://docs.gluster.org/en/latest/Administrator%20Guide/"
+                    "Split%20brain%20and%20ways%20to%20deal%20with%20it/."
+                    "\nDo you still want to continue?\n";
+                answer = cli_cmd_get_confirmation(state, question);
+                if (GF_ANSWER_NO == answer) {
+                    gf_log("cli", GF_LOG_ERROR,
+                           "Add brick"
+                           " cancelled, exiting");
+                    ret = -1;
+                    goto out;
                 }
-                ret = dict_set_int32 (dict, "stripe-count", count);
-                if (ret)
-                        goto out;
-                index = 5;
-        } else {
-                GF_ASSERT (!"opword mismatch");
-                ret = -1;
-                goto out;
+            }
         }
+    } else if ((strcmp(w, "stripe")) == 0) {
+        cli_err("stripe option not supported");
+        goto out;
+    } else {
+        GF_ASSERT(!"opword mismatch");
+        ret = -1;
+        goto out;
+    }
 
-        brick_index = index;
+    brick_index = index;
 
 parse_bricks:
 
-        if (strcmp (words[wordcount - 1], "force") == 0) {
-                is_force = _gf_true;
-                wc = wordcount - 1;
-        }
+    if (strcmp(words[wordcount - 1], "force") == 0) {
+        is_force = _gf_true;
+        wc = wordcount - 1;
+    }
 
-        ret = cli_cmd_bricks_parse (words, wc, brick_index, &bricks,
-                                    &brick_count);
-        if (ret)
-                goto out;
+    ret = cli_cmd_bricks_parse(words, wc, brick_index, &bricks, &brick_count);
+    if (ret)
+        goto out;
 
-        ret = dict_set_dynstr (dict, "bricks", bricks);
-        if (ret)
-                goto out;
+    ret = dict_set_dynstr(dict, "bricks", bricks);
+    if (ret)
+        goto out;
 
-        ret = dict_set_int32 (dict, "count", brick_count);
+    ret = dict_set_int32(dict, "count", brick_count);
 
-        if (ret)
-                goto out;
+    if (ret)
+        goto out;
 
-        ret = dict_set_int32 (dict, "force", is_force);
-        if (ret)
-                goto out;
+    ret = dict_set_int32(dict, "force", is_force);
+    if (ret)
+        goto out;
 
-        *options = dict;
+    *options = dict;
 
 out:
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Unable to parse add-brick CLI");
-                if (dict)
-                        dict_destroy (dict);
-        }
+    if (ret_type)
+        *ret_type = type;
 
-        return ret;
-}
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Unable to parse add-brick CLI");
+        if (dict)
+            dict_unref(dict);
+    }
 
+    return ret;
+}
 
 int32_t
-cli_cmd_volume_remove_brick_parse (const char **words, int wordcount,
-                                   dict_t **options, int *question)
+cli_cmd_volume_remove_brick_parse(struct cli_state *state, const char **words,
+                                  int wordcount, dict_t **options,
+                                  int *question, int *brick_count,
+                                  int32_t *comm)
 {
-        dict_t  *dict = NULL;
-        char    *volname = NULL;
-        char    *delimiter = NULL;
-        int     ret = -1;
-        char    key[50];
-        int     brick_count = 0, brick_index = 0;
-        int32_t tmp_index = 0;
-        int32_t j = 0;
-        char    *tmp_brick = NULL;
-        char    *tmp_brick1 = NULL;
-        char    *type_opword[] = { "replica", NULL };
-        char    *opwords[] = { "start", "commit", "stop", "status",
-                               "force", NULL };
-        char    *w = NULL;
-        int32_t  command = GF_OP_CMD_NONE;
-        long     count = 0;
-
-        GF_ASSERT (words);
-        GF_ASSERT (options);
-
-        if (wordcount < 5)
-                goto out;
+    dict_t *dict = NULL;
+    char *volname = NULL;
+    char *delimiter = NULL;
+    int ret = -1;
+    char key[50];
+    int brick_index = 0;
+    int32_t tmp_index = 0;
+    int32_t j = 0;
+    char *tmp_brick = NULL;
+    char *tmp_brick1 = NULL;
+    static char *type_opword[] = {"replica", NULL};
+    static char *opwords[] = {"start",  "commit", "stop",
+                              "status", "force",  NULL};
+    char *w = NULL;
+    int32_t command = GF_OP_CMD_NONE;
+    long count = 0;
+    gf_answer_t answer = GF_ANSWER_NO;
+    const char *ques = NULL;
+
+    GF_ASSERT(words);
+    GF_ASSERT(options);
+
+    if (wordcount < 5)
+        goto out;
 
-        dict = dict_new ();
-        if (!dict)
-                goto out;
+    dict = dict_new();
+    if (!dict)
+        goto out;
 
-        volname = (char *)words[2];
+    volname = (char *)words[2];
 
-        GF_ASSERT (volname);
+    GF_ASSERT(volname);
 
-        ret = dict_set_str (dict, "volname", volname);
-        if (ret)
-                goto out;
+    ret = dict_set_str(dict, "volname", volname);
+    if (ret)
+        goto out;
 
-        brick_index = 3;
-        w = str_getunamb (words[3], type_opword);
-        if (w && !strcmp ("replica", w)) {
-                if (wordcount < 6) {
-                        ret = -1;
-                        goto out;
-                }
-                count = strtol (words[4], NULL, 0);
-                if (count < 1) {
-                        cli_err ("replica count should be greater than 0 in "
-                                 "case of remove-brick");
-                        ret = -1;
-                        goto out;
+    brick_index = 3;
+    w = str_getunamb(words[3], type_opword);
+    if (w && !strcmp("replica", w)) {
+        if (wordcount < 6) {
+            ret = -1;
+            goto out;
+        }
+        count = strtol(words[4], NULL, 0);
+        if (count < 1) {
+            cli_err(
+                "replica count should be greater than 0 in "
+                "case of remove-brick");
+            ret = -1;
+            goto out;
+        }
+
+        if (count == 2) {
+            if (strcmp(words[wordcount - 1], "force")) {
+                ques =
+                    "Replica 2 volumes are prone to "
+                    "split-brain. Use Arbiter or Replica 3 "
+                    "to avoid this. See: "
+                    "http://docs.gluster.org/en/latest/Administrator%20Guide/"
+                    "Split%20brain%20and%20ways%20to%20deal%20with%20it/."
+                    "\nDo you still want to continue?\n";
+                answer = cli_cmd_get_confirmation(state, ques);
+                if (GF_ANSWER_NO == answer) {
+                    gf_log("cli", GF_LOG_ERROR,
+                           "Remove "
+                           "brick cancelled, exiting");
+                    ret = -1;
+                    goto out;
                 }
-
-                ret = dict_set_int32 (dict, "replica-count", count);
-                if (ret)
-                        goto out;
-                brick_index = 5;
-        } else if (w) {
-                GF_ASSERT (!"opword mismatch");
+            }
         }
 
-        w = str_getunamb (words[wordcount - 1], opwords);
-        if (!w) {
-                ret = -1;
-                goto out;
+        ret = dict_set_int32(dict, "replica-count", count);
+        if (ret)
+            goto out;
+        brick_index = 5;
+    } else if (w) {
+        GF_ASSERT(!"opword mismatch");
+    }
+
+    w = str_getunamb(words[wordcount - 1], opwords);
+    if (!w) {
+        ret = -1;
+        goto out;
+    } else {
+        /* handled this option */
+        wordcount--;
+        if (!strcmp("start", w)) {
+            command = GF_OP_CMD_START;
+            if (question)
+                *question = 1;
+        } else if (!strcmp("commit", w)) {
+            command = GF_OP_CMD_COMMIT;
+        } else if (!strcmp("stop", w)) {
+            command = GF_OP_CMD_STOP;
+        } else if (!strcmp("status", w)) {
+            command = GF_OP_CMD_STATUS;
+        } else if (!strcmp("force", w)) {
+            command = GF_OP_CMD_COMMIT_FORCE;
+            if (question)
+                *question = 1;
         } else {
-                /* handled this option */
-                wordcount--;
-                if (!strcmp ("start", w)) {
-                        command = GF_OP_CMD_START;
-                } else if (!strcmp ("commit", w)) {
-                        command = GF_OP_CMD_COMMIT;
-                        if (question)
-                                *question = 1;
-                } else if (!strcmp ("stop", w)) {
-                        command = GF_OP_CMD_STOP;
-                } else if (!strcmp ("status", w)) {
-                        command = GF_OP_CMD_STATUS;
-                } else if (!strcmp ("force", w)) {
-                        command = GF_OP_CMD_COMMIT_FORCE;
-                        if (question)
-                                *question = 1;
-                } else {
-                        GF_ASSERT (!"opword mismatch");
-                        ret = -1;
-                        goto out;
-                }
+            GF_ASSERT(!"opword mismatch");
+            ret = -1;
+            goto out;
         }
+    }
 
-        ret = dict_set_int32 (dict, "command", command);
-        if (ret)
-                gf_log ("cli", GF_LOG_INFO, "failed to set 'command' %d",
-                        command);
+    ret = dict_set_int32(dict, "command", command);
+    if (ret)
+        gf_log("cli", GF_LOG_INFO, "failed to set 'command' %d", command);
 
+    tmp_index = brick_index;
+    tmp_brick = GF_MALLOC(2048 * sizeof(*tmp_brick), gf_common_mt_char);
 
-        tmp_index = brick_index;
-        tmp_brick = GF_MALLOC(2048 * sizeof(*tmp_brick), gf_common_mt_char);
+    if (!tmp_brick) {
+        gf_log("", GF_LOG_ERROR,
+               "cli_cmd_volume_remove_brick_parse: "
+               "Unable to get memory");
+        ret = -1;
+        goto out;
+    }
 
-        if (!tmp_brick) {
-                gf_log ("",GF_LOG_ERROR,"cli_cmd_volume_remove_brick_parse: "
-                        "Unable to get memory");
+    tmp_brick1 = GF_MALLOC(2048 * sizeof(*tmp_brick1), gf_common_mt_char);
+
+    if (!tmp_brick1) {
+        gf_log("", GF_LOG_ERROR,
+               "cli_cmd_volume_remove_brick_parse: "
+               "Unable to get memory");
+        ret = -1;
+        goto out;
+    }
+
+    while (brick_index < wordcount) {
+        if (validate_brick_name((char *)words[brick_index])) {
+            cli_err(
+                "wrong brick type: %s, use <HOSTNAME>:"
+                "<export-dir-abs-path>",
+                words[brick_index]);
+            ret = -1;
+            goto out;
+        } else {
+            delimiter = strrchr(words[brick_index], ':');
+            ret = gf_canonicalize_path(delimiter + 1);
+            if (ret)
+                goto out;
+        }
+
+        j = tmp_index;
+        strcpy(tmp_brick, words[brick_index]);
+        while (j < brick_index) {
+            strcpy(tmp_brick1, words[j]);
+            if (!(strcmp(tmp_brick, tmp_brick1))) {
+                gf_log("", GF_LOG_ERROR,
+                       "Duplicate bricks"
+                       " found %s",
+                       words[brick_index]);
+                cli_err("Duplicate bricks found %s", words[brick_index]);
                 ret = -1;
                 goto out;
+            }
+            j++;
         }
+        snprintf(key, 50, "brick%d", ++(*brick_count));
+        ret = dict_set_str(dict, key, (char *)words[brick_index++]);
 
-        tmp_brick1 = GF_MALLOC(2048 * sizeof(*tmp_brick1), gf_common_mt_char);
+        if (ret)
+            goto out;
+    }
 
-        if (!tmp_brick1) {
-                gf_log ("",GF_LOG_ERROR,"cli_cmd_volume_remove_brick_parse: "
-                        "Unable to get memory");
-                ret = -1;
-                goto out;
-        }
+    if (command != GF_OP_CMD_STATUS && command != GF_OP_CMD_STOP) {
+        ret = dict_set_int32(dict, "count", *brick_count);
+        if (ret)
+            goto out;
+    }
 
-        while (brick_index < wordcount) {
-                if (validate_brick_name ((char *)words[brick_index])) {
-                        cli_err ("wrong brick type: %s, use <HOSTNAME>:"
-                                 "<export-dir-abs-path>", words[brick_index]);
-                        ret = -1;
-                        goto out;
-                } else {
-                        delimiter = strrchr(words[brick_index], ':');
-                        ret = gf_canonicalize_path (delimiter + 1);
-                        if (ret)
-                                goto out;
-                }
+    *options = dict;
 
-                j = tmp_index;
-                strcpy(tmp_brick, words[brick_index]);
-                while ( j < brick_index) {
-                        strcpy(tmp_brick1, words[j]);
-                        if (!(strcmp (tmp_brick, tmp_brick1))) {
-                                gf_log("",GF_LOG_ERROR, "Duplicate bricks"
-                                       " found %s", words[brick_index]);
-                                cli_err("Duplicate bricks found %s",
-                                        words[brick_index]);
-                                ret = -1;
-                                goto out;
-                        }
-                        j++;
-                }
-                snprintf (key, 50, "brick%d", ++brick_count);
-                ret = dict_set_str (dict, key, (char *)words[brick_index++]);
+out:
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Unable to parse remove-brick CLI");
+        if (dict)
+            dict_unref(dict);
+    }
 
-                if (ret)
-                        goto out;
-        }
+    GF_FREE(tmp_brick);
+    GF_FREE(tmp_brick1);
 
-        ret = dict_set_int32 (dict, "count", brick_count);
+    *comm = command;
+
+    return ret;
+}
+
+int32_t
+cli_cmd_brick_op_validate_bricks(const char **words, dict_t *dict, int src,
+                                 int dst)
+{
+    int ret = -1;
+    char *delimiter = NULL;
+
+    if (validate_brick_name((char *)words[src])) {
+        cli_err(
+            "wrong brick type: %s, use "
+            "<HOSTNAME>:<export-dir-abs-path>",
+            words[3]);
+        ret = -1;
+        goto out;
+    } else {
+        delimiter = strrchr((char *)words[src], '/');
+        ret = gf_canonicalize_path(delimiter);
         if (ret)
-                goto out;
+            goto out;
+    }
 
-        *options = dict;
+    ret = dict_set_str(dict, "src-brick", (char *)words[src]);
+    if (ret)
+        goto out;
 
-out:
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Unable to parse remove-brick CLI");
-                if (dict)
-                        dict_destroy (dict);
-        }
+    if (dst == -1) {
+        ret = 0;
+        goto out;
+    }
 
-        GF_FREE (tmp_brick);
-        GF_FREE (tmp_brick1);
+    if (validate_brick_name((char *)words[dst])) {
+        cli_err(
+            "wrong brick type: %s, use "
+            "<HOSTNAME>:<export-dir-abs-path>",
+            words[dst]);
+        ret = -1;
+        goto out;
+    } else {
+        delimiter = strrchr((char *)words[dst], '/');
+        ret = gf_canonicalize_path(delimiter);
+        if (ret)
+            goto out;
+    }
 
-        return ret;
+    ret = dict_set_str(dict, "dst-brick", (char *)words[dst]);
+    if (ret)
+        goto out;
+    ret = 0;
+out:
+    return ret;
 }
 
-
 int32_t
-cli_cmd_volume_replace_brick_parse (const char **words, int wordcount,
-                                   dict_t **options)
+cli_cmd_volume_reset_brick_parse(const char **words, int wordcount,
+                                 dict_t **options)
 {
-        dict_t  *dict = NULL;
-        char    *volname = NULL;
-        int     ret = -1;
-        int     op_index = 0;
-        char    *delimiter = NULL;
-        gf1_cli_replace_op replace_op = GF_REPLACE_OP_NONE;
-        char    *opwords[] = { "start", "commit", "pause", "abort", "status",
-                                NULL };
-        char    *w = NULL;
-        gf_boolean_t is_force = _gf_false;
-
-        GF_ASSERT (words);
-        GF_ASSERT (options);
-
-        dict = dict_new ();
-
-        if (!dict)
-                goto out;
+    int ret = -1;
+    char *volname = NULL;
+    dict_t *dict = NULL;
 
-        if (wordcount < 3)
-                goto out;
+    if (wordcount < 5 || wordcount > 7)
+        goto out;
 
-        volname = (char *)words[2];
+    dict = dict_new();
 
-        GF_ASSERT (volname);
+    if (!dict)
+        goto out;
 
-        ret = dict_set_str (dict, "volname", volname);
+    volname = (char *)words[2];
 
-        if (ret)
-                goto out;
+    ret = dict_set_str(dict, "volname", volname);
+    if (ret)
+        goto out;
 
-        if (wordcount < 4) {
-                ret = -1;
-                goto out;
+    if (wordcount == 5) {
+        if (strcmp(words[4], "start")) {
+            cli_err(
+                "Invalid option '%s' for reset-brick. Please "
+                "enter valid reset-brick command",
+                words[4]);
+            ret = -1;
+            goto out;
         }
 
-        if (validate_brick_name ((char *)words[3])) {
-                cli_err ("wrong brick type: %s, use "
-                         "<HOSTNAME>:<export-dir-abs-path>", words[3]);
-                ret = -1;
-                goto out;
-        } else {
-                delimiter = strrchr ((char *)words[3], ':');
-                ret = gf_canonicalize_path (delimiter + 1);
-                if (ret)
-                        goto out;
-        }
-        ret = dict_set_str (dict, "src-brick", (char *)words[3]);
+        ret = cli_cmd_brick_op_validate_bricks(words, dict, 3, -1);
+        if (ret)
+            goto out;
 
+        ret = dict_set_str(dict, "operation", "GF_RESET_OP_START");
         if (ret)
-                goto out;
+            goto out;
+    } else if (wordcount == 6) {
+        if (strcmp(words[5], "commit")) {
+            cli_err(
+                "Invalid option '%s' for reset-brick. Please "
+                "enter valid reset-brick command",
+                words[5]);
+            ret = -1;
+            goto out;
+        }
+
+        ret = cli_cmd_brick_op_validate_bricks(words, dict, 3, 4);
+        if (ret)
+            goto out;
 
-        if (wordcount < 5) {
-                ret = -1;
-                goto out;
-        }
+        ret = dict_set_str(dict, "operation", "GF_RESET_OP_COMMIT");
+        if (ret)
+            goto out;
+    } else if (wordcount == 7) {
+        if (strcmp(words[5], "commit") || strcmp(words[6], "force")) {
+            cli_err(
+                "Invalid option '%s %s' for reset-brick. Please "
+                "enter valid reset-brick command",
+                words[5], words[6]);
+            ret = -1;
+            goto out;
+        }
+
+        ret = cli_cmd_brick_op_validate_bricks(words, dict, 3, 4);
+        if (ret)
+            goto out;
 
-        if (validate_brick_name ((char *)words[4])) {
-                cli_err ("wrong brick type: %s, use "
-                         "<HOSTNAME>:<export-dir-abs-path>", words[4]);
-                ret = -1;
-                goto out;
-        } else {
-                delimiter = strrchr ((char *)words[4], ':');
-                ret = gf_canonicalize_path (delimiter + 1);
-                if (ret)
-                        goto out;
-        }
+        ret = dict_set_str(dict, "operation", "GF_RESET_OP_COMMIT_FORCE");
+        if (ret)
+            goto out;
+    }
 
+    *options = dict;
 
-        ret = dict_set_str (dict, "dst-brick", (char *)words[4]);
+out:
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Unable to parse reset-brick CLI");
+        if (dict)
+            dict_unref(dict);
+    }
 
-        if (ret)
-                goto out;
+    return ret;
+}
 
-        op_index = 5;
-        if ((wordcount < (op_index + 1))) {
-                ret = -1;
-                goto out;
-        }
+int32_t
+cli_cmd_volume_replace_brick_parse(const char **words, int wordcount,
+                                   dict_t **options)
+{
+    int ret = -1;
+    char *volname = NULL;
+    dict_t *dict = NULL;
 
-        w = str_getunamb (words[op_index], opwords);
+    GF_ASSERT(words);
+    GF_ASSERT(options);
 
-        if (!w) {
-        } else if (!strcmp ("start", w)) {
-                replace_op = GF_REPLACE_OP_START;
-        } else if (!strcmp ("commit", w)) {
-                replace_op = GF_REPLACE_OP_COMMIT;
-        } else if (!strcmp ("pause", w)) {
-                replace_op = GF_REPLACE_OP_PAUSE;
-        } else if (!strcmp ("abort", w)) {
-                replace_op = GF_REPLACE_OP_ABORT;
-        } else if (!strcmp ("status", w)) {
-                replace_op = GF_REPLACE_OP_STATUS;
-        } else
-                GF_ASSERT (!"opword mismatch");
-
-        /* commit force option */
-
-        op_index = 6;
-
-        if (wordcount > (op_index + 1)) {
-                ret = -1;
-                goto out;
-        }
+    if (wordcount != 7) {
+        ret = -1;
+        goto out;
+    }
 
-        if (wordcount == (op_index + 1)) {
-                if ((replace_op != GF_REPLACE_OP_COMMIT) &&
-                    (replace_op != GF_REPLACE_OP_START)) {
-                        ret = -1;
-                        goto out;
-                }
-                if (!strcmp ("force", words[op_index])) {
-                        if (replace_op == GF_REPLACE_OP_COMMIT)
-                                replace_op = GF_REPLACE_OP_COMMIT_FORCE;
+    dict = dict_new();
 
-                        else if (replace_op == GF_REPLACE_OP_START)
-                                is_force = _gf_true;
-                }
-        }
+    if (!dict) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to allocate dictionary");
+        goto out;
+    }
 
-        if (replace_op == GF_REPLACE_OP_NONE) {
-                ret = -1;
-                goto out;
-        }
+    volname = (char *)words[2];
 
-        ret = dict_set_int32 (dict, "operation", (int32_t) replace_op);
+    GF_ASSERT(volname);
 
-        if (ret)
-                goto out;
+    ret = dict_set_str(dict, "volname", volname);
+    if (ret)
+        goto out;
 
-        ret = dict_set_int32 (dict, "force", is_force);
-        if (ret)
-                goto out;
+    ret = cli_cmd_brick_op_validate_bricks(words, dict, 3, 4);
+    if (ret)
+        goto out;
 
-        *options = dict;
+    /* commit force option */
+    if (strcmp("commit", words[5]) || strcmp("force", words[6])) {
+        cli_err(
+            "Invalid option '%s' '%s' for replace-brick. Please "
+            "enter valid replace-brick command",
+            words[5], words[6]);
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_set_str(dict, "operation", "GF_REPLACE_OP_COMMIT_FORCE");
+    if (ret)
+        goto out;
+
+    *options = dict;
 
 out:
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Unable to parse replace-brick CLI");
-                if (dict)
-                        dict_destroy (dict);
-        }
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Unable to parse reset-brick CLI");
+        if (dict)
+            dict_unref(dict);
+    }
 
-        return ret;
+    return ret;
 }
 
 int32_t
-cli_cmd_log_filename_parse (const char **words, int wordcount, dict_t **options)
+cli_cmd_log_filename_parse(const char **words, int wordcount, dict_t **options)
 {
-        dict_t  *dict = NULL;
-        char    *volname = NULL;
-        char    *str = NULL;
-        int     ret = -1;
-        char    *delimiter = NULL;
+    dict_t *dict = NULL;
+    char *volname = NULL;
+    char *str = NULL;
+    int ret = -1;
+    char *delimiter = NULL;
 
-        GF_ASSERT (words);
-        GF_ASSERT (options);
+    GF_ASSERT(words);
+    GF_ASSERT(options);
 
-        dict = dict_new ();
-        if (!dict)
-                goto out;
+    dict = dict_new();
+    if (!dict)
+        goto out;
 
-        volname = (char *)words[3];
-        GF_ASSERT (volname);
+    volname = (char *)words[3];
+    GF_ASSERT(volname);
 
-        ret = dict_set_str (dict, "volname", volname);
-        if (ret)
-                goto out;
+    ret = dict_set_str(dict, "volname", volname);
+    if (ret)
+        goto out;
 
-        str = (char *)words[4];
-        if (strchr (str, ':')) {
-                delimiter = strchr (words[4], ':');
-                if (!delimiter || delimiter == words[4]
-                    || *(delimiter+1) != '/') {
-                        cli_err ("wrong brick type: %s, use <HOSTNAME>:"
-                                 "<export-dir-abs-path>", words[4]);
-                        ret = -1;
-                        goto out;
-                } else {
-                        ret = gf_canonicalize_path (delimiter + 1);
-                        if (ret)
-                                goto out;
-                }
-                ret = dict_set_str (dict, "brick", str);
-                if (ret)
-                        goto out;
-                /* Path */
-                str = (char *)words[5];
-                ret = dict_set_str (dict, "path", str);
-                if (ret)
-                        goto out;
+    str = (char *)words[4];
+    if (strchr(str, ':')) {
+        delimiter = strchr(words[4], ':');
+        if (!delimiter || delimiter == words[4] || *(delimiter + 1) != '/') {
+            cli_err(
+                "wrong brick type: %s, use <HOSTNAME>:"
+                "<export-dir-abs-path>",
+                words[4]);
+            ret = -1;
+            goto out;
         } else {
-                ret = dict_set_str (dict, "path", str);
-                if (ret)
-                        goto out;
+            ret = gf_canonicalize_path(delimiter + 1);
+            if (ret)
+                goto out;
         }
+        ret = dict_set_str(dict, "brick", str);
+        if (ret)
+            goto out;
+        /* Path */
+        str = (char *)words[5];
+        ret = dict_set_str(dict, "path", str);
+        if (ret)
+            goto out;
+    } else {
+        ret = dict_set_str(dict, "path", str);
+        if (ret)
+            goto out;
+    }
 
-        *options = dict;
+    *options = dict;
 
 out:
-        if (ret && dict)
-                dict_destroy (dict);
+    if (ret && dict)
+        dict_unref(dict);
 
-        return ret;
+    return ret;
 }
 
 int32_t
-cli_cmd_log_level_parse (const char **words, int worcount, dict_t **options)
+cli_cmd_log_level_parse(const char **words, int worcount, dict_t **options)
 {
-        dict_t *dict            = NULL;
-        int     ret             = -1;
-
-        GF_ASSERT (words);
-        GF_ASSERT (options);
-
-        /*
-         * loglevel command format:
-         *  > volume log level <VOL> <XLATOR[*]> <LOGLEVEL>
-         *  > volume log level colon-o posix WARNING
-         *  > volume log level colon-o replicate* DEBUG
-         *  > volume log level coon-o * TRACE
-         */
-
-        GF_ASSERT ((strncmp(words[0], "volume", 6) == 0));
-        GF_ASSERT ((strncmp(words[1], "log", 3) == 0));
-        GF_ASSERT ((strncmp(words[2], "level", 5) == 0));
-
-        ret = glusterd_check_log_level(words[5]);
-        if (ret == -1) {
-                cli_err("Invalid log level [%s] specified", words[5]);
-                cli_err("Valid values for loglevel: (DEBUG|WARNING|ERROR"
-                        "|CRITICAL|NONE|TRACE)");
-                goto out;
-        }
+    dict_t *dict = NULL;
+    int ret = -1;
+
+    GF_ASSERT(words);
+    GF_ASSERT(options);
+
+    /*
+     * loglevel command format:
+     *  > volume log level <VOL> <XLATOR[*]> <LOGLEVEL>
+     *  > volume log level colon-o posix WARNING
+     *  > volume log level colon-o replicate* DEBUG
+     *  > volume log level coon-o * TRACE
+     */
+
+    GF_ASSERT((strncmp(words[0], "volume", 6) == 0));
+    GF_ASSERT((strncmp(words[1], "log", 3) == 0));
+    GF_ASSERT((strncmp(words[2], "level", 5) == 0));
+
+    ret = glusterd_check_log_level(words[5]);
+    if (ret == -1) {
+        cli_err("Invalid log level [%s] specified", words[5]);
+        cli_err(
+            "Valid values for loglevel: (DEBUG|WARNING|ERROR"
+            "|CRITICAL|NONE|TRACE)");
+        goto out;
+    }
 
-        dict = dict_new ();
-        if (!dict)
-                goto out;
+    dict = dict_new();
+    if (!dict)
+        goto out;
 
-        GF_ASSERT(words[3]);
-        GF_ASSERT(words[4]);
+    GF_ASSERT(words[3]);
+    GF_ASSERT(words[4]);
 
-        ret = dict_set_str (dict, "volname", (char *)words[3]);
-        if (ret)
-                goto out;
+    ret = dict_set_str(dict, "volname", (char *)words[3]);
+    if (ret)
+        goto out;
 
-        ret = dict_set_str (dict, "xlator", (char *)words[4]);
-        if (ret)
-                goto out;
+    ret = dict_set_str(dict, "xlator", (char *)words[4]);
+    if (ret)
+        goto out;
 
-        ret = dict_set_str (dict, "loglevel", (char *)words[5]);
-        if (ret)
-                goto out;
+    ret = dict_set_str(dict, "loglevel", (char *)words[5]);
+    if (ret)
+        goto out;
 
-        *options = dict;
+    *options = dict;
 
- out:
-        if (ret && dict)
-                dict_destroy (dict);
+out:
+    if (ret && dict)
+        dict_unref(dict);
 
-        return ret;
+    return ret;
 }
 
 int32_t
-cli_cmd_log_locate_parse (const char **words, int wordcount, dict_t **options)
+cli_cmd_log_locate_parse(const char **words, int wordcount, dict_t **options)
 {
-        dict_t  *dict = NULL;
-        char    *volname = NULL;
-        char    *str = NULL;
-        int     ret = -1;
-        char    *delimiter = NULL;
+    dict_t *dict = NULL;
+    char *volname = NULL;
+    char *str = NULL;
+    int ret = -1;
+    char *delimiter = NULL;
 
-        GF_ASSERT (words);
-        GF_ASSERT (options);
+    GF_ASSERT(words);
+    GF_ASSERT(options);
 
-        dict = dict_new ();
-        if (!dict)
-                goto out;
+    dict = dict_new();
+    if (!dict)
+        goto out;
 
-        volname = (char *)words[3];
-        GF_ASSERT (volname);
+    volname = (char *)words[3];
+    GF_ASSERT(volname);
 
-        ret = dict_set_str (dict, "volname", volname);
-        if (ret)
-                goto out;
+    ret = dict_set_str(dict, "volname", volname);
+    if (ret)
+        goto out;
 
-        if (words[4]) {
-                delimiter = strchr (words[4], ':');
-                if (!delimiter || delimiter == words[4]
-                    || *(delimiter+1) != '/') {
-                        cli_err ("wrong brick type: %s, use <HOSTNAME>:"
-                                 "<export-dir-abs-path>", words[4]);
-                        ret = -1;
-                        goto out;
-                } else {
-                        ret = gf_canonicalize_path (delimiter + 1);
-                        if (ret)
-                                goto out;
-                }
-                str = (char *)words[4];
-                ret = dict_set_str (dict, "brick", str);
-                if (ret)
-                        goto out;
+    if (words[4]) {
+        delimiter = strchr(words[4], ':');
+        if (!delimiter || delimiter == words[4] || *(delimiter + 1) != '/') {
+            cli_err(
+                "wrong brick type: %s, use <HOSTNAME>:"
+                "<export-dir-abs-path>",
+                words[4]);
+            ret = -1;
+            goto out;
+        } else {
+            ret = gf_canonicalize_path(delimiter + 1);
+            if (ret)
+                goto out;
         }
+        str = (char *)words[4];
+        ret = dict_set_str(dict, "brick", str);
+        if (ret)
+            goto out;
+    }
 
-        *options = dict;
+    *options = dict;
 
 out:
-        if (ret && dict)
-                dict_destroy (dict);
+    if (ret && dict)
+        dict_unref(dict);
 
-        return ret;
+    return ret;
 }
 
 int32_t
-cli_cmd_log_rotate_parse (const char **words, int wordcount, dict_t **options)
+cli_cmd_log_rotate_parse(const char **words, int wordcount, dict_t **options)
 {
-        dict_t  *dict = NULL;
-        char    *volname = NULL;
-        char    *str = NULL;
-        int     ret = -1;
-        char    *delimiter = NULL;
+    dict_t *dict = NULL;
+    char *volname = NULL;
+    char *str = NULL;
+    int ret = -1;
+    char *delimiter = NULL;
 
-        GF_ASSERT (words);
-        GF_ASSERT (options);
+    GF_ASSERT(words);
+    GF_ASSERT(options);
 
-        dict = dict_new ();
-        if (!dict)
-                goto out;
+    dict = dict_new();
+    if (!dict)
+        goto out;
 
-        if (strcmp ("rotate", words[3]) == 0)
-                volname = (char *)words[2];
-        else if (strcmp ("rotate", words[2]) == 0)
-                volname = (char *)words[3];
-        GF_ASSERT (volname);
+    if (strcmp("rotate", words[3]) == 0)
+        volname = (char *)words[2];
+    GF_ASSERT(volname);
 
-        ret = dict_set_str (dict, "volname", volname);
-        if (ret)
-                goto out;
+    ret = dict_set_str(dict, "volname", volname);
+    if (ret)
+        goto out;
 
-        if (words[4]) {
-                delimiter = strchr (words[4], ':');
-                if (!delimiter || delimiter == words[4]
-                    || *(delimiter+1) != '/') {
-                        cli_err ("wrong brick type: %s, use <HOSTNAME>:"
-                                 "<export-dir-abs-path>", words[4]);
-                        ret = -1;
-                        goto out;
-                } else {
-                        ret = gf_canonicalize_path (delimiter + 1);
-                        if (ret)
-                                goto out;
-                }
-                str = (char *)words[4];
-                ret = dict_set_str (dict, "brick", str);
-                if (ret)
-                        goto out;
+    if (words[4]) {
+        delimiter = strchr(words[4], ':');
+        if (!delimiter || delimiter == words[4] || *(delimiter + 1) != '/') {
+            cli_err(
+                "wrong brick type: %s, use <HOSTNAME>:"
+                "<export-dir-abs-path>",
+                words[4]);
+            ret = -1;
+            goto out;
+        } else {
+            ret = gf_canonicalize_path(delimiter + 1);
+            if (ret)
+                goto out;
         }
+        str = (char *)words[4];
+        ret = dict_set_str(dict, "brick", str);
+        if (ret)
+            goto out;
+    }
 
-        *options = dict;
+    *options = dict;
 
 out:
-        if (ret && dict)
-                dict_destroy (dict);
+    if (ret && dict)
+        dict_unref(dict);
 
-        return ret;
+    return ret;
 }
 
 static gf_boolean_t
-gsyncd_url_check (const char *w)
+gsyncd_url_check(const char *w)
 {
-        return !!strpbrk (w, ":/");
+    return !!strpbrk(w, ":/");
 }
 
 static gf_boolean_t
-gsyncd_glob_check (const char *w)
+valid_slave_gsyncd_url(const char *w)
 {
-        return !!strpbrk (w, "*?[");
+    if (strstr(w, ":::"))
+        return _gf_false;
+    else if (strstr(w, "::"))
+        return _gf_true;
+    else
+        return _gf_false;
 }
 
-static int
-config_parse (const char **words, int wordcount, dict_t *dict,
-              unsigned cmdi, unsigned glob)
+static gf_boolean_t
+gsyncd_glob_check(const char *w)
 {
-        int32_t            ret     = -1;
-        int32_t            i       = -1;
-        char               *append_str = NULL;
-        size_t             append_len = 0;
-        char               *subop = NULL;
+    return !!strpbrk(w, "*?[");
+}
 
-        switch ((wordcount - 1) - cmdi) {
+static int
+config_parse(const char **words, int wordcount, dict_t *dict, unsigned cmdi,
+             unsigned glob)
+{
+    int32_t ret = -1;
+    int32_t i = -1;
+    char *append_str = NULL;
+    size_t append_len = 0;
+    char *subop = NULL;
+    char *ret_chkpt = NULL;
+    struct tm checkpoint_time;
+    char chkpt_buf[20] = "";
+
+    switch ((wordcount - 1) - cmdi) {
         case 0:
-                subop = gf_strdup ("get-all");
-                break;
+            subop = gf_strdup("get-all");
+            break;
         case 1:
-                if (words[cmdi + 1][0] == '!') {
-                        (words[cmdi + 1])++;
-                        if (gf_asprintf (&subop, "del%s",
-                                         glob ? "-glob" : "") == -1)
-                                subop = NULL;
-                } else
-                        subop = gf_strdup ("get");
+            if (words[cmdi + 1][0] == '!') {
+                (words[cmdi + 1])++;
+                if (gf_asprintf(&subop, "del%s", glob ? "-glob" : "") == -1)
+                    subop = NULL;
+            } else
+                subop = gf_strdup("get");
 
-                ret = dict_set_str (dict, "op_name", ((char *)words[cmdi + 1]));
-                if (ret < 0)
-                        goto out;
-                break;
+            ret = dict_set_str(dict, "op_name", ((char *)words[cmdi + 1]));
+            if (ret < 0)
+                goto out;
+            break;
         default:
-                if (gf_asprintf (&subop, "set%s", glob ? "-glob" : "") == -1)
-                        subop = NULL;
+            if (gf_asprintf(&subop, "set%s", glob ? "-glob" : "") == -1)
+                subop = NULL;
 
-                ret = dict_set_str (dict, "op_name", ((char *)words[cmdi + 1]));
-                if (ret < 0)
-                        goto out;
+            ret = dict_set_str(dict, "op_name", ((char *)words[cmdi + 1]));
+            if (ret < 0)
+                goto out;
 
-                /* join the varargs by spaces to get the op_value */
+            /* join the varargs by spaces to get the op_value */
 
-                for (i = cmdi + 2; i < wordcount; i++)
-                        append_len += (strlen (words[i]) + 1);
-                /* trailing strcat will add two bytes, make space for that */
-                append_len++;
+            for (i = cmdi + 2; i < wordcount; i++)
+                append_len += (strlen(words[i]) + 1);
+            /* trailing strcat will add two bytes, make space for that */
+            append_len++;
 
-                append_str = GF_CALLOC (1, append_len, cli_mt_append_str);
-                if (!append_str) {
-                        ret = -1;
-                        goto out;
-                }
+            /* strcat is used on this allocation and hence expected to be
+             * initiatlized to 0. So GF_CALLOC is used.
+             */
+            append_str = GF_CALLOC(1, append_len, cli_mt_append_str);
+            if (!append_str) {
+                ret = -1;
+                goto out;
+            }
+
+            for (i = cmdi + 2; i < wordcount; i++) {
+                strcat(append_str, words[i]);
+                strcat(append_str, " ");
+            }
+            append_str[append_len - 2] = '\0';
+            /* "checkpoint now" is special: we resolve that "now" */
+            if ((strcmp(words[cmdi + 1], "checkpoint") == 0) &&
+                (strcmp(append_str, "now") == 0)) {
+                struct timeval tv = {
+                    0,
+                };
+
+                ret = gettimeofday(&tv, NULL);
+                if (ret == -1)
+                    goto out;
 
-                for (i = cmdi + 2; i < wordcount; i++) {
-                        strcat (append_str, words[i]);
-                        strcat (append_str, " ");
-                }
-                append_str[append_len - 2] = '\0';
-                /* "checkpoint now" is special: we resolve that "now" */
-                if ((strcmp (words[cmdi + 1], "checkpoint") == 0) &&
-                    (strcmp (append_str, "now") == 0)) {
-                        struct timeval tv = {0,};
-
-                        ret = gettimeofday (&tv, NULL);
-                        if (ret == -1)
-                                goto out;
-
-                        GF_FREE (append_str);
-                        append_str = GF_CALLOC (1, 300, cli_mt_append_str);
-                        if (!append_str) {
-                                ret = -1;
-                                goto out;
-                        }
-                        snprintf (append_str, 300, "now:%" GF_PRI_SECOND ".%06"GF_PRI_SUSECONDS,
-                                  tv.tv_sec, tv.tv_usec);
+                GF_FREE(append_str);
+                append_str = GF_MALLOC(300, cli_mt_append_str);
+                if (!append_str) {
+                    ret = -1;
+                    goto out;
+                }
+                snprintf(append_str, 300, "%" GF_PRI_SECOND, tv.tv_sec);
+            } else if ((strcmp(words[cmdi + 1], "checkpoint") == 0) &&
+                       (strcmp(append_str, "now") != 0)) {
+                memset(&checkpoint_time, 0, sizeof(struct tm));
+                ret_chkpt = strptime(append_str, "%Y-%m-%d %H:%M:%S",
+                                     &checkpoint_time);
+
+                if (ret_chkpt == NULL || *ret_chkpt != '\0') {
+                    ret = -1;
+                    cli_err(
+                        "Invalid Checkpoint label. Use format "
+                        "\"Y-m-d H:M:S\", Example: 2016-10-25 15:30:45");
+                    goto out;
+                }
+                GF_FREE(append_str);
+                append_str = GF_MALLOC(300, cli_mt_append_str);
+                if (!append_str) {
+                    ret = -1;
+                    goto out;
                 }
+                strftime(chkpt_buf, sizeof(chkpt_buf), "%s", &checkpoint_time);
+                snprintf(append_str, 300, "%s", chkpt_buf);
+            }
 
-                ret = dict_set_dynstr (dict, "op_value", append_str);
-        }
+            ret = dict_set_dynstr(dict, "op_value", append_str);
+            if (ret != 0) {
+                goto out;
+            }
+            append_str = NULL;
+    }
 
-        ret = -1;
-        if (subop) {
-                ret = dict_set_dynstr (dict, "subop", subop);
-                if (!ret)
-                      subop = NULL;
-        }
+    ret = -1;
+    if (subop) {
+        ret = dict_set_dynstr(dict, "subop", subop);
+        if (!ret)
+            subop = NULL;
+    }
 
 out:
-        if (ret && append_str)
-                GF_FREE (append_str);
+    GF_FREE(append_str);
+    GF_FREE(subop);
 
-        GF_FREE (subop);
-
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
 }
 
+/* ssh_port_parse: Parses and validates when ssh_port is given.
+ *                 ssh_index refers to index of ssh_port and
+ *                 type refers to either push-pem or no-verify
+ */
+
 static int32_t
-force_push_pem_parse (const char **words, int wordcount,
-                      dict_t *dict, unsigned *cmdi)
+parse_ssh_port(const char **words, int wordcount, dict_t *dict, unsigned *cmdi,
+               int ssh_index, char *type)
 {
-        int32_t            ret     = 0;
-
-        if (!strcmp ((char *)words[wordcount-1], "force")) {
-                if ((strcmp ((char *)words[wordcount-2], "start")) &&
-                    (strcmp ((char *)words[wordcount-2], "stop")) &&
-                    (strcmp ((char *)words[wordcount-2], "create")) &&
-                    (strcmp ((char *)words[wordcount-2], "push-pem")) &&
-                    (strcmp ((char *)words[wordcount-2], "pause")) &&
-                    (strcmp ((char *)words[wordcount-2], "resume"))) {
-                        ret = -1;
-                        goto out;
-                }
-                ret = dict_set_uint32 (dict, "force",
-                                       _gf_true);
-                if (ret)
-                        goto out;
-                (*cmdi)++;
+    int ret = 0;
+    char *end_ptr = NULL;
+    int64_t limit = 0;
 
-                if (!strcmp ((char *)words[wordcount-2], "push-pem")) {
-                        if (strcmp ((char *)words[wordcount-3], "create")) {
-                                ret = -1;
-                                goto out;
-                        }
-                        ret = dict_set_int32 (dict, "push_pem", 1);
-                        if (ret)
-                                goto out;
-                        (*cmdi)++;
-                }
-        } else if (!strcmp ((char *)words[wordcount-1], "push-pem")) {
-                if (strcmp ((char *)words[wordcount-2], "create")) {
-                        ret = -1;
-                        goto out;
-                }
-                ret = dict_set_int32 (dict, "push_pem", 1);
-                if (ret)
-                        goto out;
-                (*cmdi)++;
+    if (!strcmp((char *)words[ssh_index], "ssh-port")) {
+        if (strcmp((char *)words[ssh_index - 1], "create")) {
+            ret = -1;
+            goto out;
         }
+        (*cmdi)++;
+        limit = strtol(words[ssh_index + 1], &end_ptr, 10);
+        if (errno == ERANGE || errno == EINVAL || limit <= 0 ||
+            strcmp(end_ptr, "") != 0) {
+            ret = -1;
+            cli_err("Please enter an integer value for ssh_port ");
+            goto out;
+        }
+
+        ret = dict_set_int32(dict, "ssh_port", limit);
+        if (ret)
+            goto out;
+        (*cmdi)++;
+    } else if (strcmp((char *)words[ssh_index + 1], "create")) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_set_int32(dict, type, 1);
+    if (ret)
+        goto out;
+    (*cmdi)++;
 
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    return ret;
 }
 
-
-int32_t
-cli_cmd_gsync_set_parse (const char **words, int wordcount, dict_t **options)
+static int32_t
+force_push_pem_no_verify_parse(const char **words, int wordcount, dict_t *dict,
+                               unsigned *cmdi)
 {
-        int32_t            ret     = -1;
-        dict_t             *dict   = NULL;
-        gf1_cli_gsync_set  type    = GF_GSYNC_OPTION_TYPE_NONE;
-        int                i       = 0;
-        unsigned           masteri = 0;
-        unsigned           slavei  = 0;
-        unsigned           glob    = 0;
-        unsigned           cmdi    = 0;
-        char               *opwords[] = { "create", "status", "start", "stop",
-                                          "config", "force", "delete",
-                                          "push-pem", "detail", "pause",
-                                          "resume", NULL };
-        char               *w = NULL;
-
-        GF_ASSERT (words);
-        GF_ASSERT (options);
-
-        dict = dict_new ();
-        if (!dict)
-                goto out;
-
-        /* new syntax:
-         *
-         * volume geo-replication $m $s create [push-pem] [force]
-         * volume geo-replication [$m [$s]] status [detail]
-         * volume geo-replication [$m] $s config [[!]$opt [$val]]
-         * volume geo-replication $m $s start|stop [force]
-         * volume geo-replication $m $s delete
-         * volume geo-replication $m $s pause [force]
-         * volume geo-replication $m $s resume [force]
-         */
+    int32_t ret = 0;
+
+    if (!strcmp((char *)words[wordcount - 1], "force")) {
+        if ((strcmp((char *)words[wordcount - 2], "start")) &&
+            (strcmp((char *)words[wordcount - 2], "stop")) &&
+            (strcmp((char *)words[wordcount - 2], "create")) &&
+            (strcmp((char *)words[wordcount - 2], "no-verify")) &&
+            (strcmp((char *)words[wordcount - 2], "push-pem")) &&
+            (strcmp((char *)words[wordcount - 2], "pause")) &&
+            (strcmp((char *)words[wordcount - 2], "resume"))) {
+            ret = -1;
+            goto out;
+        }
+        ret = dict_set_int32n(dict, "force", SLEN("force"), 1);
+        if (ret)
+            goto out;
+        (*cmdi)++;
 
-        if (wordcount < 3)
+        if (!strcmp((char *)words[wordcount - 2], "push-pem")) {
+            ret = parse_ssh_port(words, wordcount, dict, cmdi, wordcount - 4,
+                                 "push_pem");
+            if (ret)
                 goto out;
-
-        for (i = 2; i <= 3 && i < wordcount - 1; i++) {
-                if (gsyncd_glob_check (words[i]))
-                        glob = i;
-                if (gsyncd_url_check (words[i])) {
-                        slavei = i;
-                        break;
-                }
-        }
-
-        if (glob && !slavei)
-                /* glob is allowed only for config, thus it implies there is a
-                 * slave argument; but that might have not been recognized on
-                 * the first scan as it's url characteristics has been covered
-                 * by the glob syntax.
-                 *
-                 * In this case, the slave is perforce the last glob-word -- the
-                 * upcoming one is neither glob, nor url, so it's definitely not
-                 * the slave.
-                 */
-                slavei = glob;
-        if (slavei) {
-                cmdi = slavei + 1;
-                if (slavei == 3)
-                        masteri = 2;
-        } else if (i <= 3) {
-                if (!strcmp ((char *)words[wordcount-1], "detail")) {
-                        /* For status detail it is mandatory to provide
-                         * both master and slave */
-                        ret = -1;
-                        goto out;
-                }
-
-                /* no $s, can only be status cmd
-                 * (with either a single $m before it or nothing)
-                 * -- these conditions imply that i <= 3 after
-                 * the iteration and that i is the successor of
-                 * the (0 or 1 length) sequence of $m-s.
-                 */
-                cmdi = i;
-                if (i == 3)
-                        masteri = 2;
-        } else
+        } else if (!strcmp((char *)words[wordcount - 2], "no-verify")) {
+            ret = parse_ssh_port(words, wordcount, dict, cmdi, wordcount - 4,
+                                 "no_verify");
+            if (ret)
                 goto out;
+        }
+    } else if (!strcmp((char *)words[wordcount - 1], "push-pem")) {
+        ret = parse_ssh_port(words, wordcount, dict, cmdi, wordcount - 3,
+                             "push_pem");
+        if (ret)
+            goto out;
+    } else if (!strcmp((char *)words[wordcount - 1], "no-verify")) {
+        ret = parse_ssh_port(words, wordcount, dict, cmdi, wordcount - 3,
+                             "no_verify");
+        if (ret)
+            goto out;
+    }
 
-        /* now check if input really complies syntax
-         * (in a somewhat redundant way, in favor
-         * transparent soundness)
-         */
-
-        if (masteri && gsyncd_url_check (words[masteri]))
-                goto out;
-        if (slavei && !glob && !gsyncd_url_check (words[slavei]))
-                goto out;
+out:
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
+}
 
-        w = str_getunamb (words[cmdi], opwords);
-        if (!w)
-                goto out;
+int32_t
+cli_cmd_gsync_set_parse(struct cli_state *state, const char **words,
+                        int wordcount, dict_t **options, char **errstr)
+{
+    int32_t ret = -1;
+    dict_t *dict = NULL;
+    gf1_cli_gsync_set type = GF_GSYNC_OPTION_TYPE_NONE;
+    int i = 0;
+    unsigned masteri = 0;
+    unsigned slavei = 0;
+    unsigned glob = 0;
+    unsigned cmdi = 0;
+    static char *opwords[] = {"create",    "status",   "start",  "stop",
+                              "config",    "force",    "delete", "ssh-port",
+                              "no-verify", "push-pem", "detail", "pause",
+                              "resume",    NULL};
+    char *w = NULL;
+    char *save_ptr = NULL;
+    char *slave_temp = NULL;
+    char *token = NULL;
+    gf_answer_t answer = GF_ANSWER_NO;
+    const char *question = NULL;
+
+    GF_ASSERT(words);
+    GF_ASSERT(options);
+
+    dict = dict_new();
+    if (!dict)
+        goto out;
 
-        if (strcmp (w, "create") == 0) {
-                type = GF_GSYNC_OPTION_TYPE_CREATE;
+    /* new syntax:
+     *
+     * volume geo-replication $m $s create [[ssh-port n] [[no-verify] |
+     * [push-pem]]] [force] volume geo-replication [$m [$s]] status [detail]
+     * volume geo-replication [$m] $s config [[!]$opt [$val]]
+     * volume geo-replication $m $s start|stop [force]
+     * volume geo-replication $m $s delete [reset-sync-time]
+     * volume geo-replication $m $s pause [force]
+     * volume geo-replication $m $s resume [force]
+     */
+
+    if (wordcount < 3)
+        goto out;
 
-                if (!masteri || !slavei)
-                        goto out;
-        } else if (strcmp (w, "status") == 0) {
-                type = GF_GSYNC_OPTION_TYPE_STATUS;
+    for (i = 2; i <= 3 && i < wordcount - 1; i++) {
+        if (gsyncd_glob_check(words[i]))
+            glob = i;
+        if (gsyncd_url_check(words[i])) {
+            slavei = i;
+            break;
+        }
+    }
 
-                if (slavei && !masteri)
-                        goto out;
-        } else if (strcmp (w, "config") == 0) {
-                type = GF_GSYNC_OPTION_TYPE_CONFIG;
+    if (glob && !slavei)
+        /* glob is allowed only for config, thus it implies there is a
+         * slave argument; but that might have not been recognized on
+         * the first scan as it's url characteristics has been covered
+         * by the glob syntax.
+         *
+         * In this case, the slave is perforce the last glob-word -- the
+         * upcoming one is neither glob, nor url, so it's definitely not
+         * the slave.
+         */
+        slavei = glob;
+    if (slavei) {
+        cmdi = slavei + 1;
+        if (slavei == 3)
+            masteri = 2;
+    } else if (i <= 4) {
+        if (strtail("detail", (char *)words[wordcount - 1])) {
+            cmdi = wordcount - 2;
+            if (i == 4)
+                masteri = 2;
+        } else {
+            /* no $s, can only be status cmd
+             * (with either a single $m before it or nothing)
+             * -- these conditions imply that i <= 3 after
+             * the iteration and that i is the successor of
+             * the (0 or 1 length) sequence of $m-s.
+             */
+            cmdi = i;
+            if (i == 3)
+                masteri = 2;
+        }
+    } else
+        goto out;
 
-                if (!slavei)
-                        goto out;
-        } else if (strcmp (w, "start") == 0) {
-                type = GF_GSYNC_OPTION_TYPE_START;
+    /* now check if input really complies syntax
+     * (in a somewhat redundant way, in favor
+     * transparent soundness)
+     */
 
-                if (!masteri || !slavei)
-                        goto out;
-        } else if (strcmp (w, "stop") == 0) {
-                type = GF_GSYNC_OPTION_TYPE_STOP;
+    if (masteri && gsyncd_url_check(words[masteri]))
+        goto out;
 
-                if (!masteri || !slavei)
-                        goto out;
-        } else if (strcmp (w, "delete") == 0) {
-                type = GF_GSYNC_OPTION_TYPE_DELETE;
+    if (slavei && !glob && !valid_slave_gsyncd_url(words[slavei])) {
+        gf_asprintf(errstr, "Invalid slave url: %s", words[slavei]);
+        goto out;
+    }
 
-                if (!masteri || !slavei)
-                        goto out;
-        } else if (strcmp (w, "pause") == 0) {
-                type = GF_GSYNC_OPTION_TYPE_PAUSE;
+    w = str_getunamb(words[cmdi], opwords);
+    if (!w)
+        goto out;
 
-                if (!masteri || !slavei)
-                        goto out;
-        } else if (strcmp (w, "resume") == 0) {
-                type = GF_GSYNC_OPTION_TYPE_RESUME;
+    if (strcmp(w, "create") == 0) {
+        type = GF_GSYNC_OPTION_TYPE_CREATE;
+
+        if (!masteri || !slavei)
+            goto out;
+    } else if (strcmp(w, "status") == 0) {
+        type = GF_GSYNC_OPTION_TYPE_STATUS;
+
+        if (slavei && !masteri)
+            goto out;
+    } else if (strcmp(w, "config") == 0) {
+        type = GF_GSYNC_OPTION_TYPE_CONFIG;
+
+        if (!slavei)
+            goto out;
+    } else if (strcmp(w, "start") == 0) {
+        type = GF_GSYNC_OPTION_TYPE_START;
+
+        if (!masteri || !slavei)
+            goto out;
+    } else if (strcmp(w, "stop") == 0) {
+        type = GF_GSYNC_OPTION_TYPE_STOP;
+
+        if (!masteri || !slavei)
+            goto out;
+    } else if (strcmp(w, "delete") == 0) {
+        type = GF_GSYNC_OPTION_TYPE_DELETE;
+
+        if (!masteri || !slavei)
+            goto out;
+    } else if (strcmp(w, "pause") == 0) {
+        type = GF_GSYNC_OPTION_TYPE_PAUSE;
+
+        if (!masteri || !slavei)
+            goto out;
+    } else if (strcmp(w, "resume") == 0) {
+        type = GF_GSYNC_OPTION_TYPE_RESUME;
+
+        if (!masteri || !slavei)
+            goto out;
+    } else
+        GF_ASSERT(!"opword mismatch");
+
+    ret = force_push_pem_no_verify_parse(words, wordcount, dict, &cmdi);
+    if (ret)
+        goto out;
 
-                if (!masteri || !slavei)
-                        goto out;
-        } else
-                GF_ASSERT (!"opword mismatch");
+    if (strtail("detail", (char *)words[wordcount - 1])) {
+        if (!strtail("status", (char *)words[wordcount - 2])) {
+            ret = -1;
+            goto out;
+        }
 
-        ret = force_push_pem_parse (words, wordcount, dict, &cmdi);
+        ret = dict_set_uint32(dict, "status-detail", _gf_true);
         if (ret)
-                goto out;
+            goto out;
+        cmdi++;
+    }
 
-        if (!strcmp ((char *)words[wordcount-1], "detail")) {
-                if (strcmp ((char *)words[wordcount-2], "status")) {
-                        ret = -1;
-                        goto out;
-                }
-                if (!slavei || !masteri) {
-                        ret = -1;
-                        goto out;
-                }
-                ret = dict_set_uint32 (dict, "status-detail", _gf_true);
-                if (ret)
-                        goto out;
-                cmdi++;
+    if (type == GF_GSYNC_OPTION_TYPE_DELETE &&
+        !strcmp((char *)words[wordcount - 1], "reset-sync-time")) {
+        if (strcmp((char *)words[wordcount - 2], "delete")) {
+            ret = -1;
+            goto out;
         }
+        ret = dict_set_uint32(dict, "reset-sync-time", _gf_true);
+        if (ret)
+            goto out;
+        cmdi++;
+    }
 
-        if (type != GF_GSYNC_OPTION_TYPE_CONFIG &&
-            (cmdi < wordcount - 1 || glob))
-                goto out;
+    if (type != GF_GSYNC_OPTION_TYPE_CONFIG && (cmdi < wordcount - 1 || glob)) {
+        ret = -1;
+        goto out;
+    }
 
-        /* If got so far, input is valid, assemble the message */
+    /* If got so far, input is valid, assemble the message */
 
-        ret = 0;
+    ret = 0;
 
-        if (masteri) {
-                ret = dict_set_str (dict, "master", (char *)words[masteri]);
-                if (!ret)
-                        ret = dict_set_str (dict, "volname",
-                                            (char *)words[masteri]);
-        }
-        if (!ret && slavei)
-                ret = dict_set_str (dict, "slave", (char *)words[slavei]);
+    if (masteri) {
+        ret = dict_set_str(dict, "master", (char *)words[masteri]);
         if (!ret)
-                ret = dict_set_int32 (dict, "type", type);
-        if (!ret && type == GF_GSYNC_OPTION_TYPE_CONFIG)
-                ret = config_parse (words, wordcount, dict, cmdi, glob);
+            ret = dict_set_str(dict, "volname", (char *)words[masteri]);
+    }
+    if (!ret && slavei) {
+        /* If geo-rep is created with root user using the syntax
+         * gluster vol geo-rep <mastervol> root@<slavehost> ...
+         * pass down only <slavehost> else pass as it is.
+         */
+        slave_temp = gf_strdup(words[slavei]);
+        if (slave_temp == NULL) {
+            ret = -1;
+            goto out;
+        }
+        token = strtok_r(slave_temp, "@", &save_ptr);
+        if (token && !strcmp(token, "root")) {
+            ret = dict_set_str(dict, "slave", (char *)words[slavei] + 5);
+        } else {
+            ret = dict_set_str(dict, "slave", (char *)words[slavei]);
+        }
+    }
+    if (!ret)
+        ret = dict_set_int32(dict, "type", type);
+    if (!ret && type == GF_GSYNC_OPTION_TYPE_CONFIG) {
+        if (!strcmp((char *)words[wordcount - 2], "ignore-deletes") &&
+            !strcmp((char *)words[wordcount - 1], "true")) {
+            question =
+                "There exists ~15 seconds delay for the option to take"
+                " effect from stime of the corresponding brick. Please"
+                " check the log for the time, the option is effective."
+                " Proceed";
+
+            answer = cli_cmd_get_confirmation(state, question);
+
+            if (GF_ANSWER_NO == answer) {
+                gf_log("cli", GF_LOG_INFO,
+                       "Operation "
+                       "cancelled, exiting");
+                *errstr = gf_strdup("Aborted by user.");
+                ret = -1;
+                goto out;
+            }
+        }
 
-out:
-        if (ret) {
-                if (dict)
-                        dict_destroy (dict);
-        } else
-                *options = dict;
+        ret = config_parse(words, wordcount, dict, cmdi, glob);
+    }
 
+out:
+    if (slave_temp)
+        GF_FREE(slave_temp);
+    if (ret && dict)
+        dict_unref(dict);
+    else
+        *options = dict;
 
-        return ret;
+    return ret;
 }
 
 int32_t
-cli_cmd_volume_profile_parse (const char **words, int wordcount,
-                              dict_t **options)
+cli_cmd_volume_profile_parse(const char **words, int wordcount,
+                             dict_t **options)
 {
-        dict_t              *dict       = NULL;
-        char                *volname    = NULL;
-        int                 ret         = -1;
-        gf1_cli_stats_op    op          = GF_CLI_STATS_NONE;
-        gf1_cli_info_op     info_op     = GF_CLI_INFO_NONE;
-        gf_boolean_t        is_peek     = _gf_false;
+    dict_t *dict = NULL;
+    char *volname = NULL;
+    int ret = -1;
+    gf1_cli_stats_op op = GF_CLI_STATS_NONE;
+    gf1_cli_info_op info_op = GF_CLI_INFO_NONE;
+    gf_boolean_t is_peek = _gf_false;
 
-        char      *opwords[] = { "start", "stop", "info", NULL };
-        char      *w = NULL;
+    static char *opwords[] = {"start", "stop", "info", NULL};
+    char *w = NULL;
 
-        GF_ASSERT (words);
-        GF_ASSERT (options);
+    GF_ASSERT(words);
+    GF_ASSERT(options);
 
-        dict = dict_new ();
-        if (!dict)
-                goto out;
+    dict = dict_new();
+    if (!dict)
+        goto out;
 
-        if (wordcount < 4)
-                goto out;
+    if (wordcount < 4)
+        goto out;
 
-        volname = (char *)words[2];
+    volname = (char *)words[2];
 
-        ret = dict_set_str (dict, "volname", volname);
-        if (ret)
-                goto out;
-
-        w = str_getunamb (words[3], opwords);
-        if (!w) {
-                ret = -1;
-                goto out;
-        }
+    ret = dict_set_str(dict, "volname", volname);
+    if (ret)
+        goto out;
 
-        if ((strcmp (w, "start") == 0 || strcmp (w, "stop") == 0) &&
-            wordcount > 5)
-                goto out;
+    w = str_getunamb(words[3], opwords);
+    if (!w) {
+        ret = -1;
+        goto out;
+    }
 
-        if (strcmp (w, "info") == 0 && wordcount > 7)
-                goto out;
+    if ((strcmp(w, "start") == 0 || strcmp(w, "stop") == 0) && wordcount > 5) {
+        ret = -1;
+        goto out;
+    }
 
-        if (strcmp (w, "start") == 0) {
-                op = GF_CLI_STATS_START;
-        } else if (strcmp (w, "stop") == 0) {
-                op = GF_CLI_STATS_STOP;
-        } else if (strcmp (w, "info") == 0) {
-                op = GF_CLI_STATS_INFO;
-                info_op = GF_CLI_INFO_ALL;
-                if (wordcount > 4) {
-                        if (strcmp (words[4], "incremental") == 0) {
-                                info_op = GF_CLI_INFO_INCREMENTAL;
-                                if (wordcount > 5 &&
-                                    strcmp (words[5], "peek") == 0) {
-                                        is_peek = _gf_true;
-                                }
-                        } else if (strcmp (words[4], "cumulative") == 0) {
-                                info_op = GF_CLI_INFO_CUMULATIVE;
-                        } else if (strcmp (words[4], "clear") == 0) {
-                                info_op = GF_CLI_INFO_CLEAR;
-                        } else if (strcmp (words[4], "peek") == 0) {
-                                is_peek = _gf_true;
-                        }
-                }
-        } else
-                GF_ASSERT (!"opword mismatch");
+    if (strcmp(w, "info") == 0 && wordcount > 7) {
+        ret = -1;
+        goto out;
+    }
+
+    if (strcmp(w, "start") == 0) {
+        op = GF_CLI_STATS_START;
+    } else if (strcmp(w, "stop") == 0) {
+        op = GF_CLI_STATS_STOP;
+    } else if (strcmp(w, "info") == 0) {
+        op = GF_CLI_STATS_INFO;
+        info_op = GF_CLI_INFO_ALL;
+        if (wordcount > 4) {
+            if (strcmp(words[4], "incremental") == 0) {
+                info_op = GF_CLI_INFO_INCREMENTAL;
+                if (wordcount > 5 && strcmp(words[5], "peek") == 0) {
+                    is_peek = _gf_true;
+                }
+            } else if (strcmp(words[4], "cumulative") == 0) {
+                info_op = GF_CLI_INFO_CUMULATIVE;
+            } else if (strcmp(words[4], "clear") == 0) {
+                info_op = GF_CLI_INFO_CLEAR;
+            } else if (strcmp(words[4], "peek") == 0) {
+                is_peek = _gf_true;
+            }
+        }
+    } else
+        GF_ASSERT(!"opword mismatch");
+
+    ret = dict_set_int32(dict, "op", (int32_t)op);
+    if (ret)
+        goto out;
 
-        ret = dict_set_int32 (dict, "op", (int32_t)op);
-        if (ret)
-                goto out;
+    ret = dict_set_int32(dict, "info-op", (int32_t)info_op);
+    if (ret)
+        goto out;
 
-        ret = dict_set_int32 (dict, "info-op", (int32_t)info_op);
-        if (ret)
-                goto out;
+    ret = dict_set_int32(dict, "peek", is_peek);
+    if (ret)
+        goto out;
 
-        ret = dict_set_int32 (dict, "peek", is_peek);
+    if (!strcmp(words[wordcount - 1], "nfs")) {
+        ret = dict_set_int32(dict, "nfs", _gf_true);
         if (ret)
-                goto out;
+            goto out;
+    }
 
-        if (!strcmp (words[wordcount - 1], "nfs")) {
-                ret = dict_set_int32 (dict, "nfs", _gf_true);
-                if (ret)
-                        goto out;
-        }
-
-        *options = dict;
+    *options = dict;
 out:
-        if (ret && dict)
-                dict_destroy (dict);
-        return ret;
+    if (ret && dict)
+        dict_unref(dict);
+    return ret;
 }
 
 int32_t
-cli_cmd_volume_top_parse (const char **words, int wordcount,
-                              dict_t **options)
+cli_cmd_volume_top_parse(const char **words, int wordcount, dict_t **options)
 {
-        dict_t  *dict           = NULL;
-        char    *volname        = NULL;
-        char    *value          = NULL;
-        char    *key            = NULL;
-        int      ret            = -1;
-        gf1_cli_stats_op op = GF_CLI_STATS_NONE;
-        gf1_cli_top_op    top_op = GF_CLI_TOP_NONE;
-        int32_t  list_cnt       = -1;
-        int      index          = 0;
-        int      perf           = 0;
-        uint32_t  blk_size      = 0;
-        uint32_t  count         = 0;
-        gf_boolean_t nfs        = _gf_false;
-        char    *delimiter      = NULL;
-        char    *opwords[]      = { "open", "read", "write", "opendir",
-                                    "readdir", "read-perf", "write-perf",
-                                    "clear", NULL };
-        char    *w = NULL;
-
-        GF_ASSERT (words);
-        GF_ASSERT (options);
-
-        dict = dict_new ();
-        if (!dict)
-                goto out;
+    dict_t *dict = NULL;
+    char *volname = NULL;
+    char *value = NULL;
+    char *key = NULL;
+    int ret = -1;
+    gf1_cli_stats_op op = GF_CLI_STATS_NONE;
+    gf1_cli_top_op top_op = GF_CLI_TOP_NONE;
+    int32_t list_cnt = -1;
+    int index = 0;
+    int perf = 0;
+    int32_t blk_size = 0;
+    int count = 0;
+    gf_boolean_t nfs = _gf_false;
+    char *delimiter = NULL;
+    static char *opwords[] = {"open",       "read",    "write",
+                              "opendir",    "readdir", "read-perf",
+                              "write-perf", "clear",   NULL};
+    char *w = NULL;
+
+    GF_ASSERT(words);
+    GF_ASSERT(options);
+
+    dict = dict_new();
+    if (!dict)
+        goto out;
 
-        if (wordcount < 4)
-                goto out;
+    if (wordcount < 4)
+        goto out;
 
-        volname = (char *)words[2];
+    volname = (char *)words[2];
 
-        ret = dict_set_str (dict, "volname", volname);
-        if (ret)
-                goto out;
+    ret = dict_set_str(dict, "volname", volname);
+    if (ret)
+        goto out;
 
-        op = GF_CLI_STATS_TOP;
-        ret = dict_set_int32 (dict, "op", (int32_t)op);
-        if (ret)
-                goto out;
+    op = GF_CLI_STATS_TOP;
+    ret = dict_set_int32(dict, "op", (int32_t)op);
+    if (ret)
+        goto out;
 
-        w = str_getunamb (words[3], opwords);
-        if (!w) {
-                ret = -1;
-                goto out;
+    w = str_getunamb(words[3], opwords);
+    if (!w) {
+        ret = -1;
+        goto out;
+    }
+    if (strcmp(w, "open") == 0) {
+        top_op = GF_CLI_TOP_OPEN;
+    } else if (strcmp(w, "read") == 0) {
+        top_op = GF_CLI_TOP_READ;
+    } else if (strcmp(w, "write") == 0) {
+        top_op = GF_CLI_TOP_WRITE;
+    } else if (strcmp(w, "opendir") == 0) {
+        top_op = GF_CLI_TOP_OPENDIR;
+    } else if (strcmp(w, "readdir") == 0) {
+        top_op = GF_CLI_TOP_READDIR;
+    } else if (strcmp(w, "read-perf") == 0) {
+        top_op = GF_CLI_TOP_READ_PERF;
+        perf = 1;
+    } else if (strcmp(w, "write-perf") == 0) {
+        top_op = GF_CLI_TOP_WRITE_PERF;
+        perf = 1;
+    } else if (strcmp(w, "clear") == 0) {
+        ret = dict_set_int32(dict, "clear-stats", 1);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR, "Could not set clear-stats in dict");
+            goto out;
         }
-        if (strcmp (w, "open") == 0) {
-                top_op = GF_CLI_TOP_OPEN;
-        } else if (strcmp (w, "read") == 0) {
-                top_op = GF_CLI_TOP_READ;
-        } else if (strcmp (w, "write") == 0) {
-                top_op = GF_CLI_TOP_WRITE;
-        } else if (strcmp (w, "opendir") == 0) {
-                top_op = GF_CLI_TOP_OPENDIR;
-        } else if (strcmp (w, "readdir") == 0) {
-                top_op = GF_CLI_TOP_READDIR;
-        } else if (strcmp (w, "read-perf") == 0) {
-                top_op = GF_CLI_TOP_READ_PERF;
-                perf = 1;
-        } else if (strcmp (w, "write-perf") == 0) {
-                top_op = GF_CLI_TOP_WRITE_PERF;
-                perf = 1;
-        } else if (strcmp (w, "clear") == 0) {
-                ret = dict_set_int32 (dict, "clear-stats", 1);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Could not set clear-stats in dict");
-                        goto out;
-                }
-        } else
-                GF_ASSERT (!"opword mismatch");
-        ret = dict_set_int32 (dict, "top-op", (int32_t)top_op);
+    } else
+        GF_ASSERT(!"opword mismatch");
+    ret = dict_set_int32(dict, "top-op", (int32_t)top_op);
+    if (ret)
+        goto out;
+
+    if ((wordcount > 4) && !strcmp(words[4], "nfs")) {
+        nfs = _gf_true;
+        ret = dict_set_int32(dict, "nfs", nfs);
         if (ret)
+            goto out;
+        index = 5;
+    } else {
+        index = 4;
+    }
+
+    for (; index < wordcount; index += 2) {
+        key = (char *)words[index];
+        value = (char *)words[index + 1];
+
+        if (!key || !value) {
+            ret = -1;
+            goto out;
+        }
+        if (!strcmp(key, "brick")) {
+            delimiter = strchr(value, ':');
+            if (!delimiter || delimiter == value || *(delimiter + 1) != '/') {
+                cli_err(
+                    "wrong brick type: %s, use <HOSTNAME>:"
+                    "<export-dir-abs-path>",
+                    value);
+                ret = -1;
                 goto out;
-
-        if ((wordcount > 4) && !strcmp (words[4], "nfs")) {
-                nfs = _gf_true;
-                ret = dict_set_int32 (dict, "nfs", nfs);
+            } else {
+                ret = gf_canonicalize_path(delimiter + 1);
                 if (ret)
-                        goto out;
-                index = 5;
-        } else {
-                index = 4;
-        }
-
-        for (; index < wordcount; index+=2) {
-
-                key = (char *) words[index];
-                value = (char *) words[index+1];
-
-                if ( key && !value ) {
-                        ret = -1;
-                        goto out;
-                }
-                if (!strcmp (key, "brick")) {
-                        delimiter = strchr (value, ':');
-                        if (!delimiter || delimiter == value
-                            || *(delimiter+1) != '/') {
-                                cli_err ("wrong brick type: %s, use <HOSTNAME>:"
-                                         "<export-dir-abs-path>", value);
-                                ret = -1;
-                                goto out;
-                        } else {
-                                ret = gf_canonicalize_path (delimiter + 1);
-                                if (ret)
-                                        goto out;
-                        }
-                        ret = dict_set_str (dict, "brick", value);
-
-                } else if (!strcmp (key, "list-cnt")) {
-                        ret = gf_is_str_int (value);
-                        if (!ret)
-                                list_cnt = atoi (value);
-                        if (ret || (list_cnt < 0) || (list_cnt > 100)) {
-                                cli_err ("list-cnt should be between 0 to 100");
-                                ret = -1;
-                                goto out;
-                        }
-                } else if (perf && !nfs && !strcmp (key, "bs")) {
-                        ret = gf_is_str_int (value);
-                        if (!ret)
-                                blk_size = atoi (value);
-                        if (ret || (blk_size <= 0)) {
-                                if (blk_size < 0)
-                                        cli_err ("block size is an invalid"
-                                                 " number");
-                                else
-                                        cli_err ("block size should be an "
-                                                 "integer greater than zero");
-                                ret = -1;
-                                goto out;
-                        }
-                        ret = dict_set_uint32 (dict, "blk-size", blk_size);
-                } else if (perf && !nfs && !strcmp (key, "count")) {
-                        ret = gf_is_str_int (value);
-                        if (!ret)
-                                count = atoi(value);
-                        if (ret || (count <= 0)) {
-                                if (count < 0)
-                                        cli_err ("count is an invalid number");
-                                else
-                                        cli_err ("count should be an integer "
-                                                 "greater than zero");
-
-                                ret = -1;
-                                goto out;
-                        }
-                        ret = dict_set_uint32 (dict, "blk-cnt", count);
-                } else {
-                        ret = -1;
-                        goto out;
-                }
-                if (ret) {
-                        gf_log ("", GF_LOG_WARNING, "Dict set failed for "
-                                "key %s", key);
-                        goto out;
-                }
-        }
-        if (list_cnt == -1)
-                list_cnt = 100;
-        ret = dict_set_int32 (dict, "list-cnt", list_cnt);
-        if (ret) {
-                gf_log ("", GF_LOG_WARNING, "Dict set failed for list_cnt");
+                    goto out;
+            }
+            ret = dict_set_str(dict, "brick", value);
+
+        } else if (!strcmp(key, "list-cnt")) {
+            ret = gf_is_str_int(value);
+            if (!ret)
+                list_cnt = atoi(value);
+            if (ret || (list_cnt < 0) || (list_cnt > 100)) {
+                cli_err("list-cnt should be between 0 to 100");
+                ret = -1;
                 goto out;
-        }
-
-        if ((blk_size > 0) ^ (count > 0)) {
-                cli_err ("Need to give both 'bs' and 'count'");
+            }
+        } else if (perf && !nfs && !strcmp(key, "bs")) {
+            ret = gf_is_str_int(value);
+            if (!ret)
+                blk_size = atoi(value);
+            if (ret || (blk_size <= 0)) {
+                if (blk_size < 0)
+                    cli_err(
+                        "block size is an invalid"
+                        " number");
+                else
+                    cli_err(
+                        "block size should be an "
+                        "integer greater than zero");
                 ret = -1;
                 goto out;
-        } else if (((uint64_t)blk_size * count) > (10 * GF_UNIT_GB)) {
-                cli_err ("'bs * count' value %"PRIu64" is greater than "
-                         "maximum allowed value of 10GB",
-                         ((uint64_t)blk_size * count));
+            }
+            ret = dict_set_uint32(dict, "blk-size", (uint32_t)blk_size);
+        } else if (perf && !nfs && !strcmp(key, "count")) {
+            ret = gf_is_str_int(value);
+            if (!ret)
+                count = atoi(value);
+            if (ret || (count <= 0)) {
+                if (count < 0)
+                    cli_err("count is an invalid number");
+                else
+                    cli_err(
+                        "count should be an integer "
+                        "greater than zero");
+
                 ret = -1;
                 goto out;
+            }
+            ret = dict_set_uint32(dict, "blk-cnt", count);
+        } else {
+            ret = -1;
+            goto out;
         }
+        if (ret) {
+            gf_log("", GF_LOG_WARNING,
+                   "Dict set failed for "
+                   "key %s",
+                   key);
+            goto out;
+        }
+    }
+    if (list_cnt == -1)
+        list_cnt = 100;
+    ret = dict_set_int32(dict, "list-cnt", list_cnt);
+    if (ret) {
+        gf_log("", GF_LOG_WARNING, "Dict set failed for list_cnt");
+        goto out;
+    }
 
-        *options = dict;
+    if ((blk_size > 0) ^ (count > 0)) {
+        cli_err("Need to give both 'bs' and 'count'");
+        ret = -1;
+        goto out;
+    } else if (((uint64_t)blk_size * count) > (10 * GF_UNIT_GB)) {
+        cli_err("'bs * count' value %" PRIu64
+                " is greater than "
+                "maximum allowed value of 10GB",
+                ((uint64_t)blk_size * count));
+        ret = -1;
+        goto out;
+    }
+
+    *options = dict;
 out:
-        if (ret && dict)
-                dict_destroy (dict);
-        return ret;
+    if (ret && dict)
+        dict_unref(dict);
+    return ret;
 }
 
 uint32_t
-cli_cmd_get_statusop (const char *arg)
+cli_cmd_get_statusop(const char *arg)
 {
-        int        i         = 0;
-        uint32_t   ret       = GF_CLI_STATUS_NONE;
-        char      *w         = NULL;
-        char      *opwords[] = {"detail", "mem", "clients", "fd",
-                                "inode", "callpool", "tasks", NULL};
-        struct {
-                char      *opname;
-                uint32_t   opcode;
-        } optable[] = {
-                { "detail",   GF_CLI_STATUS_DETAIL   },
-                { "mem",      GF_CLI_STATUS_MEM      },
-                { "clients",  GF_CLI_STATUS_CLIENTS  },
-                { "fd",       GF_CLI_STATUS_FD       },
-                { "inode",    GF_CLI_STATUS_INODE    },
-                { "callpool", GF_CLI_STATUS_CALLPOOL },
-                { "tasks",    GF_CLI_STATUS_TASKS    },
-                { NULL }
-        };
-
-        w = str_getunamb (arg, opwords);
-        if (!w) {
-                gf_log ("cli", GF_LOG_DEBUG,
-                        "Not a status op  %s", arg);
-                goto out;
-        }
+    int i = 0;
+    uint32_t ret = GF_CLI_STATUS_NONE;
+    char *w = NULL;
+    static char *opwords[] = {"detail",   "mem",   "clients",     "fd", "inode",
+                              "callpool", "tasks", "client-list", NULL};
+    static struct {
+        char *opname;
+        uint32_t opcode;
+    } optable[] = {{"detail", GF_CLI_STATUS_DETAIL},
+                   {"mem", GF_CLI_STATUS_MEM},
+                   {"clients", GF_CLI_STATUS_CLIENTS},
+                   {"fd", GF_CLI_STATUS_FD},
+                   {"inode", GF_CLI_STATUS_INODE},
+                   {"callpool", GF_CLI_STATUS_CALLPOOL},
+                   {"tasks", GF_CLI_STATUS_TASKS},
+                   {"client-list", GF_CLI_STATUS_CLIENT_LIST},
+                   {NULL}};
+
+    w = str_getunamb(arg, opwords);
+    if (!w) {
+        gf_log("cli", GF_LOG_DEBUG, "Not a status op  %s", arg);
+        goto out;
+    }
 
-        for (i = 0; optable[i].opname; i++) {
-                if (!strcmp (w, optable[i].opname)) {
-                        ret = optable[i].opcode;
-                        break;
-                }
+    for (i = 0; optable[i].opname; i++) {
+        if (!strcmp(w, optable[i].opname)) {
+            ret = optable[i].opcode;
+            break;
         }
+    }
 
- out:
-        return ret;
+out:
+    return ret;
 }
 
 int
-cli_cmd_volume_status_parse (const char **words, int wordcount,
-                             dict_t **options)
+cli_cmd_volume_status_parse(const char **words, int wordcount, dict_t **options)
 {
-        dict_t    *dict            = NULL;
-        int        ret             = -1;
-        uint32_t   cmd             = 0;
+    dict_t *dict = NULL;
+    int ret = -1;
+    uint32_t cmd = 0;
 
-        GF_ASSERT (options);
+    GF_ASSERT(options);
 
-        dict = dict_new ();
-        if (!dict)
-                goto out;
-
-        switch (wordcount) {
+    dict = dict_new();
+    if (!dict)
+        goto out;
 
+    switch (wordcount) {
         case 2:
+            cmd = GF_CLI_STATUS_ALL;
+            ret = 0;
+            break;
+
+        case 3:
+            if (!strcmp(words[2], "all")) {
                 cmd = GF_CLI_STATUS_ALL;
                 ret = 0;
-                break;
 
-        case 3:
-                if (!strcmp (words[2], "all")) {
-                        cmd = GF_CLI_STATUS_ALL;
-                        ret = 0;
+            } else {
+                cmd = GF_CLI_STATUS_VOL;
+                ret = dict_set_str(dict, "volname", (char *)words[2]);
+            }
 
-                } else {
-                        cmd = GF_CLI_STATUS_VOL;
-                        ret = dict_set_str (dict, "volname", (char *)words[2]);
-                }
-
-                break;
+            break;
 
         case 4:
-                cmd = cli_cmd_get_statusop (words[3]);
-
-                if (!strcmp (words[2], "all")) {
-                        if (cmd == GF_CLI_STATUS_NONE) {
-                                cli_err ("%s is not a valid status option",
-                                         words[3]);
-                                ret = -1;
-                                goto out;
-                        }
-                        cmd |= GF_CLI_STATUS_ALL;
-                        ret  = 0;
-
-                } else {
-                        ret = dict_set_str (dict, "volname",
-                                            (char *)words[2]);
-                        if (ret)
-                                goto out;
-
-                        if (cmd == GF_CLI_STATUS_NONE) {
-                                if (!strcmp (words[3], "nfs")) {
-                                        cmd |= GF_CLI_STATUS_NFS;
-                                } else if (!strcmp (words[3], "shd")) {
-                                        cmd |= GF_CLI_STATUS_SHD;
-                                } else if (!strcmp (words[3], "quotad")) {
-                                        cmd |= GF_CLI_STATUS_QUOTAD;
-                                } else if (!strcmp (words[3], "snapd")) {
-                                        cmd |= GF_CLI_STATUS_SNAPD;
-                                } else {
-                                        cmd = GF_CLI_STATUS_BRICK;
-                                        ret = dict_set_str (dict, "brick",
-                                                            (char *)words[3]);
-                                }
-
-                        } else {
-                                cmd |= GF_CLI_STATUS_VOL;
-                                ret  = 0;
-                        }
-                }
-
-                break;
+            cmd = cli_cmd_get_statusop(words[3]);
 
-        case 5:
-                if (!strcmp (words[2], "all")) {
-                        cli_err ("Cannot specify brick/nfs for \"all\"");
-                        ret = -1;
-                        goto out;
-                }
-
-                cmd = cli_cmd_get_statusop (words[4]);
+            if (!strcmp(words[2], "all")) {
                 if (cmd == GF_CLI_STATUS_NONE) {
-                        cli_err ("%s is not a valid status option",
-                                 words[4]);
-                        ret = -1;
-                        goto out;
+                    cli_err("%s is not a valid status option", words[3]);
+                    ret = -1;
+                    goto out;
                 }
+                cmd |= GF_CLI_STATUS_ALL;
+                ret = 0;
 
-
-                ret = dict_set_str (dict, "volname", (char *)words[2]);
+            } else {
+                ret = dict_set_str(dict, "volname", (char *)words[2]);
                 if (ret)
-                        goto out;
+                    goto out;
 
-                if (!strcmp (words[3], "nfs")) {
-                        if (cmd == GF_CLI_STATUS_FD ||
-                            cmd == GF_CLI_STATUS_DETAIL ||
-                            cmd == GF_CLI_STATUS_TASKS) {
-                                cli_err ("Detail/FD/Tasks status not available"
-                                         " for NFS Servers");
-                                ret = -1;
-                                goto out;
-                        }
+                if (cmd == GF_CLI_STATUS_NONE) {
+                    if (!strcmp(words[3], "nfs")) {
                         cmd |= GF_CLI_STATUS_NFS;
-                } else if (!strcmp (words[3], "shd")){
-                        if (cmd == GF_CLI_STATUS_FD ||
-                            cmd == GF_CLI_STATUS_CLIENTS ||
-                            cmd == GF_CLI_STATUS_DETAIL ||
-                            cmd == GF_CLI_STATUS_TASKS) {
-                                cli_err ("Detail/FD/Clients/Tasks status not "
-                                         "available for Self-heal Daemons");
-                                ret = -1;
-                                goto out;
-                        }
+                    } else if (!strcmp(words[3], "shd")) {
                         cmd |= GF_CLI_STATUS_SHD;
-                } else if (!strcmp (words[3], "quotad")) {
-                        if (cmd == GF_CLI_STATUS_FD ||
-                            cmd == GF_CLI_STATUS_CLIENTS ||
-                            cmd == GF_CLI_STATUS_DETAIL ||
-                            cmd == GF_CLI_STATUS_INODE) {
-                                cli_err ("Detail/FD/Clients/Inode status not "
-                                         "available for Quota Daemon");
-                                ret = -1;
-                                goto out;
-                        }
+                    } else if (!strcmp(words[3], "quotad")) {
                         cmd |= GF_CLI_STATUS_QUOTAD;
-                } else if  (!strcmp (words[3], "snapd")) {
-                        if (cmd == GF_CLI_STATUS_FD ||
-                            cmd == GF_CLI_STATUS_CLIENTS ||
-                            cmd == GF_CLI_STATUS_DETAIL ||
-                            cmd == GF_CLI_STATUS_INODE) {
-                                cli_err ("Detail/FD/Clients/Inode status not "
-                                         "available for snap daemon");
-                                ret = -1;
-                                goto out;
-                        }
+                    } else if (!strcmp(words[3], "snapd")) {
                         cmd |= GF_CLI_STATUS_SNAPD;
+                    } else if (!strcmp(words[3], "bitd")) {
+                        cmd |= GF_CLI_STATUS_BITD;
+                    } else if (!strcmp(words[3], "scrub")) {
+                        cmd |= GF_CLI_STATUS_SCRUB;
+                    } else {
+                        cmd = GF_CLI_STATUS_BRICK;
+                        ret = dict_set_str(dict, "brick", (char *)words[3]);
+                    }
+
                 } else {
-                        if (cmd == GF_CLI_STATUS_TASKS) {
-                                cli_err ("Tasks status not available for "
-                                         "bricks");
-                                ret = -1;
-                                goto out;
-                        }
-                        cmd |= GF_CLI_STATUS_BRICK;
-                        ret = dict_set_str (dict, "brick", (char *)words[3]);
+                    cmd |= GF_CLI_STATUS_VOL;
+                    ret = 0;
                 }
-                break;
+            }
 
-        default:
-                goto out;
-        }
+            break;
 
-        if (ret)
+        case 5:
+            if (!strcmp(words[2], "all")) {
+                cli_err("Cannot specify brick/nfs for \"all\"");
+                ret = -1;
                 goto out;
+            }
 
-        ret = dict_set_int32 (dict, "cmd", cmd);
-        if (ret)
+            cmd = cli_cmd_get_statusop(words[4]);
+            if (cmd == GF_CLI_STATUS_NONE) {
+                cli_err("%s is not a valid status option", words[4]);
+                ret = -1;
                 goto out;
+            }
+
+            ret = dict_set_str(dict, "volname", (char *)words[2]);
+            if (ret)
+                goto out;
+
+            if (!strcmp(words[3], "nfs")) {
+                if (cmd == GF_CLI_STATUS_FD || cmd == GF_CLI_STATUS_DETAIL ||
+                    cmd == GF_CLI_STATUS_TASKS) {
+                    cli_err(
+                        "Detail/FD/Tasks status not available"
+                        " for NFS Servers");
+                    ret = -1;
+                    goto out;
+                }
+                cmd |= GF_CLI_STATUS_NFS;
+            } else if (!strcmp(words[3], "shd")) {
+                if (cmd == GF_CLI_STATUS_FD || cmd == GF_CLI_STATUS_CLIENTS ||
+                    cmd == GF_CLI_STATUS_DETAIL || cmd == GF_CLI_STATUS_TASKS) {
+                    cli_err(
+                        "Detail/FD/Clients/Tasks status not "
+                        "available for Self-heal Daemons");
+                    ret = -1;
+                    goto out;
+                }
+                cmd |= GF_CLI_STATUS_SHD;
+            } else if (!strcmp(words[3], "quotad")) {
+                if (cmd == GF_CLI_STATUS_FD || cmd == GF_CLI_STATUS_CLIENTS ||
+                    cmd == GF_CLI_STATUS_DETAIL || cmd == GF_CLI_STATUS_INODE) {
+                    cli_err(
+                        "Detail/FD/Clients/Inode status not "
+                        "available for Quota Daemon");
+                    ret = -1;
+                    goto out;
+                }
+                cmd |= GF_CLI_STATUS_QUOTAD;
+            } else if (!strcmp(words[3], "snapd")) {
+                if (cmd == GF_CLI_STATUS_FD || cmd == GF_CLI_STATUS_CLIENTS ||
+                    cmd == GF_CLI_STATUS_DETAIL || cmd == GF_CLI_STATUS_INODE) {
+                    cli_err(
+                        "Detail/FD/Clients/Inode status not "
+                        "available for snap daemon");
+                    ret = -1;
+                    goto out;
+                }
+                cmd |= GF_CLI_STATUS_SNAPD;
+            } else {
+                if (cmd == GF_CLI_STATUS_TASKS) {
+                    cli_err(
+                        "Tasks status not available for "
+                        "bricks");
+                    ret = -1;
+                    goto out;
+                }
+                cmd |= GF_CLI_STATUS_BRICK;
+                ret = dict_set_str(dict, "brick", (char *)words[3]);
+            }
+            break;
 
-        *options = dict;
+        default:
+            goto out;
+    }
+
+    if (ret)
+        goto out;
+
+    ret = dict_set_int32(dict, "cmd", cmd);
+    if (ret)
+        goto out;
 
- out:
-        if (ret && dict)
-                dict_destroy (dict);
+    *options = dict;
 
-        return ret;
+out:
+    if (ret && dict)
+        dict_unref(dict);
+
+    return ret;
 }
 
 gf_boolean_t
-cli_cmd_validate_dumpoption (const char *arg, char **option)
+cli_cmd_validate_dumpoption(const char *arg, char **option)
 {
-        char    *opwords[] = {"all", "nfs", "mem", "iobuf", "callpool", "priv",
-                              "fd", "inode", "history", "inodectx", "fdctx",
-                              "quotad", NULL};
-        char    *w = NULL;
-
-        w = str_getunamb (arg, opwords);
-        if (!w) {
-                gf_log ("cli", GF_LOG_DEBUG, "Unknown statedump option  %s",
-                        arg);
-                return _gf_false;
-        }
-        *option = w;
-        return _gf_true;
+    static char *opwords[] = {"all",   "nfs",    "mem",   "iobuf",   "callpool",
+                              "priv",  "fd",     "inode", "history", "inodectx",
+                              "fdctx", "quotad", NULL};
+    char *w = NULL;
+
+    w = str_getunamb(arg, opwords);
+    if (!w) {
+        gf_log("cli", GF_LOG_DEBUG, "Unknown statedump option  %s", arg);
+        return _gf_false;
+    }
+    *option = w;
+    return _gf_true;
 }
 
 int
-cli_cmd_volume_statedump_options_parse (const char **words, int wordcount,
-                                        dict_t **options)
+cli_cmd_volume_statedump_options_parse(const char **words, int wordcount,
+                                       dict_t **options)
 {
-        int     ret = 0;
-        int     i = 0;
-        dict_t  *dict = NULL;
-        int     option_cnt = 0;
-        char    *option = NULL;
-        char    option_str[100] = {0,};
-
-        for (i = 3; i < wordcount; i++, option_cnt++) {
-                if (!cli_cmd_validate_dumpoption (words[i], &option)) {
-                        ret = -1;
-                        goto out;
-                }
-                strncat (option_str, option, strlen (option));
-                strncat (option_str, " ", 1);
+    int ret = 0;
+    int i = 0;
+    dict_t *dict = NULL;
+    int option_cnt = 0;
+    char *option = NULL;
+    char *option_str = NULL;
+    char *tmp_str = NULL;
+    char *tmp = NULL;
+    char *ip_addr = NULL;
+    char *pid = NULL;
+
+    if ((wordcount >= 5) && ((strcmp(words[3], "client")) == 0)) {
+        tmp = gf_strdup(words[4]);
+        if (!tmp) {
+            ret = -1;
+            goto out;
+        }
+        ip_addr = strtok(tmp, ":");
+        pid = strtok(NULL, ":");
+        if (valid_internet_address(ip_addr, _gf_true, _gf_false) && pid &&
+            gf_valid_pid(pid, strlen(pid))) {
+            ret = gf_asprintf(&option_str, "%s %s %s", words[3], ip_addr, pid);
+            if (ret < 0) {
+                goto out;
+            }
+            option_cnt = 3;
+        } else {
+            ret = -1;
+            goto out;
         }
-        if((strstr (option_str, "nfs")) && strstr (option_str, "quotad")) {
+    } else {
+        for (i = 3; i < wordcount; i++, option_cnt++) {
+            if (!cli_cmd_validate_dumpoption(words[i], &option)) {
                 ret = -1;
                 goto out;
+            }
+            tmp_str = option_str;
+            option_str = NULL;
+            ret = gf_asprintf(&option_str, "%s%s ", tmp_str ? tmp_str : "",
+                              option);
+            GF_FREE(tmp_str);
+            if (ret < 0) {
+                goto out;
+            }
         }
+        if (option_str && (strstr(option_str, "nfs")) &&
+            strstr(option_str, "quotad")) {
+            ret = -1;
+            goto out;
+        }
+    }
 
-        dict = dict_new ();
-        if (!dict)
-                goto out;
+    dict = dict_new();
+    if (!dict) {
+        ret = -1;
+        goto out;
+    }
 
-        ret = dict_set_dynstr (dict, "options", gf_strdup (option_str));
-        if (ret)
-                goto out;
+    /* dynamic string in dict is freed up when dict is freed up, and hence
+    if option_str is NULL pass in an duplicate empty string to the same */
+    ret = dict_set_dynstr(dict, "options",
+                          (option_str ? option_str : gf_strdup("")));
+    if (ret)
+        goto out;
+    option_str = NULL;
 
-        ret = dict_set_int32 (dict, "option_cnt", option_cnt);
-        if (ret)
-                goto out;
+    ret = dict_set_int32(dict, "option_cnt", option_cnt);
+    if (ret)
+        goto out;
 
-        *options = dict;
+    *options = dict;
 out:
-        if (ret && dict)
-                dict_destroy (dict);
-        if (ret)
-                gf_log ("cli", GF_LOG_ERROR, "Error parsing dumpoptions");
-        return ret;
+    GF_FREE(tmp);
+    GF_FREE(option_str);
+    if (ret && dict)
+        dict_unref(dict);
+    if (ret)
+        gf_log("cli", GF_LOG_ERROR, "Error parsing dumpoptions");
+    return ret;
 }
 
 int
-cli_cmd_volume_clrlks_opts_parse (const char **words, int wordcount,
-                                  dict_t **options)
+cli_cmd_volume_clrlks_opts_parse(const char **words, int wordcount,
+                                 dict_t **options)
 {
-        int     ret = -1;
-        int     i = 0;
-        dict_t  *dict = NULL;
-        char    *kind_opts[4] = {"blocked", "granted", "all", NULL};
-        char    *types[4] = {"inode", "entry", "posix", NULL};
-        char    *free_ptr = NULL;
-
-        dict = dict_new ();
-        if (!dict)
-                goto out;
+    int ret = -1;
+    int i = 0;
+    dict_t *dict = NULL;
+    char *kind_opts[4] = {"blocked", "granted", "all", NULL};
+    char *types[4] = {"inode", "entry", "posix", NULL};
+    char *free_ptr = NULL;
+
+    dict = dict_new();
+    if (!dict)
+        goto out;
 
-        if (strcmp (words[4], "kind"))
-                goto out;
+    if (strcmp(words[4], "kind"))
+        goto out;
 
-        for (i = 0; kind_opts[i]; i++) {
-               if (!strcmp (words[5], kind_opts[i])) {
-                       free_ptr = gf_strdup (words[5]);
-                       ret = dict_set_dynstr (dict, "kind", free_ptr);
-                       if (ret)
-                               goto out;
-                       free_ptr = NULL;
-                       break;
-               }
-        }
-        if (i == 3)
+    for (i = 0; kind_opts[i]; i++) {
+        if (!strcmp(words[5], kind_opts[i])) {
+            free_ptr = gf_strdup(words[5]);
+            ret = dict_set_dynstr(dict, "kind", free_ptr);
+            if (ret)
                 goto out;
+            free_ptr = NULL;
+            break;
+        }
+    }
+    if (i == 3)
+        goto out;
 
-        ret = -1;
-        for (i = 0; types[i]; i++) {
-               if (!strcmp (words[6], types[i])) {
-                       free_ptr = gf_strdup (words[6]);
-                       ret = dict_set_dynstr (dict, "type", free_ptr);
-                       if (ret)
-                               goto out;
-                       free_ptr = NULL;
-                       break;
-               }
-        }
-        if (i == 3)
+    ret = -1;
+    for (i = 0; types[i]; i++) {
+        if (!strcmp(words[6], types[i])) {
+            free_ptr = gf_strdup(words[6]);
+            ret = dict_set_dynstr(dict, "type", free_ptr);
+            if (ret)
                 goto out;
-
-        if (wordcount == 8) {
-                free_ptr = gf_strdup (words[7]);
-                ret = dict_set_dynstr (dict, "opts", free_ptr);
-                if (ret)
-                        goto out;
-                free_ptr = NULL;
+            free_ptr = NULL;
+            break;
         }
+    }
+    if (i == 3)
+        goto out;
 
-        ret = 0;
-        *options = dict;
+    if (wordcount == 8) {
+        free_ptr = gf_strdup(words[7]);
+        ret = dict_set_dynstr(dict, "opts", free_ptr);
+        if (ret)
+            goto out;
+        free_ptr = NULL;
+    }
+
+    ret = 0;
+    *options = dict;
 out:
-       if (ret) {
-               GF_FREE (free_ptr);
-               dict_unref (dict);
-       }
+    if (ret) {
+        GF_FREE(free_ptr);
+        dict_unref(dict);
+    }
 
-       return ret;
+    return ret;
 }
 
 static int
-extract_hostname_path_from_token (const char *tmp_words, char **hostname,
-                                  char **path)
+extract_hostname_path_from_token(const char *tmp_words, char **hostname,
+                                 char **path)
 {
-        int ret = 0;
-        char *delimiter = NULL;
-        char *tmp_host = NULL;
-        char *host_name = NULL;
-        char *words = NULL;
+    int ret = 0;
+    char *delimiter = NULL;
+    char *tmp_host = NULL;
+    char *host_name = NULL;
+    char *words = NULL;
+    int str_len = 0;
+    *hostname = NULL;
+    *path = NULL;
+
+    str_len = strlen(tmp_words) + 1;
+    words = GF_MALLOC(str_len, gf_common_mt_char);
+    if (!words) {
+        ret = -1;
+        goto out;
+    }
 
-        *hostname = NULL;
-        *path = NULL;
+    snprintf(words, str_len, "%s", tmp_words);
 
-        words = GF_CALLOC (1, strlen (tmp_words) + 1, gf_common_mt_char);
-        if (!words){
+    if (validate_brick_name(words)) {
+        cli_err(
+            "Wrong brick type: %s, use <HOSTNAME>:"
+            "<export-dir-abs-path>",
+            words);
+        ret = -1;
+        goto out;
+    } else {
+        delimiter = strrchr(words, ':');
+        ret = gf_canonicalize_path(delimiter + 1);
+        if (ret) {
+            goto out;
+        } else {
+            str_len = strlen(delimiter + 1) + 1;
+            *path = GF_MALLOC(str_len, gf_common_mt_char);
+            if (!*path) {
                 ret = -1;
                 goto out;
+            }
+            snprintf(*path, str_len, "%s", delimiter + 1);
         }
+    }
 
-        strncpy (words, tmp_words, strlen (tmp_words) + 1);
+    tmp_host = gf_strdup(words);
+    if (!tmp_host) {
+        gf_log("cli", GF_LOG_ERROR, "Out of memory");
+        ret = -1;
+        goto out;
+    }
+    get_host_name(tmp_host, &host_name);
+    if (!host_name) {
+        ret = -1;
+        gf_log("cli", GF_LOG_ERROR,
+               "Unable to allocate "
+               "memory");
+        goto out;
+    }
+    if (!(strcmp(host_name, "localhost") && strcmp(host_name, "127.0.0.1") &&
+          strncmp(host_name, "0.", 2))) {
+        cli_err(
+            "Please provide a valid hostname/ip other "
+            "than localhost, 127.0.0.1 or loopback "
+            "address (0.0.0.0 to 0.255.255.255).");
+        ret = -1;
+        goto out;
+    }
+    if (!valid_internet_address(host_name, _gf_false, _gf_false)) {
+        cli_err(
+            "internet address '%s' does not conform to "
+            "standards",
+            host_name);
+        ret = -1;
+        goto out;
+    }
 
-        if (validate_brick_name (words)) {
-                cli_err ("Wrong brick type: %s, use <HOSTNAME>:"
-                        "<export-dir-abs-path>", words);
-                ret = -1;
-                goto out;
-        } else {
-                delimiter = strrchr (words, ':');
-                ret = gf_canonicalize_path (delimiter + 1);
-                if (ret) {
-                        goto out;
-                } else {
-                        *path = GF_CALLOC (1, strlen (delimiter+1) +1,
-                                           gf_common_mt_char);
-                        if (!*path) {
-                           ret = -1;
-                                goto out;
+    str_len = strlen(host_name) + 1;
+    *hostname = GF_MALLOC(str_len, gf_common_mt_char);
+    if (!*hostname) {
+        ret = -1;
+        goto out;
+    }
+    snprintf(*hostname, str_len, "%s", host_name);
+    ret = 0;
 
-                        }
-                        strncpy (*path, delimiter +1,
-                                 strlen(delimiter + 1) + 1);
-                }
-        }
+out:
+    GF_FREE(words);
+    GF_FREE(tmp_host);
+    return ret;
+}
 
-        tmp_host = gf_strdup (words);
-        if (!tmp_host) {
-                gf_log ("cli", GF_LOG_ERROR, "Out of memory");
-                ret = -1;
-                goto out;
-        }
-        get_host_name (tmp_host, &host_name);
-        if (!host_name) {
-                ret = -1;
-                gf_log("cli",GF_LOG_ERROR, "Unable to allocate "
-                        "memory");
-                goto out;
-        }
-        if (!(strcmp (host_name, "localhost") &&
-            strcmp (host_name, "127.0.0.1") &&
-            strncmp (host_name, "0.", 2))) {
-                cli_err ("Please provide a valid hostname/ip other "
-                         "than localhost, 127.0.0.1 or loopback "
-                         "address (0.0.0.0 to 0.255.255.255).");
-                ret = -1;
-                goto out;
-        }
-        if (!valid_internet_address (host_name, _gf_false)) {
-                cli_err ("internet address '%s' does not conform to "
-                          "standards", host_name);
-                ret = -1;
-                goto out;
-        }
+static int
+set_hostname_path_in_dict(const char *token, dict_t *dict, int heal_op)
+{
+    char *hostname = NULL;
+    char *path = NULL;
+    int ret = 0;
 
-        *hostname = GF_CALLOC (1, strlen (host_name) + 1,
-                                       gf_common_mt_char);
-        if (!*hostname) {
-                ret = -1;
-                goto out;
-        }
-        strncpy (*hostname, host_name, strlen (host_name) + 1);
-        ret = 0;
+    ret = extract_hostname_path_from_token(token, &hostname, &path);
+    if (ret)
+        goto out;
+
+    switch (heal_op) {
+        case GF_SHD_OP_SBRAIN_HEAL_FROM_BRICK:
+            ret = dict_set_dynstr(dict, "heal-source-hostname", hostname);
+            if (ret)
+                goto out;
+            hostname = NULL;
+            ret = dict_set_dynstr(dict, "heal-source-brickpath", path);
+            if (ret) {
+                goto out;
+            }
+            path = NULL;
+            break;
+        case GF_SHD_OP_STATISTICS_HEAL_COUNT_PER_REPLICA:
+            ret = dict_set_dynstr(dict, "per-replica-cmd-hostname", hostname);
+            if (ret)
+                goto out;
+            hostname = NULL;
+            ret = dict_set_dynstr(dict, "per-replica-cmd-path", path);
+            if (ret) {
+                goto out;
+            }
+            path = NULL;
+            break;
+        default:
+            ret = -1;
+            break;
+    }
 
 out:
-        GF_FREE (words);
-        GF_FREE (tmp_host);
-        return ret;
+    GF_FREE(hostname);
+    GF_FREE(path);
+    return ret;
 }
 
+static int
+heal_command_type_get(const char *command)
+{
+    int i = 0;
+    /* subcommands are set as NULL */
+    char *heal_cmds[GF_SHD_OP_HEAL_DISABLE + 1] = {
+        [GF_SHD_OP_INVALID] = NULL,
+        [GF_SHD_OP_HEAL_INDEX] = NULL,
+        [GF_SHD_OP_HEAL_FULL] = "full",
+        [GF_SHD_OP_INDEX_SUMMARY] = "info",
+        [GF_SHD_OP_SPLIT_BRAIN_FILES] = NULL,
+        [GF_SHD_OP_STATISTICS] = "statistics",
+        [GF_SHD_OP_STATISTICS_HEAL_COUNT] = NULL,
+        [GF_SHD_OP_STATISTICS_HEAL_COUNT_PER_REPLICA] = NULL,
+        [GF_SHD_OP_SBRAIN_HEAL_FROM_BIGGER_FILE] = NULL,
+        [GF_SHD_OP_SBRAIN_HEAL_FROM_BRICK] = NULL,
+        [GF_SHD_OP_HEAL_ENABLE] = "enable",
+        [GF_SHD_OP_HEAL_DISABLE] = "disable",
+    };
+
+    for (i = 0; i <= GF_SHD_OP_HEAL_DISABLE; i++) {
+        if (heal_cmds[i] && (strcmp(heal_cmds[i], command) == 0))
+            return i;
+    }
+
+    return GF_SHD_OP_INVALID;
+}
 
 int
-cli_cmd_volume_heal_options_parse (const char **words, int wordcount,
-                                   dict_t **options)
+cli_cmd_volume_heal_options_parse(const char **words, int wordcount,
+                                  dict_t **options)
 {
-        int     ret = 0;
-        dict_t  *dict = NULL;
-        char    *hostname = NULL;
-        char    *path = NULL;
+    int ret = 0;
+    dict_t *dict = NULL;
+    gf_xl_afr_op_t op = GF_SHD_OP_INVALID;
 
-        dict = dict_new ();
-        if (!dict)
-                goto out;
+    dict = dict_new();
+    if (!dict) {
+        gf_log(THIS->name, GF_LOG_ERROR, "Failed to create the dict");
+        ret = -1;
+        goto out;
+    }
 
-        ret = dict_set_str (dict, "volname", (char *) words[2]);
-        if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "failed to set volname");
-                goto out;
+    ret = dict_set_str(dict, "volname", (char *)words[2]);
+    if (ret) {
+        gf_log(THIS->name, GF_LOG_ERROR, "failed to set volname");
+        goto out;
+    }
+
+    if (wordcount == 3) {
+        ret = dict_set_int32(dict, "heal-op", GF_SHD_OP_HEAL_INDEX);
+        goto done;
+    }
+
+    if (wordcount == 4) {
+        op = heal_command_type_get(words[3]);
+        if (op == GF_SHD_OP_INVALID) {
+            ret = -1;
+            goto out;
+        }
+
+        ret = dict_set_int32(dict, "heal-op", op);
+        goto done;
+    }
+
+    if (wordcount == 5) {
+        if (strcmp(words[3], "info") && strcmp(words[3], "statistics") &&
+            strcmp(words[3], "granular-entry-heal")) {
+            ret = -1;
+            goto out;
         }
 
-        if (wordcount == 3) {
-                ret = dict_set_int32 (dict, "heal-op", GF_AFR_OP_HEAL_INDEX);
+        if (!strcmp(words[3], "info")) {
+            if (!strcmp(words[4], "split-brain")) {
+                ret = dict_set_int32(dict, "heal-op",
+                                     GF_SHD_OP_SPLIT_BRAIN_FILES);
                 goto done;
+            }
+            if (!strcmp(words[4], "summary")) {
+                ret = dict_set_int32(dict, "heal-op", GF_SHD_OP_HEAL_SUMMARY);
+                goto done;
+            }
         }
 
-        if (wordcount == 4) {
-                if (!strcmp (words[3], "full")) {
-                        ret = dict_set_int32 (dict, "heal-op",
-                                              GF_AFR_OP_HEAL_FULL);
-                        goto done;
-                } else if (!strcmp (words[3], "statistics")) {
-                        ret = dict_set_int32 (dict, "heal-op",
-                                              GF_AFR_OP_STATISTICS);
-                        goto done;
-
-                } else if (!strcmp (words[3], "info")) {
-                        ret = dict_set_int32 (dict, "heal-op",
-                                              GF_AFR_OP_INDEX_SUMMARY);
-                        goto done;
-                } else {
-                        ret = -1;
-                        goto out;
-                }
+        if (!strcmp(words[3], "statistics")) {
+            if (!strcmp(words[4], "heal-count")) {
+                ret = dict_set_int32(dict, "heal-op",
+                                     GF_SHD_OP_STATISTICS_HEAL_COUNT);
+                goto done;
+            }
         }
-        if (wordcount == 5) {
-                if (strcmp (words[3], "info") &&
-                    strcmp (words[3], "statistics")) {
-                        ret = -1;
-                        goto out;
-                }
 
-                if (!strcmp (words[3], "info")) {
-                        if (!strcmp (words[4], "healed")) {
-                                ret = dict_set_int32 (dict, "heal-op",
-                                                      GF_AFR_OP_HEALED_FILES);
-                                goto done;
-                        }
-                        if (!strcmp (words[4], "heal-failed")) {
-                                ret = dict_set_int32 (dict, "heal-op",
-                                                   GF_AFR_OP_HEAL_FAILED_FILES);
-                                goto done;
-                        }
-                        if (!strcmp (words[4], "split-brain")) {
-                                ret = dict_set_int32 (dict, "heal-op",
-                                                   GF_AFR_OP_SPLIT_BRAIN_FILES);
-                                goto done;
-                        }
-                }
+        if (!strcmp(words[3], "granular-entry-heal")) {
+            if (!strcmp(words[4], "enable")) {
+                ret = dict_set_int32(dict, "heal-op",
+                                     GF_SHD_OP_GRANULAR_ENTRY_HEAL_ENABLE);
+                goto done;
+            } else if (!strcmp(words[4], "disable")) {
+                ret = dict_set_int32(dict, "heal-op",
+                                     GF_SHD_OP_GRANULAR_ENTRY_HEAL_DISABLE);
+                goto done;
+            }
+        }
 
-                if (!strcmp (words[3], "statistics")) {
-                        if (!strcmp (words[4], "heal-count")) {
-                                ret = dict_set_int32 (dict, "heal-op",
-                                               GF_AFR_OP_STATISTICS_HEAL_COUNT);
-                                goto done;
-                        }
-                }
-                ret = -1;
+        ret = -1;
+        goto out;
+    }
+    if (wordcount == 6) {
+        if (strcmp(words[3], "split-brain")) {
+            ret = -1;
+            goto out;
+        }
+        if (!strcmp(words[4], "bigger-file")) {
+            ret = dict_set_int32(dict, "heal-op",
+                                 GF_SHD_OP_SBRAIN_HEAL_FROM_BIGGER_FILE);
+            if (ret)
+                goto out;
+            ret = dict_set_str(dict, "file", (char *)words[5]);
+            if (ret)
                 goto out;
+            goto done;
         }
-        if (wordcount == 7) {
-                if (!strcmp (words[3], "statistics")
-                    && !strcmp (words[4], "heal-count")
-                    && !strcmp (words[5], "replica")) {
-
-                        ret = dict_set_int32 (dict, "heal-op",
-                                   GF_AFR_OP_STATISTICS_HEAL_COUNT_PER_REPLICA);
-                        if (ret)
-                                goto out;
-                        ret = extract_hostname_path_from_token (words[6],
-                                                              &hostname, &path);
-                        if (ret)
-                                goto out;
-                        ret = dict_set_dynstr (dict, "per-replica-cmd-hostname",
-                                               hostname);
-                        if (ret)
-                                goto out;
-                        ret = dict_set_dynstr (dict, "per-replica-cmd-path",
-                                               path);
-                        if (ret)
-                                goto out;
-                        else
-                                goto done;
-
-                }
+        if (!strcmp(words[4], "latest-mtime")) {
+            ret = dict_set_int32(dict, "heal-op",
+                                 GF_SHD_OP_SBRAIN_HEAL_FROM_LATEST_MTIME);
+            if (ret)
+                goto out;
+            ret = dict_set_str(dict, "file", (char *)words[5]);
+            if (ret)
+                goto out;
+            goto done;
+        }
+        if (!strcmp(words[4], "source-brick")) {
+            ret = dict_set_int32(dict, "heal-op",
+                                 GF_SHD_OP_SBRAIN_HEAL_FROM_BRICK);
+            if (ret)
+                goto out;
+            ret = set_hostname_path_in_dict(words[5], dict,
+                                            GF_SHD_OP_SBRAIN_HEAL_FROM_BRICK);
+            if (ret)
+                goto out;
+            goto done;
         }
         ret = -1;
         goto out;
+    }
+    if (wordcount == 7) {
+        if (!strcmp(words[3], "statistics") &&
+            !strcmp(words[4], "heal-count") && !strcmp(words[5], "replica")) {
+            ret = dict_set_int32(dict, "heal-op",
+                                 GF_SHD_OP_STATISTICS_HEAL_COUNT_PER_REPLICA);
+            if (ret)
+                goto out;
+            ret = set_hostname_path_in_dict(
+                words[6], dict, GF_SHD_OP_STATISTICS_HEAL_COUNT_PER_REPLICA);
+            if (ret)
+                goto out;
+            goto done;
+        }
+        if (!strcmp(words[3], "split-brain") &&
+            !strcmp(words[4], "source-brick")) {
+            ret = dict_set_int32(dict, "heal-op",
+                                 GF_SHD_OP_SBRAIN_HEAL_FROM_BRICK);
+            ret = set_hostname_path_in_dict(words[5], dict,
+                                            GF_SHD_OP_SBRAIN_HEAL_FROM_BRICK);
+            if (ret)
+                goto out;
+            ret = dict_set_str(dict, "file", (char *)words[6]);
+            if (ret)
+                goto out;
+            goto done;
+        }
+    }
+    ret = -1;
+    goto out;
 done:
-        *options = dict;
+    *options = dict;
 out:
-        if (ret && dict) {
-                dict_unref (dict);
-                *options = NULL;
-        }
+    if (ret && dict) {
+        dict_unref(dict);
+        *options = NULL;
+    }
 
-        return ret;
+    return ret;
 }
 
 int
-cli_cmd_volume_defrag_parse (const char **words, int wordcount,
-                             dict_t **options)
+cli_cmd_volume_defrag_parse(const char **words, int wordcount, dict_t **options)
 {
-        dict_t                 *dict = NULL;
-        int                      ret = -1;
-        char                *option  = NULL;
-        char                *volname = NULL;
-        char                *command = NULL;
-        gf_cli_defrag_type       cmd = 0;
-
-        GF_ASSERT (words);
-        GF_ASSERT (options);
-
-        dict = dict_new ();
-        if (!dict)
-                goto out;
-
-        if (!((wordcount == 4) || (wordcount == 5)))
-                goto out;
-
-        if (wordcount == 4) {
-                if (strcmp (words[3], "start") && strcmp (words[3], "stop") &&
-                    strcmp (words[3], "status"))
-                            goto out;
-        } else {
-                if (strcmp (words[3], "fix-layout") &&
-                    strcmp (words[3], "start"))
-                        goto out;
-        }
-
-        volname = (char *) words[2];
-
-        if (wordcount == 4) {
-                command = (char *) words[3];
-        }
-        if (wordcount == 5) {
-               if ((strcmp (words[3], "fix-layout") ||
-                    strcmp (words[4], "start")) &&
-                    (strcmp (words[3], "start") ||
-                    strcmp (words[4], "force"))) {
-                        ret = -1;
-                        goto out;
-                }
-                command = (char *) words[3];
-                option = (char *) words[4];
-        }
+    dict_t *dict = NULL;
+    int ret = -1;
+    char *option = NULL;
+    char *volname = NULL;
+    char *command = NULL;
+    gf_cli_defrag_type cmd = 0;
+
+    GF_ASSERT(words);
+    GF_ASSERT(options);
+
+    dict = dict_new();
+    if (!dict)
+        goto out;
 
-        if (strcmp (command, "start") == 0) {
-                cmd = GF_DEFRAG_CMD_START;
-                if (option && strcmp (option, "force") == 0) {
-                                cmd = GF_DEFRAG_CMD_START_FORCE;
-                        }
-                goto done;
-        }
+    if (!((wordcount == 4) || (wordcount == 5)))
+        goto out;
 
-        if (strcmp (command, "fix-layout") == 0) {
-                cmd = GF_DEFRAG_CMD_START_LAYOUT_FIX;
-                goto done;
-        }
-        if (strcmp (command, "stop") == 0) {
-                cmd = GF_DEFRAG_CMD_STOP;
-                goto done;
-        }
-        if (strcmp (command, "status") == 0) {
-                cmd = GF_DEFRAG_CMD_STATUS;
-        }
+    if (wordcount == 4) {
+        if (strcmp(words[3], "start") && strcmp(words[3], "stop") &&
+            strcmp(words[3], "status"))
+            goto out;
+    } else {
+        if (strcmp(words[3], "fix-layout") && strcmp(words[3], "start"))
+            goto out;
+    }
+
+    volname = (char *)words[2];
+
+    if (wordcount == 4) {
+        command = (char *)words[3];
+    }
+    if (wordcount == 5) {
+        if ((strcmp(words[3], "fix-layout") || strcmp(words[4], "start")) &&
+            (strcmp(words[3], "start") || strcmp(words[4], "force"))) {
+            ret = -1;
+            goto out;
+        }
+        command = (char *)words[3];
+        option = (char *)words[4];
+    }
+
+    if (strcmp(command, "start") == 0) {
+        cmd = GF_DEFRAG_CMD_START;
+        if (option && strcmp(option, "force") == 0) {
+            cmd = GF_DEFRAG_CMD_START_FORCE;
+        }
+        goto done;
+    }
+
+    if (strcmp(command, "fix-layout") == 0) {
+        cmd = GF_DEFRAG_CMD_START_LAYOUT_FIX;
+        goto done;
+    }
+    if (strcmp(command, "stop") == 0) {
+        cmd = GF_DEFRAG_CMD_STOP;
+        goto done;
+    }
+    if (strcmp(command, "status") == 0) {
+        cmd = GF_DEFRAG_CMD_STATUS;
+    }
 
 done:
-        ret = dict_set_str (dict, "volname", volname);
+    ret = dict_set_str(dict, "volname", volname);
 
-        if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "failed to set dict");
-                goto out;
-        }
+    if (ret) {
+        gf_log(THIS->name, GF_LOG_ERROR, "failed to set dict");
+        goto out;
+    }
 
-        ret = dict_set_int32 (dict, "rebalance-command", (int32_t) cmd);
+    ret = dict_set_int32(dict, "rebalance-command", (int32_t)cmd);
 
-        if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "failed to set dict");
-                goto out;
-        }
+    if (ret) {
+        gf_log(THIS->name, GF_LOG_ERROR, "failed to set dict");
+        goto out;
+    }
 
-        *options = dict;
+    *options = dict;
 
 out:
-        if (ret && dict)
-                dict_destroy (dict);
+    if (ret && dict)
+        dict_unref(dict);
 
-        return ret;
+    return ret;
 }
 
 int32_t
-cli_snap_create_desc_parse (dict_t *dict, const char **words,
-                            size_t wordcount, int32_t desc_opt_loc)
+cli_snap_create_desc_parse(dict_t *dict, const char **words, size_t wordcount,
+                           int32_t desc_opt_loc)
 {
-        int32_t        ret      = -1;
-        char          *desc     = NULL;
-        int32_t        desc_len = 0;
+    int32_t ret = -1;
+    char *desc = NULL;
+    int32_t desc_len = 0;
+    int len;
 
-        desc = GF_CALLOC (MAX_SNAP_DESCRIPTION_LEN + 1, sizeof(char),
-                          gf_common_mt_char);
-        if (!desc) {
-                ret = -1;
-                goto out;
-        }
+    desc = GF_MALLOC(MAX_SNAP_DESCRIPTION_LEN + 1, gf_common_mt_char);
+    if (!desc) {
+        ret = -1;
+        goto out;
+    }
+
+    len = strlen(words[desc_opt_loc]);
+    if (len >= MAX_SNAP_DESCRIPTION_LEN) {
+        cli_out(
+            "snapshot create: description truncated: "
+            "Description provided is longer than 1024 characters");
+        desc_len = MAX_SNAP_DESCRIPTION_LEN;
+    } else {
+        desc_len = len;
+    }
+
+    snprintf(desc, desc_len + 1, "%s", words[desc_opt_loc]);
+    /* Calculating the size of the description as given by the user */
+
+    ret = dict_set_dynstr(dict, "description", desc);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR,
+               "Unable to save snap "
+               "description");
+        goto out;
+    }
 
+    ret = 0;
+out:
+    if (ret && desc)
+        GF_FREE(desc);
 
-        if (strlen (words[desc_opt_loc]) >= MAX_SNAP_DESCRIPTION_LEN) {
-                cli_out ("snapshot create: description truncated: "
-                         "Description provided is longer than 1024 characters");
-                desc_len = MAX_SNAP_DESCRIPTION_LEN;
-        } else {
-                desc_len = strlen (words[desc_opt_loc]);
-        }
+    return ret;
+}
 
-        strncpy (desc, words[desc_opt_loc], desc_len);
-        desc[desc_len] = '\0';
-        /* Calculating the size of the description as given by the user */
+/* Function to check whether the Volume name is repeated */
+int
+cli_check_if_volname_repeated(const char **words, unsigned int start_index,
+                              uint64_t cur_index)
+{
+    uint64_t i = -1;
+    int ret = 0;
 
-        ret = dict_set_dynstr (dict, "description", desc);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Unable to save snap "
-                        "description");
-                goto out;
-        }
+    GF_ASSERT(words);
 
-        ret = 0;
+    for (i = start_index; i < cur_index; i++) {
+        if (strcmp(words[i], words[cur_index]) == 0) {
+            ret = -1;
+            goto out;
+        }
+    }
 out:
-        if (ret && desc)
-                GF_FREE (desc);
-
-        return ret;
+    return ret;
 }
 
-/* Function to check whether the Volume name is repeated */
+/* snapshot clone <clonename> <snapname>
+ * @arg-0, dict     : Request Dictionary to be sent to server side.
+ * @arg-1, words    : Contains individual words of CLI command.
+ * @arg-2, wordcount: Contains number of words present in the CLI command.
+ *
+ * return value : -1 on failure
+ *                 0 on success
+ */
 int
-cli_check_if_volname_repeated (const char **words, unsigned int start_index,
-                           uint64_t cur_index) {
-        uint64_t        i       =       -1;
-        int             ret     =        0;
+cli_snap_clone_parse(dict_t *dict, const char **words, int wordcount)
+{
+    uint64_t i = 0;
+    int ret = -1;
+    char *clonename = NULL;
+    unsigned int cmdi = 2;
+    /* cmdi is command index, here cmdi is "2" (gluster snapshot clone)*/
+
+    GF_ASSERT(words);
+    GF_ASSERT(dict);
+
+    if (wordcount == cmdi + 1) {
+        cli_err("Invalid Syntax.");
+        gf_log("cli", GF_LOG_ERROR,
+               "Invalid number of  words for snap clone command");
+        goto out;
+    }
 
-        GF_ASSERT (words);
+    if (strlen(words[cmdi]) >= GLUSTERD_MAX_SNAP_NAME) {
+        cli_err(
+            "snapshot clone: failed: clonename cannot exceed "
+            "255 characters.");
+        gf_log("cli", GF_LOG_ERROR, "Clone name too long");
+
+        goto out;
+    }
+
+    clonename = (char *)words[cmdi];
+    for (i = 0; i < strlen(clonename); i++) {
+        /* Following volume name convention */
+        if (!isalnum(clonename[i]) &&
+            (clonename[i] != '_' && (clonename[i] != '-'))) {
+            /* TODO : Is this message enough?? */
+            cli_err(
+                "Clonename can contain only alphanumeric, "
+                "\"-\" and \"_\" characters");
+            goto out;
+        }
+    }
+
+    ret = dict_set_int32(dict, "volcount", 1);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Could not save volcount");
+        goto out;
+    }
+
+    ret = dict_set_str(dict, "clonename", (char *)words[cmdi]);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR,
+               "Could not save clone "
+               "name(%s)",
+               (char *)words[cmdi]);
+        goto out;
+    }
+
+    /* Filling snap name in the dictionary */
+    ret = dict_set_str(dict, "snapname", (char *)words[cmdi + 1]);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR,
+               "Could not "
+               "save snap name(%s)",
+               (char *)words[cmdi + 1]);
+        goto out;
+    }
+
+    ret = 0;
 
-        for (i = start_index ; i < cur_index ; i++) {
-                if (strcmp (words[i], words[cur_index]) == 0) {
-                        ret = -1;
-                        goto out;
-                }
-        }
 out:
-        return ret;
+    return ret;
 }
 
 /* snapshot create <snapname> <vol-name(s)> [description <description>]
@@ -3211,159 +4324,181 @@ out:
  *                 0 on success
  */
 int
-cli_snap_create_parse (dict_t *dict, const char **words, int wordcount) {
-        uint64_t        i               =       0;
-        int             ret             =       -1;
-        uint64_t        volcount        =       0;
-        char            key[PATH_MAX]   =       "";
-        char            *snapname       =       NULL;
-        unsigned int    cmdi            =       2;
-        int             flags           =       0;
-        /* cmdi is command index, here cmdi is "2" (gluster snapshot create)*/
-
-        GF_ASSERT (words);
-        GF_ASSERT (dict);
-
-        if (wordcount <= cmdi + 1) {
-                cli_err ("Invalid Syntax.");
-                gf_log ("cli", GF_LOG_ERROR,
-                        "Too less words for snap create command");
-                goto out;
-        }
+cli_snap_create_parse(dict_t *dict, const char **words, int wordcount)
+{
+    uint64_t i = 0;
+    int ret = -1;
+    uint64_t volcount = 0;
+    char key[PATH_MAX] = "";
+    char *snapname = NULL;
+    unsigned int cmdi = 2;
+    int flags = 0;
+    /* cmdi is command index, here cmdi is "2" (gluster snapshot create)*/
+
+    GF_ASSERT(words);
+    GF_ASSERT(dict);
+
+    if (wordcount <= cmdi + 1) {
+        cli_err("Invalid Syntax.");
+        gf_log("cli", GF_LOG_ERROR, "Too less words for snap create command");
+        goto out;
+    }
 
-        if (strlen(words[cmdi]) >= GLUSTERD_MAX_SNAP_NAME) {
-                cli_err ("snapshot create: failed: snapname cannot exceed "
-                         "255 characters.");
-                gf_log ("cli", GF_LOG_ERROR, "Snapname too long");
+    if (strlen(words[cmdi]) >= GLUSTERD_MAX_SNAP_NAME) {
+        cli_err(
+            "snapshot create: failed: snapname cannot exceed "
+            "255 characters.");
+        gf_log("cli", GF_LOG_ERROR, "Snapname too long");
 
-                goto out;
+        goto out;
+    }
+
+    snapname = (char *)words[cmdi];
+    for (i = 0; i < strlen(snapname); i++) {
+        /* Following volume name convention */
+        if (!isalnum(snapname[i]) &&
+            (snapname[i] != '_' && (snapname[i] != '-'))) {
+            /* TODO : Is this message enough?? */
+            cli_err(
+                "Snapname can contain only alphanumeric, "
+                "\"-\" and \"_\" characters");
+            goto out;
+        }
+    }
+
+    ret = dict_set_str(dict, "snapname", (char *)words[cmdi]);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR,
+               "Could not save snap "
+               "name(%s)",
+               (char *)words[cmdi]);
+        goto out;
+    }
+
+    /* Filling volume name in the dictionary */
+    for (i = cmdi + 1;
+         i < wordcount && (strcmp(words[i], "description")) != 0 &&
+         (strcmp(words[i], "force") != 0) &&
+         (strcmp(words[i], "no-timestamp") != 0);
+         i++) {
+        volcount++;
+        /* volume index starts from 1 */
+        ret = snprintf(key, sizeof(key), "volname%" PRIu64, volcount);
+        if (ret < 0) {
+            goto out;
         }
 
-        snapname = (char *) words[cmdi];
-        for (i = 0 ; i < strlen (snapname); i++) {
-                /* Following volume name convention */
-                if (!isalnum (snapname[i]) && (snapname[i] != '_'
-                                           && (snapname[i] != '-'))) {
-                        /* TODO : Is this message enough?? */
-                        cli_err ("Snapname can contain only alphanumeric, "
-                                 "\"-\" and \"_\" characters");
-                        goto out;
-                }
+        ret = dict_set_str(dict, key, (char *)words[i]);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Could not "
+                   "save volume name(%s)",
+                   (char *)words[i]);
+            goto out;
         }
 
-        ret = dict_set_str (dict, "snapname", (char *)words[cmdi]);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Could not save snap "
-                        "name(%s)", (char *)words[cmdi]);
-                goto out;
+        if (i >= cmdi + 2) {
+            ret = -1;
+            cli_err(
+                "Creating multiple volume snapshot is not "
+                "supported as of now");
+            goto out;
         }
+        /* TODO : remove this above condition check once
+         * multiple volume snapshot is supported */
+    }
 
-        /* Filling volume name in the dictionary */
-        for (i = cmdi + 1 ; i < wordcount
-                            && (strcmp (words[i], "description")) != 0
-                            && (strcmp (words[i], "force") != 0); i++) {
-                volcount++;
-                /* volume index starts from 1 */
-                ret = snprintf (key, sizeof (key), "volname%"PRIu64, volcount);
-                if (ret < 0) {
-                        goto out;
-                }
+    if (volcount == 0) {
+        ret = -1;
+        cli_err("Please provide the volume name");
+        gf_log("cli", GF_LOG_ERROR, "Invalid Syntax");
+        goto out;
+    }
 
-                ret = dict_set_str (dict, key, (char *)words[i]);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Could not "
-                                "save volume name(%s)", (char *)words[i]);
-                        goto out;
-                }
+    ret = dict_set_int32(dict, "volcount", volcount);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Could not save volcount");
+        goto out;
+    }
+
+    /* Verify how we got out of "for" loop,
+     * if it is by reaching wordcount limit then goto "out",
+     * because we need not parse for "description","force" and
+     * "no-timestamp" after this.
+     */
+    if (i == wordcount) {
+        goto out;
+    }
 
-                if (i >= cmdi + 2) {
-                        ret = -1;
-                        cli_err("Creating multiple volume snapshot is not "
-                                "supported as of now");
-                        goto out;
-                }
-                /* TODO : remove this above condition check once
-                 * multiple volume snapshot is supported */
+    if (strcmp(words[i], "no-timestamp") == 0) {
+        ret = dict_set_int32n(dict, "no-timestamp", SLEN("no-timestamp"), 1);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Could not save "
+                   "time-stamp option");
         }
+        if (i == (wordcount - 1))
+            goto out;
+        i++;
+    }
 
-        if (volcount == 0) {
-                ret = -1;
-                cli_err ("Please provide the volume name");
-                gf_log ("cli", GF_LOG_ERROR, "Invalid Syntax");
-                goto out;
+    if ((strcmp(words[i], "description")) == 0) {
+        ++i;
+        if (i > (wordcount - 1)) {
+            ret = -1;
+            cli_err("Please provide a description");
+            gf_log("cli", GF_LOG_ERROR, "Description not provided");
+            goto out;
         }
 
-        ret = dict_set_int32 (dict, "volcount", volcount);
+        ret = cli_snap_create_desc_parse(dict, words, wordcount, i);
         if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Could not save volcount");
-                goto out;
-        }
-
-        /* Verify how we got out of "for" loop,
-         * if it is by reaching wordcount limit then goto "out",
-         * because we need not parse for "description" and "force"
-         * after this.
+            gf_log("cli", GF_LOG_ERROR,
+                   "Could not save snap "
+                   "description");
+            goto out;
+        }
+
+        if (i == (wordcount - 1))
+            goto out;
+        i++;
+        /* point the index to next word.
+         * As description might be follwed by force option.
+         * Before that, check if wordcount limit is reached
          */
-        if (i == wordcount) {
-                goto out;
-        }
-
-        if ((strcmp (words[i], "description")) == 0) {
-                ++i;
-                if (i > (wordcount - 1)) {
-                        ret = -1;
-                        cli_err ("Please provide a description");
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Description not provided");
-                        goto out;
-                }
+    }
 
-                ret = cli_snap_create_desc_parse(dict, words, wordcount, i);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Could not save snap "
-                                "description");
-                        goto out;
-                }
-
-                if (i == (wordcount - 1))
-                        goto out;
-                i++;
-                /* point the index to next word.
-                 * As description might be follwed by force option.
-                 * Before that, check if wordcount limit is reached
-                 */
-        }
+    if (strcmp(words[i], "force") == 0) {
+        flags = GF_CLI_FLAG_OP_FORCE;
 
-        if (strcmp (words[i], "force") == 0) {
-                flags = GF_CLI_FLAG_OP_FORCE;
-
-        } else {
-                ret = -1;
-                cli_err ("Invalid Syntax.");
-                gf_log ("cli", GF_LOG_ERROR, "Invalid Syntax");
-                goto out;
-        }
+    } else {
+        ret = -1;
+        cli_err("Invalid Syntax.");
+        gf_log("cli", GF_LOG_ERROR, "Invalid Syntax");
+        goto out;
+    }
 
-        /* Check if the command has anything after "force" keyword */
-        if (++i < wordcount) {
-                ret = -1;
-                gf_log ("cli", GF_LOG_ERROR, "Invalid Syntax");
-                goto out;
-        }
+    /* Check if the command has anything after "force" keyword */
+    if (++i < wordcount) {
+        ret = -1;
+        gf_log("cli", GF_LOG_ERROR, "Invalid Syntax");
+        goto out;
+    }
 
-        ret = 0;
+    ret = 0;
 
 out:
-        if(ret == 0) {
-                /*Adding force flag in either of the case i.e force set
-                 * or unset*/
-                ret = dict_set_int32 (dict, "flags", flags);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Could not save "
-                                "snap force option");
-                }
+    if (ret == 0) {
+        /*Adding force flag in either of the case i.e force set
+         * or unset*/
+        ret = dict_set_int32(dict, "flags", flags);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Could not save "
+                   "snap force option");
         }
-        return ret;
+    }
+    return ret;
 }
 
 /* snapshot list [volname]
@@ -3375,30 +4510,30 @@ out:
  *                 0 on success
  */
 int
-cli_snap_list_parse (dict_t *dict, const char **words, int wordcount) {
-        int             ret     =       -1;
+cli_snap_list_parse(dict_t *dict, const char **words, int wordcount)
+{
+    int ret = -1;
 
-        GF_ASSERT (words);
-        GF_ASSERT (dict);
+    GF_ASSERT(words);
+    GF_ASSERT(dict);
 
-        if (wordcount < 2 || wordcount > 3) {
-                gf_log ("cli", GF_LOG_ERROR, "Invalid Syntax");
-                goto out;
-        }
+    if (wordcount < 2 || wordcount > 3) {
+        gf_log("cli", GF_LOG_ERROR, "Invalid Syntax");
+        goto out;
+    }
 
-        if (wordcount == 2) {
-                ret = 0;
-                goto out;
-        }
+    if (wordcount == 2) {
+        ret = 0;
+        goto out;
+    }
 
-        ret = dict_set_str (dict, "volname", (char *)words[2]);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR,
-                        "Failed to save volname in dictionary");
-                goto out;
-        }
+    ret = dict_set_str(dict, "volname", (char *)words[2]);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to save volname in dictionary");
+        goto out;
+    }
 out:
-        return ret;
+    return ret;
 }
 
 /* snapshot info [(snapname |  volume <volname>)]
@@ -3410,87 +4545,88 @@ out:
  *                 0 on success
  */
 int
-cli_snap_info_parse (dict_t *dict, const char **words, int wordcount)
+cli_snap_info_parse(dict_t *dict, const char **words, int wordcount)
 {
+    int ret = -1;
+    int32_t cmd = GF_SNAP_INFO_TYPE_ALL;
+    unsigned int cmdi = 2;
+    /* cmdi is command index, here cmdi is "2" (gluster snapshot info)*/
 
-        int             ret             =       -1;
-        int32_t         cmd             =       GF_SNAP_INFO_TYPE_ALL;
-        unsigned int    cmdi            =        2;
-        /* cmdi is command index, here cmdi is "2" (gluster snapshot info)*/
+    GF_ASSERT(words);
+    GF_ASSERT(dict);
 
-        GF_ASSERT (words);
-        GF_ASSERT (dict);
+    if (wordcount > 4 || wordcount < cmdi) {
+        gf_log("cli", GF_LOG_ERROR, "Invalid syntax");
+        goto out;
+    }
 
-        if (wordcount > 4 || wordcount < cmdi) {
-                gf_log ("cli", GF_LOG_ERROR, "Invalid syntax");
-                goto out;
-        }
+    if (wordcount == cmdi) {
+        ret = 0;
+        goto out;
+    }
 
-        if (wordcount == cmdi) {
-                ret = 0;
-                goto out;
+    /* If 3rd word is not "volume", then it must
+     * be snapname.
+     */
+    if (strcmp(words[cmdi], "volume") != 0) {
+        ret = dict_set_str(dict, "snapname", (char *)words[cmdi]);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Unable to save "
+                   "snapname %s",
+                   words[cmdi]);
+            goto out;
         }
 
-        /* If 3rd word is not "volume", then it must
-         * be snapname.
+        /* Once snap name is parsed, if we encounter any other
+         * word then fail it. Invalid Syntax.
+         * example : snapshot info <snapname> word
          */
-        if (strcmp (words[cmdi], "volume") != 0) {
-                ret = dict_set_str (dict, "snapname",
-                                   (char *)words[cmdi]);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Unable to save "
-                                "snapname %s", words[cmdi]);
-                        goto out;
-                }
-
-                /* Once snap name is parsed, if we encounter any other
-                 * word then fail it. Invalid Syntax.
-                 * example : snapshot info <snapname> word
-                 */
-                if ((cmdi + 1) != wordcount) {
-                        ret = -1;
-                        gf_log ("cli", GF_LOG_ERROR, "Invalid Syntax");
-                        goto out;
-                }
-
-                cmd = GF_SNAP_INFO_TYPE_SNAP;
-                ret = 0;
-                goto out;
-                /* No need to continue the parsing once we
-                 * get the snapname
-                 */
+        if ((cmdi + 1) != wordcount) {
+            ret = -1;
+            gf_log("cli", GF_LOG_ERROR, "Invalid Syntax");
+            goto out;
         }
 
-        /* If 3rd word is "volume", then check if next word
-         * is present. As, "snapshot info volume" is an
-         * invalid command.
+        cmd = GF_SNAP_INFO_TYPE_SNAP;
+        ret = 0;
+        goto out;
+        /* No need to continue the parsing once we
+         * get the snapname
          */
-        if ((cmdi + 1) == wordcount) {
-                ret = -1;
-                gf_log ("cli", GF_LOG_ERROR, "Invalid Syntax");
-                goto out;
-        }
+    }
 
-        ret = dict_set_str (dict, "volname", (char *)words[wordcount - 1]);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Could not save "
-                        "volume name %s", words[wordcount - 1]);
-                goto out;
-        }
-        cmd = GF_SNAP_INFO_TYPE_VOL;
+    /* If 3rd word is "volume", then check if next word
+     * is present. As, "snapshot info volume" is an
+     * invalid command.
+     */
+    if ((cmdi + 1) == wordcount) {
+        ret = -1;
+        gf_log("cli", GF_LOG_ERROR, "Invalid Syntax");
+        goto out;
+    }
+
+    ret = dict_set_str(dict, "volname", (char *)words[wordcount - 1]);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR,
+               "Could not save "
+               "volume name %s",
+               words[wordcount - 1]);
+        goto out;
+    }
+    cmd = GF_SNAP_INFO_TYPE_VOL;
 out:
-        if (ret == 0) {
-                ret = dict_set_int32 (dict, "cmd", cmd);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Could not save "
-                                "type of snapshot info");
-                }
+    if (ret == 0) {
+        ret = dict_set_int32(dict, "sub-cmd", cmd);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Could not save "
+                   "type of snapshot info");
         }
-        return ret;
+    }
+    return ret;
 }
 
-
-
 /* snapshot restore <snapname>
  * @arg-0, dict     : Request Dictionary to be sent to server side.
  * @arg-1, words    : Contains individual words of CLI command.
@@ -3500,42 +4636,43 @@ out:
  *                 0 on success
  */
 int
-cli_snap_restore_parse (dict_t *dict, const char **words, int wordcount,
-                        struct cli_state *state)
+cli_snap_restore_parse(dict_t *dict, const char **words, int wordcount,
+                       struct cli_state *state)
 {
+    int ret = -1;
+    const char *question = NULL;
+    gf_answer_t answer = GF_ANSWER_NO;
 
-        int             ret             =       -1;
-        const char      *question       =       NULL;
-        gf_answer_t     answer          =       GF_ANSWER_NO;
-
-        GF_ASSERT (words);
-        GF_ASSERT (dict);
-
-        if (wordcount != 3) {
-                gf_log ("cli", GF_LOG_ERROR, "Invalid Syntax");
-                goto out;
-        }
+    GF_ASSERT(words);
+    GF_ASSERT(dict);
 
-        ret = dict_set_str (dict, "snapname", (char *)words[2]);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Unable to save snap-name %s",
-                        words[2]);
-                goto out;
-        }
-
-        question = "Restore operation will replace the "
-                   "original volume with the snapshotted volume. "
-                   "Do you still want to continue?";
+    if (wordcount != 3) {
+        gf_log("cli", GF_LOG_ERROR, "Invalid Syntax");
+        goto out;
+    }
 
-        answer = cli_cmd_get_confirmation (state, question);
-        if (GF_ANSWER_NO == answer) {
-                ret = 1;
-                gf_log ("cli", GF_LOG_ERROR, "User cancelled a snapshot "
-                        "restore operation for snap %s", (char *)words[2]);
-                goto out;
-        }
+    ret = dict_set_str(dict, "snapname", (char *)words[2]);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Unable to save snap-name %s", words[2]);
+        goto out;
+    }
+
+    question =
+        "Restore operation will replace the "
+        "original volume with the snapshotted volume. "
+        "Do you still want to continue?";
+
+    answer = cli_cmd_get_confirmation(state, question);
+    if (GF_ANSWER_NO == answer) {
+        ret = 1;
+        gf_log("cli", GF_LOG_ERROR,
+               "User cancelled a snapshot "
+               "restore operation for snap %s",
+               (char *)words[2]);
+        goto out;
+    }
 out:
-        return ret;
+    return ret;
 }
 
 /* snapshot activate <snapname> [force]
@@ -3547,43 +4684,41 @@ out:
  *                 0 on success
  */
 int
-cli_snap_activate_parse (dict_t *dict, const char **words, int wordcount)
+cli_snap_activate_parse(dict_t *dict, const char **words, int wordcount)
 {
+    int ret = -1;
+    int flags = 0;
 
-        int ret = -1;
-        int flags = 0;
+    GF_ASSERT(words);
+    GF_ASSERT(dict);
 
-        GF_ASSERT (words);
-        GF_ASSERT (dict);
-
-        if ((wordcount < 3) || (wordcount > 4)) {
-                gf_log ("cli", GF_LOG_ERROR, "Invalid Syntax");
-                goto out;
-        }
+    if ((wordcount < 3) || (wordcount > 4)) {
+        gf_log("cli", GF_LOG_ERROR, "Invalid Syntax");
+        goto out;
+    }
 
-        ret = dict_set_str (dict, "snapname", (char *)words[2]);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Unable to save snap-name %s",
-                        words[2]);
-                goto out;
-        }
+    ret = dict_set_str(dict, "snapname", (char *)words[2]);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Unable to save snap-name %s", words[2]);
+        goto out;
+    }
 
-        if (wordcount == 4) {
-                if (!strcmp("force", (char *)words[3])) {
-                        flags = GF_CLI_FLAG_OP_FORCE;
-                } else {
-                        gf_log ("cli", GF_LOG_ERROR, "Invalid option");
-                        ret = -1;
-                        goto out;
-                }
-        }
-        ret = dict_set_int32 (dict, "flags", flags);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Unable to save force option");
-                goto out;
-        }
+    if (wordcount == 4) {
+        if (!strcmp("force", (char *)words[3])) {
+            flags = GF_CLI_FLAG_OP_FORCE;
+        } else {
+            gf_log("cli", GF_LOG_ERROR, "Invalid option");
+            ret = -1;
+            goto out;
+        }
+    }
+    ret = dict_set_int32(dict, "flags", flags);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Unable to save force option");
+        goto out;
+    }
 out:
-        return ret;
+    return ret;
 }
 
 /* snapshot deactivate <snapname>
@@ -3596,42 +4731,41 @@ out:
  *                 1 if user cancelled the request
  */
 int
-cli_snap_deactivate_parse (dict_t *dict, const char **words, int wordcount,
-                        struct cli_state *state)
+cli_snap_deactivate_parse(dict_t *dict, const char **words, int wordcount,
+                          struct cli_state *state)
 {
+    int ret = -1;
+    gf_answer_t answer = GF_ANSWER_NO;
+    const char *question =
+        "Deactivating snap will make its "
+        "data inaccessible. Do you want to "
+        "continue?";
+
+    GF_ASSERT(words);
+    GF_ASSERT(dict);
+
+    if ((wordcount != 3)) {
+        gf_log("cli", GF_LOG_ERROR, "Invalid Syntax");
+        goto out;
+    }
 
-        int             ret             = -1;
-        gf_answer_t     answer          = GF_ANSWER_NO;
-        const char     *question        = "Deactivating snap will make its "
-                                          "data inaccessible. Do you want to "
-                                          "continue?";
-
-
-        GF_ASSERT (words);
-        GF_ASSERT (dict);
-
-        if ((wordcount != 3)) {
-                gf_log ("cli", GF_LOG_ERROR, "Invalid Syntax");
-                goto out;
-        }
-
-        ret = dict_set_str (dict, "snapname", (char *)words[2]);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Unable to save snap-name %s",
-                        words[2]);
-                goto out;
-        }
-
-        answer = cli_cmd_get_confirmation (state, question);
-        if (GF_ANSWER_NO == answer) {
-                ret = 1;
-                gf_log ("cli", GF_LOG_DEBUG, "User cancelled "
-                        "snapshot deactivate operation");
-                goto out;
-        }
+    ret = dict_set_str(dict, "snapname", (char *)words[2]);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Unable to save snap-name %s", words[2]);
+        goto out;
+    }
+
+    answer = cli_cmd_get_confirmation(state, question);
+    if (GF_ANSWER_NO == answer) {
+        ret = 1;
+        gf_log("cli", GF_LOG_DEBUG,
+               "User cancelled "
+               "snapshot deactivate operation");
+        goto out;
+    }
 
 out:
-        return ret;
+    return ret;
 }
 
 /* snapshot delete (all | snapname | volume <volname>)
@@ -3644,78 +4778,84 @@ out:
  *                 1 if user cancel the operation
  */
 int
-cli_snap_delete_parse (dict_t *dict, const char **words, int wordcount,
-                       struct cli_state *state) {
-
-        int             ret             =       -1;
-        const char      *question       =       NULL;
-        int32_t         cmd             =       -1;
-        unsigned int    cmdi            =       2;
-        gf_answer_t     answer          =       GF_ANSWER_NO;
-
-        GF_ASSERT (words);
-        GF_ASSERT (dict);
+cli_snap_delete_parse(dict_t *dict, const char **words, int wordcount,
+                      struct cli_state *state)
+{
+    int ret = -1;
+    const char *question = NULL;
+    int32_t cmd = -1;
+    unsigned int cmdi = 2;
+    gf_answer_t answer = GF_ANSWER_NO;
 
-        if (wordcount > 4 || wordcount <= cmdi) {
-                gf_log ("cli", GF_LOG_ERROR, "Invalid Syntax");
-                goto out;
-        }
+    GF_ASSERT(words);
+    GF_ASSERT(dict);
 
-        question = "Deleting snap will erase all the information about "
-                   "the snap. Do you still want to continue?";
+    if (wordcount > 4 || wordcount <= cmdi) {
+        gf_log("cli", GF_LOG_ERROR, "Invalid Syntax");
+        goto out;
+    }
 
-        if (strcmp (words [cmdi], "all") == 0) {
-                ret = 0;
-                cmd = GF_SNAP_DELETE_TYPE_ALL;
-        } else if (strcmp (words [cmdi], "volume") == 0) {
-                if (++cmdi == wordcount) {
-                        ret = -1;
-                        gf_log ("cli", GF_LOG_ERROR, "Invalid Syntax");
-                        goto out;
-                }
+    question =
+        "Deleting snap will erase all the information about "
+        "the snap. Do you still want to continue?";
 
-                ret = dict_set_str (dict, "volname",
-                                    (char *)words[cmdi]);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Could not save "
-                                "volume name %s", words[wordcount - 1]);
-                        goto out;
-                }
-                cmd = GF_SNAP_DELETE_TYPE_VOL;
-        } else {
-                ret = dict_set_str (dict, "snapname", (char *)words[cmdi]);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Unable to save "
-                                "snapname %s", words[2]);
-                        goto out;
-                }
-                cmd = GF_SNAP_DELETE_TYPE_SNAP;
+    if (strcmp(words[cmdi], "all") == 0) {
+        ret = 0;
+        cmd = GF_SNAP_DELETE_TYPE_ALL;
+    } else if (strcmp(words[cmdi], "volume") == 0) {
+        if (++cmdi == wordcount) {
+            ret = -1;
+            gf_log("cli", GF_LOG_ERROR, "Invalid Syntax");
+            goto out;
         }
 
-        if ((cmdi + 1) != wordcount) {
-                ret = -1;
-                gf_log ("cli", GF_LOG_ERROR, "Invalid Syntax");
-                goto out;
+        ret = dict_set_str(dict, "volname", (char *)words[cmdi]);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Could not save "
+                   "volume name %s",
+                   words[wordcount - 1]);
+            goto out;
+        }
+        cmd = GF_SNAP_DELETE_TYPE_VOL;
+    } else {
+        ret = dict_set_str(dict, "snapname", (char *)words[cmdi]);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Unable to save "
+                   "snapname %s",
+                   words[2]);
+            goto out;
         }
+        cmd = GF_SNAP_DELETE_TYPE_SNAP;
+    }
 
-        if (cmd == GF_SNAP_DELETE_TYPE_SNAP) {
-                answer = cli_cmd_get_confirmation (state, question);
-                if (GF_ANSWER_NO == answer) {
-                        ret = 1;
-                        gf_log ("cli", GF_LOG_DEBUG, "User cancelled "
-                                "snapshot delete operation for snap %s",
-                                (char *)words[2]);
-                        goto out;
-                }
-        }
+    if ((cmdi + 1) != wordcount) {
+        ret = -1;
+        gf_log("cli", GF_LOG_ERROR, "Invalid Syntax");
+        goto out;
+    }
 
-        ret = dict_set_int32 (dict, "delete-cmd", cmd);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Could not save "
-                        "type of snapshot delete");
-        }
+    if (cmd == GF_SNAP_DELETE_TYPE_SNAP) {
+        answer = cli_cmd_get_confirmation(state, question);
+        if (GF_ANSWER_NO == answer) {
+            ret = 1;
+            gf_log("cli", GF_LOG_DEBUG,
+                   "User cancelled "
+                   "snapshot delete operation for snap %s",
+                   (char *)words[2]);
+            goto out;
+        }
+    }
+
+    ret = dict_set_int32(dict, "sub-cmd", cmd);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR,
+               "Could not save "
+               "type of snapshot delete");
+    }
 out:
-        return ret;
+    return ret;
 }
 
 /* snapshot status [(snapname | volume <volname>)]
@@ -3727,121 +4867,138 @@ out:
  *                 0 on success
  */
 int
-cli_snap_status_parse (dict_t *dict, const char **words, int wordcount)
+cli_snap_status_parse(dict_t *dict, const char **words, int wordcount)
 {
+    int ret = -1;
+    int32_t cmd = GF_SNAP_STATUS_TYPE_ALL;
+    unsigned int cmdi = 2;
+    /* cmdi is command index, here cmdi is "2" (gluster snapshot status)*/
 
-        int             ret  =        -1;
-        int32_t         cmd  =       GF_SNAP_STATUS_TYPE_ALL;
-        unsigned int    cmdi =        2;
-        /* cmdi is command index, here cmdi is "2" (gluster snapshot status)*/
+    GF_ASSERT(words);
+    GF_ASSERT(dict);
 
-        GF_ASSERT (words);
-        GF_ASSERT (dict);
+    if (wordcount > 4 || wordcount < cmdi) {
+        gf_log("cli", GF_LOG_ERROR, "Invalid Syntax");
+        goto out;
+    }
 
-        if (wordcount > 4 || wordcount < cmdi) {
-                gf_log ("cli", GF_LOG_ERROR, "Invalid Syntax");
-                goto out;
-        }
+    if (wordcount == cmdi) {
+        ret = 0;
+        goto out;
+    }
 
-        if (wordcount == cmdi) {
-                ret = 0;
-                goto out;
+    /* if 3rd word is not "volume", then it must be "snapname"
+     */
+    if (strcmp(words[cmdi], "volume") != 0) {
+        ret = dict_set_str(dict, "snapname", (char *)words[cmdi]);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Count not save "
+                   "snap name %s",
+                   words[cmdi]);
+            goto out;
         }
 
-        /* if 3rd word is not "volume", then it must be "snapname"
-        */
-        if (strcmp (words[cmdi], "volume") != 0) {
-                ret = dict_set_str (dict, "snapname",
-                                   (char *)words[cmdi]);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Count not save "
-                                "snap name %s", words[cmdi]);
-                        goto out;
-                }
-
-                if ((cmdi + 1) != wordcount) {
-                        ret = -1;
-                        gf_log ("cli", GF_LOG_ERROR, "Invalid Syntax");
-                        goto out;
-                }
-
-                ret = 0;
-                cmd = GF_SNAP_STATUS_TYPE_SNAP;
-                goto out;
+        if ((cmdi + 1) != wordcount) {
+            ret = -1;
+            gf_log("cli", GF_LOG_ERROR, "Invalid Syntax");
+            goto out;
         }
 
-        /* If 3rd word is "volume", then check if next word is present.
-         * As, "snapshot info volume" is an invalid command
-         */
-        if ((cmdi + 1) == wordcount) {
-                ret = -1;
-                gf_log ("cli", GF_LOG_ERROR, "Invalid Syntax");
-                goto out;
-        }
+        ret = 0;
+        cmd = GF_SNAP_STATUS_TYPE_SNAP;
+        goto out;
+    }
 
-        ret = dict_set_str (dict, "volname", (char *)words [wordcount - 1]);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Count not save "
-                        "volume name %s", words[wordcount - 1]);
-                goto out;
-        }
-        cmd = GF_SNAP_STATUS_TYPE_VOL;
+    /* If 3rd word is "volume", then check if next word is present.
+     * As, "snapshot info volume" is an invalid command
+     */
+    if ((cmdi + 1) == wordcount) {
+        ret = -1;
+        gf_log("cli", GF_LOG_ERROR, "Invalid Syntax");
+        goto out;
+    }
+
+    ret = dict_set_str(dict, "volname", (char *)words[wordcount - 1]);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR,
+               "Count not save "
+               "volume name %s",
+               words[wordcount - 1]);
+        goto out;
+    }
+    cmd = GF_SNAP_STATUS_TYPE_VOL;
 
 out:
-        if (ret == 0) {
-                ret = dict_set_int32 (dict, "status-cmd", cmd);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Could not save cmd "
-                                "of snapshot status");
-                }
+    if (ret == 0) {
+        ret = dict_set_int32(dict, "sub-cmd", cmd);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Could not save cmd "
+                   "of snapshot status");
         }
+    }
 
-        return ret;
+    return ret;
 }
 
-
 /* return value:
  *      -1  in case of failure.
  *       0  in case of success.
  */
 int32_t
-cli_snap_config_limit_parse (const char **words, dict_t *dict,
-                             unsigned int wordcount, unsigned int index,
-                             char *key)
+cli_snap_config_limit_parse(const char **words, dict_t *dict,
+                            unsigned int wordcount, unsigned int index,
+                            char *key)
 {
-        int             ret             = -1;
-        int             limit           = 0;
-        char            *end_ptr        = NULL;
+    int ret = -1;
+    int limit = 0;
+    char *end_ptr = NULL;
 
-        GF_ASSERT (words);
-        GF_ASSERT (dict);
-        GF_ASSERT (key);
+    GF_ASSERT(words);
+    GF_ASSERT(dict);
+    GF_ASSERT(key);
 
-        if (index >= wordcount) {
-                ret = -1;
-                cli_err ("Please provide a value for %s.", key);
-                gf_log ("cli", GF_LOG_ERROR, "Value not provided for %s", key);
-                goto out;
-        }
+    if (index >= wordcount) {
+        ret = -1;
+        cli_err("Please provide a value for %s.", key);
+        gf_log("cli", GF_LOG_ERROR, "Value not provided for %s", key);
+        goto out;
+    }
 
-        limit = strtol (words[index], &end_ptr, 10);
+    limit = strtol(words[index], &end_ptr, 10);
 
-        if (limit <= 0 || strcmp (end_ptr, "") != 0) {
-                ret = -1;
-                cli_err("Please enter an integer value "
-                        "greater than zero for %s", key);
-                goto out;
-        }
+    if (limit <= 0 || strcmp(end_ptr, "") != 0) {
+        ret = -1;
+        cli_err(
+            "Please enter an integer value "
+            "greater than zero for %s",
+            key);
+        goto out;
+    }
+
+    ret = dict_set_int32(dict, key, limit);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR,
+               "Could not set "
+               "%s in dictionary",
+               key);
+        goto out;
+    }
 
-        ret = dict_set_int32 (dict, key, limit);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Could not set "
-                        "%s in dictionary", key);
-                goto out;
-        }
+    ret = dict_set_dynstr_with_alloc(dict, "globalname", "All");
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Could not set global key");
+        goto out;
+    }
+    ret = dict_set_int32(dict, "hold_global_locks", _gf_true);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Could not set global locks");
+        goto out;
+    }
 
 out:
-        return ret;
+    return ret;
 }
 
 /* function cli_snap_config_parse
@@ -3857,466 +5014,933 @@ out:
   NOTE : snap-max-soft-limit can only be set for system.
 */
 int32_t
-cli_snap_config_parse (const char **words, int wordcount, dict_t *dict,
-                       struct cli_state *state)
+cli_snap_config_parse(const char **words, int wordcount, dict_t *dict,
+                      struct cli_state *state)
 {
-        int                            ret                 = -1;
-        gf_answer_t                    answer              = GF_ANSWER_NO;
-        gf_boolean_t                   vol_presence        = _gf_false;
-        struct snap_config_opt_vals_  *conf_vals           = NULL;
-        int8_t                         hard_limit          = 0;
-        int8_t                         soft_limit          = 0;
-        int8_t                         config_type         = -1;
-        const char                    *question            = NULL;
-        unsigned int                   cmdi                = 2;
-        /* cmdi is command index, here cmdi is "2" (gluster snapshot config)*/
-
-        GF_ASSERT (words);
-        GF_ASSERT (dict);
-        GF_ASSERT (state);
-
-        if ((wordcount < 2) || (wordcount > 7)) {
-                gf_log ("cli", GF_LOG_ERROR,
-                        "Invalid wordcount(%d)", wordcount);
-                goto out;
-        }
+    int ret = -1;
+    gf_answer_t answer = GF_ANSWER_NO;
+    gf_boolean_t vol_presence = _gf_false;
+    struct snap_config_opt_vals_ *conf_vals = NULL;
+    int8_t hard_limit = 0;
+    int8_t soft_limit = 0;
+    int8_t config_type = -1;
+    const char *question = NULL;
+    unsigned int cmdi = 2;
+    /* cmdi is command index, here cmdi is "2" (gluster snapshot config)*/
+
+    GF_ASSERT(words);
+    GF_ASSERT(dict);
+    GF_ASSERT(state);
+
+    if ((wordcount < 2) || (wordcount > 7)) {
+        gf_log("cli", GF_LOG_ERROR, "Invalid wordcount(%d)", wordcount);
+        goto out;
+    }
 
-        if (wordcount == 2) {
-                config_type = GF_SNAP_CONFIG_DISPLAY;
-                ret = 0;
-                goto set;
+    if (wordcount == 2) {
+        config_type = GF_SNAP_CONFIG_DISPLAY;
+        ret = 0;
+        goto set;
+    }
+
+    /* auto-delete cannot be a volume name */
+    /* Check whether the 3rd word is volname */
+    if (strcmp(words[cmdi], "snap-max-hard-limit") != 0 &&
+        strcmp(words[cmdi], "snap-max-soft-limit") != 0 &&
+        strcmp(words[cmdi], "auto-delete") != 0 &&
+        strcmp(words[cmdi], "activate-on-create") != 0) {
+        ret = dict_set_str(dict, "volname", (char *)words[cmdi]);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR, "Failed to set volname");
+            goto out;
         }
+        cmdi++;
+        vol_presence = _gf_true;
 
-        /* auto-delete cannot be a volume name */
-        /* Check whether the 3rd word is volname */
-        if (strcmp (words[cmdi], "snap-max-hard-limit") != 0
-             && strcmp (words[cmdi], "snap-max-soft-limit") != 0
-             && strcmp (words[cmdi], "auto-delete") != 0
-             && strcmp (words[cmdi], "activate-on-create") != 0) {
-                ret = dict_set_str (dict, "volname", (char *)words[cmdi]);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Failed to set volname");
-                        goto out;
-                }
-                cmdi++;
-                vol_presence = _gf_true;
-
-                if (cmdi == wordcount) {
-                        config_type = GF_SNAP_CONFIG_DISPLAY;
-                        ret = 0;
-                        goto set;
-                }
+        if (cmdi == wordcount) {
+            config_type = GF_SNAP_CONFIG_DISPLAY;
+            ret = 0;
+            goto set;
         }
+    }
 
-        config_type = GF_SNAP_CONFIG_TYPE_SET;
-
-        if (strcmp (words[cmdi], "snap-max-hard-limit") == 0) {
-                ret = cli_snap_config_limit_parse (words, dict, wordcount,
-                                                ++cmdi, "snap-max-hard-limit");
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Failed to parse snap "
-                                "config hard limit");
-                        goto out;
-                }
-                hard_limit = 1;
+    config_type = GF_SNAP_CONFIG_TYPE_SET;
 
-                if (++cmdi == wordcount) {
-                        ret = 0;
-                        goto set;
-                }
+    if (strcmp(words[cmdi], "snap-max-hard-limit") == 0) {
+        ret = cli_snap_config_limit_parse(words, dict, wordcount, ++cmdi,
+                                          "snap-max-hard-limit");
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Failed to parse snap "
+                   "config hard limit");
+            goto out;
         }
+        hard_limit = 1;
 
-        if (strcmp (words[cmdi], "snap-max-soft-limit") == 0) {
-                if (vol_presence == 1) {
-                        ret = -1;
-                        cli_err ("Soft limit cannot be set to individual "
-                                  "volumes.");
-                        gf_log ("cli", GF_LOG_ERROR, "Soft limit cannot be "
-                                "set to volumes");
-                        goto out;
-                }
-
-                ret = cli_snap_config_limit_parse (words, dict, wordcount,
-                                                ++cmdi, "snap-max-soft-limit");
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Failed to parse snap "
-                                "config soft limit");
-                        goto out;
-                }
-
-                if (++cmdi != wordcount) {
-                        ret = -1;
-                        gf_log ("cli", GF_LOG_ERROR, "Invalid Syntax");
-                        goto out;
-                }
-                soft_limit = 1;
+        if (++cmdi == wordcount) {
+            ret = 0;
+            goto set;
         }
+    }
 
-        if (hard_limit || soft_limit)
-                goto set;
+    if (strcmp(words[cmdi], "snap-max-soft-limit") == 0) {
+        if (vol_presence == 1) {
+            ret = -1;
+            cli_err(
+                "Soft limit cannot be set to individual "
+                "volumes.");
+            gf_log("cli", GF_LOG_ERROR,
+                   "Soft limit cannot be "
+                   "set to volumes");
+            goto out;
+        }
 
-        if (strcmp(words[cmdi], "auto-delete") == 0) {
-                if (vol_presence == 1) {
-                        ret = -1;
-                        cli_err ("As of now, auto-delete option cannot be set "
-                                "to volumes");
-                        gf_log ("cli", GF_LOG_ERROR, "auto-delete option "
-                                "cannot be set to volumes");
-                        goto out;
-                }
+        ret = cli_snap_config_limit_parse(words, dict, wordcount, ++cmdi,
+                                          "snap-max-soft-limit");
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Failed to parse snap "
+                   "config soft limit");
+            goto out;
+        }
 
-                if (++cmdi >= wordcount) {
-                        ret = -1;
-                        gf_log ("cli", GF_LOG_ERROR, "Invalid Syntax");
-                        goto out;
-                }
+        if (++cmdi != wordcount) {
+            ret = -1;
+            gf_log("cli", GF_LOG_ERROR, "Invalid Syntax");
+            goto out;
+        }
+        soft_limit = 1;
+    }
 
-                ret = dict_set_str (dict, "auto-delete", (char *)words[cmdi]);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Failed to set "
-                                "value of auto-delete in request "
-                                "dictionary");
-                        goto out;
-                }
+    if (hard_limit || soft_limit)
+        goto set;
 
-                if (++cmdi != wordcount) {
-                        ret = -1;
-                        gf_log ("cli", GF_LOG_ERROR, "Invalid Syntax");
-                        goto out;
-                }
-        } else if (strcmp(words[cmdi], "activate-on-create") == 0) {
-                if (vol_presence == 1) {
-                        ret = -1;
-                        cli_err ("As of now, activate-on-create option "
-                                 "cannot be set to volumes");
-                        gf_log ("cli", GF_LOG_ERROR, "activate-on-create "
-                                "option cannot be set to volumes");
-                        goto out;
-                }
+    if (strcmp(words[cmdi], "auto-delete") == 0) {
+        if (vol_presence == 1) {
+            ret = -1;
+            cli_err(
+                "As of now, auto-delete option cannot be set "
+                "to volumes");
+            gf_log("cli", GF_LOG_ERROR,
+                   "auto-delete option "
+                   "cannot be set to volumes");
+            goto out;
+        }
 
-                if (++cmdi >= wordcount) {
-                        ret = -1;
-                        gf_log ("cli", GF_LOG_ERROR, "Invalid Syntax");
-                        goto out;
-                }
+        if (++cmdi >= wordcount) {
+            ret = -1;
+            gf_log("cli", GF_LOG_ERROR, "Invalid Syntax");
+            goto out;
+        }
 
-                ret = dict_set_str (dict, "snap-activate-on-create",
-                                    (char *)words[cmdi]);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Failed to set value "
-                                "of activate-on-create in request dictionary");
-                        goto out;
-                }
+        ret = dict_set_str(dict, "auto-delete", (char *)words[cmdi]);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Failed to set "
+                   "value of auto-delete in request "
+                   "dictionary");
+            goto out;
+        }
+
+        if (++cmdi != wordcount) {
+            ret = -1;
+            gf_log("cli", GF_LOG_ERROR, "Invalid Syntax");
+            goto out;
+        }
+    } else if (strcmp(words[cmdi], "activate-on-create") == 0) {
+        if (vol_presence == 1) {
+            ret = -1;
+            cli_err(
+                "As of now, activate-on-create option "
+                "cannot be set to volumes");
+            gf_log("cli", GF_LOG_ERROR,
+                   "activate-on-create "
+                   "option cannot be set to volumes");
+            goto out;
+        }
+
+        if (++cmdi >= wordcount) {
+            ret = -1;
+            gf_log("cli", GF_LOG_ERROR, "Invalid Syntax");
+            goto out;
+        }
+
+        ret = dict_set_str(dict, "snap-activate-on-create",
+                           (char *)words[cmdi]);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Failed to set value "
+                   "of activate-on-create in request dictionary");
+            goto out;
+        }
 
-                if (++cmdi != wordcount) {
-                        ret = -1;
-                        gf_log ("cli", GF_LOG_ERROR, "Invalid Syntax");
-                        goto out;
-                }
-        } else {
-                ret = -1;
-                gf_log ("cli", GF_LOG_ERROR, "Invalid Syntax");
-                goto out;
+        if (++cmdi != wordcount) {
+            ret = -1;
+            gf_log("cli", GF_LOG_ERROR, "Invalid Syntax");
+            goto out;
         }
+    } else {
+        ret = -1;
+        gf_log("cli", GF_LOG_ERROR, "Invalid Syntax");
+        goto out;
+    }
 
-        ret = 0; /* Success */
+    ret = 0; /* Success */
 
 set:
-        ret = dict_set_int32 (dict, "config-command", config_type);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Unable to set "
-                        "config-command");
-                goto out;
-        }
+    ret = dict_set_int32(dict, "config-command", config_type);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR,
+               "Unable to set "
+               "config-command");
+        goto out;
+    }
 
-        if (config_type == GF_SNAP_CONFIG_TYPE_SET &&
-           (hard_limit || soft_limit)) {
-                conf_vals = snap_confopt_vals;
-                if (hard_limit && soft_limit) {
-                        question = conf_vals[GF_SNAP_CONFIG_SET_BOTH].question;
-                } else if (soft_limit) {
-                        question = conf_vals[GF_SNAP_CONFIG_SET_SOFT].question;
-                } else if (hard_limit) {
-                        question = conf_vals[GF_SNAP_CONFIG_SET_HARD].question;
-                }
+    if (config_type == GF_SNAP_CONFIG_TYPE_SET && (hard_limit || soft_limit)) {
+        conf_vals = snap_confopt_vals;
+        if (hard_limit && soft_limit) {
+            question = conf_vals[GF_SNAP_CONFIG_SET_BOTH].question;
+        } else if (soft_limit) {
+            question = conf_vals[GF_SNAP_CONFIG_SET_SOFT].question;
+        } else if (hard_limit) {
+            question = conf_vals[GF_SNAP_CONFIG_SET_HARD].question;
+        }
 
-                answer = cli_cmd_get_confirmation (state, question);
-                if (GF_ANSWER_NO == answer) {
-                        ret = 1;
-                        gf_log ("cli", GF_LOG_DEBUG, "User cancelled "
-                        "snapshot config operation");
-                }
+        answer = cli_cmd_get_confirmation(state, question);
+        if (GF_ANSWER_NO == answer) {
+            ret = 1;
+            gf_log("cli", GF_LOG_DEBUG,
+                   "User cancelled "
+                   "snapshot config operation");
         }
+    }
 
 out:
-        return ret;
+    return ret;
 }
 
 int
-validate_snapname (const char *snapname, char **opwords) {
-        int     ret     =       -1;
-        int     i       =       0;
+validate_op_name(const char *op, const char *opname, char **opwords)
+{
+    int ret = -1;
+    int i = 0;
 
-        GF_ASSERT (snapname);
-        GF_ASSERT (opwords);
+    GF_ASSERT(opname);
+    GF_ASSERT(opwords);
 
-        for (i = 0 ; opwords[i] != NULL; i++) {
-                if (strcmp (opwords[i], snapname) == 0) {
-                        cli_out ("\"%s\" cannot be a snapname", snapname);
-                        goto out;
-                }
+    for (i = 0; opwords[i] != NULL; i++) {
+        if (strcmp(opwords[i], opname) == 0) {
+            cli_out("\"%s\" cannot be a %s", opname, op);
+            goto out;
         }
-        ret = 0;
+    }
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 int32_t
-cli_cmd_snapshot_parse (const char **words, int wordcount, dict_t **options,
-                        struct cli_state *state)
+cli_cmd_snapshot_parse(const char **words, int wordcount, dict_t **options,
+                       struct cli_state *state)
 {
-        int32_t            ret        = -1;
-        dict_t             *dict      = NULL;
-        gf1_cli_snapshot   type       = GF_SNAP_OPTION_TYPE_NONE;
-        char               *w         = NULL;
-        char               *opwords[] = {"create", "delete", "restore",
-                                        "activate", "deactivate", "list",
-                                        "status", "config", "info", NULL};
-        char               *invalid_snapnames[] = {"description", "force",
-                                                  "volume", "all", NULL};
-
-        GF_ASSERT (words);
-        GF_ASSERT (options);
-        GF_ASSERT (state);
-
-        dict = dict_new ();
-        if (!dict)
-                goto out;
+    int32_t ret = -1;
+    dict_t *dict = NULL;
+    gf1_cli_snapshot type = GF_SNAP_OPTION_TYPE_NONE;
+    char *w = NULL;
+    static char *opwords[] = {"create",     "delete", "restore", "activate",
+                              "deactivate", "list",   "status",  "config",
+                              "info",       "clone",  NULL};
+    static char *invalid_snapnames[] = {"description", "force", "volume", "all",
+                                        NULL};
+    static char *invalid_volnames[] = {"volume",
+                                       "type",
+                                       "subvolumes",
+                                       "option",
+                                       "end-volume",
+                                       "all",
+                                       "volume_not_in_ring",
+                                       "description",
+                                       "force",
+                                       "snap-max-hard-limit",
+                                       "snap-max-soft-limit",
+                                       "auto-delete",
+                                       "activate-on-create",
+                                       NULL};
 
-        /* Lowest wordcount possible */
-        if (wordcount < 2) {
-                gf_log ("", GF_LOG_ERROR,
-                        "Invalid command: Not enough arguments");
-                goto out;
-        }
+    GF_ASSERT(words);
+    GF_ASSERT(options);
+    GF_ASSERT(state);
 
-        w = str_getunamb (words[1], opwords);
-        if (!w) {
-                /* Checks if the operation is a valid operation */
-                gf_log ("", GF_LOG_ERROR, "Opword Mismatch");
-                goto out;
-        }
+    dict = dict_new();
+    if (!dict)
+        goto out;
 
-        if (!strcmp (w, "create")) {
-                type = GF_SNAP_OPTION_TYPE_CREATE;
-        } else if (!strcmp (w, "list")) {
-                type = GF_SNAP_OPTION_TYPE_LIST;
-        } else if (!strcmp (w, "info")) {
-                type = GF_SNAP_OPTION_TYPE_INFO;
-        } else if (!strcmp (w, "delete")) {
-                type = GF_SNAP_OPTION_TYPE_DELETE;
-        } else if (!strcmp (w, "config")) {
-                type = GF_SNAP_OPTION_TYPE_CONFIG;
-        } else if (!strcmp (w, "restore")) {
-                type = GF_SNAP_OPTION_TYPE_RESTORE;
-        } else if (!strcmp (w, "status")) {
-                type = GF_SNAP_OPTION_TYPE_STATUS;
-        } else if (!strcmp (w, "activate")) {
-                type = GF_SNAP_OPTION_TYPE_ACTIVATE;
-        } else if (!strcmp (w, "deactivate")) {
-                type = GF_SNAP_OPTION_TYPE_DEACTIVATE;
-        }
-
-        if (type != GF_SNAP_OPTION_TYPE_CONFIG &&
-            type != GF_SNAP_OPTION_TYPE_STATUS) {
-                ret = dict_set_int32 (dict, "hold_snap_locks", _gf_true);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Unable to set hold-snap-locks value "
-                                "as _gf_true");
-                        goto out;
-                }
-        }
+    /* Lowest wordcount possible */
+    if (wordcount < 2) {
+        gf_log("", GF_LOG_ERROR, "Invalid command: Not enough arguments");
+        goto out;
+    }
 
-        /* Following commands does not require volume locks  */
-        if (type == GF_SNAP_OPTION_TYPE_STATUS ||
-            type == GF_SNAP_OPTION_TYPE_ACTIVATE ||
-            type == GF_SNAP_OPTION_TYPE_DEACTIVATE) {
-                ret = dict_set_int32 (dict, "hold_vol_locks", _gf_false);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Setting volume lock "
-                                "flag failed");
-                        goto out;
-                }
+    w = str_getunamb(words[1], opwords);
+    if (!w) {
+        /* Checks if the operation is a valid operation */
+        gf_log("", GF_LOG_ERROR, "Opword Mismatch");
+        goto out;
+    }
+
+    if (!strcmp(w, "create")) {
+        type = GF_SNAP_OPTION_TYPE_CREATE;
+    } else if (!strcmp(w, "list")) {
+        type = GF_SNAP_OPTION_TYPE_LIST;
+    } else if (!strcmp(w, "info")) {
+        type = GF_SNAP_OPTION_TYPE_INFO;
+    } else if (!strcmp(w, "delete")) {
+        type = GF_SNAP_OPTION_TYPE_DELETE;
+    } else if (!strcmp(w, "config")) {
+        type = GF_SNAP_OPTION_TYPE_CONFIG;
+    } else if (!strcmp(w, "restore")) {
+        type = GF_SNAP_OPTION_TYPE_RESTORE;
+    } else if (!strcmp(w, "status")) {
+        type = GF_SNAP_OPTION_TYPE_STATUS;
+    } else if (!strcmp(w, "activate")) {
+        type = GF_SNAP_OPTION_TYPE_ACTIVATE;
+    } else if (!strcmp(w, "deactivate")) {
+        type = GF_SNAP_OPTION_TYPE_DEACTIVATE;
+    } else if (!strcmp(w, "clone")) {
+        type = GF_SNAP_OPTION_TYPE_CLONE;
+    }
+
+    if (type != GF_SNAP_OPTION_TYPE_CONFIG &&
+        type != GF_SNAP_OPTION_TYPE_STATUS) {
+        ret = dict_set_int32(dict, "hold_snap_locks", _gf_true);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Unable to set hold-snap-locks value "
+                   "as _gf_true");
+            goto out;
+        }
+    }
+
+    /* Following commands does not require volume locks  */
+    if (type == GF_SNAP_OPTION_TYPE_STATUS ||
+        type == GF_SNAP_OPTION_TYPE_ACTIVATE ||
+        type == GF_SNAP_OPTION_TYPE_DEACTIVATE) {
+        ret = dict_set_int32(dict, "hold_vol_locks", _gf_false);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Setting volume lock "
+                   "flag failed");
+            goto out;
         }
+    }
 
-        /* Check which op is intended */
-        switch (type) {
+    /* Check which op is intended */
+    switch (type) {
         case GF_SNAP_OPTION_TYPE_CREATE:
-                /* Syntax :
-                 * gluster snapshot create <snapname> <vol-name(s)>
-                 *                         [description <description>]
-                 *                         [force]
-                 */
-                /* In cases where the snapname is not given then
-                 * parsing fails & snapname cannot be "description",
-                 * "force" and "volume", that check is made here
-                 */
-                if (wordcount == 2){
-                        ret = -1;
-                        gf_log ("cli", GF_LOG_ERROR, "Invalid Syntax");
-                        goto out;
-                }
+            /* Syntax :
+             * gluster snapshot create <snapname> <vol-name(s)>
+             *                         [no-timestamp]
+             *                         [description <description>]
+             *                         [force]
+             */
+            /* In cases where the snapname is not given then
+             * parsing fails & snapname cannot be "description",
+             * "force" and "volume", that check is made here
+             */
+            if (wordcount == 2) {
+                ret = -1;
+                gf_log("cli", GF_LOG_ERROR, "Invalid Syntax");
+                goto out;
+            }
 
-                ret = validate_snapname (words[2], invalid_snapnames);
-                if (ret) {
-                        goto out;
-                }
+            ret = validate_op_name("snapname", words[2], invalid_snapnames);
+            if (ret) {
+                goto out;
+            }
 
-                ret = cli_snap_create_parse (dict, words, wordcount);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "create command parsing failed.");
-                        goto out;
-                }
-                break;
+            ret = cli_snap_create_parse(dict, words, wordcount);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR, "create command parsing failed.");
+                goto out;
+            }
+            break;
+
+        case GF_SNAP_OPTION_TYPE_CLONE:
+            /* Syntax :
+             * gluster snapshot clone <clonename> <snapname>
+             */
+            /* In cases where the clonename is not given then
+             * parsing fails & snapname cannot be "description",
+             * "force" and "volume", that check is made here
+             */
+            if (wordcount == 2) {
+                ret = -1;
+                gf_log("cli", GF_LOG_ERROR, "Invalid Syntax");
+                goto out;
+            }
+
+            ret = validate_op_name("clonename", words[2], invalid_volnames);
+            if (ret) {
+                goto out;
+            }
+
+            ret = cli_snap_clone_parse(dict, words, wordcount);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR, "clone command parsing failed.");
+                goto out;
+            }
+            break;
 
         case GF_SNAP_OPTION_TYPE_INFO:
-                /* Syntax :
-                 * gluster snapshot info [(snapname] | [vol <volname>)]
-                 */
-                ret = cli_snap_info_parse (dict, words, wordcount);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Failed to parse "
-                                "snapshot info command");
-                        goto out;
-                }
-                break;
+            /* Syntax :
+             * gluster snapshot info [(snapname] | [vol <volname>)]
+             */
+            ret = cli_snap_info_parse(dict, words, wordcount);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR,
+                       "Failed to parse "
+                       "snapshot info command");
+                goto out;
+            }
+            break;
 
         case GF_SNAP_OPTION_TYPE_LIST:
-                /* Syntax :
-                 * gluster snaphsot list [volname]
-                 */
+            /* Syntax :
+             * gluster snaphsot list [volname]
+             */
 
-                ret = cli_snap_list_parse (dict, words, wordcount);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Failed to parse "
-                                "snapshot list command");
-                        goto out;
-                }
-                break;
+            ret = cli_snap_list_parse(dict, words, wordcount);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR,
+                       "Failed to parse "
+                       "snapshot list command");
+                goto out;
+            }
+            break;
 
         case GF_SNAP_OPTION_TYPE_DELETE:
-                /* Syntax :
-                 * snapshot delete (all | snapname | volume <volname>)
-                 */
-                ret = cli_snap_delete_parse (dict, words, wordcount, state);
-                if (ret) {
-                        /* A positive ret value means user cancelled
-                        * the command */
-                        if (ret < 0) {
-                                gf_log ("cli", GF_LOG_ERROR, "Failed to parse "
-                                        "snapshot delete command");
-                        }
-                        goto out;
+            /* Syntax :
+             * snapshot delete (all | snapname | volume <volname>)
+             */
+            ret = cli_snap_delete_parse(dict, words, wordcount, state);
+            if (ret) {
+                /* A positive ret value means user cancelled
+                 * the command */
+                if (ret < 0) {
+                    gf_log("cli", GF_LOG_ERROR,
+                           "Failed to parse "
+                           "snapshot delete command");
                 }
-                break;
+                goto out;
+            }
+            break;
 
         case GF_SNAP_OPTION_TYPE_CONFIG:
-                /* snapshot config [volname]  [snap-max-hard-limit <count>]
-                 *                            [snap-max-soft-limit <percent>] */
-                ret = cli_snap_config_parse (words, wordcount, dict, state);
-                if (ret) {
-                        if (ret < 0)
-                                gf_log ("cli", GF_LOG_ERROR,
-                                        "config command parsing failed.");
-                        goto out;
+            /* snapshot config [volname]  [snap-max-hard-limit <count>]
+             *                            [snap-max-soft-limit <percent>] */
+            ret = cli_snap_config_parse(words, wordcount, dict, state);
+            if (ret) {
+                if (ret < 0)
+                    gf_log("cli", GF_LOG_ERROR,
+                           "config command parsing failed.");
+                goto out;
+            }
+
+            ret = dict_set_int32(dict, "type", GF_SNAP_OPTION_TYPE_CONFIG);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR,
+                       "Unable to set "
+                       "config type");
+                ret = -1;
+                goto out;
+            }
+            break;
+
+        case GF_SNAP_OPTION_TYPE_STATUS: {
+            /* Syntax :
+             * gluster snapshot status [(snapname |
+             *                         volume <volname>)]
+             */
+            ret = cli_snap_status_parse(dict, words, wordcount);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR,
+                       "Failed to parse "
+                       "snapshot status command");
+                goto out;
+            }
+            break;
+        }
+
+        case GF_SNAP_OPTION_TYPE_RESTORE:
+            /* Syntax:
+             * snapshot restore <snapname>
+             */
+            ret = cli_snap_restore_parse(dict, words, wordcount, state);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR,
+                       "Failed to parse "
+                       "restore command");
+                goto out;
+            }
+            break;
+
+        case GF_SNAP_OPTION_TYPE_ACTIVATE:
+            /* Syntax:
+             * snapshot activate <snapname> [force]
+             */
+            ret = cli_snap_activate_parse(dict, words, wordcount);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR,
+                       "Failed to parse "
+                       "start command");
+                goto out;
+            }
+            break;
+        case GF_SNAP_OPTION_TYPE_DEACTIVATE:
+            /* Syntax:
+             * snapshot deactivate <snapname>
+             */
+            ret = cli_snap_deactivate_parse(dict, words, wordcount, state);
+            if (ret) {
+                /* A positive ret value means user cancelled
+                 * the command */
+                if (ret < 0) {
+                    gf_log("cli", GF_LOG_ERROR,
+                           "Failed to parse deactivate "
+                           "command");
                 }
+                goto out;
+            }
+            break;
+
+        default:
+            ret = -1;
+            gf_log("", GF_LOG_ERROR, "Opword Mismatch");
+            goto out;
+    }
+
+    ret = dict_set_int32(dict, "type", type);
+    if (ret) {
+        gf_log("", GF_LOG_ERROR, "Failed to set type.");
+        goto out;
+    }
+    /* If you got so far, input is valid */
+    ret = 0;
+out:
+    if (ret) {
+        if (dict)
+            dict_unref(dict);
+    } else
+        *options = dict;
+
+    return ret;
+}
+
+int
+cli_cmd_validate_volume(char *volname)
+{
+    int i = 0;
+    int ret = -1;
+    int volname_len;
+
+    if (volname[0] == '-')
+        return ret;
+
+    if (!strcmp(volname, "all")) {
+        cli_err("\"all\" cannot be the name of a volume.");
+        return ret;
+    }
+
+    if (strchr(volname, '/')) {
+        cli_err("Volume name should not contain \"/\" character.");
+        return ret;
+    }
+
+    volname_len = strlen(volname);
+    if (volname_len > GD_VOLUME_NAME_MAX) {
+        cli_err("Volname can not exceed %d characters.", GD_VOLUME_NAME_MAX);
+        return ret;
+    }
+
+    for (i = 0; i < volname_len; i++)
+        if (!isalnum(volname[i]) && (volname[i] != '_') &&
+            (volname[i] != '-')) {
+            cli_err(
+                "Volume name should not contain \"%c\""
+                " character.\nVolume names can only"
+                "contain alphanumeric, '-' and '_' "
+                "characters.",
+                volname[i]);
+            return ret;
+        }
+
+    ret = 0;
 
-                ret = dict_set_int32 (dict, "type", GF_SNAP_OPTION_TYPE_CONFIG);
+    return ret;
+}
+
+int32_t
+cli_cmd_bitrot_parse(const char **words, int wordcount, dict_t **options)
+{
+    int32_t ret = -1;
+    char *w = NULL;
+    char *volname = NULL;
+    static char *opwords[] = {"enable",          "disable", "scrub-throttle",
+                              "scrub-frequency", "scrub",   "signing-time",
+                              "signer-threads",  NULL};
+    static char *scrub_throt_values[] = {"lazy", "normal", "aggressive", NULL};
+    static char *scrub_freq_values[] = {
+        "hourly", "daily", "weekly", "biweekly", "monthly", "minute", NULL};
+    static char *scrub_values[] = {"pause", "resume", "status", "ondemand",
+                                   NULL};
+    dict_t *dict = NULL;
+    gf_bitrot_type type = GF_BITROT_OPTION_TYPE_NONE;
+    int32_t expiry_time = 0;
+    int32_t signer_th_count = 0;
+
+    GF_ASSERT(words);
+    GF_ASSERT(options);
+
+    /* Hack to print out bitrot help properly */
+    if ((wordcount == 3) && !(strcmp(words[2], "help"))) {
+        ret = 1;
+        return ret;
+    }
+
+    if (wordcount < 4 || wordcount > 5) {
+        gf_log("cli", GF_LOG_ERROR, "Invalid syntax");
+        goto out;
+    }
+
+    dict = dict_new();
+    if (!dict)
+        goto out;
+
+    volname = (char *)words[2];
+    if (!volname) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = cli_cmd_validate_volume(volname);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to validate volume name");
+        goto out;
+    }
+
+    ret = dict_set_str(dict, "volname", volname);
+    if (ret) {
+        cli_out("Failed to set volume name in dictionary ");
+        goto out;
+    }
+
+    w = str_getunamb(words[3], opwords);
+    if (!w) {
+        cli_out("Invalid bit rot option : %s", words[3]);
+        ret = -1;
+        goto out;
+    }
+
+    if (strcmp(w, "enable") == 0) {
+        if (wordcount == 4) {
+            type = GF_BITROT_OPTION_TYPE_ENABLE;
+            ret = 0;
+            goto set_type;
+        } else {
+            ret = -1;
+            goto out;
+        }
+    }
+
+    if (strcmp(w, "disable") == 0) {
+        if (wordcount == 4) {
+            type = GF_BITROT_OPTION_TYPE_DISABLE;
+            ret = 0;
+            goto set_type;
+        } else {
+            ret = -1;
+            goto out;
+        }
+    }
+
+    if (!strcmp(w, "scrub-throttle")) {
+        if (!words[4]) {
+            cli_err(
+                "Missing scrub-throttle value for bitrot "
+                "option");
+            ret = -1;
+            goto out;
+        } else {
+            w = str_getunamb(words[4], scrub_throt_values);
+            if (!w) {
+                cli_err(
+                    "Invalid scrub-throttle option for "
+                    "bitrot");
+                ret = -1;
+                goto out;
+            } else {
+                type = GF_BITROT_OPTION_TYPE_SCRUB_THROTTLE;
+                ret = dict_set_str(dict, "scrub-throttle-value",
+                                   (char *)words[4]);
                 if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Unable to set "
-                                "config type");
-                        ret = -1;
-                        goto out;
+                    cli_out(
+                        "Failed to set scrub-throttle "
+                        "value in the dict");
+                    goto out;
                 }
-                break;
-
-        case GF_SNAP_OPTION_TYPE_STATUS:
-                {
-                        /* Syntax :
-                         * gluster snapshot status [(snapname |
-                         *                         volume <volname>)]
-                         */
-                        ret = cli_snap_status_parse (dict, words, wordcount);
-                        if (ret) {
-                                gf_log ("cli", GF_LOG_ERROR, "Failed to parse "
-                                        "snapshot status command");
-                                goto out;
-                        }
-                        break;
+                goto set_type;
+            }
+        }
+    }
+
+    if (!strcmp(words[3], "scrub-frequency")) {
+        if (!words[4]) {
+            cli_err("Missing scrub-frequency value");
+            ret = -1;
+            goto out;
+        } else {
+            w = str_getunamb(words[4], scrub_freq_values);
+            if (!w) {
+                cli_err("Invalid frequency option for bitrot");
+                ret = -1;
+                goto out;
+            } else {
+                type = GF_BITROT_OPTION_TYPE_SCRUB_FREQ;
+                ret = dict_set_str(dict, "scrub-frequency-value",
+                                   (char *)words[4]);
+                if (ret) {
+                    cli_out(
+                        "Failed to set dict for "
+                        "bitrot");
+                    goto out;
                 }
+                goto set_type;
+            }
+        }
+    }
 
-        case GF_SNAP_OPTION_TYPE_RESTORE:
-                /* Syntax:
-                 * snapshot restore <snapname>
-                 */
-                ret = cli_snap_restore_parse (dict, words, wordcount, state);
+    if (!strcmp(words[3], "scrub")) {
+        if (!words[4]) {
+            cli_err("Missing scrub value for bitrot option");
+            ret = -1;
+            goto out;
+        } else {
+            w = str_getunamb(words[4], scrub_values);
+            if (!w) {
+                cli_err("Invalid scrub option for bitrot");
+                ret = -1;
+                goto out;
+            } else {
+                if (strcmp(words[4], "status") == 0) {
+                    type = GF_BITROT_CMD_SCRUB_STATUS;
+                } else if (strcmp(words[4], "ondemand") == 0) {
+                    type = GF_BITROT_CMD_SCRUB_ONDEMAND;
+                } else {
+                    type = GF_BITROT_OPTION_TYPE_SCRUB;
+                }
+                ret = dict_set_str(dict, "scrub-value", (char *)words[4]);
                 if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Failed to parse "
-                                "restore command");
-                        goto out;
+                    cli_out(
+                        "Failed to set dict for "
+                        "bitrot");
+                    goto out;
                 }
-                break;
-
-                case GF_SNAP_OPTION_TYPE_ACTIVATE:
-                        /* Syntax:
-                        * snapshot activate <snapname> [force]
-                        */
-                        ret = cli_snap_activate_parse (dict, words, wordcount);
-                        if (ret) {
-                                gf_log ("cli", GF_LOG_ERROR, "Failed to parse "
-                                        "start command");
-                                goto out;
-                        }
-                        break;
-                case GF_SNAP_OPTION_TYPE_DEACTIVATE:
-                        /* Syntax:
-                        * snapshot deactivate <snapname>
-                        */
-                        ret = cli_snap_deactivate_parse (dict, words, wordcount,
-                                state);
-                        if (ret) {
-                                /* A positive ret value means user cancelled
-                                 * the command */
-                                if (ret < 0) {
-                                        gf_log ("cli", GF_LOG_ERROR,
-                                                "Failed to parse deactivate "
-                                                "command");
-                                }
-                                goto out;
-                        }
-                        break;
+                goto set_type;
+            }
+        }
+    }
 
-        default:
-                gf_log ("", GF_LOG_ERROR, "Opword Mismatch");
+    if (!strcmp(words[3], "signing-time")) {
+        if (!words[4]) {
+            cli_err(
+                "Missing signing-time value for bitrot "
+                "option");
+            ret = -1;
+            goto out;
+        } else {
+            type = GF_BITROT_OPTION_TYPE_EXPIRY_TIME;
+
+            expiry_time = strtol(words[4], NULL, 0);
+            if (expiry_time < 1) {
+                cli_err(
+                    "Expiry time  value should not be less"
+                    " than 1");
+                ret = -1;
+                goto out;
+            }
+
+            ret = dict_set_uint32(dict, "expiry-time",
+                                  (unsigned int)expiry_time);
+            if (ret) {
+                cli_out("Failed to set dict for bitrot");
                 goto out;
+            }
+            goto set_type;
         }
+    } else if (!strcmp(words[3], "signer-threads")) {
+        if (!words[4]) {
+            cli_err(
+                "Missing signer-thread value for bitrot "
+                "option");
+            ret = -1;
+            goto out;
+        } else {
+            type = GF_BITROT_OPTION_TYPE_SIGNER_THREADS;
 
-        ret = dict_set_int32 (dict, "type", type);
-        if (ret) {
-                gf_log ("", GF_LOG_ERROR,
-                        "Failed to set type.");
+            signer_th_count = strtol(words[4], NULL, 0);
+            if (signer_th_count < 1) {
+                cli_err("signer-thread count should not be less than 1");
+                ret = -1;
                 goto out;
+            }
+
+            ret = dict_set_uint32(dict, "signer-threads",
+                                  (unsigned int)signer_th_count);
+            if (ret) {
+                cli_out("Failed to set dict for bitrot");
+                goto out;
+            }
+            goto set_type;
         }
-        /* If you got so far, input is valid */
-        ret = 0;
+    } else {
+        cli_err(
+            "Invalid option %s for bitrot. Please enter valid "
+            "bitrot option",
+            words[3]);
+        ret = -1;
+        goto out;
+    }
+set_type:
+    ret = dict_set_int32(dict, "type", type);
+    if (ret < 0)
+        goto out;
+
+    *options = dict;
+
 out:
-        if (ret) {
-                if (dict)
-                        dict_destroy (dict);
-        } else
-                *options = dict;
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Unable to parse bitrot command");
+        if (dict)
+            dict_unref(dict);
+    }
 
-        return ret;
+    return ret;
+}
+
+/* Parsing global option for NFS-Ganesha config
+ *  gluster nfs-ganesha enable/disable */
+
+int32_t
+cli_cmd_ganesha_parse(struct cli_state *state, const char **words,
+                      int wordcount, dict_t **options, char **op_errstr)
+{
+    dict_t *dict = NULL;
+    int ret = -1;
+    char *key = NULL;
+    char *value = NULL;
+    char *w = NULL;
+    static char *opwords[] = {"enable", "disable", NULL};
+    const char *question = NULL;
+    gf_answer_t answer = GF_ANSWER_NO;
+
+    GF_ASSERT(words);
+    GF_ASSERT(options);
+
+    dict = dict_new();
+
+    if (!dict)
+        goto out;
+
+    if (wordcount != 2)
+        goto out;
+
+    key = (char *)words[0];
+    value = (char *)words[1];
+
+    if (!key || !value) {
+        cli_out("Usage : nfs-ganesha <enable/disable>");
+        ret = -1;
+        goto out;
+    }
+
+    ret = gf_strip_whitespace(value, strlen(value));
+    if (ret == -1)
+        goto out;
+
+    if (strcmp(key, "nfs-ganesha")) {
+        gf_asprintf(op_errstr,
+                    "Global option: error: ' %s '"
+                    "is not a valid global option.",
+                    key);
+        ret = -1;
+        goto out;
+    }
+
+    w = str_getunamb(value, opwords);
+    if (!w) {
+        cli_out(
+            "Invalid global option \n"
+            "Usage : nfs-ganesha <enable/disable>");
+        ret = -1;
+        goto out;
+    }
+
+    if (strcmp(value, "enable") == 0) {
+        question =
+            "Enabling NFS-Ganesha requires Gluster-NFS to be "
+            "disabled across the trusted pool. Do you "
+            "still want to continue?\n";
+    } else if (strcmp(value, "disable") == 0) {
+        question =
+            "Disabling NFS-Ganesha will tear down the entire "
+            "ganesha cluster across the trusted pool. Do you "
+            "still want to continue?\n";
+    } else {
+        ret = -1;
+        goto out;
+    }
+    answer = cli_cmd_get_confirmation(state, question);
+    if (GF_ANSWER_NO == answer) {
+        gf_log("cli", GF_LOG_ERROR,
+               "Global operation "
+               "cancelled, exiting");
+        ret = -1;
+        goto out;
+    }
+    cli_out("This will take a few minutes to complete. Please wait ..");
+
+    ret = dict_set_str(dict, "key", key);
+    if (ret) {
+        gf_log(THIS->name, GF_LOG_ERROR, "dict set on key failed");
+        goto out;
+    }
+
+    ret = dict_set_str(dict, "value", value);
+    if (ret) {
+        gf_log(THIS->name, GF_LOG_ERROR, "dict set on value failed");
+        goto out;
+    }
+
+    ret = dict_set_str(dict, "globalname", "All");
+    if (ret) {
+        gf_log(THIS->name, GF_LOG_ERROR,
+               "dict set on global"
+               " key failed.");
+        goto out;
+    }
+
+    ret = dict_set_int32(dict, "hold_global_locks", _gf_true);
+    if (ret) {
+        gf_log(THIS->name, GF_LOG_ERROR,
+               "dict set on global key "
+               "failed.");
+        goto out;
+    }
+
+    *options = dict;
+out:
+    if (ret)
+        dict_unref(dict);
+
+    return ret;
 }
diff --git a/cli/src/cli-cmd-peer.c b/cli/src/cli-cmd-peer.c
index 551312411a0..084998701d8 100644
--- a/cli/src/cli-cmd-peer.c
+++ b/cli/src/cli-cmd-peer.c
@@ -13,288 +13,305 @@
 #include <stdint.h>
 #include <pthread.h>
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 #include "cli.h"
 #include "cli-cmd.h"
 #include "cli-mem-types.h"
 #include "cli1-xdr.h"
 #include "protocol-common.h"
+#include <glusterfs/events.h>
 
-extern struct rpc_clnt *global_rpc;
-
-extern rpc_clnt_prog_t *cli_rpc_prog;
-
-int cli_cmd_peer_help_cbk (struct cli_state *state, struct cli_cmd_word *in_word,
+int
+cli_cmd_peer_help_cbk(struct cli_state *state, struct cli_cmd_word *in_word,
                       const char **words, int wordcount);
 
 int
-cli_cmd_peer_probe_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                   const char **words, int wordcount)
+cli_cmd_peer_probe_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                       const char **words, int wordcount)
 {
-        int                     ret = -1;
-        rpc_clnt_procedure_t    *proc = NULL;
-        call_frame_t            *frame = NULL;
-        dict_t                  *dict = NULL;
-        int                     sent = 0;
-        int                     parse_error = 0;
-        cli_local_t             *local = NULL;
-
-        if (!(wordcount == 3)) {
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-                goto out;
-        }
-
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_PROBE];
-
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
-
-        dict = dict_new ();
-        if (!dict)
-                goto out;
-
-        ret = dict_set_str (dict, "hostname", (char *)words[2]);
-        if (ret)
-                goto out;
-
-        ret = valid_internet_address ((char *) words[2], _gf_false);
-        if (ret == 1) {
-                ret = 0;
-        } else {
-                cli_out ("%s is an invalid address", words[2]);
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-                ret = -1;
-                goto out;
-        }
-/*        if (words[3]) {
-                ret = dict_set_str (dict, "port", (char *)words[3]);
-                if (ret)
-                        goto out;
-        }
-*/
-
-        CLI_LOCAL_INIT (local, words, frame, dict);
-
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, dict);
-        }
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *dict = NULL;
+    int sent = 0;
+    int parse_error = 0;
+    cli_local_t *local = NULL;
+
+    if (!(wordcount == 3)) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
+
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_PROBE];
+
+    dict = dict_new();
+    if (!dict)
+        goto out;
+
+    ret = dict_set_str(dict, "hostname", (char *)words[2]);
+    if (ret)
+        goto out;
+
+    ret = valid_internet_address((char *)words[2], _gf_false, _gf_false);
+    if (ret == 1) {
+        ret = 0;
+    } else {
+        cli_out("%s is an invalid address", words[2]);
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        ret = -1;
+        goto out;
+    }
+    /*        if (words[3]) {
+                    ret = dict_set_str (dict, "port", (char *)words[3]);
+                    if (ret)
+                            goto out;
+            }
+    */
+
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
+
+    CLI_LOCAL_INIT(local, words, frame, dict);
+
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, dict);
+    }
 
 out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_out ("Peer probe failed");
-        }
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("Peer probe failed");
+    }
 
-        CLI_STACK_DESTROY (frame);
+    CLI_STACK_DESTROY(frame);
 
-        return ret;
-}
+    if (ret == 0) {
+        gf_event(EVENT_PEER_ATTACH, "host=%s", (char *)words[2]);
+    }
 
+    return ret;
+}
 
 int
-cli_cmd_peer_deprobe_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                          const char **words, int wordcount)
+cli_cmd_peer_deprobe_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                         const char **words, int wordcount)
 {
-        int                   ret   = -1;
-        rpc_clnt_procedure_t *proc  = NULL;
-        call_frame_t         *frame = NULL;
-        dict_t               *dict  = NULL;
-        int                  flags = 0;
-        int                  sent = 0;
-        int                  parse_error = 0;
-        cli_local_t          *local = NULL;
-
-        if ((wordcount < 3) || (wordcount > 4)) {
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-                goto out;
-        }
-
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_DEPROBE];
-
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
-
-        dict = dict_new ();
-
-        ret = dict_set_str (dict, "hostname", (char *)words[2]);
-        if (ret)
-                goto out;
-
-/*        if (words[3]) {
-                ret = dict_set_str (dict, "port", (char *)words[3]);
-                if (ret)
-                        goto out;
-        }
-*/
-        if (wordcount == 4) {
-                if (!strcmp("force", words[3]))
-                        flags |= GF_CLI_FLAG_OP_FORCE;
-                else {
-                        ret = -1;
-                        cli_usage_out (word->pattern);
-                        parse_error = 1;
-                        goto out;
-                }
-        }
-        ret = dict_set_int32 (dict, "flags", flags);
-        if (ret)
-                goto out;
-
-        CLI_LOCAL_INIT (local, words, frame, dict);
-
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, dict);
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *dict = NULL;
+    int flags = 0;
+    int sent = 0;
+    int parse_error = 0;
+    cli_local_t *local = NULL;
+    gf_answer_t answer = GF_ANSWER_NO;
+    const char *question = NULL;
+
+    if ((wordcount < 3) || (wordcount > 4)) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
+    question =
+        "All clients mounted through the peer which is getting detached need "
+        "to be remounted using one of the other active peers in the trusted "
+        "storage pool to ensure client gets notification on any changes done "
+        "on the gluster configuration and if the same has been done do you "
+        "want to proceed?";
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_DEPROBE];
+
+    dict = dict_new();
+
+    ret = dict_set_str(dict, "hostname", (char *)words[2]);
+    if (ret)
+        goto out;
+
+    /*        if (words[3]) {
+                    ret = dict_set_str (dict, "port", (char *)words[3]);
+                    if (ret)
+                            goto out;
+            }
+    */
+    if (wordcount == 4) {
+        if (!strcmp("force", words[3]))
+            flags |= GF_CLI_FLAG_OP_FORCE;
+        else {
+            ret = -1;
+            cli_usage_out(word->pattern);
+            parse_error = 1;
+            goto out;
         }
+    }
+    ret = dict_set_int32(dict, "flags", flags);
+    if (ret)
+        goto out;
+    answer = cli_cmd_get_confirmation(state, question);
+    if (GF_ANSWER_NO == answer) {
+        ret = 0;
+        goto out;
+    }
+
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
+
+    CLI_LOCAL_INIT(local, words, frame, dict);
+
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, dict);
+    }
 
 out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_out ("Peer detach failed");
-        }
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("Peer detach failed");
+    }
 
-        CLI_STACK_DESTROY (frame);
+    CLI_STACK_DESTROY(frame);
 
-        return ret;
+    if (ret == 0) {
+        gf_event(EVENT_PEER_DETACH, "host=%s", (char *)words[2]);
+    }
+
+    return ret;
 }
 
 int
-cli_cmd_peer_status_cbk (struct cli_state *state, struct cli_cmd_word *word,
+cli_cmd_peer_status_cbk(struct cli_state *state, struct cli_cmd_word *word,
                         const char **words, int wordcount)
 {
-        int                     ret = -1;
-        rpc_clnt_procedure_t    *proc = NULL;
-        call_frame_t            *frame = NULL;
-        int                     sent = 0;
-        int                     parse_error = 0;
-
-        if (wordcount != 2) {
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-                goto out;
-        }
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    int sent = 0;
+    int parse_error = 0;
 
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_LIST_FRIENDS];
+    if (wordcount != 2) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
 
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_LIST_FRIENDS];
 
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, (void *)GF_CLI_LIST_PEERS);
-        }
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame)
+        goto out;
+
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, (void *)GF_CLI_LIST_PEERS);
+    }
 
 out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_out ("Peer status failed");
-        }
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("Peer status failed");
+    }
 
-        CLI_STACK_DESTROY (frame);
+    CLI_STACK_DESTROY(frame);
 
-        return ret;
+    return ret;
 }
 
 int
-cli_cmd_pool_list_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                       const char **words, int wordcount)
+cli_cmd_pool_list_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                      const char **words, int wordcount)
 {
-        int                     ret = -1;
-        rpc_clnt_procedure_t    *proc = NULL;
-        call_frame_t            *frame = NULL;
-        int                     sent = 0;
-        int                     parse_error = 0;
-
-        if (wordcount != 2) {
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-                goto out;
-        }
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    int sent = 0;
+    int parse_error = 0;
 
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_LIST_FRIENDS];
+    if (wordcount != 2) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
 
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_LIST_FRIENDS];
 
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS,
-                                (void *)GF_CLI_LIST_POOL_NODES);
-        }
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame)
+        goto out;
+
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, (void *)GF_CLI_LIST_POOL_NODES);
+    }
 
 out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_err ("pool list: command execution failed");
-        }
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_err("pool list: command execution failed");
+    }
 
-        CLI_STACK_DESTROY (frame);
+    CLI_STACK_DESTROY(frame);
 
-        return ret;
+    return ret;
 }
 
 struct cli_cmd cli_probe_cmds[] = {
-        { "peer probe <HOSTNAME>",
-          cli_cmd_peer_probe_cbk,
-          "probe peer specified by <HOSTNAME>"},
+    {"peer probe { <HOSTNAME> | <IP-address> }", cli_cmd_peer_probe_cbk,
+     "probe peer specified by <HOSTNAME>"},
 
-        { "peer detach <HOSTNAME> [force]",
-          cli_cmd_peer_deprobe_cbk,
-          "detach peer specified by <HOSTNAME>"},
+    {"peer detach { <HOSTNAME> | <IP-address> } [force]",
+     cli_cmd_peer_deprobe_cbk, "detach peer specified by <HOSTNAME>"},
 
-        { "peer status",
-          cli_cmd_peer_status_cbk,
-          "list status of peers"},
+    {"peer status", cli_cmd_peer_status_cbk, "list status of peers"},
 
-	{ "peer help",
-           cli_cmd_peer_help_cbk,
-           "Help command for peer "},
+    {"peer help", cli_cmd_peer_help_cbk, "display help for peer commands"},
 
-        { "pool list",
-          cli_cmd_pool_list_cbk,
-          "list all the nodes in the pool (including localhost)"},
+    {"pool list", cli_cmd_pool_list_cbk,
+     "list all the nodes in the pool (including localhost)"},
 
-        { NULL, NULL, NULL }
-};
+    {NULL, NULL, NULL}};
 
 int
-cli_cmd_peer_help_cbk (struct cli_state *state, struct cli_cmd_word *in_word,
+cli_cmd_peer_help_cbk(struct cli_state *state, struct cli_cmd_word *in_word,
                       const char **words, int wordcount)
 {
-        struct cli_cmd        *cmd = NULL;
+    struct cli_cmd *cmd = NULL;
+    struct cli_cmd *probe_cmd = NULL;
+    int count = 0;
+
+    cli_out("\ngluster peer commands");
+    cli_out("======================\n");
 
+    cmd = GF_MALLOC(sizeof(cli_probe_cmds), cli_mt_cli_cmd);
+    memcpy(cmd, cli_probe_cmds, sizeof(cli_probe_cmds));
+    count = (sizeof(cli_probe_cmds) / sizeof(struct cli_cmd));
+    cli_cmd_sort(cmd, count);
 
+    for (probe_cmd = cmd; probe_cmd->pattern; probe_cmd++)
+        cli_out("%s - %s", probe_cmd->pattern, probe_cmd->desc);
 
-        for (cmd = cli_probe_cmds; cmd->pattern; cmd++)
-                cli_out ("%s - %s", cmd->pattern, cmd->desc);
+    GF_FREE(cmd);
 
-        return 0;
+    cli_out("\n");
+    return 0;
 }
 
 int
-cli_cmd_probe_register (struct cli_state *state)
+cli_cmd_probe_register(struct cli_state *state)
 {
-        int  ret = 0;
-        struct cli_cmd *cmd = NULL;
+    int ret = 0;
+    struct cli_cmd *cmd = NULL;
 
-        for (cmd = cli_probe_cmds; cmd->pattern; cmd++) {
-
-                ret = cli_cmd_register (&state->tree, cmd);
-                if (ret)
-                        goto out;
-        }
+    for (cmd = cli_probe_cmds; cmd->pattern; cmd++) {
+        ret = cli_cmd_register(&state->tree, cmd);
+        if (ret)
+            goto out;
+    }
 out:
-        return ret;
+    return ret;
 }
diff --git a/cli/src/cli-cmd-snapshot.c b/cli/src/cli-cmd-snapshot.c
index d03b5b99dd0..859d6b2e40d 100644
--- a/cli/src/cli-cmd-snapshot.c
+++ b/cli/src/cli-cmd-snapshot.c
@@ -13,144 +13,123 @@
 #include <stdint.h>
 #include <pthread.h>
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 #include "cli.h"
 #include "cli-cmd.h"
-
-extern rpc_clnt_prog_t *cli_rpc_prog;
+#include "cli-mem-types.h"
 
 int
-cli_cmd_snapshot_help_cbk (struct cli_state *state, struct cli_cmd_word *in_word,
-                           const char **words, int wordcount);
+cli_cmd_snapshot_help_cbk(struct cli_state *state, struct cli_cmd_word *in_word,
+                          const char **words, int wordcount);
 
 int
-cli_cmd_snapshot_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                      const char **words, int wordcount)
+cli_cmd_snapshot_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                     const char **words, int wordcount)
 {
-        int                      ret       = 0;
-        int                      parse_err = 0;
-        dict_t                  *options   = NULL;
-        rpc_clnt_procedure_t    *proc      = NULL;
-        call_frame_t            *frame     = NULL;
-        cli_local_t             *local     = NULL;
-
-        proc = &cli_rpc_prog->proctable [GLUSTER_CLI_SNAP];
-        if (proc == NULL) {
-               ret = -1;
-                goto out;
+    int ret = 0;
+    int parse_err = 0;
+    dict_t *options = NULL;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    cli_local_t *local = NULL;
+
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_SNAP];
+
+    /* Parses the command entered by the user */
+    ret = cli_cmd_snapshot_parse(words, wordcount, &options, state);
+    if (ret) {
+        if (ret < 0) {
+            cli_usage_out(word->pattern);
+            parse_err = 1;
+        } else {
+            /* User might have cancelled the snapshot operation */
+            ret = 0;
         }
+        goto out;
+    }
 
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (frame == NULL) {
-                ret = -1;
-                goto out;
-        }
-
-        /* Parses the command entered by the user */
-        ret = cli_cmd_snapshot_parse (words, wordcount, &options, state);
-        if (ret) {
-                if (ret < 0) {
-                        cli_usage_out (word->pattern);
-                        parse_err = 1;
-                } else {
-                        /* User might have cancelled the snapshot operation */
-                        ret = 0;
-                }
-                goto out;
-        }
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (frame == NULL) {
+        ret = -1;
+        goto out;
+    }
 
-        CLI_LOCAL_INIT (local, words, frame, options);
+    CLI_LOCAL_INIT(local, words, frame, options);
 
-        if (proc->fn)
-                ret = proc->fn (frame, THIS, options);
+    if (proc->fn)
+        ret = proc->fn(frame, THIS, options);
 
 out:
-        if (ret && parse_err == 0)
-                cli_out ("Snapshot command failed");
+    if (ret && parse_err == 0)
+        cli_out("Snapshot command failed");
 
-        CLI_STACK_DESTROY (frame);
+    CLI_STACK_DESTROY(frame);
 
-        return ret;
+    return ret;
 }
 
 struct cli_cmd snapshot_cmds[] = {
-        { "snapshot help",
-          cli_cmd_snapshot_help_cbk,
-          "display help for snapshot commands"
-        },
-        { "snapshot create <snapname> <volname(s)> [description <description>] [force]",
-          cli_cmd_snapshot_cbk,
-          "Snapshot Create."
-        },
-        { "snapshot restore <snapname>",
-          cli_cmd_snapshot_cbk,
-          "Snapshot Restore."
-        },
-        { "snapshot status [(snapname | volume <volname>)]",
-          cli_cmd_snapshot_cbk,
-          "Snapshot Status."
-        },
-        { "snapshot info [(snapname | volume <volname>)]",
-          cli_cmd_snapshot_cbk,
-          "Snapshot Info."
-        },
-        { "snapshot list [volname]",
-          cli_cmd_snapshot_cbk,
-          "Snapshot List."
-        },
-        {"snapshot config [volname] ([snap-max-hard-limit <count>] "
-         "[snap-max-soft-limit <percent>]) "
-         "| ([auto-delete <enable|disable>])"
-         "| ([activate-on-create <enable|disable>])",
-          cli_cmd_snapshot_cbk,
-          "Snapshot Config."
-        },
-        {"snapshot delete (all | snapname | volume <volname>)",
-          cli_cmd_snapshot_cbk,
-          "Snapshot Delete."
-        },
-        {"snapshot activate <snapname> [force]",
-          cli_cmd_snapshot_cbk,
-          "Activate snapshot volume."
-        },
-        {"snapshot deactivate <snapname>",
-          cli_cmd_snapshot_cbk,
-          "Deactivate snapshot volume."
-        },
-        { NULL, NULL, NULL }
-};
+    {"snapshot help", cli_cmd_snapshot_help_cbk,
+     "display help for snapshot commands"},
+    {"snapshot create <snapname> <volname> [no-timestamp] "
+     "[description <description>] [force]",
+     cli_cmd_snapshot_cbk, "Snapshot Create."},
+    {"snapshot clone <clonename> <snapname>", cli_cmd_snapshot_cbk,
+     "Snapshot Clone."},
+    {"snapshot restore <snapname>", cli_cmd_snapshot_cbk, "Snapshot Restore."},
+    {"snapshot status [(snapname | volume <volname>)]", cli_cmd_snapshot_cbk,
+     "Snapshot Status."},
+    {"snapshot info [(snapname | volume <volname>)]", cli_cmd_snapshot_cbk,
+     "Snapshot Info."},
+    {"snapshot list [volname]", cli_cmd_snapshot_cbk, "Snapshot List."},
+    {"snapshot config [volname] ([snap-max-hard-limit <count>] "
+     "[snap-max-soft-limit <percent>]) "
+     "| ([auto-delete <enable|disable>])"
+     "| ([activate-on-create <enable|disable>])",
+     cli_cmd_snapshot_cbk, "Snapshot Config."},
+    {"snapshot delete (all | snapname | volume <volname>)",
+     cli_cmd_snapshot_cbk, "Snapshot Delete."},
+    {"snapshot activate <snapname> [force]", cli_cmd_snapshot_cbk,
+     "Activate snapshot volume."},
+    {"snapshot deactivate <snapname>", cli_cmd_snapshot_cbk,
+     "Deactivate snapshot volume."},
+    {NULL, NULL, NULL}};
 
 int
-cli_cmd_snapshot_help_cbk (struct cli_state *state,
-                           struct cli_cmd_word *in_word,
-                           const char **words,
-                           int wordcount)
+cli_cmd_snapshot_help_cbk(struct cli_state *state, struct cli_cmd_word *in_word,
+                          const char **words, int wordcount)
 {
-        struct cli_cmd        *cmd = NULL;
+    struct cli_cmd *cmd = NULL;
+    struct cli_cmd *snap_cmd = NULL;
+    int count = 0;
 
-        for (cmd = snapshot_cmds; cmd->pattern; cmd++)
-                if (_gf_false == cmd->disable)
-                        cli_out ("%s - %s", cmd->pattern, cmd->desc);
+    cmd = GF_MALLOC(sizeof(snapshot_cmds), cli_mt_cli_cmd);
+    memcpy(cmd, snapshot_cmds, sizeof(snapshot_cmds));
+    count = (sizeof(snapshot_cmds) / sizeof(struct cli_cmd));
+    cli_cmd_sort(cmd, count);
 
-        return 0;
+    cli_out("\ngluster snapshot commands");
+    cli_out("=========================\n");
+
+    for (snap_cmd = cmd; snap_cmd->pattern; snap_cmd++)
+        if (_gf_false == snap_cmd->disable)
+            cli_out("%s - %s", snap_cmd->pattern, snap_cmd->desc);
+    cli_out("\n");
+
+    GF_FREE(cmd);
+    return 0;
 }
 
 int
-cli_cmd_snapshot_register (struct cli_state *state)
+cli_cmd_snapshot_register(struct cli_state *state)
 {
-        int  ret = 0;
-        struct cli_cmd *cmd = NULL;
-
-        for (cmd = snapshot_cmds; cmd->pattern; cmd++) {
-
-                ret = cli_cmd_register (&state->tree, cmd);
-                if (ret)
-                        goto out;
-        }
+    int ret = 0;
+    struct cli_cmd *cmd = NULL;
+
+    for (cmd = snapshot_cmds; cmd->pattern; cmd++) {
+        ret = cli_cmd_register(&state->tree, cmd);
+        if (ret)
+            goto out;
+    }
 out:
-        return ret;
+    return ret;
 }
diff --git a/cli/src/cli-cmd-system.c b/cli/src/cli-cmd-system.c
index 8cfa5e70c3c..801e8f4efed 100644
--- a/cli/src/cli-cmd-system.c
+++ b/cli/src/cli-cmd-system.c
@@ -13,590 +13,612 @@
 #include <stdint.h>
 #include <pthread.h>
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 #include "cli.h"
 #include "cli-cmd.h"
 #include "cli-mem-types.h"
 #include "protocol-common.h"
 
+int
+cli_cmd_system_help_cbk(struct cli_state *state, struct cli_cmd_word *in_word,
+                        const char **words, int wordcount);
 
-extern struct rpc_clnt *global_rpc;
-
-extern rpc_clnt_prog_t *cli_rpc_prog;
-
-int cli_cmd_system_help_cbk (struct cli_state *state, struct cli_cmd_word *in_word,
-                             const char **words, int wordcount);
-
-int cli_cmd_copy_file_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                           const char **words, int wordcount);
+int
+cli_cmd_copy_file_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                      const char **words, int wordcount);
 
-int cli_cmd_sys_exec_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                          const char **words, int wordcount);
+int
+cli_cmd_sys_exec_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                     const char **words, int wordcount);
 
 int
-cli_cmd_getspec_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                     const char **words, int wordcount)
+cli_cmd_getspec_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                    const char **words, int wordcount)
 {
-        int                     ret = -1;
-        rpc_clnt_procedure_t    *proc = NULL;
-        call_frame_t            *frame = NULL;
-        dict_t                  *dict = NULL;
-
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
-
-        dict = dict_new ();
-        if (!dict)
-                goto out;
-
-        if (wordcount != 3) {
-                cli_usage_out (word->pattern);
-                goto out;
-        }
-
-        ret = dict_set_str (dict, "volid", (char *)words[2]);
-        if (ret)
-                goto out;
-
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_GETSPEC];
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, dict);
-        }
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *dict = NULL;
+
+    if (wordcount != 3) {
+        cli_usage_out(word->pattern);
+        goto out;
+    }
+
+    dict = dict_new();
+    if (!dict)
+        goto out;
+
+    ret = dict_set_str(dict, "volid", (char *)words[2]);
+    if (ret)
+        goto out;
+
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_GETSPEC];
+    if (proc->fn) {
+        frame = create_frame(THIS, THIS->ctx->pool);
+        if (!frame) {
+            ret = -1;
+            goto out;
+        }
+        ret = proc->fn(frame, THIS, dict);
+    }
 
 out:
-        if (!proc && ret) {
-                if (dict)
-                        dict_destroy (dict);
-                if (wordcount > 1)
-                        cli_out ("Fetching spec for volume %s failed",
-                                 (char *)words[2]);
-        }
+    if (!proc && ret) {
+        if (wordcount > 1)
+            cli_out("Fetching spec for volume %s failed", (char *)words[2]);
+    }
+
+    if (dict)
+        dict_unref(dict);
 
-        return ret;
+    CLI_STACK_DESTROY(frame);
+    return ret;
 }
 
 int
-cli_cmd_pmap_b2p_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                 const char **words, int wordcount)
+cli_cmd_pmap_b2p_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                     const char **words, int wordcount)
 {
-        int                     ret = -1;
-        rpc_clnt_procedure_t    *proc = NULL;
-        call_frame_t            *frame = NULL;
-        dict_t                  *dict = NULL;
-
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
-
-        dict = dict_new ();
-        if (!dict)
-                goto out;
-
-        if (wordcount != 4) {
-                cli_usage_out (word->pattern);
-                goto out;
-        }
-
-        ret = dict_set_str (dict, "brick", (char *)words[3]);
-        if (ret)
-                goto out;
-
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_PMAP_PORTBYBRICK];
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, dict);
-        }
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *dict = NULL;
+
+    if (wordcount != 4) {
+        cli_usage_out(word->pattern);
+        goto out;
+    }
+
+    dict = dict_new();
+    if (!dict)
+        goto out;
+
+    ret = dict_set_str(dict, "brick", (char *)words[3]);
+    if (ret)
+        goto out;
+
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_PMAP_PORTBYBRICK];
+    if (proc->fn) {
+        frame = create_frame(THIS, THIS->ctx->pool);
+        if (!frame) {
+            ret = -1;
+            goto out;
+        }
+        ret = proc->fn(frame, THIS, dict);
+    }
 
 out:
-        if (!proc && ret) {
-                if (dict)
-                        dict_destroy (dict);
-                if (wordcount > 1)
-                        cli_out ("Fetching spec for volume %s failed",
-                                 (char *)words[3]);
-        }
+    if (!proc && ret) {
+        if (wordcount > 1)
+            cli_out("Fetching spec for volume %s failed", (char *)words[3]);
+    }
+
+    if (dict)
+        dict_unref(dict);
 
-        return ret;
+    CLI_STACK_DESTROY(frame);
+    return ret;
 }
 
 int
-cli_cmd_fsm_log_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                     const char **words, int wordcount)
+cli_cmd_fsm_log_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                    const char **words, int wordcount)
 {
-        int                             ret = -1;
-        rpc_clnt_procedure_t            *proc = NULL;
-        call_frame_t                    *frame = NULL;
-        char                            *name = "";
-
-        if ((wordcount != 4) && (wordcount != 3)) {
-                cli_usage_out (word->pattern);
-                goto out;
-        }
-
-        if (wordcount == 4)
-                name = (char*)words[3];
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_FSM_LOG];
-        if (proc && proc->fn) {
-                frame = create_frame (THIS, THIS->ctx->pool);
-                if (!frame)
-                        goto out;
-                ret = proc->fn (frame, THIS, (void*)name);
-        }
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    char *name = "";
+
+    if ((wordcount != 4) && (wordcount != 3)) {
+        cli_usage_out(word->pattern);
+        goto out;
+    }
+
+    if (wordcount == 4)
+        name = (char *)words[3];
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_FSM_LOG];
+    if (proc && proc->fn) {
+        frame = create_frame(THIS, THIS->ctx->pool);
+        if (!frame)
+            goto out;
+        ret = proc->fn(frame, THIS, (void *)name);
+    }
 out:
-        return ret;
+    return ret;
 }
 
 int
-cli_cmd_getwd_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                   const char **words, int wordcount)
+cli_cmd_getwd_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                  const char **words, int wordcount)
 {
-        int                             ret = -1;
-        rpc_clnt_procedure_t            *proc = NULL;
-        call_frame_t                    *frame = NULL;
-
-        if (wordcount != 2) {
-                cli_usage_out (word->pattern);
-                goto out;
-        }
-
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_GETWD];
-        if (proc && proc->fn) {
-                frame = create_frame (THIS, THIS->ctx->pool);
-                if (!frame)
-                        goto out;
-                ret = proc->fn (frame, THIS, NULL);
-        }
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+
+    if (wordcount != 2) {
+        cli_usage_out(word->pattern);
+        goto out;
+    }
+
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_GETWD];
+    if (proc && proc->fn) {
+        frame = create_frame(THIS, THIS->ctx->pool);
+        if (!frame)
+            goto out;
+        ret = proc->fn(frame, THIS, NULL);
+    }
 out:
-        return ret;
+    return ret;
 }
 
 static dict_t *
-make_seq_dict (int argc, char **argv)
+make_seq_dict(int argc, char **argv)
 {
-        char index[] = "4294967296"; // 1<<32
-        int i        = 0;
-        int ret      = 0;
-        dict_t *dict = dict_new ();
-
-        if (!dict)
-                return NULL;
-
-        for (i = 0; i < argc; i++) {
-                snprintf(index, sizeof(index), "%d", i);
-                ret = dict_set_str (dict, index, argv[i]);
-                if (ret == -1)
-                        break;
-        }
-
-        if (ret) {
-                dict_destroy (dict);
-                dict = NULL;
-        }
-
-        return dict;
+    char index[] = "4294967296";  // 1<<32
+    int i = 0;
+    int len;
+    int ret = 0;
+    dict_t *dict = dict_new();
+
+    if (!dict)
+        return NULL;
+
+    for (i = 0; i < argc; i++) {
+        len = snprintf(index, sizeof(index), "%d", i);
+        ret = dict_set_strn(dict, index, len, argv[i]);
+        if (ret == -1)
+            break;
+    }
+
+    if (ret) {
+        dict_unref(dict);
+        dict = NULL;
+    }
+
+    return dict;
 }
 
 int
-cli_cmd_mount_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                   const char **words, int wordcount)
+cli_cmd_mount_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                  const char **words, int wordcount)
 {
-        rpc_clnt_procedure_t *proc = NULL;
-        call_frame_t *frame        = NULL;
-        int ret                    = -1;
-        dict_t *dict               = NULL;
-        void *dataa[]              = {NULL, NULL};
-
-        if (wordcount < 4) {
-                cli_usage_out (word->pattern);
-                goto out;
-        }
-
-        dict = make_seq_dict (wordcount - 3, (char **)words + 3);
-        if (!dict)
-                goto out;
-
-        dataa[0] = (void *)words[2];
-        dataa[1] = dict;
-
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_MOUNT];
-        if (proc && proc->fn) {
-                frame = create_frame (THIS, THIS->ctx->pool);
-                if (!frame)
-                        goto out;
-                ret = proc->fn (frame, THIS, dataa);
-        }
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    int ret = -1;
+    dict_t *dict = NULL;
+    void *dataa[] = {NULL, NULL};
+
+    if (wordcount < 4) {
+        cli_usage_out(word->pattern);
+        goto out;
+    }
+
+    dict = make_seq_dict(wordcount - 3, (char **)words + 3);
+    if (!dict)
+        goto out;
+
+    dataa[0] = (void *)words[2];
+    dataa[1] = dict;
+
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_MOUNT];
+    if (proc && proc->fn) {
+        frame = create_frame(THIS, THIS->ctx->pool);
+        if (!frame)
+            goto out;
+        ret = proc->fn(frame, THIS, dataa);
+    }
 
- out:
-        if (dict)
-                dict_unref (dict);
+out:
+    if (dict)
+        dict_unref(dict);
 
-        if (!proc && ret)
-                cli_out ("Mount command failed");
+    if (!proc && ret)
+        cli_out("Mount command failed");
 
-        return ret;
+    return ret;
 }
 
 int
-cli_cmd_umount_cbk (struct cli_state *state, struct cli_cmd_word *word,
+cli_cmd_umount_cbk(struct cli_state *state, struct cli_cmd_word *word,
                    const char **words, int wordcount)
 {
-        rpc_clnt_procedure_t *proc = NULL;
-        call_frame_t *frame        = NULL;
-        int ret                    = -1;
-        dict_t *dict               = NULL;
-
-        if (!(wordcount == 3 ||
-              (wordcount == 4 && strcmp (words[3], "lazy") == 0))) {
-                cli_usage_out (word->pattern);
-                goto out;
-        }
-
-        dict = dict_new ();
-        if (!dict)
-                goto out;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    int ret = -1;
+    dict_t *dict = NULL;
+
+    if (!(wordcount == 3 ||
+          (wordcount == 4 && strcmp(words[3], "lazy") == 0))) {
+        cli_usage_out(word->pattern);
+        goto out;
+    }
+
+    dict = dict_new();
+    if (!dict)
+        goto out;
+
+    ret = dict_set_str(dict, "path", (char *)words[2]);
+    if (ret != 0)
+        goto out;
+    ret = dict_set_int32(dict, "lazy", wordcount == 4);
+    if (ret != 0)
+        goto out;
+
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_UMOUNT];
+    if (proc && proc->fn) {
+        frame = create_frame(THIS, THIS->ctx->pool);
+        if (!frame) {
+            gf_log(THIS->name, GF_LOG_ERROR, "failed to create frame");
+            ret = -1;
+            goto out;
+        }
+        ret = proc->fn(frame, THIS, dict);
+    }
 
-        ret = dict_set_str (dict, "path", (char *)words[2]);
-        if (ret != 0)
-                goto out;
-        ret = dict_set_int32 (dict, "lazy", wordcount == 4);
-        if (ret != 0)
-                goto out;
-
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_UMOUNT];
-        if (proc && proc->fn) {
-                frame = create_frame (THIS, THIS->ctx->pool);
-                if (!frame)
-                        goto out;
-                ret = proc->fn (frame, THIS, dict);
-        }
-
- out:
-        if (dict)
-                dict_unref (dict);
+out:
+    if (dict)
+        dict_unref(dict);
 
-        if (!proc && ret)
-                cli_out ("Umount command failed");
+    if (!proc && ret)
+        cli_out("Umount command failed");
 
-        return ret;
+    return ret;
 }
 
 int
-cli_cmd_uuid_get_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                      const char **words, int wordcount)
+cli_cmd_uuid_get_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                     const char **words, int wordcount)
 {
-        int                     ret = -1;
-        int                     sent = 0;
-        int                     parse_error = 0;
-        dict_t                  *dict  = NULL;
-        rpc_clnt_procedure_t    *proc = NULL;
-        call_frame_t            *frame = NULL;
-        cli_local_t             *local = NULL;
-        xlator_t                *this  = NULL;
-
-        this = THIS;
-        if (wordcount != 3) {
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-                goto out;
-        }
-
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_UUID_GET];
-        frame = create_frame (this, this->ctx->pool);
-        if (!frame)
-                goto out;
-
-        dict = dict_new ();
-        if (!dict)
-                goto out;
-
-        CLI_LOCAL_INIT (local, words, frame, dict);
-        if (proc->fn)
-                ret = proc->fn (frame, this, dict);
+    int ret = -1;
+    int sent = 0;
+    int parse_error = 0;
+    dict_t *dict = NULL;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    cli_local_t *local = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    if (wordcount != 3) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
+
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_UUID_GET];
+    frame = create_frame(this, this->ctx->pool);
+    if (!frame)
+        goto out;
+
+    dict = dict_new();
+    if (!dict)
+        goto out;
+
+    CLI_LOCAL_INIT(local, words, frame, dict);
+    if (proc->fn)
+        ret = proc->fn(frame, this, dict);
 
 out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_out ("uuid get failed");
-        }
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("uuid get failed");
+    }
 
-        if (dict)
-                dict_unref (dict);
+    if (dict)
+        dict_unref(dict);
 
-        CLI_STACK_DESTROY (frame);
-        return ret;
+    CLI_STACK_DESTROY(frame);
+    return ret;
 }
 
 int
-cli_cmd_uuid_reset_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                        const char **words, int wordcount)
+cli_cmd_uuid_reset_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                       const char **words, int wordcount)
 {
-        int                     ret = -1;
-        rpc_clnt_procedure_t    *proc = NULL;
-        call_frame_t            *frame = NULL;
-        int                     sent = 0;
-        int                     parse_error = 0;
-        gf_answer_t             answer = GF_ANSWER_NO;
-        char                    *question = NULL;
-        cli_local_t             *local = NULL;
-        dict_t                  *dict  = NULL;
-        xlator_t                *this  = NULL;
-
-        question = "Resetting uuid changes the uuid of local glusterd. "
-                   "Do you want to continue?";
-
-        if (wordcount != 3) {
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-                goto out;
-        }
-
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_UUID_RESET];
-
-        this = THIS;
-        frame = create_frame (this, this->ctx->pool);
-        if (!frame)
-                goto out;
-
-        dict = dict_new ();
-        if (!dict) {
-                ret = -1;
-                goto out;
-        }
-        CLI_LOCAL_INIT (local, words, frame, dict);
-        answer = cli_cmd_get_confirmation (state, question);
-
-        if (GF_ANSWER_NO == answer) {
-                ret = 0;
-                goto out;
-        }
-
-        //send NULL as argument since no dictionary is sent to glusterd
-        if (proc->fn) {
-                ret = proc->fn (frame, this, dict);
-        }
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    int sent = 0;
+    int parse_error = 0;
+    gf_answer_t answer = GF_ANSWER_NO;
+    char *question = NULL;
+    cli_local_t *local = NULL;
+    dict_t *dict = NULL;
+    xlator_t *this = NULL;
+
+    question =
+        "Resetting uuid changes the uuid of local glusterd. "
+        "Do you want to continue?";
+
+    if (wordcount != 3) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
+
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_UUID_RESET];
+
+    this = THIS;
+    frame = create_frame(this, this->ctx->pool);
+    if (!frame)
+        goto out;
+
+    dict = dict_new();
+    if (!dict) {
+        ret = -1;
+        goto out;
+    }
+    CLI_LOCAL_INIT(local, words, frame, dict);
+    answer = cli_cmd_get_confirmation(state, question);
+
+    if (GF_ANSWER_NO == answer) {
+        ret = 0;
+        goto out;
+    }
+
+    // send NULL as argument since no dictionary is sent to glusterd
+    if (proc->fn) {
+        ret = proc->fn(frame, this, dict);
+    }
 
 out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_out ("uuid reset failed");
-        }
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("uuid reset failed");
+    }
+
+    if (dict)
+        dict_unref(dict);
 
-        CLI_STACK_DESTROY (frame);
+    CLI_STACK_DESTROY(frame);
 
-        return ret;
+    return ret;
 }
 
-struct cli_cmd cli_system_cmds[] = {
-        { "system:: getspec <VOLID>",
-          cli_cmd_getspec_cbk,
-          "fetch spec for volume <VOLID>"},
+static struct cli_cmd cli_system_cmds[] = {
+    {"system:: getspec <VOLNAME>", cli_cmd_getspec_cbk,
+     "fetch the volume file for the volume <VOLNAME>"},
 
-        { "system:: portmap brick2port <BRICK>",
-          cli_cmd_pmap_b2p_cbk,
-          "query which port <BRICK> listens on"},
+    {"system:: portmap brick2port <BRICK>", cli_cmd_pmap_b2p_cbk,
+     "query which port <BRICK> listens on"},
 
-        { "system:: fsm log [<peer-name>]",
-          cli_cmd_fsm_log_cbk,
-          "display fsm transitions"},
+    {"system:: fsm log [<peer-name>]", cli_cmd_fsm_log_cbk,
+     "display fsm transitions"},
 
-        { "system:: getwd",
-          cli_cmd_getwd_cbk,
-          "query glusterd work directory"},
+    {"system:: getwd", cli_cmd_getwd_cbk, "query glusterd work directory"},
 
-        { "system:: mount <label> <args...>",
-          cli_cmd_mount_cbk,
-          "request a mount"},
+    {"system:: mount <label> <args...>", cli_cmd_mount_cbk, "request a mount"},
 
-        { "system:: umount <path> [lazy]",
-          cli_cmd_umount_cbk,
-          "request an umount"},
+    {"system:: umount <path> [lazy]", cli_cmd_umount_cbk, "request an umount"},
 
-        { "system:: uuid get",
-          cli_cmd_uuid_get_cbk,
-          "get uuid of glusterd"},
+    {"system:: uuid get", cli_cmd_uuid_get_cbk, "get uuid of glusterd"},
 
-        { "system:: uuid reset",
-          cli_cmd_uuid_reset_cbk,
-          "reset the uuid of glusterd"},
+    {"system:: uuid reset", cli_cmd_uuid_reset_cbk,
+     "reset the uuid of glusterd"},
 
-        { "system:: help",
-           cli_cmd_system_help_cbk,
-           "display help for system commands"},
+    {"system:: help", cli_cmd_system_help_cbk,
+     "display help for system commands"},
 
-        { "system:: copy file [<filename>]",
-           cli_cmd_copy_file_cbk,
-           "Copy file from current node's $working_dir to "
-           "$working_dir of all cluster nodes"},
+    {"system:: copy file [<filename>]", cli_cmd_copy_file_cbk,
+     "Copy file from current node's $working_dir to "
+     "$working_dir of all cluster nodes"},
 
-        { "system:: execute <command> <args>",
-           cli_cmd_sys_exec_cbk,
-           "Execute the command on all the nodes "
-           "in the cluster and display their output."},
+    {"system:: execute <command> <args>", cli_cmd_sys_exec_cbk,
+     "Execute the command on all the nodes "
+     "in the cluster and display their output."},
 
-        { NULL, NULL, NULL }
-};
+    {NULL, NULL, NULL}};
 
 int
-cli_cmd_sys_exec_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                      const char **words, int wordcount)
+cli_cmd_sys_exec_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                     const char **words, int wordcount)
 {
-        char                   cmd_arg_name[PATH_MAX] = "";
-        char                  *command                = NULL;
-        char                  *saveptr                = NULL;
-        char                  *tmp                    = NULL;
-        int                    ret                    = -1;
-        int                    i                      = -1;
-        int                    cmd_args_count         = 0;
-        int                    in_cmd_args_count      = 0;
-        rpc_clnt_procedure_t  *proc                   = NULL;
-        call_frame_t          *frame                  = NULL;
-        dict_t                *dict                   = NULL;
-        cli_local_t           *local                  = NULL;
-
-        if (wordcount < 3) {
-                cli_usage_out (word->pattern);
-                goto out;
-        }
-
-        dict = dict_new ();
-        if (!dict)
-                goto out;
-
-        command = strtok_r ((char *)words[2], " ", &saveptr);
-        do {
-                tmp = strtok_r (NULL, " ", &saveptr);
-                if (tmp) {
-                        in_cmd_args_count++;
-                        memset (cmd_arg_name, '\0', sizeof(cmd_arg_name));
-                        snprintf (cmd_arg_name, sizeof(cmd_arg_name),
-                                  "cmd_arg_%d", in_cmd_args_count);
-                        ret = dict_set_str (dict, cmd_arg_name, tmp);
-                        if (ret) {
-                                gf_log ("", GF_LOG_ERROR, "Unable to set "
-                                        "%s in dict", cmd_arg_name);
-                                goto out;
-                        }
-                }
-        } while (tmp);
-
-        cmd_args_count = wordcount - 3;
-
-        ret = dict_set_str (dict, "command", command);
-        if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to set command in dict");
+    char cmd_arg_name[PATH_MAX] = "";
+    char *command = NULL;
+    char *saveptr = NULL;
+    char *tmp = NULL;
+    int ret = -1;
+    int i = -1;
+    int len;
+    int cmd_args_count = 0;
+    int in_cmd_args_count = 0;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *dict = NULL;
+    cli_local_t *local = NULL;
+
+    if ((wordcount < 3) || (words[2] == NULL)) {
+        cli_usage_out(word->pattern);
+        goto out;
+    }
+
+    command = strtok_r((char *)words[2], " ", &saveptr);
+    if (command == NULL) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to parse command");
+        goto out;
+    }
+
+    dict = dict_new();
+    if (!dict)
+        goto out;
+
+    do {
+        tmp = strtok_r(NULL, " ", &saveptr);
+        if (tmp) {
+            in_cmd_args_count++;
+            snprintf(cmd_arg_name, sizeof(cmd_arg_name), "cmd_arg_%d",
+                     in_cmd_args_count);
+            ret = dict_set_str(dict, cmd_arg_name, tmp);
+            if (ret) {
+                gf_log("", GF_LOG_ERROR,
+                       "Unable to set "
+                       "%s in dict",
+                       cmd_arg_name);
                 goto out;
+            }
         }
+    } while (tmp);
 
-        for (i=1; i <= cmd_args_count; i++) {
-                in_cmd_args_count++;
-                memset (cmd_arg_name, '\0', sizeof(cmd_arg_name));
-                snprintf (cmd_arg_name, sizeof(cmd_arg_name),
-                          "cmd_arg_%d", in_cmd_args_count);
-                ret = dict_set_str (dict, cmd_arg_name,
-                                    (char *)words[2+i]);
-                if (ret) {
-                        gf_log ("", GF_LOG_ERROR, "Unable to set %s in dict",
-                               cmd_arg_name);
-                        goto out;
-                }
-        }
+    cmd_args_count = wordcount - 3;
 
-        ret = dict_set_int32 (dict, "cmd_args_count", in_cmd_args_count);
-        if (ret) {
-                gf_log ("", GF_LOG_ERROR,
-                        "Unable to set cmd_args_count in dict");
-                goto out;
-        }
+    ret = dict_set_str(dict, "command", command);
+    if (ret) {
+        gf_log("", GF_LOG_ERROR, "Unable to set command in dict");
+        goto out;
+    }
 
-        ret = dict_set_str (dict, "volname", "N/A");
+    for (i = 1; i <= cmd_args_count; i++) {
+        in_cmd_args_count++;
+        len = snprintf(cmd_arg_name, sizeof(cmd_arg_name), "cmd_arg_%d",
+                       in_cmd_args_count);
+        ret = dict_set_strn(dict, cmd_arg_name, len, (char *)words[2 + i]);
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to set volname in dict");
-                goto out;
-        }
-
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_SYS_EXEC];
-        if (proc && proc->fn) {
-                frame = create_frame (THIS, THIS->ctx->pool);
-                if (!frame)
-                        goto out;
-                CLI_LOCAL_INIT (local, words, frame, dict);
-                ret = proc->fn (frame, THIS, (void*)dict);
-        }
+            gf_log("", GF_LOG_ERROR, "Unable to set %s in dict", cmd_arg_name);
+            goto out;
+        }
+    }
+
+    ret = dict_set_int32(dict, "cmd_args_count", in_cmd_args_count);
+    if (ret) {
+        gf_log("", GF_LOG_ERROR, "Unable to set cmd_args_count in dict");
+        goto out;
+    }
+
+    ret = dict_set_str(dict, "volname", "N/A");
+    if (ret) {
+        gf_log("", GF_LOG_ERROR, "Unable to set volname in dict");
+        goto out;
+    }
+
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_SYS_EXEC];
+    if (proc->fn) {
+        frame = create_frame(THIS, THIS->ctx->pool);
+        if (!frame) {
+            gf_log(THIS->name, GF_LOG_ERROR, "failed to create frame");
+            ret = -1;
+            goto out;
+        }
+        CLI_LOCAL_INIT(local, words, frame, dict);
+        ret = proc->fn(frame, THIS, (void *)dict);
+
+        /* proc->fn is processed synchronously, which means that the
+         * execution flow won't return here until the operation is
+         * fully processed, including any related callback. For this
+         * reason, it's safe to destroy the stack here, since no one
+         * can still be using it. Additionally, it's not easy to move
+         * the stack destroy to the callback executed after completion
+         * of the operation because there are multiple things than can
+         * fail even before having queued the callback, so we would
+         * still need to destroy the stack if proc->fn returns an
+         * error. */
+        CLI_STACK_DESTROY(frame);
+        dict = NULL;
+    }
 out:
-        return ret;
+    if (dict != NULL) {
+        dict_unref(dict);
+    }
+
+    return ret;
 }
 
 int
-cli_cmd_copy_file_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                       const char **words, int wordcount)
+cli_cmd_copy_file_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                      const char **words, int wordcount)
 {
-        int                    ret      = -1;
-        rpc_clnt_procedure_t  *proc     = NULL;
-        call_frame_t          *frame    = NULL;
-        char                  *filename = "";
-        dict_t                *dict     = NULL;
-        cli_local_t           *local    = NULL;
-
-        if (wordcount != 4) {
-                cli_usage_out (word->pattern);
-                goto out;
-        }
-
-        dict = dict_new ();
-        if (!dict)
-                goto out;
-
-        filename = (char*)words[3];
-        ret = dict_set_str (dict, "source", filename);
-        if (ret)
-                 gf_log ("", GF_LOG_ERROR, "Unable to set filename in dict");
-
-        ret = dict_set_str (dict, "volname", "N/A");
-        if (ret)
-                 gf_log ("", GF_LOG_ERROR, "Unable to set volname in dict");
-
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_COPY_FILE];
-        if (proc && proc->fn) {
-                frame = create_frame (THIS, THIS->ctx->pool);
-                if (!frame)
-                        goto out;
-                CLI_LOCAL_INIT (local, words, frame, dict);
-                ret = proc->fn (frame, THIS, (void*)dict);
-        }
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    char *filename = "";
+    dict_t *dict = NULL;
+    cli_local_t *local = NULL;
+
+    if (wordcount != 4) {
+        cli_usage_out(word->pattern);
+        goto out;
+    }
+
+    dict = dict_new();
+    if (!dict)
+        goto out;
+
+    filename = (char *)words[3];
+    ret = dict_set_str(dict, "source", filename);
+    if (ret)
+        gf_log("", GF_LOG_ERROR, "Unable to set filename in dict");
+
+    ret = dict_set_str(dict, "volname", "N/A");
+    if (ret)
+        gf_log("", GF_LOG_ERROR, "Unable to set volname in dict");
+
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_COPY_FILE];
+    if (proc && proc->fn) {
+        frame = create_frame(THIS, THIS->ctx->pool);
+        if (!frame) {
+            gf_log(THIS->name, GF_LOG_ERROR, "failed to create frame");
+            ret = -1;
+            goto out;
+        }
+        CLI_LOCAL_INIT(local, words, frame, dict);
+        ret = proc->fn(frame, THIS, (void *)dict);
+    }
 out:
-        return ret;
+    return ret;
 }
 
 int
-cli_cmd_system_help_cbk (struct cli_state *state, struct cli_cmd_word *in_word,
-                         const char **words, int wordcount)
+cli_cmd_system_help_cbk(struct cli_state *state, struct cli_cmd_word *in_word,
+                        const char **words, int wordcount)
 {
-        struct cli_cmd *cmd = NULL;
+    struct cli_cmd *cmd = NULL;
+    struct cli_cmd *system_cmd = NULL;
+    int count = 0;
 
-        for (cmd = cli_system_cmds; cmd->pattern; cmd++)
-                cli_out ("%s - %s", cmd->pattern, cmd->desc);
+    cmd = GF_MALLOC(sizeof(cli_system_cmds), cli_mt_cli_cmd);
+    memcpy(cmd, cli_system_cmds, sizeof(cli_system_cmds));
+    count = (sizeof(cli_system_cmds) / sizeof(struct cli_cmd));
+    cli_cmd_sort(cmd, count);
 
-        return 0;
+    for (system_cmd = cmd; system_cmd->pattern; system_cmd++)
+        cli_out("%s - %s", system_cmd->pattern, system_cmd->desc);
+
+    GF_FREE(cmd);
+    return 0;
 }
 
 int
-cli_cmd_system_register (struct cli_state *state)
+cli_cmd_system_register(struct cli_state *state)
 {
-        int  ret = 0;
-        struct cli_cmd *cmd = NULL;
-
-        for (cmd = cli_system_cmds; cmd->pattern; cmd++) {
+    int ret = 0;
+    struct cli_cmd *cmd = NULL;
 
-                ret = cli_cmd_register (&state->tree, cmd);
-                if (ret)
-                        goto out;
-        }
+    for (cmd = cli_system_cmds; cmd->pattern; cmd++) {
+        ret = cli_cmd_register(&state->tree, cmd);
+        if (ret)
+            goto out;
+    }
 out:
-        return ret;
+    return ret;
 }
diff --git a/cli/src/cli-cmd-volume.c b/cli/src/cli-cmd-volume.c
index 84209adf936..f238851586e 100644
--- a/cli/src/cli-cmd-volume.c
+++ b/cli/src/cli-cmd-volume.c
@@ -19,2502 +19,3260 @@
 #include <sys/wait.h>
 #include <netinet/in.h>
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 #include "cli.h"
 #include "cli-cmd.h"
 #include "cli-mem-types.h"
 #include "cli1-xdr.h"
-#include "run.h"
-#include "syscall.h"
-#include "common-utils.h"
+#include <glusterfs/run.h>
+#include <glusterfs/syscall.h>
+#include <glusterfs/common-utils.h>
+#include <glusterfs/events.h>
 
-extern struct rpc_clnt *global_rpc;
-extern struct rpc_clnt *global_quotad_rpc;
-
-extern rpc_clnt_prog_t *cli_rpc_prog;
 extern rpc_clnt_prog_t cli_quotad_clnt;
 
-int
-cli_cmd_volume_help_cbk (struct cli_state *state, struct cli_cmd_word *in_word,
-                      const char **words, int wordcount);
+static int
+gf_asprintf_append(char **string_ptr, const char *format, ...);
 
 int
-cli_cmd_volume_info_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                         const char **words, int wordcount)
-{
-        int                             ret = -1;
-        rpc_clnt_procedure_t            *proc = NULL;
-        call_frame_t                    *frame = NULL;
-        cli_cmd_volume_get_ctx_t        ctx = {0,};
-        cli_local_t                     *local = NULL;
-        int                             sent = 0;
-        int                             parse_error = 0;
+cli_cmd_volume_help_cbk(struct cli_state *state, struct cli_cmd_word *in_word,
+                        const char **words, int wordcount);
 
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_GET_VOLUME];
+int
+cli_cmd_bitrot_help_cbk(struct cli_state *state, struct cli_cmd_word *in_word,
+                        const char **words, int wordcount);
 
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
+int
+cli_cmd_quota_help_cbk(struct cli_state *state, struct cli_cmd_word *in_word,
+                       const char **words, int wordcount);
 
-        if ((wordcount == 2)  || (wordcount == 3 &&
-                                  !strcmp (words[2], "all"))) {
-                ctx.flags = GF_CLI_GET_NEXT_VOLUME;
-                proc = &cli_rpc_prog->proctable[GLUSTER_CLI_GET_NEXT_VOLUME];
-        } else if (wordcount == 3) {
-                ctx.flags = GF_CLI_GET_VOLUME;
-                ctx.volname = (char *)words[2];
-                if (strlen (ctx.volname) > GD_VOLUME_NAME_MAX) {
-                        cli_out ("Invalid volume name");
-                        goto out;
-                }
-                proc = &cli_rpc_prog->proctable[GLUSTER_CLI_GET_VOLUME];
-        } else {
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-                return -1;
+int
+cli_cmd_volume_info_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                        const char **words, int wordcount)
+{
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    cli_cmd_volume_get_ctx_t ctx = {
+        0,
+    };
+    cli_local_t *local = NULL;
+    int sent = 0;
+    int parse_error = 0;
+
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_GET_VOLUME];
+
+    if ((wordcount == 2) || (wordcount == 3 && !strcmp(words[2], "all"))) {
+        ctx.flags = GF_CLI_GET_NEXT_VOLUME;
+        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_GET_NEXT_VOLUME];
+    } else if (wordcount == 3) {
+        ctx.flags = GF_CLI_GET_VOLUME;
+        ctx.volname = (char *)words[2];
+        if (strlen(ctx.volname) > GD_VOLUME_NAME_MAX) {
+            cli_out("Invalid volume name");
+            goto out;
         }
+        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_GET_VOLUME];
+    } else {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        return -1;
+    }
 
-        local = cli_local_get ();
+    local = cli_local_get();
 
-        if (!local)
-                goto out;
+    if (!local)
+        goto out;
 
-        local->get_vol.flags = ctx.flags;
-        if (ctx.volname)
-                local->get_vol.volname = gf_strdup (ctx.volname);
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame)
+        goto out;
 
-        frame->local = local;
+    local->get_vol.flags = ctx.flags;
+    if (ctx.volname)
+        local->get_vol.volname = gf_strdup(ctx.volname);
 
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, &ctx);
-        }
+    frame->local = local;
 
-out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_out ("Getting Volume information failed!");
-        }
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, &ctx);
+    }
 
-        CLI_STACK_DESTROY (frame);
+out:
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("Getting Volume information failed!");
+    }
 
-        return ret;
+    CLI_STACK_DESTROY(frame);
 
+    return ret;
 }
 
 int
-cli_cmd_sync_volume_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                         const char **words, int wordcount)
+cli_cmd_sync_volume_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                        const char **words, int wordcount)
 {
-        int                     ret = -1;
-        rpc_clnt_procedure_t    *proc = NULL;
-        call_frame_t            *frame = NULL;
-        int                     sent = 0;
-        int                     parse_error = 0;
-        dict_t                  *dict = NULL;
-        cli_local_t             *local = NULL;
-        gf_answer_t             answer = GF_ANSWER_NO;
-        const char              *question = "Sync volume may make data "
-                                            "inaccessible while the sync "
-                                            "is in progress. Do you want "
-                                            "to continue?";
-
-        if ((wordcount < 3) || (wordcount > 4)) {
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-                goto out;
-        }
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    int sent = 0;
+    int parse_error = 0;
+    dict_t *dict = NULL;
+    cli_local_t *local = NULL;
+    gf_answer_t answer = GF_ANSWER_NO;
+    const char *question =
+        "Sync volume may make data "
+        "inaccessible while the sync "
+        "is in progress. Do you want "
+        "to continue?";
+
+    if ((wordcount < 3) || (wordcount > 4)) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
 
-        dict = dict_new ();
-        if (!dict)
-                goto out;
+    dict = dict_new();
+    if (!dict)
+        goto out;
 
-        if ((wordcount == 3) || !strcmp(words[3], "all")) {
-                ret = dict_set_int32 (dict, "flags", (int32_t)
-                                      GF_CLI_SYNC_ALL);
-                if (ret) {
-                        gf_log (THIS->name, GF_LOG_ERROR, "failed to set"
-                                "flag");
-                        goto out;
-                }
-        } else {
-                ret = dict_set_str (dict, "volname", (char *) words[3]);
-                if (ret) {
-                        gf_log (THIS->name, GF_LOG_ERROR, "failed to set "
-                                "volume");
-                        goto out;
-                }
+    if ((wordcount == 3) || !strcmp(words[3], "all")) {
+        ret = dict_set_int32(dict, "flags", (int32_t)GF_CLI_SYNC_ALL);
+        if (ret) {
+            gf_log(THIS->name, GF_LOG_ERROR,
+                   "failed to set"
+                   "flag");
+            goto out;
         }
-
-        ret = dict_set_str (dict, "hostname", (char *) words[2]);
+    } else {
+        ret = dict_set_str(dict, "volname", (char *)words[3]);
         if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "failed to set hostname");
-                goto out;
+            gf_log(THIS->name, GF_LOG_ERROR,
+                   "failed to set "
+                   "volume");
+            goto out;
         }
+    }
 
-        if (!(state->mode & GLUSTER_MODE_SCRIPT)) {
-                answer = cli_cmd_get_confirmation (state, question);
-                if (GF_ANSWER_NO == answer) {
-                        ret = 0;
-                        goto out;
-                }
+    ret = dict_set_str(dict, "hostname", (char *)words[2]);
+    if (ret) {
+        gf_log(THIS->name, GF_LOG_ERROR, "failed to set hostname");
+        goto out;
+    }
+
+    if (!(state->mode & GLUSTER_MODE_SCRIPT)) {
+        answer = cli_cmd_get_confirmation(state, question);
+        if (GF_ANSWER_NO == answer) {
+            ret = 0;
+            goto out;
         }
+    }
 
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_SYNC_VOLUME];
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_SYNC_VOLUME];
 
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        gf_log(THIS->name, GF_LOG_ERROR, "failed to create frame");
+        ret = -1;
+        goto out;
+    }
 
-        CLI_LOCAL_INIT (local, words, frame, dict);
+    CLI_LOCAL_INIT(local, words, frame, dict);
 
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, dict);
-        }
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, dict);
+    }
 
 out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_out ("Volume sync failed");
-        }
-
-        CLI_STACK_DESTROY (frame);
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("Volume sync failed");
+    }
 
-        return ret;
-}
+    CLI_STACK_DESTROY(frame);
 
-gf_ai_compare_t
-cli_cmd_compare_addrinfo (struct addrinfo *first, struct addrinfo *next)
-{
-        int             ret = -1;
-        struct addrinfo *tmp1 = NULL;
-        struct addrinfo *tmp2 = NULL;
-        char            firstip[NI_MAXHOST] = {0.};
-        char            nextip[NI_MAXHOST] = {0,};
-
-        for (tmp1 = first; tmp1 != NULL; tmp1 = tmp1->ai_next) {
-                ret = getnameinfo (tmp1->ai_addr, tmp1->ai_addrlen, firstip,
-                                   NI_MAXHOST, NULL, 0, NI_NUMERICHOST);
-                if (ret)
-                        return GF_AI_COMPARE_ERROR;
-                for (tmp2 = next; tmp2 != NULL; tmp2 = tmp2->ai_next) {
-                        ret = getnameinfo (tmp2->ai_addr, tmp2->ai_addrlen, nextip,
-                                           NI_MAXHOST, NULL, 0, NI_NUMERICHOST);
-                        if (ret)
-                                return GF_AI_COMPARE_ERROR;
-                        if (!strcmp (firstip, nextip)) {
-                                return GF_AI_COMPARE_MATCH;
-                        }
-                }
-        }
-        return GF_AI_COMPARE_NO_MATCH;
+    return ret;
 }
 
-/* Check for non optimal brick order for replicate :
- * Checks if bricks belonging to a replicate volume
- * are present on the same server
- */
-int32_t
-cli_cmd_check_brick_order (struct cli_state *state, const char *bricks,
-                           int brick_count, int sub_count)
+int
+cli_cmd_volume_create_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                          const char **words, int wordcount)
 {
-        int             ret = -1;
-        int             i = 0;
-        int             j = 0;
-        int             k = 0;
-        addrinfo_list_t *ai_list = NULL;
-        addrinfo_list_t *ai_list_tmp1 = NULL;
-        addrinfo_list_t *ai_list_tmp2 = NULL;
-        char            *brick = NULL;
-        char            *brick_list = NULL;
-        char            *brick_list_dup = NULL;
-        char            *tmpptr = NULL;
-        struct addrinfo *ai_info = NULL;
-        gf_answer_t     answer = GF_ANSWER_NO;
-        const char      *failed_question = NULL;
-        const char      *found_question = NULL;
-        failed_question = "Failed to perform brick order check. "
-                          "Do you want to continue creating the volume? ";
-        found_question =  "Multiple bricks of a replicate volume are present"
-                          " on the same server. This setup is not optimal.\n"
-                          "Do you still want to continue creating the volume? ";
-
-        GF_ASSERT (bricks);
-        GF_ASSERT (brick_count > 0);
-        GF_ASSERT (sub_count > 0);
-
-        ai_list = malloc (sizeof (addrinfo_list_t));
-        ai_list->info = NULL;
-        INIT_LIST_HEAD (&ai_list->list);
-        brick_list = gf_strdup (bricks);
-        if (brick_list == NULL) {
-                gf_log ("cli", GF_LOG_DEBUG, "failed to allocate memory");
-                goto check_failed;
-        }
-        brick_list_dup = brick_list;
-        /* Resolve hostnames and get addrinfo */
-        while (i < brick_count) {
-                ++i;
-                brick = strtok_r (brick_list, " \n", &tmpptr);
-                brick_list = tmpptr;
-                if (brick == NULL)
-                        goto check_failed;
-                brick = strtok_r (brick, ":", &tmpptr);
-                if (brick == NULL)
-                        goto check_failed;
-                ret = getaddrinfo (brick, NULL, NULL, &ai_info);
-                if (ret)
-                        goto check_failed;
-                ai_list_tmp1 = malloc (sizeof (addrinfo_list_t));
-                if (ai_list_tmp1 == NULL)
-                        goto check_failed;
-                ai_list_tmp1->info = ai_info;
-                list_add_tail (&ai_list_tmp1->list, &ai_list->list);
-                ai_list_tmp1 = NULL;
-        }
-
-        i = 0;
-        ai_list_tmp1 = list_entry (ai_list->list.next, addrinfo_list_t, list);
-
-        /* Check for bad brick order */
-        while (i < brick_count) {
-                ++i;
-                ai_info = ai_list_tmp1->info;
-                ai_list_tmp1 = list_entry (ai_list_tmp1->list.next,
-                                           addrinfo_list_t, list);
-                if ( 0 == i % sub_count) {
-                        j = 0;
-                        continue;
-                }
-                ai_list_tmp2 = ai_list_tmp1;
-                k = j;
-                while (k < sub_count - 1) {
-                        ++k;
-                        ret = cli_cmd_compare_addrinfo (ai_info,
-                                                        ai_list_tmp2->info);
-                        if (GF_AI_COMPARE_ERROR == ret)
-                                goto check_failed;
-                        if (GF_AI_COMPARE_MATCH == ret)
-                                goto found_bad_brick_order;
-                        ai_list_tmp2 = list_entry (ai_list_tmp2->list.next,
-                                                   addrinfo_list_t, list);
-                }
-                ++j;
-        }
-        gf_log ("cli", GF_LOG_INFO, "Brick order okay");
-        ret = 0;
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *options = NULL;
+    int sent = 0;
+    int parse_error = 0;
+    cli_local_t *local = NULL;
+    char *trans_type = NULL;
+    char *bricks = NULL;
+
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_CREATE_VOLUME];
+
+    ret = cli_cmd_volume_create_parse(state, words, wordcount, &options,
+                                      &bricks);
+
+    if (ret) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
         goto out;
+    }
 
-check_failed:
-        gf_log ("cli", GF_LOG_INFO, "Failed bad brick order check");
-        answer = cli_cmd_get_confirmation(state, failed_question);
-        if (GF_ANSWER_YES == answer)
-                ret = 0;
+    ret = dict_get_str(options, "transport", &trans_type);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Unable to get transport type");
         goto out;
+    }
 
-found_bad_brick_order:
-        gf_log ("cli", GF_LOG_INFO, "Bad brick order found");
-        answer = cli_cmd_get_confirmation (state, found_question);
-        if (GF_ANSWER_YES == answer)
-                ret = 0;
-out:
-        ai_list_tmp2 = NULL;
-        i = 0;
-        GF_FREE (brick_list_dup);
-        list_for_each_entry (ai_list_tmp1, &ai_list->list, list) {
-                if (ai_list_tmp1->info)
-                        freeaddrinfo (ai_list_tmp1->info);
-                free (ai_list_tmp2);
-                ai_list_tmp2 = ai_list_tmp1;
-        }
-        free (ai_list_tmp2);
-        return ret;
-}
-
-int
-cli_cmd_volume_create_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                           const char **words, int wordcount)
-{
-        int                     ret = -1;
-        rpc_clnt_procedure_t    *proc = NULL;
-        call_frame_t            *frame = NULL;
-        dict_t                  *options = NULL;
-        int                     sent = 0;
-        int                     parse_error = 0;
-        char                    *brick_list = NULL;
-        int32_t                 brick_count = 0;
-        int32_t                 sub_count = 0;
-        int32_t                 type = GF_CLUSTER_TYPE_NONE;
-        cli_local_t             *local = NULL;
-        char                    *trans_type = NULL;
-
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_CREATE_VOLUME];
-
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
-
-        ret = cli_cmd_volume_create_parse (state, words, wordcount, &options);
-
+    if (state->mode & GLUSTER_MODE_WIGNORE) {
+        ret = dict_set_int32(options, "force", _gf_true);
         if (ret) {
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-                goto out;
+            gf_log("cli", GF_LOG_ERROR,
+                   "Failed to set force "
+                   "option");
+            goto out;
         }
-        /*Check brick order if type is replicate*/
-        ret = dict_get_int32 (options, "type", &type);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Could not get brick type");
-                goto out;
-        }
-        if ((type == GF_CLUSTER_TYPE_REPLICATE) ||
-            (type == GF_CLUSTER_TYPE_STRIPE_REPLICATE) ||
-            (type == GF_CLUSTER_TYPE_DISPERSE)) {
-                if ((ret = dict_get_str (options, "bricks",
-                                         &brick_list)) != 0) {
-                        gf_log ("cli", GF_LOG_ERROR, "Bricks check : Could "
-                                                     "not retrieve bricks "
-                                                     "list");
-                        goto out;
-                }
-                if ((ret = dict_get_int32 (options, "count",
-                                           &brick_count)) != 0) {
-                        gf_log ("cli", GF_LOG_ERROR, "Bricks check : Could "
-                                                     "not retrieve brick "
-                                                     "count");
-                        goto out;
-                }
+    }
 
-                if (type != GF_CLUSTER_TYPE_DISPERSE) {
-                    if ((ret = dict_get_int32 (options, "replica-count",
-                                               &sub_count)) != 0) {
-                            gf_log ("cli", GF_LOG_ERROR, "Bricks check : "
-                                                         "Could not retrieve "
-                                                         "replica count");
-                            goto out;
-                    }
-                    gf_log ("cli", GF_LOG_INFO, "Replicate cluster type found."
-                                                " Checking brick order.");
-                } else {
-                    ret = dict_get_int32 (options, "disperse-count",
-                                          &sub_count);
-                    if (ret) {
-                            gf_log ("cli", GF_LOG_ERROR, "Bricks check : "
-                                                         "Could not retrieve "
-                                                         "disperse count");
-                            goto out;
-                    }
-                    gf_log ("cli", GF_LOG_INFO, "Disperse cluster type found. "
-                                                "Checking brick order.");
-                }
-                ret = cli_cmd_check_brick_order (state, brick_list,
-                                                 brick_count, sub_count);
-                if (ret) {
-                        gf_log("cli", GF_LOG_INFO, "Not creating volume "
-                                                   "because of bad brick "
-                                                   "order");
-                        goto out;
-                }
-        }
-
-        ret = dict_get_str (options, "transport", &trans_type);
-        if (ret) {
-                gf_log("cli", GF_LOG_ERROR, "Unable to get transport type");
-                goto out;
-        }
-
-        if (state->mode & GLUSTER_MODE_WIGNORE) {
-                ret = dict_set_int32 (options, "force", _gf_true);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Failed to set force "
-                                "option");
-                        goto out;
-                }
-        }
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
 
-        CLI_LOCAL_INIT (local, words, frame, options);
+    CLI_LOCAL_INIT(local, words, frame, options);
 
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, options);
-        }
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, options);
+    }
 
 out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_out ("Volume create failed");
-        }
-
-        CLI_STACK_DESTROY (frame);
-
-        return ret;
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("Volume create failed");
+    }
+
+    if (ret == 0) {
+        gf_event(EVENT_VOLUME_CREATE, "name=%s;bricks=%s", (char *)words[2],
+                 bricks);
+    }
+
+    CLI_STACK_DESTROY(frame);
+    return ret;
 }
 
-
 int
-cli_cmd_volume_delete_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                           const char **words, int wordcount)
+cli_cmd_volume_delete_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                          const char **words, int wordcount)
 {
-        int                     ret = -1;
-        rpc_clnt_procedure_t    *proc = NULL;
-        call_frame_t            *frame = NULL;
-        char                    *volname = NULL;
-        gf_answer_t             answer = GF_ANSWER_NO;
-        const char              *question = NULL;
-        int                     sent = 0;
-        int                     parse_error = 0;
-        cli_local_t             *local = NULL;
-        dict_t                  *dict = NULL;
-
-        question = "Deleting volume will erase all information about the volume. "
-                   "Do you want to continue?";
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_DELETE_VOLUME];
-
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
-
-        dict = dict_new ();
-        if (!dict)
-                goto out;
-
-        if (wordcount != 3) {
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-                goto out;
-        }
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    char *volname = NULL;
+    gf_answer_t answer = GF_ANSWER_NO;
+    const char *question = NULL;
+    int sent = 0;
+    int parse_error = 0;
+    cli_local_t *local = NULL;
+    dict_t *dict = NULL;
+
+    question =
+        "Deleting volume will erase all information about the volume. "
+        "Do you want to continue?";
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_DELETE_VOLUME];
+
+    if (wordcount != 3) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
 
-        answer = cli_cmd_get_confirmation (state, question);
+    volname = (char *)words[2];
 
-        if (GF_ANSWER_NO == answer) {
-                ret = 0;
-                goto out;
-        }
-
-        volname = (char *)words[2];
+    dict = dict_new();
+    if (!dict)
+        goto out;
 
-        ret = dict_set_str (dict, "volname", volname);
+    ret = dict_set_str(dict, "volname", volname);
+    if (ret) {
+        gf_log(THIS->name, GF_LOG_WARNING, "dict set failed");
+        goto out;
+    }
+
+    if (!strcmp(volname, GLUSTER_SHARED_STORAGE)) {
+        question =
+            "Deleting the shared storage volume"
+            "(gluster_shared_storage), will affect features "
+            "like snapshot scheduler, geo-replication "
+            "and NFS-Ganesha. Do you still want to "
+            "continue?";
+    }
+
+    answer = cli_cmd_get_confirmation(state, question);
+    if (GF_ANSWER_NO == answer) {
+        ret = 0;
+        goto out;
+    }
 
-        if (ret) {
-                gf_log (THIS->name, GF_LOG_WARNING, "dict set failed");
-                goto out;
-        }
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
 
-        CLI_LOCAL_INIT (local, words, frame, dict);
+    CLI_LOCAL_INIT(local, words, frame, dict);
 
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, dict);
-        }
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, dict);
+    }
 
 out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_out ("Volume delete failed");
-        }
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("Volume delete failed");
+    }
+
+    CLI_STACK_DESTROY(frame);
 
-        CLI_STACK_DESTROY (frame);
+    if (ret == 0 && GF_ANSWER_YES == answer) {
+        gf_event(EVENT_VOLUME_DELETE, "name=%s", (char *)words[2]);
+    }
 
-        return ret;
+    return ret;
 }
 
 int
-cli_cmd_volume_start_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                          const char **words, int wordcount)
+cli_cmd_volume_start_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                         const char **words, int wordcount)
 {
-        int                     ret = -1;
-        rpc_clnt_procedure_t    *proc = NULL;
-        call_frame_t            *frame = NULL;
-        int                     sent = 0;
-        int                     parse_error = 0;
-        dict_t                  *dict = NULL;
-        int                     flags = 0;
-        cli_local_t             *local = NULL;
-
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    int sent = 0;
+    int parse_error = 0;
+    dict_t *dict = NULL;
+    int flags = 0;
+    cli_local_t *local = NULL;
+
+    if (wordcount < 3 || wordcount > 4) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
 
-        if (wordcount < 3 || wordcount > 4) {
-               cli_usage_out (word->pattern);
-                parse_error = 1;
-               goto out;
-        }
+    if (!words[2])
+        goto out;
 
-        dict = dict_new ();
-        if (!dict) {
-                goto out;
+    if (wordcount == 4) {
+        if (!strcmp("force", words[3])) {
+            flags |= GF_CLI_FLAG_OP_FORCE;
+        } else {
+            ret = -1;
+            cli_usage_out(word->pattern);
+            parse_error = 1;
+            goto out;
         }
+    }
 
-        if (!words[2])
-                goto out;
+    dict = dict_new();
+    if (!dict) {
+        goto out;
+    }
 
-        ret = dict_set_str (dict, "volname", (char *)words[2]);
-        if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "dict set failed");
-                goto out;
-        }
+    ret = dict_set_str(dict, "volname", (char *)words[2]);
+    if (ret) {
+        gf_log(THIS->name, GF_LOG_ERROR, "dict set failed");
+        goto out;
+    }
 
-        if (wordcount == 4) {
-                if (!strcmp("force", words[3])) {
-                        flags |= GF_CLI_FLAG_OP_FORCE;
-                } else {
-                        ret = -1;
-                        cli_usage_out (word->pattern);
-                        parse_error = 1;
-                        goto out;
-                }
-        }
-        ret = dict_set_int32 (dict, "flags", flags);
-        if (ret) {
-                 gf_log (THIS->name, GF_LOG_ERROR,
-                         "dict set failed");
-                 goto out;
-        }
+    ret = dict_set_int32(dict, "flags", flags);
+    if (ret) {
+        gf_log(THIS->name, GF_LOG_ERROR, "dict set failed");
+        goto out;
+    }
 
-        if (ret < 0) {
-                gf_log (THIS->name, GF_LOG_ERROR,
-                        "failed to serialize dict");
-                goto out;
-        }
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_START_VOLUME];
 
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_START_VOLUME];
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
 
-        CLI_LOCAL_INIT (local, words, frame, dict);
+    CLI_LOCAL_INIT(local, words, frame, dict);
 
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, dict);
-        }
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, dict);
+    }
 
 out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_out ("Volume start failed");
-        }
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("Volume start failed");
+    }
+
+    CLI_STACK_DESTROY(frame);
 
-        CLI_STACK_DESTROY (frame);
+    if (ret == 0) {
+        gf_event(EVENT_VOLUME_START, "name=%s;force=%d", (char *)words[2],
+                 (flags & GF_CLI_FLAG_OP_FORCE));
+    }
 
-        return ret;
+    return ret;
 }
 
 gf_answer_t
-cli_cmd_get_confirmation (struct cli_state *state, const char *question)
+cli_cmd_get_confirmation(struct cli_state *state, const char *question)
 {
-        char                    answer[5] = {'\0', };
-        char                    flush = '\0';
-        size_t			len;
+    char answer[5] = {
+        '\0',
+    };
+    int flush = '\0';
+    size_t len;
 
-        if (state->mode & GLUSTER_MODE_SCRIPT)
-                return GF_ANSWER_YES;
+    if (state->mode & GLUSTER_MODE_SCRIPT)
+        return GF_ANSWER_YES;
 
-	printf ("%s (y/n) ", question);
+    printf("%s (y/n) ", question);
 
-        if (fgets (answer, 4, stdin) == NULL) {
-                cli_out("gluster cli read error");
-                goto out;
-        }
+    if (fgets(answer, 4, stdin) == NULL) {
+        cli_out("gluster cli read error");
+        goto out;
+    }
 
-	len = strlen (answer);
+    len = strlen(answer);
 
-	if (len && answer [len - 1] == '\n'){
-		answer [--len] = '\0';
-	} else {
-		do{
-			flush = getchar ();
-		}while (flush != '\n');
-	}
+    if (len && answer[len - 1] == '\n') {
+        answer[--len] = '\0';
+    } else {
+        do {
+            flush = getchar();
+        } while (flush != '\n');
+    }
 
-	if (len > 3)
-		goto out;
+    if (len > 3)
+        goto out;
 
-	if (!strcasecmp (answer, "y") || !strcasecmp (answer, "yes"))
-		return GF_ANSWER_YES;
+    if (!strcasecmp(answer, "y") || !strcasecmp(answer, "yes"))
+        return GF_ANSWER_YES;
 
-	else if (!strcasecmp (answer, "n") || !strcasecmp (answer, "no"))
-		return GF_ANSWER_NO;
+    else if (!strcasecmp(answer, "n") || !strcasecmp(answer, "no"))
+        return GF_ANSWER_NO;
 
 out:
-	cli_out ("Invalid input, please enter y/n");
+    cli_out("Invalid input, please enter y/n");
 
-	return GF_ANSWER_NO;
+    return GF_ANSWER_NO;
 }
 
 int
-cli_cmd_volume_stop_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                           const char **words, int wordcount)
+cli_cmd_volume_stop_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                        const char **words, int wordcount)
 {
-        int                     ret = -1;
-        rpc_clnt_procedure_t    *proc = NULL;
-        call_frame_t            *frame = NULL;
-        int                     flags   = 0;
-        gf_answer_t             answer = GF_ANSWER_NO;
-        int                     sent = 0;
-        int                     parse_error = 0;
-        dict_t                  *dict = NULL;
-        char                    *volname = NULL;
-        cli_local_t             *local = NULL;
-
-        const char *question = "Stopping volume will make its data inaccessible. "
-                               "Do you want to continue?";
-
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
-
-        if (wordcount < 3 || wordcount > 4) {
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-                goto out;
-        }
-
-        volname = (char*) words[2];
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    int flags = 0;
+    gf_answer_t answer = GF_ANSWER_NO;
+    int sent = 0;
+    int parse_error = 0;
+    dict_t *dict = NULL;
+    char *volname = NULL;
+    cli_local_t *local = NULL;
+
+    const char *question =
+        "Stopping volume will make its data inaccessible. "
+        "Do you want to continue?";
+
+    if (wordcount < 3 || wordcount > 4) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
 
-        dict = dict_new ();
-        ret = dict_set_str (dict, "volname", volname);
-        if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "dict set failed");
-                goto out;
-        }
+    volname = (char *)words[2];
 
-        if (wordcount == 4) {
-                if (!strcmp("force", words[3])) {
-                        flags |= GF_CLI_FLAG_OP_FORCE;
-                } else {
-                        ret = -1;
-                        cli_usage_out (word->pattern);
-                        parse_error = 1;
-                        goto out;
-                }
-        }
-        ret = dict_set_int32 (dict, "flags", flags);
-        if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR,
-                        "dict set failed");
-                goto out;
+    dict = dict_new();
+    ret = dict_set_str(dict, "volname", volname);
+    if (ret) {
+        gf_log(THIS->name, GF_LOG_ERROR, "dict set failed");
+        goto out;
+    }
+
+    if (!strcmp(volname, GLUSTER_SHARED_STORAGE)) {
+        question =
+            "Stopping the shared storage volume"
+            "(gluster_shared_storage), will affect features "
+            "like snapshot scheduler, geo-replication "
+            "and NFS-Ganesha. Do you still want to "
+            "continue?";
+    }
+
+    if (wordcount == 4) {
+        if (!strcmp("force", words[3])) {
+            flags |= GF_CLI_FLAG_OP_FORCE;
+        } else {
+            ret = -1;
+            cli_usage_out(word->pattern);
+            parse_error = 1;
+            goto out;
         }
+    }
 
-        answer = cli_cmd_get_confirmation (state, question);
+    ret = dict_set_int32(dict, "flags", flags);
+    if (ret) {
+        gf_log(THIS->name, GF_LOG_ERROR, "dict set failed");
+        goto out;
+    }
 
-        if (GF_ANSWER_NO == answer) {
-                ret = 0;
-                goto out;
-        }
+    answer = cli_cmd_get_confirmation(state, question);
 
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_STOP_VOLUME];
+    if (GF_ANSWER_NO == answer) {
+        ret = 0;
+        goto out;
+    }
 
-        CLI_LOCAL_INIT (local, words, frame, dict);
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_STOP_VOLUME];
 
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, dict);
-        }
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
 
-out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_out ("Volume stop on '%s' failed", volname);
-        }
+    CLI_LOCAL_INIT(local, words, frame, dict);
 
-        CLI_STACK_DESTROY (frame);
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, dict);
+    }
 
-        return ret;
+out:
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("Volume stop on '%s' failed", volname);
+    }
+
+    CLI_STACK_DESTROY(frame);
+    if (dict)
+        dict_unref(dict);
+
+    if (ret == 0 && GF_ANSWER_YES == answer) {
+        gf_event(EVENT_VOLUME_STOP, "name=%s;force=%d", (char *)words[2],
+                 (flags & GF_CLI_FLAG_OP_FORCE));
+    }
+
+    return ret;
 }
 
-
 int
-cli_cmd_volume_rename_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                           const char **words, int wordcount)
+cli_cmd_volume_rename_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                          const char **words, int wordcount)
 {
-        int                     ret = -1;
-        rpc_clnt_procedure_t    *proc = NULL;
-        call_frame_t            *frame = NULL;
-        dict_t                  *dict = NULL;
-        int                     sent = 0;
-        int                     parse_error = 0;
-
-
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
-
-        dict = dict_new ();
-        if (!dict)
-                goto out;
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *dict = NULL;
+    int sent = 0;
+    int parse_error = 0;
+
+    if (wordcount != 4) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
 
-        if (wordcount != 4) {
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-                goto out;
-        }
+    dict = dict_new();
+    if (!dict)
+        goto out;
 
-        ret = dict_set_str (dict, "old-volname", (char *)words[2]);
+    ret = dict_set_str(dict, "old-volname", (char *)words[2]);
 
-        if (ret)
-                goto out;
+    if (ret)
+        goto out;
 
-        ret = dict_set_str (dict, "new-volname", (char *)words[3]);
+    ret = dict_set_str(dict, "new-volname", (char *)words[3]);
 
-        if (ret)
-                goto out;
+    if (ret)
+        goto out;
 
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_RENAME_VOLUME];
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_RENAME_VOLUME];
 
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, dict);
+    if (proc->fn) {
+        frame = create_frame(THIS, THIS->ctx->pool);
+        if (!frame) {
+            ret = -1;
+            goto out;
         }
+        ret = proc->fn(frame, THIS, dict);
+    }
 
 out:
-        if (dict)
-                dict_destroy (dict);
+    if (dict)
+        dict_unref(dict);
 
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_out ("Volume rename on '%s' failed", (char *)words[2]);
-        }
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("Volume rename on '%s' failed", (char *)words[2]);
+    }
 
-        CLI_STACK_DESTROY (frame);
+    CLI_STACK_DESTROY(frame);
 
-        return ret;
+    return ret;
 }
 
 int
-cli_cmd_volume_defrag_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                           const char **words, int wordcount)
+cli_cmd_volume_defrag_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                          const char **words, int wordcount)
 {
-        int                   ret     = -1;
-        rpc_clnt_procedure_t *proc    = NULL;
-        call_frame_t         *frame   = NULL;
-        dict_t               *dict = NULL;
-        int                     sent = 0;
-        int                     parse_error = 0;
-        cli_local_t          *local = NULL;
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *dict = NULL;
+    int sent = 0;
+    int parse_error = 0;
+    cli_local_t *local = NULL;
+#if (USE_EVENTS)
+    eventtypes_t event = EVENT_LAST;
+#endif
+
 #ifdef GF_SOLARIS_HOST_OS
-        cli_out ("Command not supported on Solaris");
-        goto out;
+    cli_out("Command not supported on Solaris");
+    goto out;
 #endif
 
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
+    ret = cli_cmd_volume_defrag_parse(words, wordcount, &dict);
 
-        ret = cli_cmd_volume_defrag_parse (words, wordcount, &dict);
+    if (ret) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+    }
 
-        if (ret) {
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-        }
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_DEFRAG_VOLUME];
 
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_DEFRAG_VOLUME];
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
 
-        CLI_LOCAL_INIT (local, words, frame, dict);
+    CLI_LOCAL_INIT(local, words, frame, dict);
 
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, dict);
-        }
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, dict);
+    }
 
 out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_out ("Volume rebalance failed");
-        }
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("Volume rebalance failed");
+    } else {
+#if (USE_EVENTS)
+        if (!(strcmp(words[wordcount - 1], "start")) ||
+            !(strcmp(words[wordcount - 1], "force"))) {
+            event = EVENT_VOLUME_REBALANCE_START;
+        } else if (!strcmp(words[wordcount - 1], "stop")) {
+            event = EVENT_VOLUME_REBALANCE_STOP;
+        }
+
+        if (event != EVENT_LAST)
+            gf_event(event, "volume=%s", (char *)words[2]);
+#endif
+    }
 
-        CLI_STACK_DESTROY (frame);
+    CLI_STACK_DESTROY(frame);
 
-        return ret;
+    return ret;
 }
 
 int
-cli_cmd_volume_reset_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                          const char **words, int wordcount)
+cli_cmd_volume_reset_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                         const char **words, int wordcount)
 {
-        int                     sent = 0;
-        int                     parse_error = 0;
-        int                     ret = -1;
-        rpc_clnt_procedure_t    *proc = NULL;
-        call_frame_t            *frame = NULL;
-        dict_t                  *options = NULL;
-        cli_local_t             *local = NULL;
+    int sent = 0;
+    int parse_error = 0;
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *options = NULL;
+    cli_local_t *local = NULL;
+#if (USE_EVENTS)
+    int ret1 = -1;
+    char *tmp_opt = NULL;
+#endif
 
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_RESET_VOLUME];
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_RESET_VOLUME];
 
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
+    ret = cli_cmd_volume_reset_parse(words, wordcount, &options);
+    if (ret) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
 
-        ret = cli_cmd_volume_reset_parse (words, wordcount, &options);
-        if (ret) {
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-                goto out;
-        }
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
 
-        CLI_LOCAL_INIT (local, words, frame, options);
+    CLI_LOCAL_INIT(local, words, frame, options);
 
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, options);
-        }
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, options);
+    }
 
 out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_out ("Volume reset failed");
-        }
-
-        CLI_STACK_DESTROY (frame);
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("Volume reset failed");
+    }
+
+#if (USE_EVENTS)
+    if (ret == 0) {
+        ret1 = dict_get_str(options, "key", &tmp_opt);
+        if (ret1)
+            tmp_opt = "";
+
+        gf_event(EVENT_VOLUME_RESET, "name=%s;option=%s", (char *)words[2],
+                 tmp_opt);
+    }
+#endif
 
-        return ret;
+    CLI_STACK_DESTROY(frame);
 
+    return ret;
 }
 
 int
-cli_cmd_volume_profile_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                          const char **words, int wordcount)
+cli_cmd_volume_profile_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                           const char **words, int wordcount)
 {
-        int                     sent = 0;
-        int                     parse_error = 0;
+    int sent = 0;
+    int parse_error = 0;
 
-        int                     ret      = -1;
-        rpc_clnt_procedure_t    *proc    = NULL;
-        call_frame_t            *frame   = NULL;
-        dict_t                  *options = NULL;
-        cli_local_t             *local = NULL;
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *options = NULL;
+    cli_local_t *local = NULL;
 
-        ret = cli_cmd_volume_profile_parse (words, wordcount, &options);
+    ret = cli_cmd_volume_profile_parse(words, wordcount, &options);
 
-        if (ret) {
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-                goto out;
-        }
+    if (ret) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
 
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_PROFILE_VOLUME];
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_PROFILE_VOLUME];
 
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        gf_log(THIS->name, GF_LOG_ERROR, "failed to create frame");
+        ret = -1;
+        goto out;
+    }
 
-        CLI_LOCAL_INIT (local, words, frame, options);
+    CLI_LOCAL_INIT(local, words, frame, options);
 
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, options);
-        }
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, options);
+    }
 
 out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_out ("Volume profile failed");
-        }
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("Volume profile failed");
+    }
 
-        CLI_STACK_DESTROY (frame);
-
-        return ret;
+    CLI_STACK_DESTROY(frame);
 
+    return ret;
 }
 
 int
-cli_cmd_volume_set_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                        const char **words, int wordcount)
+cli_cmd_volume_set_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                       const char **words, int wordcount)
 {
-        int                     sent = 0;
-        int                     parse_error = 0;
-
-	int                     ret = -1;
-        rpc_clnt_procedure_t    *proc = NULL;
-        call_frame_t            *frame = NULL;
-        dict_t                  *options = NULL;
-        cli_local_t             *local = NULL;
-        char                    *op_errstr = NULL;
+    int sent = 0;
+    int parse_error = 0;
+
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *options = NULL;
+    cli_local_t *local = NULL;
+    char *op_errstr = NULL;
+
+#if (USE_EVENTS)
+    int ret1 = -1;
+    int i = 1;
+    char dict_key[50] = {
+        0,
+    };
+    char *tmp_opt = NULL;
+    char *opts_str = NULL;
+    int num_options = 0;
+#endif
 
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_SET_VOLUME];
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_SET_VOLUME];
 
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
+    ret = cli_cmd_volume_set_parse(state, words, wordcount, &options,
+                                   &op_errstr);
+    if (ret) {
+        if (op_errstr) {
+            cli_err("%s", op_errstr);
+            GF_FREE(op_errstr);
+        } else
+            cli_usage_out(word->pattern);
 
-        ret = cli_cmd_volume_set_parse (words, wordcount, &options, &op_errstr);
-        if (ret) {
-                if (op_errstr) {
-                    cli_err ("%s", op_errstr);
-                    GF_FREE (op_errstr);
-                } else
-                    cli_usage_out (word->pattern);
+        parse_error = 1;
+        goto out;
+    }
 
-                parse_error = 1;
-                goto out;
-        }
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
 
-        CLI_LOCAL_INIT (local, words, frame, options);
+    CLI_LOCAL_INIT(local, words, frame, options);
 
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, options);
-        }
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, options);
+    }
 
 out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_out ("Volume set failed");
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("Volume set failed");
+    }
+
+#if (USE_EVENTS)
+    if (ret == 0 && strcmp(words[2], "help") != 0) {
+        ret1 = dict_get_int32(options, "count", &num_options);
+        if (ret1) {
+            num_options = 0;
+            goto end;
+        } else {
+            num_options = num_options / 2;
         }
 
-        CLI_STACK_DESTROY (frame);
-
-        return ret;
-
-}
-
-int
-cli_cmd_volume_add_brick_cbk (struct cli_state *state,
-                              struct cli_cmd_word *word, const char **words,
-                              int wordcount)
-{
-        int                     ret = -1;
-        rpc_clnt_procedure_t    *proc = NULL;
-        call_frame_t            *frame = NULL;
-        dict_t                  *options = NULL;
-        int                     sent = 0;
-        int                     parse_error = 0;
-        gf_answer_t             answer = GF_ANSWER_NO;
-        cli_local_t             *local = NULL;
-
-        const char *question = "Changing the 'stripe count' of the volume is "
-                "not a supported feature. In some cases it may result in data "
-                "loss on the volume. Also there may be issues with regular "
-                "filesystem operations on the volume after the change. Do you "
-                "really want to continue with 'stripe' count option ? ";
-
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
-
-        ret = cli_cmd_volume_add_brick_parse (words, wordcount, &options);
-        if (ret) {
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-                goto out;
+        char *free_list_key[num_options];
+        char *free_list_val[num_options];
+        for (i = 0; i < num_options; i++) {
+            free_list_key[i] = NULL;
+            free_list_val[i] = NULL;
         }
+        /* Initialize opts_str */
+        opts_str = "";
 
-        /* TODO: there are challenges in supporting changing of
-           stripe-count, until it is properly supported give warning to user */
-        if (dict_get (options, "stripe-count")) {
-                answer = cli_cmd_get_confirmation (state, question);
-
-                if (GF_ANSWER_NO == answer) {
-                        ret = 0;
-                        goto out;
-                }
-        }
+        /* Prepare String in format options=KEY1,VALUE1,KEY2,VALUE2 */
+        for (i = 1; i <= num_options; i++) {
+            sprintf(dict_key, "key%d", i);
+            ret1 = dict_get_str(options, dict_key, &tmp_opt);
+            if (ret1)
+                tmp_opt = "";
 
-        if (state->mode & GLUSTER_MODE_WIGNORE) {
-                ret = dict_set_int32 (options, "force", _gf_true);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Failed to set force "
-                                "option");
-                        goto out;
-                }
-        }
+            gf_asprintf(&opts_str, "%s,%s", opts_str, tmp_opt);
+            free_list_key[i - 1] = opts_str;
 
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_ADD_BRICK];
+            sprintf(dict_key, "value%d", i);
+            ret1 = dict_get_str(options, dict_key, &tmp_opt);
+            if (ret1)
+                tmp_opt = "";
 
-        CLI_LOCAL_INIT (local, words, frame, options);
-
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, options);
+            gf_asprintf(&opts_str, "%s,%s", opts_str, tmp_opt);
+            free_list_val[i - 1] = opts_str;
         }
 
-out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_out ("Volume add-brick failed");
+        gf_event(EVENT_VOLUME_SET, "name=%s;options=%s", (char *)words[2],
+                 opts_str);
+
+        /* Allocated by gf_strdup and gf_asprintf */
+        for (i = 0; i < num_options; i++) {
+            GF_FREE(free_list_key[i]);
+            GF_FREE(free_list_val[i]);
         }
+    }
+#endif
 
-        CLI_STACK_DESTROY (frame);
+end:
+    CLI_STACK_DESTROY(frame);
 
-        return ret;
+    return ret;
 }
 
 static int
-gf_cli_create_auxiliary_mount (char *volname)
+cli_event_remove_brick_str(dict_t *options, char **event_str,
+                           eventtypes_t *event)
 {
-        int      ret                     = -1;
-        char     mountdir[PATH_MAX]      = {0,};
-        char     pidfile_path[PATH_MAX]  = {0,};
-        char     logfile[PATH_MAX]       = {0,};
-        char     qpid [16]               = {0,};
-
-        GLUSTERFS_GET_AUX_MOUNT_PIDFILE (pidfile_path, volname);
-
-        if (gf_is_service_running (pidfile_path, NULL)) {
-                gf_log ("cli", GF_LOG_DEBUG, "Aux mount of volume %s is running"
-                        " already", volname);
-                ret = 0;
-                goto out;
-        }
+    int ret = -1;
+    char *bricklist = NULL;
+    char *brick = NULL;
+    char *volname = NULL;
+    char key[256] = {
+        0,
+    };
+    const char *eventstrformat = "volume=%s;bricks=%s";
+    int32_t command = 0;
+    int32_t i = 1;
+    int32_t count = 0;
+    int32_t eventstrlen = 1;
+    int bricklen = 0;
+    char *tmp_ptr = NULL;
+
+    if (!options || !event_str || !event)
+        goto out;
 
-        GLUSTERD_GET_QUOTA_AUX_MOUNT_PATH (mountdir, volname, "/");
-        ret = mkdir (mountdir, 0777);
-        if (ret && errno != EEXIST) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to create auxiliary mount "
-                        "directory %s. Reason : %s", mountdir,
-                        strerror (errno));
-                goto out;
-        }
+    ret = dict_get_str(options, "volname", &volname);
+    if (ret || !volname) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to fetch volname");
+        ret = -1;
+        goto out;
+    }
+    /* Get the list of bricks for the event */
+    ret = dict_get_int32(options, "command", &command);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to fetch command");
+        ret = -1;
+        goto out;
+    }
+
+    switch (command) {
+        case GF_OP_CMD_START:
+            *event = EVENT_VOLUME_REMOVE_BRICK_START;
+            break;
+        case GF_OP_CMD_COMMIT:
+            *event = EVENT_VOLUME_REMOVE_BRICK_COMMIT;
+            break;
+        case GF_OP_CMD_COMMIT_FORCE:
+            *event = EVENT_VOLUME_REMOVE_BRICK_FORCE;
+            break;
+        case GF_OP_CMD_STOP:
+            *event = EVENT_VOLUME_REMOVE_BRICK_STOP;
+            break;
+        default:
+            *event = EVENT_LAST;
+            break;
+    }
 
-        snprintf (logfile, PATH_MAX-1, "%s/quota-mount-%s.log",
-                  DEFAULT_LOG_FILE_DIRECTORY, volname);
-        snprintf(qpid, 15, "%d", GF_CLIENT_PID_QUOTA_MOUNT);
+    ret = -1;
 
-        ret = runcmd (SBIN_DIR"/glusterfs",
-                      "-s", "localhost",
-                      "--volfile-id", volname,
-                      "-l", logfile,
-                      "-p", pidfile_path,
-                      "--client-pid", qpid,
-                      mountdir,
-                      NULL);
+    if (*event == EVENT_LAST) {
+        goto out;
+    }
 
+    /* I could just get this from words[] but this is cleaner in case the
+     * format changes  */
+    while (i) {
+        snprintf(key, sizeof(key), "brick%d", i);
+        ret = dict_get_str(options, key, &brick);
         if (ret) {
-                gf_log ("cli", GF_LOG_WARNING, "failed to mount glusterfs "
-                        "client. Please check the log file %s for more details",
-                        logfile);
-                ret = -1;
-                goto out;
+            break;
         }
+        eventstrlen += strlen(brick) + 1;
+        i++;
+    }
 
-        ret = 0;
+    count = --i;
 
-out:
-        return ret;
-}
+    eventstrlen += 1;
 
-static int
-cli_stage_quota_op (char *volname, int op_code)
-{
-        int ret = -1;
-
-        switch (op_code) {
-                case GF_QUOTA_OPTION_TYPE_ENABLE:
-                case GF_QUOTA_OPTION_TYPE_LIMIT_USAGE:
-                case GF_QUOTA_OPTION_TYPE_REMOVE:
-                case GF_QUOTA_OPTION_TYPE_LIST:
-                        ret = gf_cli_create_auxiliary_mount (volname);
-                        if (ret) {
-                                cli_err ("quota: Could not start quota "
-                                         "auxiliary mount");
-                                goto out;
-                        }
-                        ret = 0;
-                        break;
-
-                default:
-                        ret = 0;
-                        break;
-        }
+    bricklist = GF_CALLOC(eventstrlen, sizeof(char), gf_common_mt_char);
+    if (!bricklist) {
+        gf_log(THIS->name, GF_LOG_ERROR,
+               "memory allocation failed for"
+               "bricklist");
+        ret = -1;
+        goto out;
+    }
 
-out:
-        return ret;
-}
+    tmp_ptr = bricklist;
 
-static void
-print_quota_list_header (void)
-{
-        //Header
-        cli_out ("                  Path                   Hard-limit "
-                 "Soft-limit   Used  Available  Soft-limit exceeded?"
-                 " Hard-limit exceeded?");
-        cli_out ("-----------------------------------------------------"
-                 "-----------------------------------------------------"
-                 "-----------------");
+    i = 1;
+    while (i <= count) {
+        snprintf(key, sizeof(key), "brick%d", i);
+        ret = dict_get_str(options, key, &brick);
+        if (ret) {
+            break;
+        }
+        snprintf(tmp_ptr, eventstrlen, "%s ", brick);
+        bricklen = strlen(brick);
+        eventstrlen -= (bricklen + 1);
+        tmp_ptr += (bricklen + 1);
+        i++;
+    }
+
+    if (!ret) {
+        gf_asprintf(event_str, eventstrformat, volname, bricklist);
+    } else {
+        gf_asprintf(event_str, eventstrformat, volname, "<unavailable>");
+    }
+
+    ret = 0;
+out:
+    GF_FREE(bricklist);
+    return ret;
 }
 
 int
-cli_get_soft_limit (dict_t *options, const char **words, dict_t *xdata)
+cli_cmd_volume_add_brick_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                             const char **words, int wordcount)
 {
-        call_frame_t            *frame          = NULL;
-        cli_local_t             *local          = NULL;
-        rpc_clnt_procedure_t    *proc           = NULL;
-        char                    *default_sl     = NULL;
-        char                    *default_sl_dup = NULL;
-        int                      ret  = -1;
-
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame) {
-                ret = -1;
-                goto out;
-        }
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *options = NULL;
+    int sent = 0;
+    int parse_error = 0;
+    gf_answer_t answer = GF_ANSWER_NO;
+    cli_local_t *local = NULL;
+
+#if (USE_EVENTS)
+    char *event_str = NULL;
+    char *bricks = NULL;
+    const char *eventstrformat = "volume=%s;bricks=%s";
+#endif
 
-        //We need a ref on @options to prevent CLI_STACK_DESTROY
-        //from destroying it prematurely.
-        dict_ref (options);
-        CLI_LOCAL_INIT (local, words, frame, options);
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_QUOTA];
-        ret = proc->fn (frame, THIS, options);
+    const char *question =
+        "Changing the 'stripe count' of the volume is "
+        "not a supported feature. In some cases it may result in data "
+        "loss on the volume. Also there may be issues with regular "
+        "filesystem operations on the volume after the change. Do you "
+        "really want to continue with 'stripe' count option ? ";
+
+    ret = cli_cmd_volume_add_brick_parse(state, words, wordcount, &options, 0);
+    if (ret) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
 
-        ret = dict_get_str (options, "default-soft-limit", &default_sl);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to get default soft limit");
-                goto out;
-        }
+    /* TODO: there are challenges in supporting changing of
+       stripe-count, until it is properly supported give warning to user */
+    if (dict_get(options, "stripe-count")) {
+        answer = cli_cmd_get_confirmation(state, question);
 
-        default_sl_dup = gf_strdup (default_sl);
-        if (!default_sl_dup) {
-                ret = -1;
-                goto out;
+        if (GF_ANSWER_NO == answer) {
+            ret = 0;
+            goto out;
         }
+    }
+
+#if (USE_EVENTS)
+    /* Get the list of bricks for the event */
 
-        ret = dict_set_dynstr (xdata, "default-soft-limit", default_sl_dup);
+    ret = dict_get_str(options, "bricks", &bricks);
+
+    if (!ret) {
+        gf_asprintf(&event_str, eventstrformat, (char *)words[2],
+                    &bricks[1] /*Skip leading space*/);
+    } else {
+        gf_asprintf(&event_str, eventstrformat, (char *)words[2],
+                    "<unavailable>");
+    }
+#endif
+
+    if (state->mode & GLUSTER_MODE_WIGNORE) {
+        ret = dict_set_int32(options, "force", _gf_true);
         if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to set default soft limit");
-                GF_FREE (default_sl_dup);
-                goto out;
+            gf_log("cli", GF_LOG_ERROR,
+                   "Failed to set force "
+                   "option");
+            goto out;
         }
+    }
+
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_ADD_BRICK];
+
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
+
+    CLI_LOCAL_INIT(local, words, frame, options);
+
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, options);
+    }
 
 out:
-        CLI_STACK_DESTROY (frame);
-        return ret;
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("Volume add-brick failed");
+    } else {
+#if (USE_EVENTS)
+        gf_event(EVENT_VOLUME_ADD_BRICK, "%s", event_str);
+#endif
+    }
+#if (USE_EVENTS)
+    GF_FREE(event_str);
+#endif
+
+    CLI_STACK_DESTROY(frame);
+    return ret;
 }
 
-#define QUOTA_CONF_HEADER                                                \
-        "GlusterFS Quota conf | version: v%d.%d\n"
 int
-cli_cmd_quota_conf_skip_header (int fd)
+cli_get_soft_limit(dict_t *options, const char **words, dict_t *xdata)
 {
-        char buf[PATH_MAX] = {0,};
-
-        snprintf (buf, sizeof(buf)-1, QUOTA_CONF_HEADER, 1, 1);
-        return gf_skip_header_section (fd, strlen (buf));
-}
+    call_frame_t *frame = NULL;
+    cli_local_t *local = NULL;
+    rpc_clnt_procedure_t *proc = NULL;
+    char *default_sl = NULL;
+    char *default_sl_dup = NULL;
+    int ret = -1;
+
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
+
+    // We need a ref on @options to prevent CLI_STACK_DESTROY
+    // from destroying it prematurely.
+    dict_ref(options);
+    CLI_LOCAL_INIT(local, words, frame, options);
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_QUOTA];
+    ret = proc->fn(frame, THIS, options);
+
+    ret = dict_get_str(options, "default-soft-limit", &default_sl);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to get default soft limit");
+        goto out;
+    }
 
-/* Checks if at least one limit has been set on the volume
- *
- * Returns true if at least one limit is set. Returns false otherwise.
- */
-gf_boolean_t
-_limits_set_on_volume (char *volname) {
-        gf_boolean_t    limits_set = _gf_false;
-        int             ret = -1;
-        char            quota_conf_file[PATH_MAX] = {0,};
-        int             fd = -1;
-        char            buf[16] = {0,};
-
-        /* TODO: fix hardcoding; Need to perform an RPC call to glusterd
-         * to fetch working directory
-         */
-        sprintf (quota_conf_file, "%s/vols/%s/quota.conf",
-                 GLUSTERD_DEFAULT_WORKDIR,
-                 volname);
-        fd = open (quota_conf_file, O_RDONLY);
-        if (fd == -1)
-                goto out;
+    default_sl_dup = gf_strdup(default_sl);
+    if (!default_sl_dup) {
+        ret = -1;
+        goto out;
+    }
 
-        ret = cli_cmd_quota_conf_skip_header (fd);
-        if (ret)
-                goto out;
+    ret = dict_set_dynstr(xdata, "default-soft-limit", default_sl_dup);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to set default soft limit");
+        GF_FREE(default_sl_dup);
+        goto out;
+    }
 
-        /* Try to read atleast one gfid */
-        ret = read (fd, (void *)buf, 16);
-        if (ret == 16)
-                limits_set = _gf_true;
 out:
-        if (fd != -1)
-                close (fd);
-        return limits_set;
+    CLI_STACK_DESTROY(frame);
+    return ret;
 }
 
-/* Checks if the mount is connected to the bricks
+/* Checks if at least one limit has been set on the volume
  *
- * Returns true if connected and false if not
+ * Returns true if at least one limit is set. Returns false otherwise.
  */
 gf_boolean_t
-_quota_aux_mount_online (char *volname)
+_limits_set_on_volume(char *volname, int type)
 {
-        int         ret = 0;
-        char        mount_path[PATH_MAX + 1] = {0,};
-        struct stat buf = {0,};
+    gf_boolean_t limits_set = _gf_false;
+    int ret = -1;
+    char quota_conf_file[PATH_MAX] = {
+        0,
+    };
+    int fd = -1;
+    char buf[16] = {
+        0,
+    };
+    float version = 0.0f;
+    char gfid_type_stored = 0;
+    char gfid_type = 0;
+
+    /* TODO: fix hardcoding; Need to perform an RPC call to glusterd
+     * to fetch working directory
+     */
+    snprintf(quota_conf_file, sizeof quota_conf_file, "%s/vols/%s/quota.conf",
+             GLUSTERD_DEFAULT_WORKDIR, volname);
+    fd = open(quota_conf_file, O_RDONLY);
+    if (fd == -1)
+        goto out;
 
-        GF_ASSERT (volname);
+    ret = quota_conf_read_version(fd, &version);
+    if (ret)
+        goto out;
 
-        /* Try to create the aux mount before checking if bricks are online */
-        ret = gf_cli_create_auxiliary_mount (volname);
-        if (ret) {
-                cli_err ("quota: Could not start quota auxiliary mount");
-                return _gf_false;
-        }
+    if (type == GF_QUOTA_OPTION_TYPE_LIST)
+        gfid_type = GF_QUOTA_CONF_TYPE_USAGE;
+    else
+        gfid_type = GF_QUOTA_CONF_TYPE_OBJECTS;
 
-        GLUSTERD_GET_QUOTA_AUX_MOUNT_PATH (mount_path, volname, "/");
+    /* Try to read at least one gfid  of type 'gfid_type' */
+    while (1) {
+        ret = quota_conf_read_gfid(fd, buf, &gfid_type_stored, version);
+        if (ret <= 0)
+            break;
 
-        ret = sys_stat (mount_path, &buf);
-        if (ret) {
-                if (ENOTCONN == errno) {
-                        cli_err ("quota: Cannot connect to bricks. Check if "
-                                 "bricks are online.");
-                } else {
-                        cli_err ("quota: Error on quota auxiliary mount (%s).",
-                                 strerror (errno));
-                }
-                return _gf_false;
+        if (gfid_type_stored == gfid_type) {
+            limits_set = _gf_true;
+            break;
         }
-        return _gf_true;
+    }
+out:
+    if (fd != -1)
+        sys_close(fd);
+
+    return limits_set;
 }
 
 int
-cli_cmd_quota_handle_list_all (const char **words, dict_t *options)
+cli_cmd_quota_handle_list_all(const char **words, dict_t *options)
 {
-        int                      all_failed = 1;
-        int                      count      = 0;
-        int                      ret        = -1;
-        rpc_clnt_procedure_t    *proc       = NULL;
-        cli_local_t             *local      = NULL;
-        call_frame_t            *frame      = NULL;
-        dict_t                  *xdata      = NULL;
-        char                    *gfid_str   = NULL;
-        char                    *volname    = NULL;
-        char                    *volname_dup = NULL;
-        unsigned char            buf[16]   = {0};
-        int                      fd        = -1;
-        char                     quota_conf_file[PATH_MAX] = {0};
-
-        xdata = dict_new ();
-        if (!xdata) {
-                ret = -1;
-                goto out;
-        }
+    int all_failed = 1;
+    int count = 0;
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    cli_local_t *local = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *xdata = NULL;
+    char gfid_str[UUID_CANONICAL_FORM_LEN + 1];
+    char *volname = NULL;
+    char *volname_dup = NULL;
+    unsigned char buf[16] = {0};
+    int fd = -1;
+    char quota_conf_file[PATH_MAX] = {0};
+    gf_boolean_t xml_err_flag = _gf_false;
+    char err_str[NAME_MAX] = {
+        0,
+    };
+    int32_t type = 0;
+    char gfid_type = 0;
+    float version = 0.0f;
+    int32_t max_count = 0;
+
+    xdata = dict_new();
+    if (!xdata) {
+        ret = -1;
+        goto out;
+    }
 
-        ret = dict_get_str (options, "volname", &volname);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to get volume name");
-                goto out;
-        }
+    ret = dict_get_str(options, "volname", &volname);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to get volume name");
+        goto out;
+    }
 
-        ret = cli_get_soft_limit (options, words, xdata);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to fetch default "
-                        "soft-limit");
-                goto out;
-        }
+    ret = dict_get_int32(options, "type", &type);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to get quota option type");
+        goto out;
+    }
 
-        /* Check if at least one limit is set on volume. No need to check for
-         * quota enabled as cli_get_soft_limit() handles that
-         */
-        if (!_limits_set_on_volume (volname)) {
-                cli_out ("quota: No quota configured on volume %s", volname);
-                ret = 0;
-                goto out;
-        }
+    ret = dict_set_int32(xdata, "type", type);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to set type in xdata");
+        goto out;
+    }
 
-        /* Check if the mount is online before doing any listing */
-        if (!_quota_aux_mount_online (volname)) {
-                ret = -1;
-                goto out;
+    ret = cli_get_soft_limit(options, words, xdata);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR,
+               "Failed to fetch default "
+               "soft-limit");
+        goto out;
+    }
+
+    /* Check if at least one limit is set on volume. No need to check for
+     * quota enabled as cli_get_soft_limit() handles that
+     */
+    if (!_limits_set_on_volume(volname, type)) {
+        snprintf(err_str, sizeof(err_str),
+                 "No%s quota configured on"
+                 " volume %s",
+                 (type == GF_QUOTA_OPTION_TYPE_LIST) ? "" : " inode", volname);
+        if (global_state->mode & GLUSTER_MODE_XML) {
+            xml_err_flag = _gf_true;
+        } else {
+            cli_out("quota: %s", err_str);
         }
+        ret = 0;
+        goto out;
+    }
 
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame) {
-                ret = -1;
-                goto out;
-        }
+    volname_dup = gf_strdup(volname);
+    if (!volname_dup) {
+        ret = -1;
+        goto out;
+    }
 
-        volname_dup = gf_strdup (volname);
-        if (!volname_dup) {
-                ret = -1;
-                goto out;
-        }
+    ret = dict_set_dynstr(xdata, "volume-uuid", volname_dup);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to set volume-uuid");
+        GF_FREE(volname_dup);
+        goto out;
+    }
+
+    // TODO: fix hardcoding; Need to perform an RPC call to glusterd
+    // to fetch working directory
+    snprintf(quota_conf_file, sizeof quota_conf_file, "%s/vols/%s/quota.conf",
+             GLUSTERD_DEFAULT_WORKDIR, volname);
+    fd = open(quota_conf_file, O_RDONLY);
+    if (fd == -1) {
+        // This may because no limits were yet set on the volume
+        gf_log("cli", GF_LOG_TRACE,
+               "Unable to open "
+               "quota.conf");
+        ret = 0;
+        goto out;
+    }
 
-        ret = dict_set_dynstr (xdata, "volume-uuid", volname_dup);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to set volume-uuid");
-                GF_FREE (volname_dup);
-                goto out;
-        }
+    ret = quota_conf_read_version(fd, &version);
+    if (ret)
+        goto out;
 
-        //TODO: fix hardcoding; Need to perform an RPC call to glusterd
-        //to fetch working directory
-        sprintf (quota_conf_file, "%s/vols/%s/quota.conf",
-                 GLUSTERD_DEFAULT_WORKDIR,
-                 volname);
-        fd = open (quota_conf_file, O_RDONLY);
-        if (fd == -1) {
-                //This may because no limits were yet set on the volume
-                gf_log ("cli", GF_LOG_TRACE, "Unable to open "
-                        "quota.conf");
-                ret = 0;
-                goto out;
-         }
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
+
+    CLI_LOCAL_INIT(local, words, frame, xdata);
+    proc = &cli_quotad_clnt.proctable[GF_AGGREGATOR_GETLIMIT];
+
+    for (count = 0;; count++) {
+        ret = quota_conf_read_gfid(fd, buf, &gfid_type, version);
+        if (ret == 0) {
+            break;
+        } else if (ret < 0) {
+            gf_log(THIS->name, GF_LOG_CRITICAL,
+                   "Quota "
+                   "configuration store may be corrupt.");
+            goto out;
+        }
+
+        if ((type == GF_QUOTA_OPTION_TYPE_LIST &&
+             gfid_type == GF_QUOTA_CONF_TYPE_OBJECTS) ||
+            (type == GF_QUOTA_OPTION_TYPE_LIST_OBJECTS &&
+             gfid_type == GF_QUOTA_CONF_TYPE_USAGE))
+            continue;
+
+        max_count++;
+    }
+    ret = dict_set_int32(xdata, "max_count", max_count);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to set max_count");
+        goto out;
+    }
+
+    ret = sys_lseek(fd, 0L, SEEK_SET);
+    if (ret < 0) {
+        gf_log(THIS->name, GF_LOG_ERROR,
+               "failed to move offset to "
+               "the beginning: %s",
+               strerror(errno));
+        goto out;
+    }
+    ret = quota_conf_read_version(fd, &version);
+    if (ret)
+        goto out;
 
-        ret = cli_cmd_quota_conf_skip_header (fd);
+    for (count = 0;; count++) {
+        ret = quota_conf_read_gfid(fd, buf, &gfid_type, version);
+        if (ret == 0) {
+            break;
+        } else if (ret < 0) {
+            gf_log(THIS->name, GF_LOG_CRITICAL,
+                   "Quota "
+                   "configuration store may be corrupt.");
+            goto out;
+        }
+
+        if ((type == GF_QUOTA_OPTION_TYPE_LIST &&
+             gfid_type == GF_QUOTA_CONF_TYPE_OBJECTS) ||
+            (type == GF_QUOTA_OPTION_TYPE_LIST_OBJECTS &&
+             gfid_type == GF_QUOTA_CONF_TYPE_USAGE))
+            continue;
+
+        uuid_utoa_r(buf, gfid_str);
+        ret = dict_set_str(xdata, "gfid", gfid_str);
         if (ret) {
-                goto out;
+            gf_log("cli", GF_LOG_ERROR, "Failed to set gfid");
+            goto out;
         }
-        CLI_LOCAL_INIT (local, words, frame, xdata);
-        proc = &cli_quotad_clnt.proctable[GF_AGGREGATOR_GETLIMIT];
 
-        print_quota_list_header ();
-        gfid_str = GF_CALLOC (1, gf_common_mt_char, 64);
-        if (!gfid_str) {
-                ret = -1;
-                goto out;
+        ret = proc->fn(frame, THIS, xdata);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Failed to get quota "
+                   "limits for %s",
+                   uuid_utoa((unsigned char *)buf));
         }
-        for (count = 0;; count++) {
-                ret = read (fd, (void*) buf, 16);
-                if (ret <= 0) {
-                        //Finished reading all entries in the conf file
-                        break;
-                }
-                if (ret < 16) {
-                        //This should never happen. We must have a multiple of
-                        //entry_sz bytes in our configuration file.
-                        gf_log (THIS->name, GF_LOG_CRITICAL, "Quota "
-                                "configuration store may be corrupt.");
-                        goto out;
-                }
-                uuid_utoa_r (buf, gfid_str);
-                ret = dict_set_str (xdata, "gfid", gfid_str);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Failed to set gfid");
-                        goto out;
-                }
 
-                ret = proc->fn (frame, THIS, xdata);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Failed to get quota "
-                                "limits for %s", uuid_utoa ((unsigned char*)buf));
-                }
+        dict_del(xdata, "gfid");
+        all_failed = all_failed && ret;
+    }
 
-                dict_del (xdata, "gfid");
-                all_failed = all_failed && ret;
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        ret = cli_xml_output_vol_quota_limit_list_end(local);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Error in printing "
+                   "xml output");
+            goto out;
         }
+    }
 
-        if (count > 0) {
-                ret = all_failed? -1: 0;
-        } else {
-                ret = 0;
-        }
-out:
-        if (fd != -1) {
-                close (fd);
-        }
+    if (count > 0) {
+        ret = all_failed ? -1 : 0;
+    } else {
+        ret = 0;
+    }
 
-        GF_FREE (gfid_str);
+out:
+    if (xml_err_flag) {
+        ret = cli_xml_output_str("volQuota", NULL, -1, 0, err_str);
         if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Could not fetch and display quota"
-                        " limits");
-        }
-        CLI_STACK_DESTROY (frame);
-        return ret;
+            gf_log("cli", GF_LOG_ERROR,
+                   "Error outputting in "
+                   "xml format");
+        }
+    }
+    if (xdata)
+        dict_unref(xdata);
+
+    if (fd != -1) {
+        sys_close(fd);
+    }
+
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR,
+               "Could not fetch and display quota"
+               " limits");
+    }
+    CLI_STACK_DESTROY(frame);
+    return ret;
 }
 
 int
-cli_cmd_quota_cbk (struct cli_state *state, struct cli_cmd_word *word,
+cli_cmd_bitrot_cbk(struct cli_state *state, struct cli_cmd_word *word,
                    const char **words, int wordcount)
 {
+    int ret = -1;
+    int parse_err = 0;
+    call_frame_t *frame = NULL;
+    dict_t *options = NULL;
+    cli_local_t *local = NULL;
+    rpc_clnt_procedure_t *proc = NULL;
+    int sent = 0;
+#if (USE_EVENTS)
+    int cmd_type = -1;
+    int ret1 = -1;
+    int event_type = -1;
+    char *tmp = NULL;
+    char *events_str = NULL;
+    char *volname = NULL;
+#endif
 
-        int                      ret       = 0;
-        int                      parse_err = 0;
-        int32_t                  type      = 0;
-        rpc_clnt_procedure_t    *proc      = NULL;
-        call_frame_t            *frame     = NULL;
-        dict_t                  *options   = NULL;
-        gf_answer_t              answer    = GF_ANSWER_NO;
-        cli_local_t             *local     = NULL;
-        int                      sent      = 0;
-        char                    *volname   = NULL;
-        const char *question = "Disabling quota will delete all the quota "
-                               "configuration. Do you want to continue?";
-
-        //parse **words into options dictionary
-        ret = cli_cmd_quota_parse (words, wordcount, &options);
-        if (ret < 0) {
-                cli_usage_out (word->pattern);
-                parse_err = 1;
-                goto out;
-        }
+    ret = cli_cmd_bitrot_parse(words, wordcount, &options);
+    if (ret < 0) {
+        cli_usage_out(word->pattern);
+        parse_err = 1;
+        goto out;
+    }
 
-        ret = dict_get_int32 (options, "type", &type);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to get opcode");
-                goto out;
+    if (ret == 1) {
+        /* this is 'volume bitrot help' */
+        cli_cmd_bitrot_help_cbk(state, word, words, wordcount);
+        ret = 0;
+        goto out2;
+    }
+
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
+
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_BITROT];
+
+    CLI_LOCAL_INIT(local, words, frame, options);
+
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, options);
+    }
+
+out:
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_err == 0))
+            cli_err(
+                "Bit rot command failed. Please check the cli "
+                "logs for more details");
+    }
+
+#if (USE_EVENTS)
+    if (ret == 0) {
+        ret1 = dict_get_int32(options, "type", &cmd_type);
+        if (ret1)
+            cmd_type = -1;
+        else {
+            ret1 = dict_get_str(options, "volname", &volname);
+            if (ret1)
+                volname = "";
         }
 
-        //handle quota-disable and quota-list-all different from others
-        switch (type) {
-        case GF_QUOTA_OPTION_TYPE_DISABLE:
-                answer = cli_cmd_get_confirmation (state, question);
-                if (answer == GF_ANSWER_NO)
-                        goto out;
+        switch (cmd_type) {
+            case GF_BITROT_OPTION_TYPE_ENABLE:
+                event_type = EVENT_BITROT_ENABLE;
                 break;
-        case GF_QUOTA_OPTION_TYPE_LIST:
-                if (wordcount != 4)
-                        break;
-                ret = cli_cmd_quota_handle_list_all (words, options);
-                goto out;
-        default:
+            case GF_BITROT_OPTION_TYPE_DISABLE:
+                event_type = EVENT_BITROT_DISABLE;
+                break;
+            case GF_BITROT_CMD_SCRUB_ONDEMAND:
+                event_type = EVENT_BITROT_SCRUB_ONDEMAND;
+                break;
+            case GF_BITROT_OPTION_TYPE_SCRUB_THROTTLE:
+                event_type = EVENT_BITROT_SCRUB_THROTTLE;
+                ret1 = dict_get_str(options, "scrub-throttle-value", &tmp);
+                if (ret1)
+                    tmp = "";
+                gf_asprintf(&events_str, "name=%s;value=%s", volname, tmp);
+                break;
+            case GF_BITROT_OPTION_TYPE_SCRUB_FREQ:
+                event_type = EVENT_BITROT_SCRUB_FREQ;
+                ret1 = dict_get_str(options, "scrub-frequency-value", &tmp);
+                if (ret1)
+                    tmp = "";
+                gf_asprintf(&events_str, "name=%s;value=%s", volname, tmp);
+                break;
+            case GF_BITROT_OPTION_TYPE_SCRUB:
+                event_type = EVENT_BITROT_SCRUB_OPTION;
+                ret1 = dict_get_str(options, "scrub-value", &tmp);
+                if (ret1)
+                    tmp = "";
+                gf_asprintf(&events_str, "name=%s;value=%s", volname, tmp);
+                break;
+            default:
                 break;
         }
 
-        ret = dict_get_str (options, "volname", &volname);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to get volume name");
-                goto out;
-        }
+        if (event_type > -1)
+            gf_event(event_type, "%s", events_str);
 
-        //create auxiliary mount need for quota commands that operate on path
-        ret = cli_stage_quota_op (volname, type);
-        if (ret)
-                goto out;
+        if (events_str)
+            GF_FREE(events_str);
+    }
+#endif
 
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame) {
-                ret = -1;
-                goto out;
+    CLI_STACK_DESTROY(frame);
+out2:
+    return ret;
+}
+
+int
+cli_cmd_quota_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                  const char **words, int wordcount)
+{
+    int ret = 0;
+    int parse_err = 0;
+    int32_t type = 0;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *options = NULL;
+    gf_answer_t answer = GF_ANSWER_NO;
+    cli_local_t *local = NULL;
+    int sent = 0;
+    char *volname = NULL;
+    const char *question =
+        "Disabling quota will delete all the quota "
+        "configuration. Do you want to continue?";
+
+    // parse **words into options dictionary
+    if (strcmp(words[1], "inode-quota") == 0) {
+        ret = cli_cmd_inode_quota_parse(words, wordcount, &options);
+        if (ret < 0) {
+            cli_usage_out(word->pattern);
+            parse_err = 1;
+            goto out;
         }
+    } else {
+        ret = cli_cmd_quota_parse(words, wordcount, &options);
 
-        CLI_LOCAL_INIT (local, words, frame, options);
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_QUOTA];
-        if (proc == NULL) {
-                ret = -1;
-                goto out;
+        if (ret == 1) {
+            cli_cmd_quota_help_cbk(state, word, words, wordcount);
+            ret = 0;
+            goto out;
+        }
+        if (ret < 0) {
+            cli_usage_out(word->pattern);
+            parse_err = 1;
+            goto out;
         }
+    }
+
+    ret = dict_get_int32(options, "type", &type);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to get opcode");
+        goto out;
+    }
+
+    // handle quota-disable and quota-list-all different from others
+    switch (type) {
+        case GF_QUOTA_OPTION_TYPE_DISABLE:
+            answer = cli_cmd_get_confirmation(state, question);
+            if (answer == GF_ANSWER_NO)
+                goto out;
+            break;
+        case GF_QUOTA_OPTION_TYPE_LIST:
+        case GF_QUOTA_OPTION_TYPE_LIST_OBJECTS:
+            if (wordcount != 4)
+                break;
+            ret = cli_cmd_quota_handle_list_all(words, options);
+            goto out;
+        default:
+            break;
+    }
+
+    ret = dict_get_str(options, "volname", &volname);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to get volume name");
+        goto out;
+    }
+
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
+
+    CLI_LOCAL_INIT(local, words, frame, options);
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_QUOTA];
 
-        if (proc->fn)
-                ret = proc->fn (frame, THIS, options);
+    if (proc->fn)
+        ret = proc->fn(frame, THIS, options);
 
 out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if (sent == 0 && parse_err == 0)
-                        cli_out ("Quota command failed. Please check the cli "
-                                 "logs for more details");
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if (sent == 0 && parse_err == 0)
+            cli_out(
+                "Quota command failed. Please check the cli "
+                "logs for more details");
+    }
+    if (options)
+        dict_unref(options);
+
+    /* Events for Quota */
+    if (ret == 0) {
+        switch (type) {
+            case GF_QUOTA_OPTION_TYPE_ENABLE:
+                gf_event(EVENT_QUOTA_ENABLE, "volume=%s", volname);
+                break;
+            case GF_QUOTA_OPTION_TYPE_DISABLE:
+                gf_event(EVENT_QUOTA_DISABLE, "volume=%s", volname);
+                break;
+            case GF_QUOTA_OPTION_TYPE_LIMIT_USAGE:
+                gf_event(EVENT_QUOTA_SET_USAGE_LIMIT,
+                         "volume=%s;"
+                         "path=%s;limit=%s",
+                         volname, words[4], words[5]);
+                break;
+            case GF_QUOTA_OPTION_TYPE_LIMIT_OBJECTS:
+                gf_event(EVENT_QUOTA_SET_OBJECTS_LIMIT,
+                         "volume=%s;"
+                         "path=%s;limit=%s",
+                         volname, words[4], words[5]);
+                break;
+            case GF_QUOTA_OPTION_TYPE_REMOVE:
+                gf_event(EVENT_QUOTA_REMOVE_USAGE_LIMIT,
+                         "volume=%s;"
+                         "path=%s",
+                         volname, words[4]);
+                break;
+            case GF_QUOTA_OPTION_TYPE_REMOVE_OBJECTS:
+                gf_event(EVENT_QUOTA_REMOVE_OBJECTS_LIMIT,
+                         "volume=%s;"
+                         "path=%s",
+                         volname, words[4]);
+                break;
+            case GF_QUOTA_OPTION_TYPE_ALERT_TIME:
+                gf_event(EVENT_QUOTA_ALERT_TIME, "volume=%s;time=%s", volname,
+                         words[4]);
+                break;
+            case GF_QUOTA_OPTION_TYPE_SOFT_TIMEOUT:
+                gf_event(EVENT_QUOTA_SOFT_TIMEOUT,
+                         "volume=%s;"
+                         "soft-timeout=%s",
+                         volname, words[4]);
+                break;
+            case GF_QUOTA_OPTION_TYPE_HARD_TIMEOUT:
+                gf_event(EVENT_QUOTA_HARD_TIMEOUT,
+                         "volume=%s;"
+                         "hard-timeout=%s",
+                         volname, words[4]);
+                break;
+            case GF_QUOTA_OPTION_TYPE_DEFAULT_SOFT_LIMIT:
+                gf_event(EVENT_QUOTA_DEFAULT_SOFT_LIMIT,
+                         "volume=%s;"
+                         "default-soft-limit=%s",
+                         volname, words[4]);
+                break;
         }
+    }
 
-        CLI_STACK_DESTROY (frame);
-        return ret;
+    CLI_STACK_DESTROY(frame);
+    return ret;
 }
 
 int
-cli_cmd_volume_remove_brick_cbk (struct cli_state *state,
-                                 struct cli_cmd_word *word, const char **words,
-                                 int wordcount)
+cli_cmd_volume_remove_brick_cbk(struct cli_state *state,
+                                struct cli_cmd_word *word, const char **words,
+                                int wordcount)
 {
-        int                     ret = -1;
-        rpc_clnt_procedure_t    *proc = NULL;
-        call_frame_t            *frame = NULL;
-        dict_t                  *options = NULL;
-        gf_answer_t             answer = GF_ANSWER_NO;
-        int                     sent = 0;
-        int                     parse_error = 0;
-        int                     need_question = 0;
-        cli_local_t             *local = NULL;
-
-        const char *question = "Removing brick(s) can result in data loss. "
-                               "Do you want to Continue?";
-
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *options = NULL;
+    gf_answer_t answer = GF_ANSWER_NO;
+    int brick_count = 0;
+    int sent = 0;
+    int parse_error = 0;
+    int need_question = 0;
+    cli_local_t *local = NULL;
+    char *volname = NULL;
+#if (USE_EVENTS)
+    eventtypes_t event = EVENT_LAST;
+    char *event_str = NULL;
+    int event_ret = -1;
+#endif
+    int32_t command = GF_OP_CMD_NONE;
+    char *question = NULL;
+
+    ret = cli_cmd_volume_remove_brick_parse(state, words, wordcount, &options,
+                                            &need_question, &brick_count,
+                                            &command);
+    if (ret) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
+
+    if (command == GF_OP_CMD_COMMIT_FORCE) {
+        question =
+            "Remove-brick force will not migrate files from the "
+            "removed bricks, so they will no longer be available"
+            " on the volume.\nDo you want to continue?";
+    } else if (command == GF_OP_CMD_START) {
+        question =
+            "It is recommended that remove-brick be run with"
+            " cluster.force-migration option disabled to prevent"
+            " possible data corruption. Doing so will ensure that"
+            " files that receive writes during migration will not"
+            " be migrated and will need to be manually copied"
+            " after the remove-brick commit operation. Please"
+            " check the value of the option and update accordingly."
+            " \nDo you want to continue with your current"
+            " cluster.force-migration settings?";
+    }
+
+    if (!brick_count) {
+        cli_err("No bricks specified");
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        ret = -1;
+        goto out;
+    }
+    ret = dict_get_str(options, "volname", &volname);
+    if (ret || !volname) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to fetch volname");
+        ret = -1;
+        goto out;
+    }
 
-        ret = cli_cmd_volume_remove_brick_parse (words, wordcount, &options,
-                                                 &need_question);
-        if (ret) {
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-                goto out;
-        }
+#if (USE_EVENTS)
+    event_ret = cli_event_remove_brick_str(options, &event_str, &event);
+#endif
 
-        if (!(state->mode & GLUSTER_MODE_SCRIPT) && need_question) {
-                /* we need to ask question only in case of 'commit or force' */
-                answer = cli_cmd_get_confirmation (state, question);
-                if (GF_ANSWER_NO == answer) {
-                        ret = 0;
-                        goto out;
-                }
+    if (!strcmp(volname, GLUSTER_SHARED_STORAGE)) {
+        question =
+            "Removing brick from the shared storage volume"
+            "(gluster_shared_storage), will affect features "
+            "like snapshot scheduler, geo-replication "
+            "and NFS-Ganesha. Do you still want to "
+            "continue?";
+        need_question = _gf_true;
+    }
+
+    if (!(state->mode & GLUSTER_MODE_SCRIPT) && need_question) {
+        answer = cli_cmd_get_confirmation(state, question);
+        if (GF_ANSWER_NO == answer) {
+            ret = 0;
+            goto out;
         }
+    }
 
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_REMOVE_BRICK];
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_REMOVE_BRICK];
 
-        CLI_LOCAL_INIT (local, words, frame, options);
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
 
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, options);
-        }
+    CLI_LOCAL_INIT(local, words, frame, options);
+
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, options);
+    }
 
 out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_out ("Volume remove-brick failed");
-        }
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("Volume remove-brick failed");
+    }
+#if (USE_EVENTS)
+    if (!ret && !event_ret)
+        gf_event(event, "%s", event_str);
+    if (event_str)
+        GF_FREE(event_str);
 
-        CLI_STACK_DESTROY (frame);
+#endif
 
-        return ret;
+    CLI_STACK_DESTROY(frame);
+    if (options)
+        dict_unref(options);
 
+    return ret;
 }
 
 int
-cli_cmd_volume_replace_brick_cbk (struct cli_state *state,
-                                  struct cli_cmd_word *word,
-                                  const char **words,
-                                  int wordcount)
+cli_cmd_volume_reset_brick_cbk(struct cli_state *state,
+                               struct cli_cmd_word *word, const char **words,
+                               int wordcount)
 {
-        int                     ret = -1;
-        rpc_clnt_procedure_t    *proc = NULL;
-        call_frame_t            *frame = NULL;
-        dict_t                  *options = NULL;
-        int                     sent = 0;
-        int                     parse_error = 0;
-        cli_local_t             *local = NULL;
-        int                     replace_op = 0;
-        char                    *q = "All replace-brick commands except "
-                                     "commit force are deprecated. "
-                                     "Do you want to continue?";
-        gf_answer_t             answer = GF_ANSWER_NO;
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *options = NULL;
+    int sent = 0;
+    int parse_error = 0;
+    cli_local_t *local = NULL;
 
 #ifdef GF_SOLARIS_HOST_OS
-        cli_out ("Command not supported on Solaris");
-        goto out;
+    cli_out("Command not supported on Solaris");
+    goto out;
 #endif
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_REPLACE_BRICK];
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_RESET_BRICK];
 
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
+    ret = cli_cmd_volume_reset_brick_parse(words, wordcount, &options);
 
-        ret = cli_cmd_volume_replace_brick_parse (words, wordcount, &options);
+    if (ret) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
 
+    if (state->mode & GLUSTER_MODE_WIGNORE_PARTITION) {
+        ret = dict_set_int32(options, "ignore-partition", _gf_true);
         if (ret) {
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-                goto out;
-        }
-
-        ret = dict_get_int32 (options, "operation", &replace_op);
-        if (replace_op != GF_REPLACE_OP_COMMIT_FORCE) {
-                answer = cli_cmd_get_confirmation (state, q);
-                if (GF_ANSWER_NO == answer) {
-                        ret = 0;
-                        goto out;
-                }
+            gf_log("cli", GF_LOG_ERROR,
+                   "Failed to set ignore-"
+                   "partition option");
+            goto out;
         }
+    }
 
-        if (state->mode & GLUSTER_MODE_WIGNORE) {
-                ret = dict_set_int32 (options, "force", _gf_true);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Failed to set force"
-                                "option");
-                        goto out;
-                }
-        }
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
 
-        CLI_LOCAL_INIT (local, words, frame, options);
+    CLI_LOCAL_INIT(local, words, frame, options);
 
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, options);
-        }
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, options);
+    }
 
 out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_out ("Volume replace-brick failed");
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("Volume reset-brick failed");
+    } else {
+        if (wordcount > 5) {
+            gf_event(EVENT_BRICK_RESET_COMMIT,
+                     "Volume=%s;source-brick=%s;"
+                     "destination-brick=%s",
+                     (char *)words[2], (char *)words[3], (char *)words[4]);
+        } else {
+            gf_event(EVENT_BRICK_RESET_START, "Volume=%s;source-brick=%s",
+                     (char *)words[2], (char *)words[3]);
         }
+    }
+    CLI_STACK_DESTROY(frame);
 
-        CLI_STACK_DESTROY (frame);
-
-        return ret;
+    return ret;
 }
 
-
 int
-cli_cmd_volume_set_transport_cbk (struct cli_state *state,
-                                  struct cli_cmd_word *word,
-                                  const char **words, int wordcount)
+cli_cmd_volume_replace_brick_cbk(struct cli_state *state,
+                                 struct cli_cmd_word *word, const char **words,
+                                 int wordcount)
 {
-        cli_cmd_broadcast_response (0);
-        return 0;
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *options = NULL;
+    int sent = 0;
+    int parse_error = 0;
+    cli_local_t *local = NULL;
+
+#ifdef GF_SOLARIS_HOST_OS
+    cli_out("Command not supported on Solaris");
+    goto out;
+#endif
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_REPLACE_BRICK];
+
+    ret = cli_cmd_volume_replace_brick_parse(words, wordcount, &options);
+
+    if (ret) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
+
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
+
+    CLI_LOCAL_INIT(local, words, frame, options);
+
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, options);
+    }
+
+out:
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("Volume replace-brick failed");
+    } else {
+        gf_event(EVENT_BRICK_REPLACE,
+                 "Volume=%s;source-brick=%s;destination-brick=%s",
+                 (char *)words[2], (char *)words[3], (char *)words[4]);
+    }
+    CLI_STACK_DESTROY(frame);
+
+    return ret;
 }
 
 int
-cli_cmd_volume_top_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                          const char **words, int wordcount)
+cli_cmd_volume_set_transport_cbk(struct cli_state *state,
+                                 struct cli_cmd_word *word, const char **words,
+                                 int wordcount)
 {
+    cli_cmd_broadcast_response(0);
+    return 0;
+}
 
-        int                     ret      = -1;
-        rpc_clnt_procedure_t    *proc    = NULL;
-        call_frame_t            *frame   = NULL;
-        dict_t                  *options = NULL;
-        int                     sent     = 0;
-        int                     parse_error = 0;
-        cli_local_t             *local = NULL;
-
-        ret = cli_cmd_volume_top_parse (words, wordcount, &options);
-
-        if (ret) {
-                parse_error = 1;
-                cli_usage_out (word->pattern);
-                goto out;
-        }
+int
+cli_cmd_volume_top_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                       const char **words, int wordcount)
+{
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *options = NULL;
+    int sent = 0;
+    int parse_error = 0;
+    cli_local_t *local = NULL;
+
+    ret = cli_cmd_volume_top_parse(words, wordcount, &options);
+
+    if (ret) {
+        parse_error = 1;
+        cli_usage_out(word->pattern);
+        goto out;
+    }
 
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_TOP_VOLUME];
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_TOP_VOLUME];
 
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        gf_log(THIS->name, GF_LOG_ERROR, "failed to create frame");
+        ret = -1;
+        goto out;
+    }
 
-        CLI_LOCAL_INIT (local, words, frame, options);
+    CLI_LOCAL_INIT(local, words, frame, options);
 
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, options);
-        }
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, options);
+    }
 
 out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_out ("Volume top failed");
-        }
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("Volume top failed");
+    }
 
-        CLI_STACK_DESTROY (frame);
-
-        return ret;
+    CLI_STACK_DESTROY(frame);
 
+    return ret;
 }
 
-
 int
-cli_cmd_log_rotate_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                        const char **words, int wordcount)
+cli_cmd_log_rotate_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                       const char **words, int wordcount)
 {
-        int                     ret = -1;
-        rpc_clnt_procedure_t    *proc = NULL;
-        call_frame_t            *frame = NULL;
-        dict_t                  *options = NULL;
-        int                     sent = 0;
-        int                     parse_error = 0;
-        cli_local_t             *local = NULL;
-
-        if (!((wordcount == 4) || (wordcount == 5))) {
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-                goto out;
-        }
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *options = NULL;
+    int sent = 0;
+    int parse_error = 0;
+    cli_local_t *local = NULL;
+
+    if (!((wordcount == 4) || (wordcount == 5))) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
 
-        if (!((strcmp ("rotate", words[2]) == 0) ||
-              (strcmp ("rotate", words[3]) == 0))) {
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-                goto out;
-        }
+    if (!(strcmp("rotate", words[3]) == 0)) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
 
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_LOG_ROTATE];
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_LOG_ROTATE];
 
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
+    ret = cli_cmd_log_rotate_parse(words, wordcount, &options);
+    if (ret)
+        goto out;
 
-        ret = cli_cmd_log_rotate_parse (words, wordcount, &options);
-        if (ret)
-                goto out;
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        gf_log(THIS->name, GF_LOG_ERROR, "failed to create frame");
+        ret = -1;
+        goto out;
+    }
 
-        CLI_LOCAL_INIT (local, words, frame, options);
+    CLI_LOCAL_INIT(local, words, frame, options);
 
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, options);
-        }
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, options);
+    }
 
 out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_out ("Volume log rotate failed");
-        }
-        CLI_STACK_DESTROY (frame);
-
-        return ret;
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("Volume log rotate failed");
+    }
+    CLI_STACK_DESTROY(frame);
+
+    return ret;
 }
 
 #if (SYNCDAEMON_COMPILE)
 static int
-cli_check_gsync_present ()
+cli_check_gsync_present()
 {
-        char                buff[PATH_MAX] = {0, };
-        runner_t            runner = {0,};
-        char                *ptr = NULL;
-        int                 ret = 0;
-
-        ret = setenv ("_GLUSTERD_CALLED_", "1", 1);
-        if (-1 == ret) {
-                gf_log ("", GF_LOG_WARNING, "setenv syscall failed, hence could"
-                        "not assert if geo-replication is installed");
-                goto out;
-        }
-
-        runinit (&runner);
-        runner_add_args (&runner, GSYNCD_PREFIX"/gsyncd", "--version", NULL);
-        runner_redir (&runner, STDOUT_FILENO, RUN_PIPE);
-        ret = runner_start (&runner);
-        if (ret == -1) {
-                gf_log ("", GF_LOG_INFO, "geo-replication not installed");
-                goto out;
-        }
+    char buff[PATH_MAX] = {
+        0,
+    };
+    runner_t runner = {
+        0,
+    };
+    char *ptr = NULL;
+    int ret = 0;
+
+    ret = setenv("_GLUSTERD_CALLED_", "1", 1);
+    if (-1 == ret) {
+        gf_log("", GF_LOG_WARNING,
+               "setenv syscall failed, hence could"
+               "not assert if geo-replication is installed");
+        goto out;
+    }
+
+    runinit(&runner);
+    runner_add_args(&runner, GSYNCD_PREFIX "/gsyncd", "--version", NULL);
+    runner_redir(&runner, STDOUT_FILENO, RUN_PIPE);
+    ret = runner_start(&runner);
+    if (ret == -1) {
+        gf_log("", GF_LOG_INFO, "geo-replication not installed");
+        goto out;
+    }
 
-        ptr = fgets(buff, sizeof(buff), runner_chio (&runner, STDOUT_FILENO));
-        if (ptr) {
-                if (!strstr (buff, "gsyncd")) {
-                        ret  = -1;
-                        goto out;
-                }
-        } else {
-                ret = -1;
-                goto out;
+    ptr = fgets(buff, sizeof(buff), runner_chio(&runner, STDOUT_FILENO));
+    if (ptr) {
+        if (!strstr(buff, "gsyncd")) {
+            ret = -1;
+            goto out;
         }
+    } else {
+        ret = -1;
+        goto out;
+    }
 
-        ret = runner_end (&runner);
+    ret = runner_end(&runner);
 
-        if (ret)
-                gf_log ("", GF_LOG_ERROR, "geo-replication not installed");
+    if (ret)
+        gf_log("", GF_LOG_ERROR, "geo-replication not installed");
 
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret ? -1 : 0;
-
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret ? -1 : 0;
 }
 
 void
-cli_cmd_check_gsync_exists_cbk (struct cli_cmd *this)
+cli_cmd_check_gsync_exists_cbk(struct cli_cmd *this)
 {
+    int ret = 0;
 
-        int             ret = 0;
-
-        ret = cli_check_gsync_present ();
-        if (ret)
-                this->disable = _gf_true;
-
+    ret = cli_check_gsync_present();
+    if (ret)
+        this->disable = _gf_true;
 }
 #endif
 
 int
-cli_cmd_volume_gsync_set_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                              const char **words, int wordcount)
+cli_cmd_volume_gsync_set_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                             const char **words, int wordcount)
 {
-        int                      ret     = 0;
-        int                      parse_err = 0;
-        dict_t                  *options = NULL;
-        rpc_clnt_procedure_t    *proc    = NULL;
-        call_frame_t            *frame   = NULL;
-        cli_local_t             *local   = NULL;
-
-        proc = &cli_rpc_prog->proctable [GLUSTER_CLI_GSYNC_SET];
-        if (proc == NULL) {
-                ret = -1;
-                goto out;
-        }
+    int ret = 0;
+    int parse_err = 0;
+    dict_t *options = NULL;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    cli_local_t *local = NULL;
+    char *errstr = NULL;
+#if (USE_EVENTS)
+    int ret1 = -1;
+    int cmd_type = -1;
+    int tmpi = 0;
+    char *tmp = NULL;
+    char *events_str = NULL;
+    int event_type = -1;
+#endif
 
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (frame == NULL) {
-                ret = -1;
-                goto out;
-        }
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_GSYNC_SET];
 
-        ret = cli_cmd_gsync_set_parse (words, wordcount, &options);
-        if (ret) {
-                cli_usage_out (word->pattern);
-                parse_err = 1;
-                goto out;
+    ret = cli_cmd_gsync_set_parse(state, words, wordcount, &options, &errstr);
+    if (ret) {
+        if (errstr) {
+            cli_err("%s", errstr);
+            GF_FREE(errstr);
+        } else {
+            cli_usage_out(word->pattern);
         }
+        parse_err = 1;
+        goto out;
+    }
 
-        CLI_LOCAL_INIT (local, words, frame, options);
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (frame == NULL) {
+        ret = -1;
+        goto out;
+    }
+
+    CLI_LOCAL_INIT(local, words, frame, options);
 
-        if (proc->fn)
-                ret = proc->fn (frame, THIS, options);
+    if (proc->fn)
+        ret = proc->fn(frame, THIS, options);
 
 out:
-        if (ret && parse_err == 0)
-                cli_out (GEOREP" command failed");
+    if (ret && parse_err == 0)
+        cli_out(GEOREP " command failed");
+
+#if (USE_EVENTS)
+    if (ret == 0) {
+        events_str = gf_strdup("");
+
+        /* Type of Geo-rep Action - Create, Start etc */
+        ret1 = dict_get_int32(options, "type", &cmd_type);
+        if (ret1)
+            cmd_type = -1;
+
+        /* Only capture Events for modification commands */
+        switch (cmd_type) {
+            case GF_GSYNC_OPTION_TYPE_CREATE:
+                event_type = EVENT_GEOREP_CREATE;
+                break;
+            case GF_GSYNC_OPTION_TYPE_START:
+                event_type = EVENT_GEOREP_START;
+                break;
+            case GF_GSYNC_OPTION_TYPE_STOP:
+                event_type = EVENT_GEOREP_STOP;
+                break;
+            case GF_GSYNC_OPTION_TYPE_PAUSE:
+                event_type = EVENT_GEOREP_PAUSE;
+                break;
+            case GF_GSYNC_OPTION_TYPE_RESUME:
+                event_type = EVENT_GEOREP_RESUME;
+                break;
+            case GF_GSYNC_OPTION_TYPE_DELETE:
+                event_type = EVENT_GEOREP_DELETE;
+                break;
+            case GF_GSYNC_OPTION_TYPE_CONFIG:
+                ret1 = dict_get_str(options, "subop", &tmp);
+                if (ret1)
+                    tmp = "";
+
+                /* For Config Set additionally capture key and value */
+                /* For Config Reset capture key */
+                if (strcmp(tmp, "set") == 0) {
+                    event_type = EVENT_GEOREP_CONFIG_SET;
+
+                    ret1 = dict_get_str(options, "op_name", &tmp);
+                    if (ret1)
+                        tmp = "";
+
+                    gf_asprintf_append(&events_str, "%soption=%s;", events_str,
+                                       tmp);
+
+                    ret1 = dict_get_str(options, "op_value", &tmp);
+                    if (ret1)
+                        tmp = "";
+
+                    gf_asprintf_append(&events_str, "%svalue=%s;", events_str,
+                                       tmp);
+                } else if (strcmp(tmp, "del") == 0) {
+                    event_type = EVENT_GEOREP_CONFIG_RESET;
+
+                    ret1 = dict_get_str(options, "op_name", &tmp);
+                    if (ret1)
+                        tmp = "";
+
+                    gf_asprintf_append(&events_str, "%soption=%s;", events_str,
+                                       tmp);
+                }
+                break;
+            default:
+                break;
+        }
 
-        CLI_STACK_DESTROY (frame);
+        if (event_type > -1) {
+            /* Capture all optional arguments used */
+            ret1 = dict_get_int32(options, "force", &tmpi);
+            if (ret1 == 0) {
+                gf_asprintf_append(&events_str, "%sforce=%d;", events_str,
+                                   tmpi);
+            }
+            ret1 = dict_get_int32(options, "push_pem", &tmpi);
+            if (ret1 == 0) {
+                gf_asprintf_append(&events_str, "%spush_pem=%d;", events_str,
+                                   tmpi);
+            }
+            ret1 = dict_get_int32(options, "no_verify", &tmpi);
+            if (ret1 == 0) {
+                gf_asprintf_append(&events_str, "%sno_verify=%d;", events_str,
+                                   tmpi);
+            }
+
+            ret1 = dict_get_int32(options, "ssh_port", &tmpi);
+            if (ret1 == 0) {
+                gf_asprintf_append(&events_str, "%sssh_port=%d;", events_str,
+                                   tmpi);
+            }
+
+            ret1 = dict_get_int32(options, "reset-sync-time", &tmpi);
+            if (ret1 == 0) {
+                gf_asprintf_append(&events_str, "%sreset_sync_time=%d;",
+                                   events_str, tmpi);
+            }
+            /* Capture Master and Slave Info */
+            ret1 = dict_get_str(options, "master", &tmp);
+            if (ret1)
+                tmp = "";
+            gf_asprintf_append(&events_str, "%smaster=%s;", events_str, tmp);
+
+            ret1 = dict_get_str(options, "slave", &tmp);
+            if (ret1)
+                tmp = "";
+            gf_asprintf_append(&events_str, "%sslave=%s", events_str, tmp);
+
+            gf_event(event_type, "%s", events_str);
+        }
+
+        /* Allocated by gf_strdup and gf_asprintf */
+        if (events_str)
+            GF_FREE(events_str);
+    }
+#endif
 
-        return ret;
+    CLI_STACK_DESTROY(frame);
+
+    return ret;
 }
 
 int
-cli_cmd_volume_status_cbk (struct cli_state *state,
-                           struct cli_cmd_word *word,
-                           const char **words, int wordcount)
+cli_cmd_volume_status_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                          const char **words, int wordcount)
 {
-        int                   ret         = -1;
-        rpc_clnt_procedure_t *proc        = NULL;
-        call_frame_t         *frame       = NULL;
-        dict_t               *dict        = NULL;
-        uint32_t              cmd         = 0;
-        cli_local_t          *local       = NULL;
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *dict = NULL;
+    uint32_t cmd = 0;
+    cli_local_t *local = NULL;
 
-        ret = cli_cmd_volume_status_parse (words, wordcount, &dict);
+    ret = cli_cmd_volume_status_parse(words, wordcount, &dict);
 
-        if (ret) {
-                cli_usage_out (word->pattern);
-                goto out;
-        }
+    if (ret) {
+        cli_usage_out(word->pattern);
+        goto out;
+    }
 
-        ret = dict_get_uint32 (dict, "cmd", &cmd);
-        if (ret)
-                goto out;
+    ret = dict_get_uint32(dict, "cmd", &cmd);
+    if (ret)
+        goto out;
 
-        if (!(cmd & GF_CLI_STATUS_ALL)) {
-                /* for one volume or brick */
-                proc = &cli_rpc_prog->proctable[GLUSTER_CLI_STATUS_VOLUME];
-        } else {
-                /* volume status all or all detail */
-                proc = &cli_rpc_prog->proctable[GLUSTER_CLI_STATUS_ALL];
-        }
+    if (!(cmd & GF_CLI_STATUS_ALL)) {
+        /* for one volume or brick */
+        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_STATUS_VOLUME];
+    } else {
+        /* volume status all or all detail */
+        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_STATUS_ALL];
+    }
 
-        if (!proc->fn)
-                goto out;
+    if (!proc->fn) {
+        ret = -1;
+        goto out;
+    }
 
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        gf_log(THIS->name, GF_LOG_ERROR, "failed to create frame");
+        ret = -1;
+        goto out;
+    }
 
-        CLI_LOCAL_INIT (local, words, frame, dict);
+    CLI_LOCAL_INIT(local, words, frame, dict);
 
-        ret = proc->fn (frame, THIS, dict);
+    ret = proc->fn(frame, THIS, dict);
 
 out:
-        CLI_STACK_DESTROY (frame);
+    CLI_STACK_DESTROY(frame);
 
-        return ret;
+    return ret;
 }
 
-
 int
-cli_get_detail_status (dict_t *dict, int i, cli_volume_status_t *status)
+cli_get_detail_status(dict_t *dict, int i, cli_volume_status_t *status)
 {
-        uint64_t                   free            = 0;
-        uint64_t                   total           = 0;
-        char                       key[1024]       = {0};
-        int                        ret             = 0;
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "brick%d.free", i);
-        ret = dict_get_uint64 (dict, key, &free);
+    uint64_t free = 0;
+    uint64_t total = 0;
+    char key[1024] = {0};
+    int ret = 0;
 
-        status->free = gf_uint64_2human_readable (free);
-        if (!status->free)
-                goto out;
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "brick%d.total", i);
-        ret = dict_get_uint64 (dict, key, &total);
-
-        status->total = gf_uint64_2human_readable (total);
-        if (!status->total)
-                goto out;
-
-#ifdef GF_LINUX_HOST_OS
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "brick%d.device", i);
-        ret = dict_get_str (dict, key, &(status->device));
-        if (ret)
-                status->device = NULL;
-#endif
+    snprintf(key, sizeof(key), "brick%d.free", i);
+    ret = dict_get_uint64(dict, key, &free);
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "brick%d.block_size", i);
-        ret = dict_get_uint64 (dict, key, &(status->block_size));
-        if (ret) {
-                ret = 0;
-                status->block_size = 0;
-        }
+    status->free = gf_uint64_2human_readable(free);
+    if (!status->free)
+        goto out;
 
-#ifdef GF_LINUX_HOST_OS
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "brick%d.mnt_options", i);
-        ret = dict_get_str (dict, key, &(status->mount_options));
-        if (ret)
-                status->mount_options = NULL;
+    snprintf(key, sizeof(key), "brick%d.total", i);
+    ret = dict_get_uint64(dict, key, &total);
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "brick%d.fs_name", i);
-        ret = dict_get_str (dict, key, &(status->fs_name));
-        if (ret) {
-                ret = 0;
-                status->fs_name = NULL;
-        }
+    status->total = gf_uint64_2human_readable(total);
+    if (!status->total)
+        goto out;
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "brick%d.inode_size", i);
-        ret = dict_get_str (dict, key, &(status->inode_size));
-        if (ret)
-                status->inode_size = NULL;
-#endif /* GF_LINUX_HOST_OS */
+    snprintf(key, sizeof(key), "brick%d.device", i);
+    ret = dict_get_str(dict, key, &(status->device));
+    if (ret)
+        status->device = NULL;
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "brick%d.total_inodes", i);
-        ret = dict_get_uint64 (dict, key,
-                        &(status->total_inodes));
-        if (ret)
-                status->total_inodes = 0;
+    snprintf(key, sizeof(key), "brick%d.block_size", i);
+    ret = dict_get_uint64(dict, key, &(status->block_size));
+    if (ret) {
+        ret = 0;
+        status->block_size = 0;
+    }
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "brick%d.free_inodes", i);
-        ret = dict_get_uint64 (dict, key, &(status->free_inodes));
-        if (ret) {
-                ret = 0;
-                status->free_inodes = 0;
-        }
+    snprintf(key, sizeof(key), "brick%d.mnt_options", i);
+    ret = dict_get_str(dict, key, &(status->mount_options));
+    if (ret)
+        status->mount_options = NULL;
 
+    snprintf(key, sizeof(key), "brick%d.fs_name", i);
+    ret = dict_get_str(dict, key, &(status->fs_name));
+    if (ret) {
+        ret = 0;
+        status->fs_name = NULL;
+    }
+
+    snprintf(key, sizeof(key), "brick%d.inode_size", i);
+    ret = dict_get_str(dict, key, &(status->inode_size));
+    if (ret)
+        status->inode_size = NULL;
+
+    snprintf(key, sizeof(key), "brick%d.total_inodes", i);
+    ret = dict_get_uint64(dict, key, &(status->total_inodes));
+    if (ret)
+        status->total_inodes = 0;
+
+    snprintf(key, sizeof(key), "brick%d.free_inodes", i);
+    ret = dict_get_uint64(dict, key, &(status->free_inodes));
+    if (ret) {
+        ret = 0;
+        status->free_inodes = 0;
+    }
 
- out:
-        return ret;
+out:
+    return ret;
 }
 
 void
-cli_print_detailed_status (cli_volume_status_t *status)
+cli_print_detailed_status(cli_volume_status_t *status)
 {
-        cli_out ("%-20s : %-20s", "Brick", status->brick);
-        if (status->online)
-                cli_out ("%-20s : %-20d", "Port", status->port);
-        else
-                cli_out ("%-20s : %-20s", "Port", "N/A");
-        cli_out ("%-20s : %-20c", "Online", (status->online) ? 'Y' : 'N');
-        cli_out ("%-20s : %-20s", "Pid", status->pid_str);
-
-#ifdef GF_LINUX_HOST_OS
-        if (status->fs_name)
-                cli_out ("%-20s : %-20s", "File System", status->fs_name);
-        else
-                cli_out ("%-20s : %-20s", "File System", "N/A");
-
-        if (status->device)
-                cli_out ("%-20s : %-20s", "Device", status->device);
-        else
-                cli_out ("%-20s : %-20s", "Device", "N/A");
-
-        if (status->mount_options) {
-                cli_out ("%-20s : %-20s", "Mount Options",
-                         status->mount_options);
-        } else {
-                cli_out ("%-20s : %-20s", "Mount Options", "N/A");
-        }
+    cli_out("%-20s : %-20s", "Brick", status->brick);
+
+    if (status->online) {
+        cli_out("%-20s : %-20d", "TCP Port", status->port);
+        cli_out("%-20s : %-20d", "RDMA Port", status->rdma_port);
+    } else {
+        cli_out("%-20s : %-20s", "TCP Port", "N/A");
+        cli_out("%-20s : %-20s", "RDMA Port", "N/A");
+    }
+
+    cli_out("%-20s : %-20c", "Online", (status->online) ? 'Y' : 'N');
+    cli_out("%-20s : %-20s", "Pid", status->pid_str);
+
+    if (status->fs_name)
+        cli_out("%-20s : %-20s", "File System", status->fs_name);
+    else
+        cli_out("%-20s : %-20s", "File System", "N/A");
+
+    if (status->device)
+        cli_out("%-20s : %-20s", "Device", status->device);
+    else
+        cli_out("%-20s : %-20s", "Device", "N/A");
+
+    if (status->mount_options) {
+        cli_out("%-20s : %-20s", "Mount Options", status->mount_options);
+    } else {
+        cli_out("%-20s : %-20s", "Mount Options", "N/A");
+    }
+
+    if (status->inode_size) {
+        cli_out("%-20s : %-20s", "Inode Size", status->inode_size);
+    } else {
+        cli_out("%-20s : %-20s", "Inode Size", "N/A");
+    }
+    if (status->free)
+        cli_out("%-20s : %-20s", "Disk Space Free", status->free);
+    else
+        cli_out("%-20s : %-20s", "Disk Space Free", "N/A");
+
+    if (status->total)
+        cli_out("%-20s : %-20s", "Total Disk Space", status->total);
+    else
+        cli_out("%-20s : %-20s", "Total Disk Space", "N/A");
+
+    if (status->total_inodes) {
+        cli_out("%-20s : %-20" GF_PRI_INODE, "Inode Count",
+                status->total_inodes);
+    } else {
+        cli_out("%-20s : %-20s", "Inode Count", "N/A");
+    }
+
+    if (status->free_inodes) {
+        cli_out("%-20s : %-20" GF_PRI_INODE, "Free Inodes",
+                status->free_inodes);
+    } else {
+        cli_out("%-20s : %-20s", "Free Inodes", "N/A");
+    }
+}
 
-        if (status->inode_size) {
-                cli_out ("%-20s : %-20s", "Inode Size",
-                         status->inode_size);
+int
+cli_print_brick_status(cli_volume_status_t *status)
+{
+    int fieldlen = CLI_VOL_STATUS_BRICK_LEN;
+    int bricklen = 0;
+    char *p = NULL;
+    int num_spaces = 0;
+
+    p = status->brick;
+    bricklen = strlen(p);
+    while (bricklen > 0) {
+        if (bricklen > fieldlen) {
+            cli_out("%.*s", fieldlen, p);
+            p += fieldlen;
+            bricklen -= fieldlen;
         } else {
-                cli_out ("%-20s : %-20s", "Inode Size", "N/A");
-        }
-#endif
-        if (status->free)
-                cli_out ("%-20s : %-20s", "Disk Space Free", status->free);
-        else
-                cli_out ("%-20s : %-20s", "Disk Space Free", "N/A");
-
-        if (status->total)
-                cli_out ("%-20s : %-20s", "Total Disk Space", status->total);
-        else
-                cli_out ("%-20s : %-20s", "Total Disk Space", "N/A");
+            num_spaces = (fieldlen - bricklen) + 1;
+            printf("%s", p);
+            while (num_spaces-- != 0)
+                printf(" ");
+            if (status->port || status->rdma_port) {
+                if (status->online)
+                    cli_out("%-10d%-11d%-8c%-5s", status->port,
+                            status->rdma_port, status->online ? 'Y' : 'N',
+                            status->pid_str);
+                else
+                    cli_out("%-10s%-11s%-8c%-5s", "N/A", "N/A",
+                            status->online ? 'Y' : 'N', status->pid_str);
+            } else
+                cli_out("%-10s%-11s%-8c%-5s", "N/A", "N/A",
+                        status->online ? 'Y' : 'N', status->pid_str);
+            bricklen = 0;
+        }
+    }
+
+    return 0;
+}
 
+#define NEEDS_GLFS_HEAL(op)                                                    \
+    ((op == GF_SHD_OP_SBRAIN_HEAL_FROM_BIGGER_FILE) ||                         \
+     (op == GF_SHD_OP_SBRAIN_HEAL_FROM_LATEST_MTIME) ||                        \
+     (op == GF_SHD_OP_SBRAIN_HEAL_FROM_BRICK) ||                               \
+     (op == GF_SHD_OP_INDEX_SUMMARY) || (op == GF_SHD_OP_SPLIT_BRAIN_FILES) || \
+     (op == GF_SHD_OP_GRANULAR_ENTRY_HEAL_ENABLE) ||                           \
+     (op == GF_SHD_OP_HEAL_SUMMARY))
 
-        if (status->total_inodes) {
-                cli_out ("%-20s : %-20"GF_PRI_INODE, "Inode Count",
-                         status->total_inodes);
-        } else {
-                cli_out ("%-20s : %-20s", "Inode Count", "N/A");
-        }
+int
+cli_launch_glfs_heal(int heal_op, dict_t *options)
+{
+    char buff[PATH_MAX] = {0};
+    runner_t runner = {0};
+    char *filename = NULL;
+    char *hostname = NULL;
+    char *path = NULL;
+    char *volname = NULL;
+    char *out = NULL;
+    int ret = 0;
+
+    runinit(&runner);
+    ret = dict_get_str(options, "volname", &volname);
+    runner_add_args(&runner, GLFSHEAL_PREFIX "/glfsheal", volname, NULL);
+    runner_redir(&runner, STDOUT_FILENO, RUN_PIPE);
+
+    switch (heal_op) {
+        case GF_SHD_OP_INDEX_SUMMARY:
+            if (global_state->mode & GLUSTER_MODE_XML) {
+                runner_add_args(&runner, "--xml", NULL);
+            }
+            break;
+        case GF_SHD_OP_SBRAIN_HEAL_FROM_BIGGER_FILE:
+            ret = dict_get_str(options, "file", &filename);
+            runner_add_args(&runner, "bigger-file", filename, NULL);
+            break;
+        case GF_SHD_OP_SBRAIN_HEAL_FROM_LATEST_MTIME:
+            ret = dict_get_str(options, "file", &filename);
+            runner_add_args(&runner, "latest-mtime", filename, NULL);
+            break;
+        case GF_SHD_OP_SBRAIN_HEAL_FROM_BRICK:
+            ret = dict_get_str(options, "heal-source-hostname", &hostname);
+            ret = dict_get_str(options, "heal-source-brickpath", &path);
+            runner_add_args(&runner, "source-brick", NULL);
+            runner_argprintf(&runner, "%s:%s", hostname, path);
+            if (dict_get_str(options, "file", &filename) == 0)
+                runner_argprintf(&runner, "%s", filename);
+            break;
+        case GF_SHD_OP_SPLIT_BRAIN_FILES:
+            runner_add_args(&runner, "split-brain-info", NULL);
+            if (global_state->mode & GLUSTER_MODE_XML) {
+                runner_add_args(&runner, "--xml", NULL);
+            }
+            break;
+        case GF_SHD_OP_GRANULAR_ENTRY_HEAL_ENABLE:
+        case GF_SHD_OP_GRANULAR_ENTRY_HEAL_DISABLE:
+            runner_add_args(&runner, "granular-entry-heal-op", NULL);
+            break;
+        case GF_SHD_OP_HEAL_SUMMARY:
+            runner_add_args(&runner, "info-summary", NULL);
+            if (global_state->mode & GLUSTER_MODE_XML) {
+                runner_add_args(&runner, "--xml", NULL);
+            }
+            break;
+        default:
+            ret = -1;
+            goto out;
+    }
+    if (global_state->mode & GLUSTER_MODE_GLFSHEAL_NOLOG)
+        runner_add_args(&runner, "--nolog", NULL);
+    ret = runner_start(&runner);
+    if (ret == -1)
+        goto out;
+    while ((
+        out = fgets(buff, sizeof(buff), runner_chio(&runner, STDOUT_FILENO)))) {
+        printf("%s", out);
+    }
+    ret = runner_end(&runner);
 
-        if (status->free_inodes) {
-                cli_out ("%-20s : %-20"GF_PRI_INODE, "Free Inodes",
-                         status->free_inodes);
-        } else {
-                cli_out ("%-20s : %-20s", "Free Inodes", "N/A");
-        }
+out:
+    return ret;
 }
 
 int
-cli_print_brick_status (cli_volume_status_t *status)
+cli_cmd_volume_heal_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                        const char **words, int wordcount)
 {
-        int  fieldlen = CLI_VOL_STATUS_BRICK_LEN;
-        int  bricklen = 0;
-        char *p = NULL;
-        int  num_tabs = 0;
-
-        p = status->brick;
-        bricklen = strlen (p);
-        while (bricklen > 0) {
-                if (bricklen > fieldlen) {
-                        cli_out ("%.*s", fieldlen, p);
-                        p += fieldlen;
-                        bricklen -= fieldlen;
-                } else {
-                        num_tabs = (fieldlen - bricklen) / CLI_TAB_LENGTH + 1;
-                        printf ("%s", p);
-                        while (num_tabs-- != 0)
-                                printf ("\t");
-                        if (status->port) {
-                                if (status->online)
-                                        cli_out ("%d\t%c\t%s",
-                                                 status->port,
-                                                 status->online?'Y':'N',
-                                                 status->pid_str);
-                                else
-                                        cli_out ("%s\t%c\t%s",
-                                                 "N/A",
-                                                 status->online?'Y':'N',
-                                                 status->pid_str);
-                        }
-                        else
-                                cli_out ("%s\t%c\t%s",
-                                         "N/A", status->online?'Y':'N',
-                                         status->pid_str);
-                        bricklen = 0;
-                }
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    int sent = 0;
+    int parse_error = 0;
+    dict_t *options = NULL;
+    xlator_t *this = NULL;
+    cli_local_t *local = NULL;
+    int heal_op = 0;
+
+    this = THIS;
+
+    if (wordcount < 3) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
+
+    ret = cli_cmd_volume_heal_options_parse(words, wordcount, &options);
+    if (ret) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
+    ret = dict_get_int32(options, "heal-op", &heal_op);
+    if (ret < 0)
+        goto out;
+    if (NEEDS_GLFS_HEAL(heal_op)) {
+        ret = cli_launch_glfs_heal(heal_op, options);
+        if (ret < 0)
+            goto out;
+        if (heal_op != GF_SHD_OP_GRANULAR_ENTRY_HEAL_ENABLE)
+            goto out;
+    }
+
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_HEAL_VOLUME];
+
+    frame = create_frame(this, this->ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
+
+    CLI_LOCAL_INIT(local, words, frame, options);
+
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, options);
+    }
+out:
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0) &&
+            !(global_state->mode & GLUSTER_MODE_XML)) {
+            cli_out("Volume heal failed.");
         }
+    }
+
+    if (options)
+        dict_unref(options);
 
-        return 0;
+    CLI_STACK_DESTROY(frame);
+
+    return ret;
 }
 
 int
-cli_cmd_volume_heal_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                          const char **words, int wordcount)
+cli_cmd_volume_statedump_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                             const char **words, int wordcount)
 {
-        int                     ret = -1;
-        rpc_clnt_procedure_t    *proc = NULL;
-        call_frame_t            *frame = NULL;
-        int                     sent = 0;
-        int                     parse_error = 0;
-        dict_t                  *options = NULL;
-        xlator_t                *this = NULL;
-        cli_local_t             *local = NULL;
-        int                     heal_op = 0;
-        runner_t                runner = {0};
-        char                    buff[PATH_MAX] = {0};
-        char                    *out = NULL;
-
-        this = THIS;
-        frame = create_frame (this, this->ctx->pool);
-        if (!frame)
-                goto out;
-
-        if (wordcount < 3) {
-               cli_usage_out (word->pattern);
-               parse_error = 1;
-               goto out;
-        }
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *options = NULL;
+    int sent = 0;
+    int parse_error = 0;
+    cli_local_t *local = NULL;
+
+    if (wordcount < 3) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
 
-        ret = cli_cmd_volume_heal_options_parse (words, wordcount, &options);
+    if (wordcount >= 3) {
+        ret = cli_cmd_volume_statedump_options_parse(words, wordcount,
+                                                     &options);
         if (ret) {
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-                goto out;
+            parse_error = 1;
+            gf_log("cli", GF_LOG_ERROR,
+                   "Error parsing "
+                   "statedump options");
+            cli_out("Error parsing options");
+            cli_usage_out(word->pattern);
         }
-        ret = dict_get_int32 (options, "heal-op", &heal_op);
-        if (ret < 0)
-                goto out;
+    }
 
-        if (heal_op == GF_AFR_OP_INDEX_SUMMARY) {
-                runinit (&runner);
-                runner_add_args (&runner, SBIN_DIR"/glfsheal", words[2], NULL);
-                runner_redir (&runner, STDOUT_FILENO, RUN_PIPE);
-                ret = runner_start (&runner);
-                if (ret == -1)
-                        goto out;
-                while ((out = fgets(buff, sizeof(buff),
-                                   runner_chio (&runner, STDOUT_FILENO)))) {
-                        printf ("%s", out);
-                }
+    ret = dict_set_str(options, "volname", (char *)words[2]);
+    if (ret)
+        goto out;
 
-                ret = runner_end (&runner);
-                ret = WEXITSTATUS (ret);
-        }
-        else {
-                proc = &cli_rpc_prog->proctable[GLUSTER_CLI_HEAL_VOLUME];
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_STATEDUMP_VOLUME];
 
-                CLI_LOCAL_INIT (local, words, frame, options);
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
 
-                if (proc->fn) {
-                        ret = proc->fn (frame, THIS, options);
-                }
-        }
+    CLI_LOCAL_INIT(local, words, frame, options);
+
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, options);
+    }
 
 out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_out ("Volume heal failed");
-        }
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("Volume statedump failed");
+    }
 
-        CLI_STACK_DESTROY (frame);
+    CLI_STACK_DESTROY(frame);
 
-        return ret;
+    return ret;
 }
 
 int
-cli_cmd_volume_statedump_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                              const char **words, int wordcount)
+cli_cmd_volume_list_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                        const char **words, int wordcount)
 {
-        int                             ret = -1;
-        rpc_clnt_procedure_t            *proc = NULL;
-        call_frame_t                    *frame = NULL;
-        dict_t                          *options = NULL;
-        int                             sent = 0;
-        int                             parse_error = 0;
-        cli_local_t                     *local = NULL;
-
-        frame = create_frame (THIS, THIS->ctx->pool);
+    int ret = -1;
+    call_frame_t *frame = NULL;
+    rpc_clnt_procedure_t *proc = NULL;
+    int sent = 0;
+
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_LIST_VOLUME];
+    if (proc->fn) {
+        frame = create_frame(THIS, THIS->ctx->pool);
         if (!frame)
-                goto out;
+            goto out;
+        ret = proc->fn(frame, THIS, NULL);
+    }
 
-        if (wordcount < 3) {
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-                goto out;
-        }
+out:
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if (sent == 0)
+            cli_out("Volume list failed");
+    }
 
-        if (wordcount >= 3) {
-               ret = cli_cmd_volume_statedump_options_parse (words, wordcount,
-                                                              &options);
-               if (ret) {
-                       parse_error = 1;
-                       gf_log ("cli", GF_LOG_ERROR, "Error parsing "
-                               "statedump options");
-                       cli_out ("Error parsing options");
-                       cli_usage_out (word->pattern);
-               }
-        }
+    CLI_STACK_DESTROY(frame);
 
-        ret = dict_set_str (options, "volname", (char *)words[2]);
-        if (ret)
-                goto out;
+    return ret;
+}
 
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_STATEDUMP_VOLUME];
+int
+cli_cmd_volume_clearlocks_cbk(struct cli_state *state,
+                              struct cli_cmd_word *word, const char **words,
+                              int wordcount)
+{
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *options = NULL;
+    int sent = 0;
+    int parse_error = 0;
+    cli_local_t *local = NULL;
+
+    if (wordcount < 7 || wordcount > 8) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
+
+    ret = cli_cmd_volume_clrlks_opts_parse(words, wordcount, &options);
+    if (ret) {
+        parse_error = 1;
+        gf_log("cli", GF_LOG_ERROR,
+               "Error parsing "
+               "clear-locks options");
+        cli_out("Error parsing options");
+        cli_usage_out(word->pattern);
+    }
+
+    ret = dict_set_str(options, "volname", (char *)words[2]);
+    if (ret)
+        goto out;
 
-        CLI_LOCAL_INIT (local, words, frame, options);
+    ret = dict_set_str(options, "path", (char *)words[3]);
+    if (ret)
+        goto out;
 
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, options);
-        }
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_CLRLOCKS_VOLUME];
+
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
+
+    CLI_LOCAL_INIT(local, words, frame, options);
+
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, options);
+    }
 
 out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error = 0))
-                        cli_out ("Volume statedump failed");
-        }
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("Volume clear-locks failed");
+    }
 
-        CLI_STACK_DESTROY (frame);
+    CLI_STACK_DESTROY(frame);
 
-        return ret;
+    return ret;
 }
 
 int
-cli_cmd_volume_list_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                         const char **words, int wordcount)
+cli_cmd_volume_barrier_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                           const char **words, int wordcount)
 {
-        int                     ret = -1;
-        call_frame_t            *frame = NULL;
-        rpc_clnt_procedure_t    *proc = NULL;
-        int                     sent = 0;
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *options = NULL;
+    int sent = 0;
+    int parse_error = 0;
+    cli_local_t *local = NULL;
+
+    if (wordcount != 4) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
 
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
+    options = dict_new();
+    if (!options) {
+        ret = -1;
+        goto out;
+    }
+    ret = dict_set_str(options, "volname", (char *)words[2]);
+    if (ret)
+        goto out;
 
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_LIST_VOLUME];
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, NULL);
-        }
+    ret = dict_set_str(options, "barrier", (char *)words[3]);
+    if (ret)
+        goto out;
 
-out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if (sent == 0)
-                        cli_out ("Volume list failed");
-        }
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_BARRIER_VOLUME];
+
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
+
+    CLI_LOCAL_INIT(local, words, frame, options);
 
-        CLI_STACK_DESTROY (frame);
+    if (proc->fn)
+        ret = proc->fn(frame, THIS, options);
 
-        return ret;
+out:
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_err("Volume barrier failed");
+    }
+    CLI_STACK_DESTROY(frame);
+
+    return ret;
 }
 
 int
-cli_cmd_volume_clearlocks_cbk (struct cli_state *state,
-                               struct cli_cmd_word *word,
-                               const char **words, int wordcount)
+cli_cmd_volume_getopt_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                          const char **words, int wordcount)
 {
-        int                             ret = -1;
-        rpc_clnt_procedure_t            *proc = NULL;
-        call_frame_t                    *frame = NULL;
-        dict_t                          *options = NULL;
-        int                             sent = 0;
-        int                             parse_error = 0;
-        cli_local_t                     *local = NULL;
-
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *options = NULL;
+    int sent = 0;
+    int parse_err = 0;
+    cli_local_t *local = NULL;
+
+    if (wordcount != 4) {
+        cli_usage_out(word->pattern);
+        parse_err = 1;
+        goto out;
+    }
 
-        if (wordcount < 7 || wordcount > 8) {
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-                goto out;
-        }
+    options = dict_new();
+    if (!options)
+        goto out;
 
-       ret = cli_cmd_volume_clrlks_opts_parse (words, wordcount, &options);
-       if (ret) {
-               parse_error = 1;
-               gf_log ("cli", GF_LOG_ERROR, "Error parsing "
-                       "clear-locks options");
-               cli_out ("Error parsing options");
-               cli_usage_out (word->pattern);
-       }
+    ret = dict_set_str(options, "volname", (char *)words[2]);
+    if (ret)
+        goto out;
 
-        ret = dict_set_str (options, "volname", (char *)words[2]);
-        if (ret)
-                goto out;
+    ret = dict_set_str(options, "key", (char *)words[3]);
+    if (ret)
+        goto out;
 
-        ret = dict_set_str (options, "path", (char *)words[3]);
-        if (ret)
-                goto out;
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_GET_VOL_OPT];
 
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_CLRLOCKS_VOLUME];
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
 
-        CLI_LOCAL_INIT (local, words, frame, options);
+    CLI_LOCAL_INIT(local, words, frame, options);
 
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, options);
-        }
+    if (proc->fn)
+        ret = proc->fn(frame, THIS, options);
 
 out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error = 0))
-                        cli_out ("Volume clear-locks failed");
-        }
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_err == 0))
+            cli_err("Volume get option failed");
+    }
+    CLI_STACK_DESTROY(frame);
+    return ret;
+}
 
-        CLI_STACK_DESTROY (frame);
+/* This is a bit of a hack to display the help. The current bitrot cmd
+ * format does not work well when registering the cmds.
+ * Ideally the should have been of the form
+ * gluster volume bitrot <subcommand> <volumename> ...
+ */
 
-        return ret;
-}
+struct cli_cmd bitrot_cmds[] = {
 
-int
-cli_cmd_volume_barrier_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                            const char **words, int wordcount)
-{
-        int ret = -1;
-        rpc_clnt_procedure_t *proc = NULL;
-        call_frame_t *frame = NULL;
-        dict_t *options = NULL;
-        int sent = 0;
-        int parse_error = 0;
-        cli_local_t *local = NULL;
-
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
+    {"volume bitrot help", cli_cmd_bitrot_help_cbk,
+     "display help for volume bitrot commands"},
 
-        if (wordcount != 4) {
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-                goto out;
-        }
+    {"volume bitrot <VOLNAME> {enable|disable}", NULL, /*cli_cmd_bitrot_cbk,*/
+     "Enable/disable bitrot for volume <VOLNAME>"},
 
-        options = dict_new();
-        if (!options) {
-                ret = -1;
-                goto out;
-        }
-        ret = dict_set_str(options, "volname", (char *)words[2]);
-        if (ret)
-                goto out;
+    {"volume bitrot <VOLNAME> signing-time <time-in-secs>",
+     NULL, /*cli_cmd_bitrot_cbk,*/
+     "Waiting time for an object after last fd is closed to start signing "
+     "process"},
 
-        ret = dict_set_str (options, "barrier", (char *)words[3]);
-        if (ret)
-                goto out;
+    {"volume bitrot <VOLNAME> signer-threads <count>",
+     NULL, /*cli_cmd_bitrot_cbk,*/
+     "Number of signing process threads. Usually set to number of available "
+     "cores"},
 
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_BARRIER_VOLUME];
+    {"volume bitrot <VOLNAME> scrub-throttle {lazy|normal|aggressive}",
+     NULL, /*cli_cmd_bitrot_cbk,*/
+     "Set the speed of the scrubber for volume <VOLNAME>"},
 
-        CLI_LOCAL_INIT (local, words, frame, options);
+    {"volume bitrot <VOLNAME> scrub-frequency {hourly|daily|weekly|biweekly"
+     "|monthly}",
+     NULL, /*cli_cmd_bitrot_cbk,*/
+     "Set the frequency of the scrubber for volume <VOLNAME>"},
 
-        if (proc->fn)
-                ret = proc->fn (frame, THIS, options);
+    {"volume bitrot <VOLNAME> scrub {pause|resume|status|ondemand}",
+     NULL, /*cli_cmd_bitrot_cbk,*/
+     "Pause/resume the scrubber for <VOLNAME>. Status displays the status of "
+     "the scrubber. ondemand starts the scrubber immediately."},
 
-out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_err ("Volume barrier failed");
-        }
-        CLI_STACK_DESTROY (frame);
-        if (options)
-                dict_unref (options);
+    {"volume bitrot <VOLNAME> {enable|disable}\n"
+     "volume bitrot <VOLNAME> signing-time <time-in-secs>\n"
+     "volume bitrot <VOLNAME> signer-threads <count>\n"
+     "volume bitrot <volname> scrub-throttle {lazy|normal|aggressive}\n"
+     "volume bitrot <volname> scrub-frequency {hourly|daily|weekly|biweekly"
+     "|monthly}\n"
+     "volume bitrot <volname> scrub {pause|resume|status|ondemand}",
+     cli_cmd_bitrot_cbk, NULL},
+
+    {NULL, NULL, NULL}};
+
+struct cli_cmd quota_cmds[] = {
+
+    /* Quota commands */
+    {"volume quota help", cli_cmd_quota_help_cbk,
+     "display help for volume quota commands"},
+
+    {"volume quota <VOLNAME> {enable|disable|list [<path> ...]| "
+     "list-objects [<path> ...] | remove <path>| remove-objects <path> | "
+     "default-soft-limit <percent>}",
+     cli_cmd_quota_cbk, "Enable/disable and configure quota for <VOLNAME>"},
+
+    {"volume quota <VOLNAME> {limit-usage <path> <size> [<percent>]}",
+     cli_cmd_quota_cbk, "Set maximum size for <path> for <VOLNAME>"},
+
+    {"volume quota <VOLNAME> {limit-objects <path> <number> [<percent>]}",
+     cli_cmd_quota_cbk,
+     "Set the maximum number of entries allowed in <path> for <VOLNAME>"},
+
+    {"volume quota <VOLNAME> {alert-time|soft-timeout|hard-timeout} {<time>}",
+     cli_cmd_quota_cbk, "Set quota timeout for <VOLNAME>"},
+
+    {"volume inode-quota <VOLNAME> enable", cli_cmd_quota_cbk,
+     "Enable/disable inode-quota for <VOLNAME>"},
+
+    {"volume quota <VOLNAME> {enable|disable|list [<path> ...]| "
+     "list-objects [<path> ...] | remove <path>| remove-objects <path> | "
+     "default-soft-limit <percent>}\n"
+     "volume quota <VOLNAME> {limit-usage <path> <size> [<percent>]}\n"
+     "volume quota <VOLNAME> {limit-objects <path> <number> [<percent>]}\n"
+     "volume quota <VOLNAME> {alert-time|soft-timeout|hard-timeout} {<time>}",
+     cli_cmd_quota_cbk, NULL},
+
+    {NULL, NULL, NULL}};
 
-        return ret;
-}
 struct cli_cmd volume_cmds[] = {
-        { "volume info [all|<VOLNAME>]",
-          cli_cmd_volume_info_cbk,
-          "list information of all volumes"},
-
-        { "volume create <NEW-VOLNAME> [stripe <COUNT>] [replica <COUNT>] "
-          "[disperse [<COUNT>]] [redundancy <COUNT>] "
-          "[transport <tcp|rdma|tcp,rdma>] <NEW-BRICK>"
-#ifdef HAVE_BD_XLATOR
-          "?<vg_name>"
-#endif
-          "... [force]",
+    {"volume help", cli_cmd_volume_help_cbk,
+     "display help for volume commands"},
+
+    {"volume info [all|<VOLNAME>]", cli_cmd_volume_info_cbk,
+     "list information of all volumes"},
 
-          cli_cmd_volume_create_cbk,
-          "create a new volume of specified type with mentioned bricks"},
+    {"volume create <NEW-VOLNAME> [stripe <COUNT>] "
+     "[[replica <COUNT> [arbiter <COUNT>]]|[replica 2 thin-arbiter 1]] "
+     "[disperse [<COUNT>]] [disperse-data <COUNT>] [redundancy <COUNT>] "
+     "[transport <tcp|rdma|tcp,rdma>] <NEW-BRICK> <TA-BRICK>"
+     "... [force]",
 
-        { "volume delete <VOLNAME>",
-          cli_cmd_volume_delete_cbk,
-          "delete volume specified by <VOLNAME>"},
+     cli_cmd_volume_create_cbk,
+     "create a new volume of specified type with mentioned bricks"},
 
-        { "volume start <VOLNAME> [force]",
-          cli_cmd_volume_start_cbk,
-          "start volume specified by <VOLNAME>"},
+    {"volume delete <VOLNAME>", cli_cmd_volume_delete_cbk,
+     "delete volume specified by <VOLNAME>"},
 
-        { "volume stop <VOLNAME> [force]",
-          cli_cmd_volume_stop_cbk,
-          "stop volume specified by <VOLNAME>"},
+    {"volume start <VOLNAME> [force]", cli_cmd_volume_start_cbk,
+     "start volume specified by <VOLNAME>"},
 
-        /*{ "volume rename <VOLNAME> <NEW-VOLNAME>",
-          cli_cmd_volume_rename_cbk,
-          "rename volume <VOLNAME> to <NEW-VOLNAME>"},*/
+    {"volume stop <VOLNAME> [force]", cli_cmd_volume_stop_cbk,
+     "stop volume specified by <VOLNAME>"},
 
-        { "volume add-brick <VOLNAME> [<stripe|replica> <COUNT>] <NEW-BRICK> ... [force]",
-          cli_cmd_volume_add_brick_cbk,
-          "add brick to volume <VOLNAME>"},
+    /*{ "volume rename <VOLNAME> <NEW-VOLNAME>",
+      cli_cmd_volume_rename_cbk,
+      "rename volume <VOLNAME> to <NEW-VOLNAME>"},*/
 
-        { "volume remove-brick <VOLNAME> [replica <COUNT>] <BRICK> ..."
-          " <start|stop|status|commit|force>",
-          cli_cmd_volume_remove_brick_cbk,
-          "remove brick from volume <VOLNAME>"},
+    {"volume add-brick <VOLNAME> [<stripe|replica> <COUNT> "
+     "[arbiter <COUNT>]] <NEW-BRICK> ... [force]",
+     cli_cmd_volume_add_brick_cbk, "add brick to volume <VOLNAME>"},
 
-        { "volume rebalance <VOLNAME> {{fix-layout start} | {start [force]|stop|status}}",
-          cli_cmd_volume_defrag_cbk,
-          "rebalance operations"},
+    {"volume remove-brick <VOLNAME> [replica <COUNT>] <BRICK> ..."
+     " <start|stop|status|commit|force>",
+     cli_cmd_volume_remove_brick_cbk, "remove brick from volume <VOLNAME>"},
 
-        { "volume replace-brick <VOLNAME> <BRICK> <NEW-BRICK> {start [force]|pause|abort|status|commit [force]}",
-          cli_cmd_volume_replace_brick_cbk,
-          "replace-brick operations"},
+    {"volume rebalance <VOLNAME> {{fix-layout start} | {start "
+     "[force]|stop|status}}",
+     cli_cmd_volume_defrag_cbk, "rebalance operations"},
 
-        /*{ "volume set-transport <VOLNAME> <TRANSPORT-TYPE> [<TRANSPORT-TYPE>] ...",
-          cli_cmd_volume_set_transport_cbk,
-          "set transport type for volume <VOLNAME>"},*/
+    {"volume replace-brick <VOLNAME> <SOURCE-BRICK> <NEW-BRICK> "
+     "{commit force}",
+     cli_cmd_volume_replace_brick_cbk, "replace-brick operations"},
 
-        { "volume set <VOLNAME> <KEY> <VALUE>",
-          cli_cmd_volume_set_cbk,
-         "set options for volume <VOLNAME>"},
+    /*{ "volume set-transport <VOLNAME> <TRANSPORT-TYPE> [<TRANSPORT-TYPE>]
+      ...", cli_cmd_volume_set_transport_cbk, "set transport type for volume
+      <VOLNAME>"},*/
 
-        { "volume help",
-          cli_cmd_volume_help_cbk,
-          "display help for the volume command"},
+    {"volume set <VOLNAME> <KEY> <VALUE>", cli_cmd_volume_set_cbk,
+     "set options for volume <VOLNAME>"},
 
-        { "volume log <VOLNAME> rotate [BRICK]",
-          cli_cmd_log_rotate_cbk,
-         "rotate the log file for corresponding volume/brick"},
+    {"volume set <VOLNAME> group <GROUP>", cli_cmd_volume_set_cbk,
+     "This option can be used for setting multiple pre-defined volume options "
+     "where group_name is a file under /var/lib/glusterd/groups containing one "
+     "key value pair per line"},
 
-        { "volume log rotate <VOLNAME> [BRICK]",
-          cli_cmd_log_rotate_cbk,
-         "rotate the log file for corresponding volume/brick"
-         " NOTE: This is an old syntax, will be deprecated from next release."},
+    {"volume log <VOLNAME> rotate [BRICK]", cli_cmd_log_rotate_cbk,
+     "rotate the log file for corresponding volume/brick"},
 
-        { "volume sync <HOSTNAME> [all|<VOLNAME>]",
-          cli_cmd_sync_volume_cbk,
-         "sync the volume information from a peer"},
+    {"volume sync <HOSTNAME> [all|<VOLNAME>]", cli_cmd_sync_volume_cbk,
+     "sync the volume information from a peer"},
 
-         { "volume reset <VOLNAME> [option] [force]",
-         cli_cmd_volume_reset_cbk,
-         "reset all the reconfigured options"},
+    {"volume reset <VOLNAME> [option] [force]", cli_cmd_volume_reset_cbk,
+     "reset all the reconfigured options"},
 
 #if (SYNCDAEMON_COMPILE)
-        {"volume "GEOREP" [<VOLNAME>] [<SLAVE-URL>] {create [push-pem] [force]"
-         "|start [force]|stop [force]|pause [force]|resume [force]|config|status [detail]|delete} [options...]",
-         cli_cmd_volume_gsync_set_cbk,
-         "Geo-sync operations",
-         cli_cmd_check_gsync_exists_cbk},
+    {"volume " GEOREP " [<MASTER-VOLNAME>] [<SLAVE-IP>]::[<SLAVE-VOLNAME>] {"
+     "\\\n create [[ssh-port n] [[no-verify] \\\n | [push-pem]]] [force] \\\n"
+     " | start [force] \\\n | stop [force] \\\n | pause [force] \\\n | resume "
+     "[force] \\\n"
+     " | config [[[\\!]<option>] [<value>]] \\\n | status "
+     "[detail] \\\n | delete [reset-sync-time]} ",
+     cli_cmd_volume_gsync_set_cbk, "Geo-sync operations",
+     cli_cmd_check_gsync_exists_cbk},
 #endif
 
-         { "volume profile <VOLNAME> {start|info [peek|incremental [peek]|cumulative|clear]|stop} [nfs]",
-           cli_cmd_volume_profile_cbk,
-           "volume profile operations"},
-
-        { "volume quota <VOLNAME> {enable|disable|list [<path> ...]|remove <path>| default-soft-limit <percent>} |\n"
-          "volume quota <VOLNAME> {limit-usage <path> <size> [<percent>]} |\n"
-          "volume quota <VOLNAME> {alert-time|soft-timeout|hard-timeout} {<time>}",
-          cli_cmd_quota_cbk,
-          "quota translator specific operations"},
-
-         { "volume top <VOLNAME> {open|read|write|opendir|readdir|clear} [nfs|brick <brick>] [list-cnt <value>] |\n"
-           "volume top <VOLNAME> {read-perf|write-perf} [bs <size> count <count>] [brick <brick>] [list-cnt <value>]",
-           cli_cmd_volume_top_cbk,
-           "volume top operations"},
-
-        { "volume status [all | <VOLNAME> [nfs|shd|<BRICK>|quotad]]"
-          " [detail|clients|mem|inode|fd|callpool|tasks]",
-          cli_cmd_volume_status_cbk,
-          "display status of all or specified volume(s)/brick"},
-
-        { "volume heal <VOLNAME> [{full | statistics {heal-count {replica <hostname:brickname>}} |info {healed | heal-failed | split-brain}}]",
-          cli_cmd_volume_heal_cbk,
-          "self-heal commands on volume specified by <VOLNAME>"},
-
-        {"volume statedump <VOLNAME> [nfs|quotad] [all|mem|iobuf|callpool|priv|fd|"
-         "inode|history]...",
-         cli_cmd_volume_statedump_cbk,
-         "perform statedump on bricks"},
-
-        {"volume list",
-         cli_cmd_volume_list_cbk,
-         "list all volumes in cluster"},
-
-        {"volume clear-locks <VOLNAME> <path> kind {blocked|granted|all}"
-          "{inode [range]|entry [basename]|posix [range]}",
-          cli_cmd_volume_clearlocks_cbk,
-          "Clear locks held on path"
-        },
-        {"volume barrier <VOLNAME> {enable|disable}",
-         cli_cmd_volume_barrier_cbk,
-         "Barrier/unbarrier file operations on a volume"},
-
-        { NULL, NULL, NULL }
-};
+    {"volume profile <VOLNAME> {start|info [peek|incremental "
+     "[peek]|cumulative|clear]|stop} [nfs]",
+     cli_cmd_volume_profile_cbk, "volume profile operations"},
+
+    {"volume top <VOLNAME> {open|read|write|opendir|readdir|clear} [nfs|brick "
+     "<brick>] [list-cnt <value>] | "
+     "{read-perf|write-perf} [bs <size> count <count>] "
+     "[brick <brick>] [list-cnt <value>]",
+     cli_cmd_volume_top_cbk, "volume top operations"},
+
+    {"volume status [all | <VOLNAME> [nfs|shd|<BRICK>|quotad]]"
+     " [detail|clients|mem|inode|fd|callpool|tasks|client-list]",
+     cli_cmd_volume_status_cbk,
+     "display status of all or specified volume(s)/brick"},
+
+    {"volume heal <VOLNAME> [enable | disable | full |"
+     "statistics [heal-count [replica <HOSTNAME:BRICKNAME>]] |"
+     "info [summary | split-brain] |"
+     "split-brain {bigger-file <FILE> | latest-mtime <FILE> |"
+     "source-brick <HOSTNAME:BRICKNAME> [<FILE>]} |"
+     "granular-entry-heal {enable | disable}]",
+     cli_cmd_volume_heal_cbk,
+     "self-heal commands on volume specified by <VOLNAME>"},
+
+    {"volume statedump <VOLNAME> [[nfs|quotad] [all|mem|iobuf|callpool|"
+     "priv|fd|inode|history]... | [client <hostname:process-id>]]",
+     cli_cmd_volume_statedump_cbk, "perform statedump on bricks"},
+
+    {"volume list", cli_cmd_volume_list_cbk, "list all volumes in cluster"},
+
+    {"volume clear-locks <VOLNAME> <path> kind {blocked|granted|all}"
+     "{inode [range]|entry [basename]|posix [range]}",
+     cli_cmd_volume_clearlocks_cbk, "Clear locks held on path"},
+    {"volume barrier <VOLNAME> {enable|disable}", cli_cmd_volume_barrier_cbk,
+     "Barrier/unbarrier file operations on a volume"},
+    {"volume get <VOLNAME|all> <key|all>", cli_cmd_volume_getopt_cbk,
+     "Get the value of the all options or given option for volume <VOLNAME>"
+     " or all option. gluster volume get all all is to get all global "
+     "options"},
+
+    {"volume reset-brick <VOLNAME> <SOURCE-BRICK> {{start} |"
+     " {<NEW-BRICK> commit}}",
+     cli_cmd_volume_reset_brick_cbk, "reset-brick operations"},
+
+    {NULL, NULL, NULL}};
 
 int
-cli_cmd_volume_help_cbk (struct cli_state *state, struct cli_cmd_word *in_word,
-                      const char **words, int wordcount)
+cli_cmd_quota_help_cbk(struct cli_state *state, struct cli_cmd_word *in_word,
+                       const char **words, int wordcount)
 {
-        struct cli_cmd        *cmd = NULL;
+    struct cli_cmd *cmd = NULL;
+    struct cli_cmd *quota_cmd = NULL;
+    int count = 0;
+
+    cmd = GF_MALLOC(sizeof(quota_cmds), cli_mt_cli_cmd);
+    memcpy(cmd, quota_cmds, sizeof(quota_cmds));
+    count = (sizeof(quota_cmds) / sizeof(struct cli_cmd));
+    cli_cmd_sort(cmd, count);
 
-        for (cmd = volume_cmds; cmd->pattern; cmd++)
-                if (_gf_false == cmd->disable)
-                        cli_out ("%s - %s", cmd->pattern, cmd->desc);
+    cli_out("\ngluster quota commands");
+    cli_out("=======================\n");
 
-        return 0;
+    for (quota_cmd = cmd; quota_cmd->pattern; quota_cmd++)
+        if ((_gf_false == quota_cmd->disable) && (quota_cmd->desc))
+            cli_out("%s - %s", quota_cmd->pattern, quota_cmd->desc);
+
+    cli_out("\n");
+    GF_FREE(cmd);
+
+    return 0;
 }
 
 int
-cli_cmd_volume_register (struct cli_state *state)
+cli_cmd_bitrot_help_cbk(struct cli_state *state, struct cli_cmd_word *in_word,
+                        const char **words, int wordcount)
 {
-        int  ret = 0;
-        struct cli_cmd *cmd = NULL;
+    struct cli_cmd *cmd = NULL;
+    struct cli_cmd *bitrot_cmd = NULL;
+    int count = 0;
 
-        for (cmd = volume_cmds; cmd->pattern; cmd++) {
+    cmd = GF_MALLOC(sizeof(bitrot_cmds), cli_mt_cli_cmd);
+    memcpy(cmd, bitrot_cmds, sizeof(bitrot_cmds));
+    count = (sizeof(bitrot_cmds) / sizeof(struct cli_cmd));
+    cli_cmd_sort(cmd, count);
+
+    cli_out("\ngluster bitrot commands");
+    cli_out("========================\n");
+
+    for (bitrot_cmd = cmd; bitrot_cmd->pattern; bitrot_cmd++)
+        if ((_gf_false == bitrot_cmd->disable) && (bitrot_cmd->desc))
+            cli_out("%s - %s", bitrot_cmd->pattern, bitrot_cmd->desc);
+
+    cli_out("\n");
+    GF_FREE(cmd);
+
+    return 0;
+}
+
+int
+cli_cmd_volume_help_cbk(struct cli_state *state, struct cli_cmd_word *in_word,
+                        const char **words, int wordcount)
+{
+    struct cli_cmd *cmd = NULL;
+    struct cli_cmd *vol_cmd = NULL;
+    int count = 0;
+
+    cmd = GF_MALLOC(sizeof(volume_cmds), cli_mt_cli_cmd);
+    memcpy(cmd, volume_cmds, sizeof(volume_cmds));
+    count = (sizeof(volume_cmds) / sizeof(struct cli_cmd));
+    cli_cmd_sort(cmd, count);
+
+    cli_out("\ngluster volume commands");
+    cli_out("========================\n");
+
+    for (vol_cmd = cmd; vol_cmd->pattern; vol_cmd++)
+        if (_gf_false == vol_cmd->disable)
+            cli_out("%s - %s", vol_cmd->pattern, vol_cmd->desc);
+
+    cli_out("\n");
+    GF_FREE(cmd);
+    return 0;
+}
+
+int
+cli_cmd_volume_register(struct cli_state *state)
+{
+    int ret = 0;
+    struct cli_cmd *cmd = NULL;
+
+    for (cmd = volume_cmds; cmd->pattern; cmd++) {
+        ret = cli_cmd_register(&state->tree, cmd);
+        if (ret)
+            goto out;
+    }
+
+    for (cmd = bitrot_cmds; cmd->pattern; cmd++) {
+        ret = cli_cmd_register(&state->tree, cmd);
+        if (ret)
+            goto out;
+    }
+
+    for (cmd = quota_cmds; cmd->pattern; cmd++) {
+        ret = cli_cmd_register(&state->tree, cmd);
+        if (ret)
+            goto out;
+    }
 
-                ret = cli_cmd_register (&state->tree, cmd);
-                if (ret)
-                        goto out;
-        }
 out:
-        return ret;
+    return ret;
+}
+
+static int
+gf_asprintf_append(char **string_ptr, const char *format, ...)
+{
+    va_list arg;
+    int rv = 0;
+    char *tmp = *string_ptr;
+
+    va_start(arg, format);
+    rv = gf_vasprintf(string_ptr, format, arg);
+    va_end(arg);
+
+    if (tmp)
+        GF_FREE(tmp);
+
+    return rv;
 }
diff --git a/cli/src/cli-cmd.c b/cli/src/cli-cmd.c
index ec2d58ae91c..2d458b16a56 100644
--- a/cli/src/cli-cmd.c
+++ b/cli/src/cli-cmd.c
@@ -13,11 +13,6 @@
 #include <stdint.h>
 #include <pthread.h>
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 #include "cli.h"
 #include "cli-cmd.h"
 #include "cli-mem-types.h"
@@ -27,360 +22,390 @@
 
 static int cmd_done;
 static int cmd_sent;
-static pthread_cond_t      cond  = PTHREAD_COND_INITIALIZER;
-static pthread_mutex_t     cond_mutex = PTHREAD_MUTEX_INITIALIZER;
-static pthread_cond_t      conn  = PTHREAD_COND_INITIALIZER;
-static pthread_mutex_t     conn_mutex = PTHREAD_MUTEX_INITIALIZER;
-
-int    cli_op_ret = 0;
-int    connected = 0;
+static pthread_cond_t cond = PTHREAD_COND_INITIALIZER;
+static pthread_mutex_t cond_mutex = PTHREAD_MUTEX_INITIALIZER;
+static pthread_cond_t conn = PTHREAD_COND_INITIALIZER;
+static pthread_mutex_t conn_mutex = PTHREAD_MUTEX_INITIALIZER;
 
-int cli_cmd_log_help_cbk (struct cli_state *state, struct cli_cmd_word *in_word,
-                      const char **words, int wordcount);
+int cli_op_ret = 0;
+static gf_boolean_t connected = _gf_false;
 
 static unsigned
-cli_cmd_needs_connection (struct cli_cmd_word *word)
+cli_cmd_needs_connection(struct cli_cmd_word *word)
 {
-        if (!strcasecmp ("quit", word->word))
-                return 0;
+    if (!strcasecmp("quit", word->word))
+        return 0;
 
-        if (!strcasecmp ("help", word->word))
-                return 0;
+    if (!strcasecmp("help", word->word))
+        return 0;
 
-        if (!strcasecmp ("getwd", word->word))
-                return 1;
+    if (!strcasecmp("getwd", word->word))
+        return 1;
 
-        if (!strcasecmp ("exit", word->word))
-                return 0;
+    if (!strcasecmp("exit", word->word))
+        return 0;
 
-        return CLI_DEFAULT_CONN_TIMEOUT;
+    return cli_default_conn_timeout;
 }
 
 int
-cli_cmd_status_reset (void)
+cli_cmd_status_reset(void)
 {
-        int ret = 0;
-
-        ret = cli_cmd_lock ();
-        {
-                if (ret == 0) {
-                        cmd_sent = 0;
-                        cmd_done = 0;
-                }
-        }
-        ret = cli_cmd_unlock ();
-        return ret;
+    int ret = 0;
 
+    ret = cli_cmd_lock();
+    {
+        if (ret == 0) {
+            cmd_sent = 0;
+            cmd_done = 0;
+        }
+    }
+    ret = cli_cmd_unlock();
+    return ret;
 }
 
 int
-cli_cmd_sent_status_get (int *status)
+cli_cmd_sent_status_get(int *status)
 {
-        int ret = 0;
-        GF_ASSERT (status);
-
-        ret = cli_cmd_lock ();
-        {
-                if (ret == 0)
-                        *status = cmd_sent;
-        }
-        ret = cli_cmd_unlock ();
-        return ret;
+    int ret = 0;
+    GF_ASSERT(status);
+
+    ret = cli_cmd_lock();
+    {
+        if (ret == 0)
+            *status = cmd_sent;
+    }
+    ret = cli_cmd_unlock();
+    return ret;
 }
 
 int
-cli_cmd_process (struct cli_state *state, int argc, char **argv)
+cli_cmd_process(struct cli_state *state, int argc, char **argv)
 {
-        int                  ret = 0;
-        struct cli_cmd_word *word = NULL;
-        struct cli_cmd_word *next = NULL;
-        int                  i = 0;
+    int ret = 0;
+    struct cli_cmd_word *word = NULL;
+    struct cli_cmd_word *next = NULL;
+    int i = 0;
 
-        word = &state->tree.root;
+    word = &state->tree.root;
 
-        if (!argc)
-                return 0;
+    if (!argc)
+        return 0;
 
-        for (i = 0; i < argc; i++) {
-                next = cli_cmd_nextword (word, argv[i]);
+    for (i = 0; i < argc; i++) {
+        next = cli_cmd_nextword(word, argv[i]);
 
-                word = next;
-                if (!word)
-                        break;
+        word = next;
+        if (!word)
+            break;
 
-                if (word->cbkfn)
-                        break;
-        }
+        if (word->cbkfn)
+            break;
+    }
 
-        if (!word) {
-                cli_out ("unrecognized word: %s (position %d)",
-                         argv[i], i);
-                return -1;
-        }
+    if (!word) {
+        cli_out("unrecognized word: %s (position %d)\n", argv[i], i);
+        usage();
+        return -1;
+    }
 
-        if (!word->cbkfn) {
-                cli_out ("unrecognized command");
-                return -1;
-        }
+    if (!word->cbkfn) {
+        cli_out("unrecognized command\n");
+        usage();
+        return -1;
+    }
 
-	if ( strcmp (word->word,"help")==0 )
-		goto callback;
+    if (strcmp(word->word, "help") == 0)
+        goto callback;
 
-        state->await_connected = cli_cmd_needs_connection (word);
+    state->await_connected = cli_cmd_needs_connection(word);
 
-        ret = cli_cmd_await_connected (state->await_connected);
-        if (ret) {
-                cli_out ("Connection failed. Please check if gluster "
-                          "daemon is operational.");
-                gf_log ("", GF_LOG_INFO, "Exiting with: %d", ret);
-                exit (ret);
-        }
+    ret = cli_cmd_await_connected(state->await_connected);
+    if (ret) {
+        cli_out(
+            "Connection failed. Please check if gluster "
+            "daemon is operational.");
+        gf_log("", GF_LOG_INFO, "Exiting with: %d", ret);
+        exit(ret);
+    }
 
 callback:
-        ret = word->cbkfn (state, word, (const char **)argv, argc);
-        (void) cli_cmd_status_reset ();
-        return ret;
+    ret = word->cbkfn(state, word, (const char **)argv, argc);
+    (void)cli_cmd_status_reset();
+    return ret;
 }
 
 int
-cli_cmd_input_token_count (const char *text)
+cli_cmd_input_token_count(const char *text)
 {
-        int          count = 0;
-        const char  *trav = NULL;
-        int          is_spc = 1;
-
-        for (trav = text; *trav; trav++) {
-                if (*trav == ' ') {
-                        is_spc = 1;
-                } else {
-                        if (is_spc) {
-                                count++;
-                                is_spc = 0;
-                        }
-                }
+    int count = 0;
+    const char *trav = NULL;
+    int is_spc = 1;
+
+    for (trav = text; *trav; trav++) {
+        if (*trav == ' ') {
+            is_spc = 1;
+        } else {
+            if (is_spc) {
+                count++;
+                is_spc = 0;
+            }
         }
+    }
 
-        return count;
+    return count;
 }
 
-
 int
-cli_cmd_process_line (struct cli_state *state, const char *text)
+cli_cmd_process_line(struct cli_state *state, const char *text)
 {
-        int     count = 0;
-        char  **tokens = NULL;
-        char  **tokenp = NULL;
-        char   *token = NULL;
-        char   *copy = NULL;
-        char   *saveptr = NULL;
-        int     i = 0;
-        int     ret = -1;
-
-        count = cli_cmd_input_token_count (text);
-
-        tokens = calloc (count + 1, sizeof (*tokens));
-        if (!tokens)
-                return -1;
-
-        copy = strdup (text);
-        if (!copy)
-                goto out;
-
-        tokenp = tokens;
-
-        for (token = strtok_r (copy, " \t\r\n", &saveptr); token;
-             token = strtok_r (NULL, " \t\r\n", &saveptr)) {
-                *tokenp = strdup (token);
-
-                if (!*tokenp)
-                        goto out;
-                tokenp++;
-                i++;
-
-        }
-
-        ret = cli_cmd_process (state, count, tokens);
+    int count = 0;
+    char **tokens = NULL;
+    char **tokenp = NULL;
+    char *token = NULL;
+    char *copy = NULL;
+    char *saveptr = NULL;
+    int i = 0;
+    int ret = -1;
+
+    count = cli_cmd_input_token_count(text);
+
+    tokens = calloc(count + 1, sizeof(*tokens));
+    if (!tokens)
+        return -1;
+
+    copy = strdup(text);
+    if (!copy)
+        goto out;
+
+    tokenp = tokens;
+
+    for (token = strtok_r(copy, " \t\r\n", &saveptr); token;
+         token = strtok_r(NULL, " \t\r\n", &saveptr)) {
+        *tokenp = strdup(token);
+
+        if (!*tokenp)
+            goto out;
+        tokenp++;
+        i++;
+    }
+
+    ret = cli_cmd_process(state, count, tokens);
 out:
-        free (copy);
+    free(copy);
 
-        if (tokens)
-                cli_cmd_tokens_destroy (tokens);
+    if (tokens)
+        cli_cmd_tokens_destroy(tokens);
 
-        return ret;
+    return ret;
 }
 
-
 int
-cli_cmds_register (struct cli_state *state)
+cli_cmds_register(struct cli_state *state)
 {
-        int  ret = 0;
-
-        ret = cli_cmd_volume_register (state);
-        if (ret)
-                goto out;
-
-        ret = cli_cmd_probe_register (state);
-        if (ret)
-                goto out;
-
-        ret = cli_cmd_system_register (state);
-        if (ret)
-                goto out;
-
-        ret = cli_cmd_misc_register (state);
-        if (ret)
-                goto out;
-
-        ret = cli_cmd_snapshot_register (state);
-        if (ret)
-                goto out;
+    int ret = 0;
+
+    ret = cli_cmd_volume_register(state);
+    if (ret)
+        goto out;
+
+    ret = cli_cmd_probe_register(state);
+    if (ret)
+        goto out;
+
+    ret = cli_cmd_system_register(state);
+    if (ret)
+        goto out;
+
+    ret = cli_cmd_misc_register(state);
+    if (ret)
+        goto out;
+
+    ret = cli_cmd_snapshot_register(state);
+    if (ret)
+        goto out;
+    ret = cli_cmd_global_register(state);
+    if (ret)
+        goto out;
 out:
-        return ret;
-}
-
-int
-cli_cmd_cond_init ()
-{
-
-       pthread_mutex_init (&cond_mutex, NULL);
-       pthread_cond_init (&cond, NULL);
-
-       pthread_mutex_init (&conn_mutex, NULL);
-       pthread_cond_init (&conn, NULL);
-
-       return 0;
+    return ret;
 }
 
 int
-cli_cmd_lock ()
+cli_cmd_lock()
 {
-       pthread_mutex_lock (&cond_mutex);
-       return 0;
+    pthread_mutex_lock(&cond_mutex);
+    return 0;
 }
 
 int
-cli_cmd_unlock ()
+cli_cmd_unlock()
 {
-        pthread_mutex_unlock (&cond_mutex);
-        return 0;
+    pthread_mutex_unlock(&cond_mutex);
+    return 0;
 }
 
 static void
-seconds_from_now (unsigned secs, struct timespec *ts)
+seconds_from_now(unsigned secs, struct timespec *ts)
 {
-        struct timeval tv = {0,};
+    struct timeval tv = {
+        0,
+    };
 
-        gettimeofday (&tv, NULL);
+    gettimeofday(&tv, NULL);
 
-        ts->tv_sec = tv.tv_sec + secs;
-        ts->tv_nsec = tv.tv_usec * 1000;
+    ts->tv_sec = tv.tv_sec + secs;
+    ts->tv_nsec = tv.tv_usec * 1000;
 }
 
 int
-cli_cmd_await_response (unsigned time)
+cli_cmd_await_response(unsigned time)
 {
-        struct  timespec        ts = {0,};
-        int                     ret = 0;
+    struct timespec ts = {
+        0,
+    };
+    int ret = 0;
 
-        cli_op_ret = -1;
-
-        seconds_from_now (time, &ts);
-        while (!cmd_done && !ret) {
-                ret = pthread_cond_timedwait (&cond, &cond_mutex,
-                                        &ts);
-        }
+    cli_op_ret = -1;
 
-        cmd_done = 0;
+    seconds_from_now(time, &ts);
+    while (!cmd_done && !ret) {
+        ret = pthread_cond_timedwait(&cond, &cond_mutex, &ts);
+    }
 
-        if (ret)
-                return ret;
+    if (!cmd_done) {
+        if (ret == ETIMEDOUT)
+            cli_out("Error : Request timed out");
+        else
+            cli_out("Error : Command returned with error code:%d", ret);
+    }
+    cmd_done = 0;
 
-        return cli_op_ret;
+    return cli_op_ret;
 }
 
+/* This function must be called _only_ after all actions associated with
+ * command processing is complete. Otherwise, gluster process may exit before
+ * reporting results to stdout/stderr. */
 int
-cli_cmd_broadcast_response (int32_t status)
+cli_cmd_broadcast_response(int32_t status)
 {
-
-        pthread_mutex_lock (&cond_mutex);
-        {
-                if (!cmd_sent)
-                        goto out;
-                cmd_done = 1;
-                cli_op_ret = status;
-                pthread_cond_broadcast (&cond);
-        }
-
+    pthread_mutex_lock(&cond_mutex);
+    {
+        if (!cmd_sent)
+            goto out;
+        cmd_done = 1;
+        cli_op_ret = status;
+        pthread_cond_broadcast(&cond);
+    }
 
 out:
-        pthread_mutex_unlock (&cond_mutex);
-        return 0;
+    pthread_mutex_unlock(&cond_mutex);
+    return 0;
 }
 
 int32_t
-cli_cmd_await_connected (unsigned conn_timo)
+cli_cmd_await_connected(unsigned conn_timo)
 {
-        int32_t                 ret = 0;
-        struct  timespec        ts = {0,};
-
-        if (!conn_timo)
-                return 0;
-
-        pthread_mutex_lock (&conn_mutex);
-        {
-                seconds_from_now (conn_timo, &ts);
-                while (!connected && !ret) {
-                        ret = pthread_cond_timedwait (&conn, &conn_mutex,
-                                                      &ts);
-                }
-        }
-        pthread_mutex_unlock (&conn_mutex);
+    int32_t ret = 0;
+    struct timespec ts = {
+        0,
+    };
+
+    if (!conn_timo)
+        return 0;
 
+    pthread_mutex_lock(&conn_mutex);
+    {
+        seconds_from_now(conn_timo, &ts);
+        while (!connected && !ret) {
+            ret = pthread_cond_timedwait(&conn, &conn_mutex, &ts);
+        }
+    }
+    pthread_mutex_unlock(&conn_mutex);
 
-        return ret;
+    return ret;
 }
 
 int32_t
-cli_cmd_broadcast_connected ()
+cli_cmd_broadcast_connected(gf_boolean_t status)
 {
-        pthread_mutex_lock (&conn_mutex);
-        {
-                connected = 1;
-                pthread_cond_broadcast (&conn);
-        }
-
-        pthread_mutex_unlock (&conn_mutex);
-
-        return 0;
+    pthread_mutex_lock(&conn_mutex);
+    {
+        connected = status;
+        pthread_cond_broadcast(&conn);
+    }
+    pthread_mutex_unlock(&conn_mutex);
+
+    return 0;
 }
 
-int
-cli_cmd_submit (struct rpc_clnt* rpc, void *req, call_frame_t *frame,
-                rpc_clnt_prog_t *prog,
-                int procnum, struct iobref *iobref,
-                xlator_t *this, fop_cbk_fn_t cbkfn, xdrproc_t xdrproc)
+gf_boolean_t
+cli_cmd_connected(void)
 {
-        int             ret = -1;
-        unsigned        timeout = 0;
+    gf_boolean_t status;
 
-        if ((GLUSTER_CLI_PROFILE_VOLUME == procnum) ||
-            (GLUSTER_CLI_HEAL_VOLUME == procnum))
-                timeout = CLI_TEN_MINUTES_TIMEOUT;
-        else
-                timeout = CLI_DEFAULT_CMD_TIMEOUT;
+    pthread_mutex_lock(&conn_mutex);
+    {
+        status = connected;
+    }
+    pthread_mutex_unlock(&conn_mutex);
 
-        cli_cmd_lock ();
-        cmd_sent = 0;
-        ret = cli_submit_request (rpc, req, frame, prog,
-                                  procnum, NULL, this, cbkfn, xdrproc);
+    return status;
+}
 
-        if (!ret) {
-                cmd_sent = 1;
-                ret = cli_cmd_await_response (timeout);
-        }
+int
+cli_cmd_submit(struct rpc_clnt *rpc, void *req, call_frame_t *frame,
+               rpc_clnt_prog_t *prog, int procnum, struct iobref *iobref,
+               xlator_t *this, fop_cbk_fn_t cbkfn, xdrproc_t xdrproc)
+{
+    int ret = -1;
+    unsigned timeout = 0;
+
+    if ((GLUSTER_CLI_PROFILE_VOLUME == procnum) ||
+        (GLUSTER_CLI_HEAL_VOLUME == procnum) ||
+        (GLUSTER_CLI_GANESHA == procnum))
+        timeout = cli_ten_minutes_timeout;
+    else
+        timeout = cli_default_conn_timeout;
+
+    cli_cmd_lock();
+    cmd_sent = 0;
+    ret = cli_submit_request(rpc, req, frame, prog, procnum, NULL, this, cbkfn,
+                             xdrproc);
+
+    if (!ret) {
+        cmd_sent = 1;
+        ret = cli_cmd_await_response(timeout);
+    }
+
+    cli_cmd_unlock();
+
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
+}
 
-        cli_cmd_unlock ();
+int
+cli_cmd_pattern_cmp(void *a, void *b)
+{
+    struct cli_cmd *ia = NULL;
+    struct cli_cmd *ib = NULL;
+    int ret = 0;
+
+    ia = a;
+    ib = b;
+    if (strcmp(ia->pattern, ib->pattern) > 0)
+        ret = 1;
+    else if (strcmp(ia->pattern, ib->pattern) < 0)
+        ret = -1;
+    else
+        ret = 0;
+    return ret;
+}
 
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+void
+cli_cmd_sort(struct cli_cmd *cmd, int count)
+{
+    gf_array_insertionsort(cmd, 1, count - 2, sizeof(struct cli_cmd),
+                           cli_cmd_pattern_cmp);
 }
diff --git a/cli/src/cli-cmd.h b/cli/src/cli-cmd.h
index 91d15b7e170..c1c068c7085 100644
--- a/cli/src/cli-cmd.h
+++ b/cli/src/cli-cmd.h
@@ -10,121 +10,120 @@
 #ifndef __CLI_CMD_H__
 #define __CLI_CMD_H__
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 #include <netdb.h>
 
 #include "cli.h"
-#include "list.h"
-
-#define CLI_LOCAL_INIT(local, words, frame, dictionary) \
-        do {                                                 \
-                local = cli_local_get ();                    \
-                                                             \
-                if (local) {                                 \
-                        local->words = words;                \
-                        if (dictionary)                      \
-                                local->dict = dictionary;    \
-                        if (frame)                           \
-                                frame->local = local;        \
-                }                                            \
-        } while (0)
-
-#define CLI_STACK_DESTROY(_frame)                                       \
-        do {                                                            \
-                if (_frame) {                                           \
-                        if (_frame->local) {                            \
-                                gf_log ("cli", GF_LOG_DEBUG, "frame->local " \
-                                        "is not NULL (%p)", _frame->local); \
-                                cli_local_wipe (_frame->local);         \
-                                _frame->local = NULL;                   \
-                        }                                               \
-                        STACK_DESTROY (_frame->root);                   \
-                }                                                       \
-        } while (0);
-
-typedef enum {
-        GF_ANSWER_YES = 1,
-        GF_ANSWER_NO  = 2
-} gf_answer_t;
+#include <glusterfs/list.h>
+
+#define GLUSTER_SHARED_STORAGE "gluster_shared_storage"
+
+#define CLI_LOCAL_INIT(local, words, frame, dictionary)                        \
+    do {                                                                       \
+        local = cli_local_get();                                               \
+                                                                               \
+        if (local) {                                                           \
+            local->words = words;                                              \
+            if (dictionary)                                                    \
+                local->dict = dictionary;                                      \
+            if (frame)                                                         \
+                frame->local = local;                                          \
+        }                                                                      \
+    } while (0)
+
+#define CLI_STACK_DESTROY(_frame)                                              \
+    do {                                                                       \
+        if (_frame) {                                                          \
+            if (_frame->local) {                                               \
+                gf_log("cli", GF_LOG_DEBUG,                                    \
+                       "frame->local "                                         \
+                       "is not NULL (%p)",                                     \
+                       _frame->local);                                         \
+                cli_local_wipe(_frame->local);                                 \
+                _frame->local = NULL;                                          \
+            }                                                                  \
+            STACK_DESTROY(_frame->root);                                       \
+        }                                                                      \
+    } while (0);
+
+typedef enum { GF_ANSWER_YES = 1, GF_ANSWER_NO = 2 } gf_answer_t;
 
 struct cli_cmd {
-        const char         *pattern;
-        cli_cmd_cbk_t      *cbk;
-        const char         *desc;
-        cli_cmd_reg_cbk_t  *reg_cbk; /* callback to check in runtime if the   *
-                                      * command should be enabled or disabled */
-        gf_boolean_t       disable;
+    const char *pattern;
+    cli_cmd_cbk_t *cbk;
+    const char *desc;
+    cli_cmd_reg_cbk_t *reg_cbk; /* callback to check in runtime if the   *
+                                 * command should be enabled or disabled */
+    gf_boolean_t disable;
 };
 
 struct cli_cmd_volume_get_ctx_ {
-        char            *volname;
-        int             flags;
+    char *volname;
+    int flags;
 };
 
 typedef struct cli_profile_info_ {
-        uint64_t fop_hits;
-        double min_latency;
-        double max_latency;
-        double avg_latency;
-        char   *fop_name;
-        double percentage_avg_latency;
+    uint64_t fop_hits;
+    double min_latency;
+    double max_latency;
+    double avg_latency;
+    char *fop_name;
+    double percentage_avg_latency;
 } cli_profile_info_t;
 
-typedef struct addrinfo_list {
-        struct list_head list;
-        struct addrinfo *info;
-} addrinfo_list_t;
-
-typedef enum {
-        GF_AI_COMPARE_NO_MATCH     = 0,
-        GF_AI_COMPARE_MATCH        = 1,
-        GF_AI_COMPARE_ERROR        = 2
-} gf_ai_compare_t;
-
 typedef struct cli_cmd_volume_get_ctx_ cli_cmd_volume_get_ctx_t;
 
-int cli_cmd_volume_register (struct cli_state *state);
+int
+cli_cmd_volume_register(struct cli_state *state);
 
-int cli_cmd_probe_register (struct cli_state *state);
+int
+cli_cmd_probe_register(struct cli_state *state);
 
-int cli_cmd_system_register (struct cli_state *state);
+int
+cli_cmd_system_register(struct cli_state *state);
 
-int cli_cmd_snapshot_register (struct cli_state *state);
+int
+cli_cmd_snapshot_register(struct cli_state *state);
 
-int cli_cmd_misc_register (struct cli_state *state);
+int
+cli_cmd_global_register(struct cli_state *state);
 
-struct cli_cmd_word *cli_cmd_nextword (struct cli_cmd_word *word,
-                                       const char *text);
-void cli_cmd_tokens_destroy (char **tokens);
+int
+cli_cmd_misc_register(struct cli_state *state);
 
-int cli_cmd_await_response (unsigned time);
+struct cli_cmd_word *
+cli_cmd_nextword(struct cli_cmd_word *word, const char *text);
+void
+cli_cmd_tokens_destroy(char **tokens);
 
-int cli_cmd_broadcast_response (int32_t status);
+int
+cli_cmd_await_response(unsigned time);
 
-int cli_cmd_cond_init ();
+int
+cli_cmd_broadcast_response(int32_t status);
 
-int cli_cmd_lock ();
+int
+cli_cmd_lock();
 
-int cli_cmd_unlock ();
+int
+cli_cmd_unlock();
 
 int
-cli_cmd_submit (struct rpc_clnt *rpc, void *req, call_frame_t *frame,
-                rpc_clnt_prog_t *prog,
-                int procnum, struct iobref *iobref,
-                xlator_t *this, fop_cbk_fn_t cbkfn, xdrproc_t xdrproc);
+cli_cmd_submit(struct rpc_clnt *rpc, void *req, call_frame_t *frame,
+               rpc_clnt_prog_t *prog, int procnum, struct iobref *iobref,
+               xlator_t *this, fop_cbk_fn_t cbkfn, xdrproc_t xdrproc);
 
-gf_answer_t
-cli_cmd_get_confirmation (struct cli_state *state, const char *question);
-int cli_cmd_sent_status_get (int *status);
+int
+cli_cmd_pattern_cmp(void *a, void *b);
 
-gf_boolean_t
-_limits_set_on_volume (char *volname);
+void
+cli_cmd_sort(struct cli_cmd *cmd, int count);
+
+gf_answer_t
+cli_cmd_get_confirmation(struct cli_state *state, const char *question);
+int
+cli_cmd_sent_status_get(int *status);
 
 gf_boolean_t
-_quota_aux_mount_online (char *volname);
+_limits_set_on_volume(char *volname, int type);
 
 #endif /* __CLI_CMD_H__ */
diff --git a/cli/src/cli-mem-types.h b/cli/src/cli-mem-types.h
index 09fcb639bd0..b42b4dd86c2 100644
--- a/cli/src/cli-mem-types.h
+++ b/cli/src/cli-mem-types.h
@@ -10,20 +10,21 @@
 #ifndef __CLI_MEM_TYPES_H__
 #define __CLI_MEM_TYPES_H__
 
-#include "mem-types.h"
+#include <glusterfs/mem-types.h>
 
 #define CLI_MEM_TYPE_START (gf_common_mt_end + 1)
 
 enum cli_mem_types_ {
-        cli_mt_xlator_list_t = CLI_MEM_TYPE_START,
-        cli_mt_xlator_t,
-        cli_mt_xlator_cmdline_option_t,
-        cli_mt_char,
-        cli_mt_call_pool_t,
-        cli_mt_cli_local_t,
-        cli_mt_cli_get_vol_ctx_t,
-        cli_mt_append_str,
-        cli_mt_end
+    cli_mt_xlator_list_t = CLI_MEM_TYPE_START,
+    cli_mt_xlator_t,
+    cli_mt_xlator_cmdline_option_t,
+    cli_mt_char,
+    cli_mt_call_pool_t,
+    cli_mt_cli_local_t,
+    cli_mt_cli_get_vol_ctx_t,
+    cli_mt_append_str,
+    cli_mt_cli_cmd,
+    cli_mt_end
 
 };
 
diff --git a/cli/src/cli-quotad-client.c b/cli/src/cli-quotad-client.c
index f0efc8640bd..772b8f75bd9 100644
--- a/cli/src/cli-quotad-client.c
+++ b/cli/src/cli-quotad-client.c
@@ -7,148 +7,140 @@
    later), or the GNU General Public License, version 2 (GPLv2), in all
    cases as published by the Free Software Foundation.
 */
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 
 #include "cli-quotad-client.h"
 
-extern struct rpc_clnt global_quotad_rpc;
-extern struct rpc_clnt_program cli_quotad_clnt;
-
 int
-cli_quotad_submit_request (void *req, call_frame_t *frame,
-                           rpc_clnt_prog_t *prog,
-                           int procnum, struct iobref *iobref,
-                           xlator_t *this, fop_cbk_fn_t cbkfn,
-                           xdrproc_t xdrproc)
+cli_quotad_submit_request(void *req, call_frame_t *frame, rpc_clnt_prog_t *prog,
+                          int procnum, struct iobref *iobref, xlator_t *this,
+                          fop_cbk_fn_t cbkfn, xdrproc_t xdrproc)
 {
-        int           ret        = -1;
-        int           count      = 0;
-        struct iovec  iov        = {0, };
-        struct iobuf *iobuf      = NULL;
-        char          new_iobref = 0;
-        ssize_t       xdr_size   = 0;
-
-        GF_ASSERT (this);
-
-        if (req) {
-                xdr_size = xdr_sizeof (xdrproc, req);
-                iobuf = iobuf_get2 (this->ctx->iobuf_pool, xdr_size);
-                if (!iobuf) {
-                        goto out;
-                };
-
-                if (!iobref) {
-                        iobref = iobref_new ();
-                        if (!iobref) {
-                                goto out;
-                        }
-
-                        new_iobref = 1;
-                }
-
-                iobref_add (iobref, iobuf);
-
-                iov.iov_base = iobuf->ptr;
-                iov.iov_len  = iobuf_size (iobuf);
-
-                /* Create the xdr payload */
-                ret = xdr_serialize_generic (iov, req, xdrproc);
-                if (ret == -1) {
-                        goto out;
-                }
-                iov.iov_len = ret;
-                count = 1;
+    int ret = -1;
+    int count = 0;
+    struct iovec iov = {
+        0,
+    };
+    struct iobuf *iobuf = NULL;
+    char new_iobref = 0;
+    ssize_t xdr_size = 0;
+
+    GF_ASSERT(this);
+
+    if (req) {
+        xdr_size = xdr_sizeof(xdrproc, req);
+        iobuf = iobuf_get2(this->ctx->iobuf_pool, xdr_size);
+        if (!iobuf) {
+            goto out;
+        };
+
+        if (!iobref) {
+            iobref = iobref_new();
+            if (!iobref) {
+                goto out;
+            }
+
+            new_iobref = 1;
+        }
+
+        iobref_add(iobref, iobuf);
+
+        iov.iov_base = iobuf->ptr;
+        iov.iov_len = iobuf_size(iobuf);
+
+        /* Create the xdr payload */
+        ret = xdr_serialize_generic(iov, req, xdrproc);
+        if (ret == -1) {
+            goto out;
         }
+        iov.iov_len = ret;
+        count = 1;
+    }
 
-        /* Send the msg */
-        ret = rpc_clnt_submit (&global_quotad_rpc, prog, procnum, cbkfn,
-                               &iov, count,
-                               NULL, 0, iobref, frame, NULL, 0, NULL, 0, NULL);
-        ret = 0;
+    /* Send the msg */
+    ret = rpc_clnt_submit(global_quotad_rpc, prog, procnum, cbkfn, &iov, count,
+                          NULL, 0, iobref, frame, NULL, 0, NULL, 0, NULL);
+    ret = 0;
 
 out:
-        if (new_iobref)
-                iobref_unref (iobref);
-        if (iobuf)
-                iobuf_unref (iobuf);
+    if (new_iobref)
+        iobref_unref(iobref);
+    if (iobuf)
+        iobuf_unref(iobuf);
 
-        return ret;
+    return ret;
 }
 
 int
-cli_quotad_notify (struct rpc_clnt *rpc, void *mydata,
-                   rpc_clnt_event_t event, void *data)
+cli_quotad_notify(struct rpc_clnt *rpc, void *mydata, rpc_clnt_event_t event,
+                  void *data)
 {
-        xlator_t                *this = NULL;
-        int                     ret = 0;
+    xlator_t *this = NULL;
+    int ret = 0;
 
-        this = mydata;
+    this = mydata;
 
-        switch (event) {
-        case RPC_CLNT_CONNECT:
-        {
-                gf_log (this->name, GF_LOG_TRACE, "got RPC_CLNT_CONNECT");
-                break;
+    switch (event) {
+        case RPC_CLNT_CONNECT: {
+            gf_log(this->name, GF_LOG_TRACE, "got RPC_CLNT_CONNECT");
+            break;
         }
 
-        case RPC_CLNT_DISCONNECT:
-        {
-                gf_log (this->name, GF_LOG_TRACE, "got RPC_CLNT_DISCONNECT");
-                break;
+        case RPC_CLNT_DISCONNECT: {
+            gf_log(this->name, GF_LOG_TRACE, "got RPC_CLNT_DISCONNECT");
+            break;
         }
 
         default:
-                gf_log (this->name, GF_LOG_TRACE,
-                        "got some other RPC event %d", event);
-                ret = 0;
-                break;
-        }
+            gf_log(this->name, GF_LOG_TRACE, "got some other RPC event %d",
+                   event);
+            ret = 0;
+            break;
+    }
 
-        return ret;
+    return ret;
 }
 
 struct rpc_clnt *
-cli_quotad_clnt_init (xlator_t *this, dict_t *options)
+cli_quotad_clnt_init(xlator_t *this, dict_t *options)
 {
-        struct rpc_clnt *rpc  = NULL;
-        int              ret  = -1;
-
-
-        ret = dict_set_str (options, "transport.address-family", "unix");
-        if (ret)
-                goto out;
-
-        ret = dict_set_str (options, "transport-type", "socket");
-        if (ret)
-                goto out;
-
-        ret = dict_set_str (options, "transport.socket.connect-path",
-                            "/tmp/quotad.socket");
-        if (ret)
-                goto out;
-
-        rpc = rpc_clnt_new (options, this->ctx, this->name, 16);
-        if (!rpc)
-                goto out;
-
-        ret = rpc_clnt_register_notify (rpc, cli_quotad_notify, this);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "failed to register notify");
-                goto out;
-        }
-
-        rpc_clnt_start (rpc);
+    struct rpc_clnt *rpc = NULL;
+    int ret = -1;
+
+    ret = dict_set_nstrn(options, "transport.address-family",
+                         SLEN("transport.address-family"), "unix",
+                         SLEN("unix"));
+    if (ret)
+        goto out;
+
+    ret = dict_set_nstrn(options, "transport-type", SLEN("transport-type"),
+                         "socket", SLEN("socket"));
+    if (ret)
+        goto out;
+
+    ret = dict_set_nstrn(options, "transport.socket.connect-path",
+                         SLEN("transport.socket.connect-path"),
+                         "/var/run/gluster/quotad.socket",
+                         SLEN("/var/run/gluster/quotad.socket"));
+    if (ret)
+        goto out;
+
+    rpc = rpc_clnt_new(options, this, this->name, 16);
+    if (!rpc)
+        goto out;
+
+    ret = rpc_clnt_register_notify(rpc, cli_quotad_notify, this);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "failed to register notify");
+        goto out;
+    }
+
+    rpc_clnt_start(rpc);
 out:
-        if (ret) {
-                if (rpc)
-                        rpc_clnt_unref (rpc);
-                rpc = NULL;
-        }
+    if (ret) {
+        if (rpc)
+            rpc_clnt_unref(rpc);
+        rpc = NULL;
+    }
 
-        return rpc;
+    return rpc;
 }
-
diff --git a/cli/src/cli-quotad-client.h b/cli/src/cli-quotad-client.h
index aa0b42af38d..71a44e5916b 100644
--- a/cli/src/cli-quotad-client.h
+++ b/cli/src/cli-quotad-client.h
@@ -8,26 +8,22 @@
    cases as published by the Free Software Foundation.
 */
 #include "cli.h"
-#include "compat-errno.h"
-#include "compat.h"
+#include <glusterfs/compat-errno.h>
+#include <glusterfs/compat.h>
 #include "cli-cmd.h"
 #include "cli1-xdr.h"
 #include "xdr-generic.h"
 #include "protocol-common.h"
 #include "cli-mem-types.h"
 
-
 int
-cli_quotad_submit_request (void *req, call_frame_t *frame,
-                           rpc_clnt_prog_t *prog,
-                           int procnum, struct iobref *iobref,
-                           xlator_t *this, fop_cbk_fn_t cbkfn,
-                           xdrproc_t xdrproc);
+cli_quotad_submit_request(void *req, call_frame_t *frame, rpc_clnt_prog_t *prog,
+                          int procnum, struct iobref *iobref, xlator_t *this,
+                          fop_cbk_fn_t cbkfn, xdrproc_t xdrproc);
 
 struct rpc_clnt *
-cli_quotad_clnt_init (xlator_t *this, dict_t *options);
+cli_quotad_clnt_init(xlator_t *this, dict_t *options);
 
 int
-cli_quotad_notify (struct rpc_clnt *rpc, void *mydata,
-                   rpc_clnt_event_t event, void *data);
-
+cli_quotad_notify(struct rpc_clnt *rpc, void *mydata, rpc_clnt_event_t event,
+                  void *data);
diff --git a/cli/src/cli-rl.c b/cli/src/cli-rl.c
index ade1c8ebb37..7a38a0b882a 100644
--- a/cli/src/cli-rl.c
+++ b/cli/src/cli-rl.c
@@ -13,16 +13,11 @@
 #include <stdint.h>
 #include <pthread.h>
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 #include "cli.h"
 #include "cli-cmd.h"
 #include "cli-mem-types.h"
 
-#include "event.h"
+#include <glusterfs/gf-event.h>
 
 #include <fnmatch.h>
 
@@ -32,384 +27,376 @@
 #include <readline/readline.h>
 #include <readline/history.h>
 
-
 int
-cli_rl_out (struct cli_state *state, const char *fmt, va_list ap)
+cli_rl_out(struct cli_state *state, const char *fmt, va_list ap)
 {
-        int tmp_rl_point = rl_point;
-        int            n = rl_end;
-        int            ret = 0;
+    int tmp_rl_point = rl_point;
+    int n = rl_end;
+    int ret = 0;
 
-        if (rl_end >= 0 ) {
-                rl_kill_text (0, rl_end);
-                rl_redisplay ();
-        }
+    if (rl_end >= 0) {
+        rl_kill_text(0, rl_end);
+        rl_redisplay();
+    }
 
-        printf ("\r%*s\r", (int)strlen (state->prompt), "");
+    printf("\r%*s\r", (int)strlen(state->prompt), "");
 
-        ret = vprintf (fmt, ap);
+    ret = vprintf(fmt, ap);
 
-        printf ("\n");
-        fflush(stdout);
+    printf("\n");
+    fflush(stdout);
 
-        if (n) {
-                rl_do_undo ();
-                rl_point = tmp_rl_point;
-                rl_reset_line_state ();
-        }
+    if (n) {
+        rl_do_undo();
+        rl_point = tmp_rl_point;
+        rl_reset_line_state();
+    }
 
-        return ret;
+    return ret;
 }
 
 int
-cli_rl_err (struct cli_state *state, const char *fmt, va_list ap)
+cli_rl_err(struct cli_state *state, const char *fmt, va_list ap)
 {
-        int tmp_rl_point = rl_point;
-        int            n = rl_end;
-        int            ret = 0;
+    int tmp_rl_point = rl_point;
+    int n = rl_end;
+    int ret = 0;
 
-        if (rl_end >= 0 ) {
-                rl_kill_text (0, rl_end);
-                rl_redisplay ();
-        }
+    if (rl_end >= 0) {
+        rl_kill_text(0, rl_end);
+        rl_redisplay();
+    }
 
-        fprintf (stderr, "\r%*s\r", (int)strlen (state->prompt), "");
+    fprintf(stderr, "\r%*s\r", (int)strlen(state->prompt), "");
 
-        ret = vfprintf (stderr, fmt, ap);
+    ret = vfprintf(stderr, fmt, ap);
 
-        fprintf (stderr, "\n");
-        fflush(stderr);
+    fprintf(stderr, "\n");
+    fflush(stderr);
 
-        if (n) {
-                rl_do_undo ();
-                rl_point = tmp_rl_point;
-                rl_reset_line_state ();
-        }
+    if (n) {
+        rl_do_undo();
+        rl_point = tmp_rl_point;
+        rl_reset_line_state();
+    }
 
-        return ret;
+    return ret;
 }
 
-
 void
-cli_rl_process_line (char *line)
+cli_rl_process_line(char *line)
 {
-        struct cli_state *state = NULL;
-        int               ret = 0;
+    struct cli_state *state = NULL;
+    int ret = 0;
 
-        state = global_state;
+    state = global_state;
 
-        state->rl_processing = 1;
-        {
-                ret = cli_cmd_process_line (state, line);
-                if (ret)
-                        gf_log (THIS->name, GF_LOG_WARNING,
-                                "failed to process line");
-
-                add_history (line);
-        }
-        state->rl_processing = 0;
+    state->rl_processing = 1;
+    {
+        ret = cli_cmd_process_line(state, line);
+        if (ret)
+            gf_log(THIS->name, GF_LOG_WARNING, "failed to process line");
 
+        add_history(line);
+    }
+    state->rl_processing = 0;
 }
 
-
-int
-cli_rl_stdin (int fd, int idx, void *data,
-              int poll_out, int poll_in, int poll_err)
+void
+cli_rl_stdin(int fd, int idx, int gen, void *data, int poll_out, int poll_in,
+             int poll_err, char event_thread_died)
 {
-        rl_callback_read_char ();
+    struct cli_state *state = NULL;
 
-        return 0;
-}
+    state = data;
 
+    rl_callback_read_char();
+
+    gf_event_handled(state->ctx->event_pool, fd, idx, gen);
+
+    return;
+}
 
 char *
-cli_rl_autocomplete_entry (const char *text, int times)
+cli_rl_autocomplete_entry(const char *text, int times)
 {
-        struct cli_state  *state = NULL;
-        char              *retp = NULL;
+    struct cli_state *state = NULL;
+    char *retp = NULL;
 
-        state = global_state;
+    state = global_state;
 
-        if (!state->matchesp)
-                return NULL;
+    if (!state->matchesp)
+        return NULL;
 
-        retp = *state->matchesp;
+    retp = *state->matchesp;
 
-        state->matchesp++;
+    state->matchesp++;
 
-        return retp ? strdup (retp) : NULL;
+    return retp ? strdup(retp) : NULL;
 }
 
-
 int
-cli_rl_token_count (const char *text)
+cli_rl_token_count(const char *text)
 {
-        int          count = 0;
-        const char  *trav = NULL;
-        int          is_spc = 1;
-
-        for (trav = text; *trav; trav++) {
-                if (*trav == ' ') {
-                        is_spc = 1;
-                } else {
-                        if (is_spc) {
-                                count++;
-                                is_spc = 0;
-                        }
-                }
+    int count = 0;
+    const char *trav = NULL;
+    int is_spc = 1;
+
+    for (trav = text; *trav; trav++) {
+        if (*trav == ' ') {
+            is_spc = 1;
+        } else {
+            if (is_spc) {
+                count++;
+                is_spc = 0;
+            }
         }
+    }
 
-        if (is_spc)
-                /* what needs to be autocompleted is a full
-                   new word, and not extend the last word
-                */
-                count++;
+    if (is_spc)
+        /* what needs to be autocompleted is a full
+           new word, and not extend the last word
+        */
+        count++;
 
-        return count;
+    return count;
 }
 
-
 char **
-cli_rl_tokenize (const char *text)
+cli_rl_tokenize(const char *text)
 {
-        int     count = 0;
-        char  **tokens = NULL;
-        char  **tokenp = NULL;
-        char   *token = NULL;
-        char   *copy = NULL;
-        char   *saveptr = NULL;
-        int     i = 0;
-
-        count = cli_rl_token_count (text);
-
-        tokens = calloc (count + 1, sizeof (*tokens));
-        if (!tokens)
-                return NULL;
-
-        copy = strdup (text);
-        if (!copy)
-                goto out;
-
-        tokenp = tokens;
-
-        for (token = strtok_r (copy, " \t\r\n", &saveptr); token;
-             token = strtok_r (NULL, " \t\r\n", &saveptr)) {
-                *tokenp = strdup (token);
-
-                if (!*tokenp)
-                        goto out;
-                tokenp++;
-                i++;
-
-        }
+    int count = 0;
+    char **tokens = NULL;
+    char **tokenp = NULL;
+    char *token = NULL;
+    char *copy = NULL;
+    char *saveptr = NULL;
+    int i = 0;
+
+    count = cli_rl_token_count(text);
+
+    tokens = calloc(count + 1, sizeof(*tokens));
+    if (!tokens)
+        return NULL;
 
-        if (i < count) {
-                /* symbolize that what needs to be autocompleted is
-                   the full set of possible nextwords, and not extend
-                   the last word
-                */
-                *tokenp = strdup ("");
-                if (!*tokenp)
-                        goto out;
-                tokenp++;
-                i++;
-        }
+    copy = strdup(text);
+    if (!copy)
+        goto out;
+
+    tokenp = tokens;
+
+    for (token = strtok_r(copy, " \t\r\n", &saveptr); token;
+         token = strtok_r(NULL, " \t\r\n", &saveptr)) {
+        *tokenp = strdup(token);
+
+        if (!*tokenp)
+            goto out;
+        tokenp++;
+        i++;
+    }
+
+    if (i < count) {
+        /* symbolize that what needs to be autocompleted is
+           the full set of possible nextwords, and not extend
+           the last word
+        */
+        *tokenp = strdup("");
+        if (!*tokenp)
+            goto out;
+        tokenp++;
+        i++;
+    }
 
 out:
-        free (copy);
+    free(copy);
 
-        if (i < count) {
-                cli_cmd_tokens_destroy (tokens);
-                tokens = NULL;
-        }
+    if (i < count) {
+        cli_cmd_tokens_destroy(tokens);
+        tokens = NULL;
+    }
 
-        return tokens;
+    return tokens;
 }
 
-
 char **
-cli_rl_get_matches (struct cli_state *state, struct cli_cmd_word *word,
-                    const char *text)
+cli_rl_get_matches(struct cli_state *state, struct cli_cmd_word *word,
+                   const char *text)
 {
-        char                 **matches = NULL;
-        char                 **matchesp = NULL;
-        struct cli_cmd_word  **next = NULL;
-        int                    count = 0;
-        int                    len = 0;
+    char **matches = NULL;
+    char **matchesp = NULL;
+    struct cli_cmd_word **next = NULL;
+    int count = 0;
+    int len = 0;
 
-        len = strlen (text);
+    len = strlen(text);
 
-        if (!word->nextwords)
-                return NULL;
+    if (!word->nextwords)
+        return NULL;
 
-        for (next = word->nextwords; *next; next++)
-                count++;
+    for (next = word->nextwords; *next; next++)
+        count++;
 
-        matches = calloc (count + 1, sizeof (*matches));
-        matchesp = matches;
+    matches = calloc(count + 1, sizeof(*matches));
+    matchesp = matches;
 
-        for (next = word->nextwords; *next; next++) {
-                if ((*next)->match) {
-                        continue;
-                }
+    for (next = word->nextwords; *next; next++) {
+        if ((*next)->match) {
+            continue;
+        }
 
-                if (strncmp ((*next)->word, text, len) == 0) {
-                        *matchesp = strdup ((*next)->word);
-                        matchesp++;
-                }
+        if (strncmp((*next)->word, text, len) == 0) {
+            *matchesp = strdup((*next)->word);
+            matchesp++;
         }
+    }
 
-        return matches;
+    return matches;
 }
 
-
 int
-cli_rl_autocomplete_prepare (struct cli_state *state, const char *text)
+cli_rl_autocomplete_prepare(struct cli_state *state, const char *text)
 {
-        struct cli_cmd_word   *word = NULL;
-        struct cli_cmd_word   *next = NULL;
-        char                 **tokens = NULL;
-        char                 **tokenp = NULL;
-        char                  *token = NULL;
-        char                 **matches = NULL;
-
-        tokens = cli_rl_tokenize (text);
-        if (!tokens)
-                return 0;
-
-        word = &state->tree.root;
-
-        for (tokenp = tokens; (token = *tokenp); tokenp++) {
-                if (!*(tokenp+1)) {
-                        /* last word */
-                        break;
-                }
-
-                next = cli_cmd_nextword (word, token);
-                word = next;
-                if (!word)
-                        break;
+    struct cli_cmd_word *word = NULL;
+    struct cli_cmd_word *next = NULL;
+    char **tokens = NULL;
+    char **tokenp = NULL;
+    char *token = NULL;
+    char **matches = NULL;
+
+    tokens = cli_rl_tokenize(text);
+    if (!tokens)
+        return 0;
+
+    word = &state->tree.root;
+
+    for (tokenp = tokens; (token = *tokenp); tokenp++) {
+        if (!*(tokenp + 1)) {
+            /* last word */
+            break;
         }
 
+        next = cli_cmd_nextword(word, token);
+        word = next;
         if (!word)
-                goto out;
+            break;
+    }
+
+    if (!word || !token)
+        goto out;
 
-        matches = cli_rl_get_matches (state, word, token);
+    matches = cli_rl_get_matches(state, word, token);
 
-        state->matches = matches;
-        state->matchesp = matches;
+    state->matches = matches;
+    state->matchesp = matches;
 
 out:
-        cli_cmd_tokens_destroy (tokens);
-        return 0;
+    cli_cmd_tokens_destroy(tokens);
+    return 0;
 }
 
-
 int
-cli_rl_autocomplete_cleanup (struct cli_state *state)
+cli_rl_autocomplete_cleanup(struct cli_state *state)
 {
-        if (state->matches)
-                cli_cmd_tokens_destroy (state->matches);
+    if (state->matches)
+        cli_cmd_tokens_destroy(state->matches);
 
-        state->matches = NULL;
-        state->matchesp = NULL;
+    state->matches = NULL;
+    state->matchesp = NULL;
 
-        return 0;
+    return 0;
 }
 
-
 char **
-cli_rl_autocomplete (const char *text, int start, int end)
+cli_rl_autocomplete(const char *text, int start, int end)
 {
-        struct cli_state  *state = NULL;
-        char             **matches = NULL;
-        char               save = 0;
+    struct cli_state *state = NULL;
+    char **matches = NULL;
+    char save = 0;
 
-        state = global_state;
+    state = global_state;
 
-        /* hack to make the autocompletion code neater */
-        /* fake it as though the cursor is at the end of line */
+    /* hack to make the autocompletion code neater */
+    /* fake it as though the cursor is at the end of line */
 
-        save = rl_line_buffer[rl_point];
-        rl_line_buffer[rl_point] = 0;
+    save = rl_line_buffer[rl_point];
+    rl_line_buffer[rl_point] = 0;
 
-        cli_rl_autocomplete_prepare (state, rl_line_buffer);
+    cli_rl_autocomplete_prepare(state, rl_line_buffer);
 
-        matches = rl_completion_matches (text, cli_rl_autocomplete_entry);
+    matches = rl_completion_matches(text, cli_rl_autocomplete_entry);
 
-        cli_rl_autocomplete_cleanup (state);
+    cli_rl_autocomplete_cleanup(state);
 
-        rl_line_buffer[rl_point] = save;
+    rl_line_buffer[rl_point] = save;
 
-        return matches;
+    return matches;
 }
 
-
 static char *
-complete_none (const char *txt, int times)
+complete_none(const char *txt, int times)
 {
-        return NULL;
+    return NULL;
 }
 
-
 void *
-cli_rl_input (void *_data)
+cli_rl_input(void *_data)
 {
-        struct cli_state *state = NULL;
-        char             *line = NULL;
+    struct cli_state *state = NULL;
+    char *line = NULL;
 
-        state = _data;
+    state = _data;
 
-        for (;;) {
-                line = readline (state->prompt);
-                if (!line)
-                        exit(0);  //break;
+    fprintf(stderr,
+            "Welcome to gluster prompt, type 'help' to see the available "
+            "commands.\n");
+    for (;;) {
+        line = readline(state->prompt);
+        if (!line)
+            exit(0);  // break;
 
-                if (*line)
-                        cli_rl_process_line (line);
+        if (*line)
+            cli_rl_process_line(line);
 
-                free (line);
-        }
+        free(line);
+    }
 
-        return NULL;
+    return NULL;
 }
 
-
 int
-cli_rl_enable (struct cli_state *state)
+cli_rl_enable(struct cli_state *state)
 {
-        int ret = 0;
-
-        rl_pre_input_hook = NULL;
-        rl_attempted_completion_function = cli_rl_autocomplete;
-        rl_completion_entry_function = complete_none;
-
-        if (!state->rl_async) {
-                ret = pthread_create (&state->input, NULL,
-                                      cli_rl_input, state);
-                if (ret == 0)
-                        state->rl_enabled = 1;
-                goto out;
-        }
+    int ret = 0;
 
-        ret = event_register (state->ctx->event_pool, 0, cli_rl_stdin, state,
-                              1, 0);
-        if (ret == -1)
-                goto out;
+    rl_pre_input_hook = NULL;
+    rl_attempted_completion_function = cli_rl_autocomplete;
+    rl_completion_entry_function = complete_none;
 
-        state->rl_enabled = 1;
-        rl_callback_handler_install (state->prompt, cli_rl_process_line);
+    if (!state->rl_async) {
+        ret = pthread_create(&state->input, NULL, cli_rl_input, state);
+        if (ret == 0)
+            state->rl_enabled = 1;
+        goto out;
+    }
+
+    ret = gf_event_register(state->ctx->event_pool, 0, cli_rl_stdin, state, 1,
+                            0, 0);
+    if (ret == -1)
+        goto out;
+
+    state->rl_enabled = 1;
+    rl_callback_handler_install(state->prompt, cli_rl_process_line);
 
 out:
-        return state->rl_enabled;
+    return state->rl_enabled;
 }
 
 #else /* HAVE_READLINE */
 
 int
-cli_rl_enable (struct cli_state *state)
+cli_rl_enable(struct cli_state *state)
 {
-        return 0;
+    return 0;
 }
 
 #endif /* HAVE_READLINE */
diff --git a/cli/src/cli-rpc-ops.c b/cli/src/cli-rpc-ops.c
index f194a76efb4..9b6b0c7fa50 100644
--- a/cli/src/cli-rpc-ops.c
+++ b/cli/src/cli-rpc-ops.c
@@ -8,68 +8,55 @@
    cases as published by the Free Software Foundation.
 */
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 /* Widths of various columns in top read/write-perf output
  * Total width of top read/write-perf should be 80 chars
  * including one space between column
  */
 #define VOL_TOP_PERF_FILENAME_DEF_WIDTH 47
 #define VOL_TOP_PERF_FILENAME_ALT_WIDTH 44
-#define VOL_TOP_PERF_SPEED_WIDTH        4
-#define VOL_TOP_PERF_TIME_WIDTH         26
+#define VOL_TOP_PERF_SPEED_WIDTH 4
+#define VOL_TOP_PERF_TIME_WIDTH 26
 
 #define INDENT_MAIN_HEAD "%-25s %s "
 
+#define RETURNING "Returning %d"
+#define XML_ERROR "Error outputting to xml"
+#define XDR_DECODE_FAIL "Failed to decode xdr response"
+#define DICT_SERIALIZE_FAIL "Failed to serialize to data to dictionary"
+#define DICT_UNSERIALIZE_FAIL "Failed to unserialize the dictionary"
+
+/* Do not show estimates if greater than this number */
+#define REBAL_ESTIMATE_SEC_UPPER_LIMIT (60 * 24 * 3600)
+#define REBAL_ESTIMATE_START_TIME 600
+
 #include "cli.h"
-#include "compat-errno.h"
+#include <glusterfs/compat-errno.h>
 #include "cli-cmd.h"
 #include <sys/uio.h>
 #include <stdlib.h>
 #include <sys/mount.h>
-#include "cli1-xdr.h"
-#include "xdr-generic.h"
-#include "protocol-common.h"
+#include <glusterfs/compat.h>
 #include "cli-mem-types.h"
-#include "compat.h"
-
-#include "syscall.h"
+#include <glusterfs/syscall.h>
 #include "glusterfs3.h"
 #include "portmap-xdr.h"
-#include "byte-order.h"
+#include <glusterfs/byte-order.h>
 
-#include "cli-quotad-client.h"
-#include "run.h"
+#include <glusterfs/run.h>
+#include <glusterfs/events.h>
 
-enum gf_task_types {
-        GF_TASK_TYPE_REBALANCE,
-        GF_TASK_TYPE_REMOVE_BRICK
-};
+enum gf_task_types { GF_TASK_TYPE_REBALANCE, GF_TASK_TYPE_REMOVE_BRICK };
 
-extern struct rpc_clnt *global_quotad_rpc;
-extern rpc_clnt_prog_t cli_quotad_clnt;
-extern rpc_clnt_prog_t *cli_rpc_prog;
-extern int              cli_op_ret;
-extern int              connected;
-
-char *cli_vol_type_str[] = {"Distribute",
-                            "Stripe",
-                            "Replicate",
-                            "Striped-Replicate",
-                            "Disperse",
-                            "Distributed-Stripe",
-                            "Distributed-Replicate",
-                            "Distributed-Striped-Replicate",
-                            "Distributed-Disperse",
-                           };
-
-char *cli_vol_status_str[] = {"Created",
-                              "Started",
-                              "Stopped",
-                             };
+rpc_clnt_prog_t cli_quotad_clnt;
+
+static int32_t
+gf_cli_remove_brick(call_frame_t *frame, xlator_t *this, void *data);
+
+char *cli_vol_status_str[] = {
+    "Created",
+    "Started",
+    "Stopped",
+};
 
 char *cli_vol_task_status_str[] = {"not started",
                                    "in progress",
@@ -80,7877 +67,8582 @@ char *cli_vol_task_status_str[] = {"not started",
                                    "fix-layout stopped",
                                    "fix-layout completed",
                                    "fix-layout failed",
-                                   "unknown"
-};
+                                   "unknown"};
 
-int32_t
-gf_cli_snapshot (call_frame_t *frame, xlator_t *this, void *data);
+static int32_t
+gf_cli_snapshot(call_frame_t *frame, xlator_t *this, void *data);
 
-int32_t
-gf_cli_get_volume (call_frame_t *frame, xlator_t *this,
-                      void *data);
+static int32_t
+gf_cli_get_volume(call_frame_t *frame, xlator_t *this, void *data);
 
-int
-cli_to_glusterd (gf_cli_req *req, call_frame_t *frame, fop_cbk_fn_t cbkfn,
-                 xdrproc_t xdrproc, dict_t *dict, int procnum, xlator_t *this,
-                 rpc_clnt_prog_t *prog, struct iobref *iobref);
-
-rpc_clnt_prog_t cli_handshake_prog = {
-        .progname  = "cli handshake",
-        .prognum   = GLUSTER_HNDSK_PROGRAM,
-        .progver   = GLUSTER_HNDSK_VERSION,
+static int
+cli_to_glusterd(gf_cli_req *req, call_frame_t *frame, fop_cbk_fn_t cbkfn,
+                xdrproc_t xdrproc, dict_t *dict, int procnum, xlator_t *this,
+                rpc_clnt_prog_t *prog, struct iobref *iobref);
+
+static int
+add_cli_cmd_timeout_to_dict(dict_t *dict);
+
+static rpc_clnt_prog_t cli_handshake_prog = {
+    .progname = "cli handshake",
+    .prognum = GLUSTER_HNDSK_PROGRAM,
+    .progver = GLUSTER_HNDSK_VERSION,
 };
 
-rpc_clnt_prog_t cli_pmap_prog = {
-        .progname   = "cli portmap",
-        .prognum    = GLUSTER_PMAP_PROGRAM,
-        .progver    = GLUSTER_PMAP_VERSION,
+static rpc_clnt_prog_t cli_pmap_prog = {
+    .progname = "cli portmap",
+    .prognum = GLUSTER_PMAP_PROGRAM,
+    .progver = GLUSTER_PMAP_VERSION,
 };
 
-int
-gf_cli_probe_cbk (struct rpc_req *req, struct iovec *iov,
-                        int count, void *myframe)
+static void
+gf_free_xdr_cli_rsp(gf_cli_rsp rsp)
 {
-        gf_cli_rsp            rsp   = {0,};
-        int                   ret   = -1;
-        char                  msg[1024] = {0,};
-
-        if (-1 == req->rpc_status) {
-                goto out;
-        }
+    if (rsp.dict.dict_val) {
+        free(rsp.dict.dict_val);
+    }
+    if (rsp.op_errstr) {
+        free(rsp.op_errstr);
+    }
+}
 
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
-        if (ret < 0) {
-                gf_log (((call_frame_t *) myframe)->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                //rsp.op_ret   = -1;
-                //rsp.op_errno = EINVAL;
-                goto out;
-        }
+static void
+gf_free_xdr_getspec_rsp(gf_getspec_rsp rsp)
+{
+    if (rsp.spec) {
+        free(rsp.spec);
+    }
+    if (rsp.xdata.xdata_val) {
+        free(rsp.xdata.xdata_val);
+    }
+}
 
-        gf_log ("cli", GF_LOG_INFO, "Received resp to probe");
+static void
+gf_free_xdr_fsm_log_rsp(gf1_cli_fsm_log_rsp rsp)
+{
+    if (rsp.op_errstr) {
+        free(rsp.op_errstr);
+    }
+    if (rsp.fsm_log.fsm_log_val) {
+        free(rsp.fsm_log.fsm_log_val);
+    }
+}
 
-        if (rsp.op_errstr && (strlen (rsp.op_errstr) > 0)) {
-                snprintf (msg, sizeof (msg), "%s", rsp.op_errstr);
-                if (rsp.op_ret)
-                        gf_log ("cli", GF_LOG_ERROR, "%s", msg);
+static int
+gf_cli_probe_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                 void *myframe)
+{
+    gf_cli_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    char msg[1024] = "success";
+
+    GF_ASSERT(myframe);
+
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,
+               XDR_DECODE_FAIL);
+        // rsp.op_ret   = -1;
+        // rsp.op_errno = EINVAL;
+        goto out;
+    }
+
+    gf_log("cli", GF_LOG_INFO, "Received resp to probe");
+
+    if (rsp.op_errstr && rsp.op_errstr[0] != '\0') {
+        snprintf(msg, sizeof(msg), "%s", rsp.op_errstr);
+        if (rsp.op_ret) {
+            gf_log("cli", GF_LOG_ERROR, "%s", msg);
         }
+    }
 
-        if (global_state->mode & GLUSTER_MODE_XML) {
-                ret = cli_xml_output_str (NULL,
-                                          (rsp.op_ret)? NULL : msg,
-                                          rsp.op_ret, rsp.op_errno,
-                                          (rsp.op_ret)? msg : NULL);
-                if (ret)
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Error outputting to xml");
-                goto out;
-        }
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        ret = cli_xml_output_str(NULL, (rsp.op_ret) ? NULL : msg, rsp.op_ret,
+                                 rsp.op_errno, (rsp.op_ret) ? msg : NULL);
+        if (ret)
+            gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+        goto out;
+    }
 
-        if (!rsp.op_ret)
-                cli_out ("peer probe: success. %s", msg);
-        else
-                cli_err ("peer probe: failed: %s", msg);
+    if (!rsp.op_ret)
+        cli_out("peer probe: %s", msg);
+    else
+        cli_err("peer probe: failed: %s", msg);
 
-        ret = rsp.op_ret;
+    ret = rsp.op_ret;
 
 out:
-        cli_cmd_broadcast_response (ret);
-        return ret;
+    cli_cmd_broadcast_response(ret);
+    gf_free_xdr_cli_rsp(rsp);
+    return ret;
 }
 
-int
-gf_cli_deprobe_cbk (struct rpc_req *req, struct iovec *iov,
-                       int count, void *myframe)
+static int
+gf_cli_deprobe_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                   void *myframe)
 {
-        gf_cli_rsp            rsp   = {0,};
-        int                   ret   = -1;
-        char              msg[1024] = {0,};
-
-        if (-1 == req->rpc_status) {
-                goto out;
-        }
-
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
-        if (ret < 0) {
-                gf_log (((call_frame_t *) myframe)->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                //rsp.op_ret   = -1;
-                //rsp.op_errno = EINVAL;
-                goto out;
-        }
-
-        gf_log ("cli", GF_LOG_INFO, "Received resp to deprobe");
-
-        if (rsp.op_ret) {
-                if (strlen (rsp.op_errstr) > 0) {
-                        snprintf (msg, sizeof (msg), "%s", rsp.op_errstr);
-                        gf_log ("cli", GF_LOG_ERROR, "%s", rsp.op_errstr);
-                }
-        } else {
-                snprintf (msg, sizeof (msg), "success");
-        }
-
-        if (global_state->mode & GLUSTER_MODE_XML) {
-                ret = cli_xml_output_str (NULL,
-                                          (rsp.op_ret)? NULL : msg,
-                                          rsp.op_ret, rsp.op_errno,
-                                          (rsp.op_ret)? msg : NULL);
-                if (ret)
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Error outputting to xml");
-                goto out;
-        }
+    gf_cli_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    char msg[1024] = "success";
+
+    GF_ASSERT(myframe);
+
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,
+               XDR_DECODE_FAIL);
+        // rsp.op_ret   = -1;
+        // rsp.op_errno = EINVAL;
+        goto out;
+    }
+
+    gf_log("cli", GF_LOG_INFO, "Received resp to deprobe");
+
+    if (rsp.op_ret) {
+        if (rsp.op_errstr[0] != '\0') {
+            snprintf(msg, sizeof(msg), "%s", rsp.op_errstr);
+            gf_log("cli", GF_LOG_ERROR, "%s", rsp.op_errstr);
+        }
+    }
+
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        ret = cli_xml_output_str(NULL, (rsp.op_ret) ? NULL : msg, rsp.op_ret,
+                                 rsp.op_errno, (rsp.op_ret) ? msg : NULL);
+        if (ret)
+            gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+        goto out;
+    }
 
-        if (!rsp.op_ret)
-                cli_out ("peer detach: %s", msg);
-        else
-                cli_err ("peer detach: failed: %s", msg);
+    if (!rsp.op_ret)
+        cli_out("peer detach: %s", msg);
+    else
+        cli_err("peer detach: failed: %s", msg);
 
-        ret = rsp.op_ret;
+    ret = rsp.op_ret;
 
 out:
-        cli_cmd_broadcast_response (ret);
-        return ret;
+    cli_cmd_broadcast_response(ret);
+    gf_free_xdr_cli_rsp(rsp);
+    return ret;
 }
 
-int
-gf_cli_output_peer_hostnames (dict_t *dict, int count, char *prefix)
+static int
+gf_cli_output_peer_hostnames(dict_t *dict, int count, const char *prefix)
 {
-        int ret = -1;
-        char key[256] = {0,};
-        int i = 0;
-        char *hostname = NULL;
-
-        cli_out ("Other names:");
-        /* Starting from friend.hostname1, as friend.hostname0 will be the same
-         * as friend.hostname
-         */
-        for (i = 1; i < count; i++) {
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%s.hostname%d", prefix, i);
-                ret = dict_get_str (dict, key, &hostname);
-                if (ret)
-                        break;
-                cli_out ("%s", hostname);
-                hostname = NULL;
-        }
+    int ret = -1;
+    char key[512] = {
+        0,
+    };
+    int i = 0;
+    char *hostname = NULL;
+
+    cli_out("Other names:");
+    /* Starting from friend.hostname1, as friend.hostname0 will be the same
+     * as friend.hostname
+     */
+    for (i = 1; i < count; i++) {
+        ret = snprintf(key, sizeof(key), "%s.hostname%d", prefix, i);
+        ret = dict_get_strn(dict, key, ret, &hostname);
+        if (ret)
+            break;
+        cli_out("%s", hostname);
+        hostname = NULL;
+    }
 
-        return ret;
+    return ret;
 }
 
-int
-gf_cli_output_peer_status (dict_t *dict, int count)
-{
-        int                        ret   = -1;
-        char                       *uuid_buf = NULL;
-        char                       *hostname_buf = NULL;
-        int32_t                    i = 1;
-        char                       key[256] = {0,};
-        char                       *state = NULL;
-        int32_t                    connected = 0;
-        char                       *connected_str = NULL;
-        int                        hostname_count = 0;
-
-        cli_out ("Number of Peers: %d", count);
-        i = 1;
-        while ( i <= count) {
-                snprintf (key, 256, "friend%d.uuid", i);
-                ret = dict_get_str (dict, key, &uuid_buf);
-                if (ret)
-                        goto out;
-
-                snprintf (key, 256, "friend%d.hostname", i);
-                ret = dict_get_str (dict, key, &hostname_buf);
-                if (ret)
-                        goto out;
+static int
+gf_cli_output_peer_status(dict_t *dict, int count)
+{
+    int ret = -1;
+    char *uuid_buf = NULL;
+    char *hostname_buf = NULL;
+    int32_t i = 1;
+    char key[256] = {
+        0,
+    };
+    int keylen;
+    char *state = NULL;
+    int32_t connected = 0;
+    const char *connected_str = NULL;
+    int hostname_count = 0;
+
+    cli_out("Number of Peers: %d", count);
+    i = 1;
+    while (i <= count) {
+        keylen = snprintf(key, sizeof(key), "friend%d.uuid", i);
+        ret = dict_get_strn(dict, key, keylen, &uuid_buf);
+        if (ret)
+            goto out;
 
-                snprintf (key, 256, "friend%d.connected", i);
-                ret = dict_get_int32 (dict, key, &connected);
-                if (ret)
-                        goto out;
-                if (connected)
-                        connected_str = "Connected";
-                else
-                        connected_str = "Disconnected";
+        keylen = snprintf(key, sizeof(key), "friend%d.hostname", i);
+        ret = dict_get_strn(dict, key, keylen, &hostname_buf);
+        if (ret)
+            goto out;
 
+        keylen = snprintf(key, sizeof(key), "friend%d.connected", i);
+        ret = dict_get_int32n(dict, key, keylen, &connected);
+        if (ret)
+            goto out;
+        if (connected)
+            connected_str = "Connected";
+        else
+            connected_str = "Disconnected";
 
-                snprintf (key, 256, "friend%d.state", i);
-                ret = dict_get_str (dict, key, &state);
-                if (ret)
-                        goto out;
+        keylen = snprintf(key, sizeof(key), "friend%d.state", i);
+        ret = dict_get_strn(dict, key, keylen, &state);
+        if (ret)
+            goto out;
 
-                cli_out ("\nHostname: %s\nUuid: %s\nState: %s (%s)",
-                         hostname_buf, uuid_buf, state, connected_str);
+        cli_out("\nHostname: %s\nUuid: %s\nState: %s (%s)", hostname_buf,
+                uuid_buf, state, connected_str);
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "friend%d.hostname_count", i);
-                ret = dict_get_int32 (dict, key, &hostname_count);
-                /* Print other addresses only if there are more than 1.
-                 */
-                if ((ret == 0) && (hostname_count > 1)) {
-                        memset (key, 0, sizeof (key));
-                        snprintf (key, sizeof (key), "friend%d", i);
-                        ret = gf_cli_output_peer_hostnames (dict,
-                                                            hostname_count,
-                                                            key);
-                }
-                i++;
+        keylen = snprintf(key, sizeof(key), "friend%d.hostname_count", i);
+        ret = dict_get_int32n(dict, key, keylen, &hostname_count);
+        /* Print other addresses only if there are more than 1.
+         */
+        if ((ret == 0) && (hostname_count > 1)) {
+            snprintf(key, sizeof(key), "friend%d", i);
+            ret = gf_cli_output_peer_hostnames(dict, hostname_count, key);
+            if (ret) {
+                gf_log("cli", GF_LOG_WARNING,
+                       "error outputting peer other names");
+                goto out;
+            }
         }
+        i++;
+    }
 
-        ret = 0;
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
-int
-gf_cli_output_pool_list (dict_t *dict, int count)
-{
-        int                         ret           = -1;
-        char                       *uuid_buf      = NULL;
-        char                       *hostname_buf  = NULL;
-        int32_t                     hostname_len  = 8; /*min len 8 chars*/
-        int32_t                     i             = 1;
-        char                        key[256]      = {0,};
-        int32_t                     connected     = 0;
-        char                       *connected_str = NULL;
-
-        if (count <= 0)
-                goto out;
-
-        while (i <= count) {
-                snprintf (key, 256, "friend%d.hostname", i);
-                ret = dict_get_str (dict, key, &hostname_buf);
-                if (ret)
-                        goto out;
+static int
+gf_cli_output_pool_list(dict_t *dict, int count)
+{
+    int ret = -1;
+    char *uuid_buf = NULL;
+    char *hostname_buf = NULL;
+    int32_t hostname_len = 8; /*min len 8 chars*/
+    int32_t i = 1;
+    char key[64] = {
+        0,
+    };
+    int keylen;
+    int32_t connected = 0;
+    const char *connected_str = NULL;
+
+    if (count <= 0)
+        goto out;
+
+    while (i <= count) {
+        keylen = snprintf(key, sizeof(key), "friend%d.hostname", i);
+        ret = dict_get_strn(dict, key, keylen, &hostname_buf);
+        if (ret)
+            goto out;
 
-                ret = strlen(hostname_buf);
-                if (ret > hostname_len)
-                        hostname_len = ret;
+        ret = strlen(hostname_buf);
+        if (ret > hostname_len)
+            hostname_len = ret;
 
-                i++;
-        }
+        i++;
+    }
 
-        cli_out ("UUID\t\t\t\t\t%-*s\tState", hostname_len, "Hostname");
+    cli_out("UUID\t\t\t\t\t%-*s\tState", hostname_len, "Hostname");
 
-        i = 1;
-        while ( i <= count) {
-                snprintf (key, 256, "friend%d.uuid", i);
-                ret = dict_get_str (dict, key, &uuid_buf);
-                if (ret)
-                        goto out;
+    i = 1;
+    while (i <= count) {
+        keylen = snprintf(key, sizeof(key), "friend%d.uuid", i);
+        ret = dict_get_strn(dict, key, keylen, &uuid_buf);
+        if (ret)
+            goto out;
 
-                snprintf (key, 256, "friend%d.hostname", i);
-                ret = dict_get_str (dict, key, &hostname_buf);
-                if (ret)
-                        goto out;
+        keylen = snprintf(key, sizeof(key), "friend%d.hostname", i);
+        ret = dict_get_strn(dict, key, keylen, &hostname_buf);
+        if (ret)
+            goto out;
 
-                snprintf (key, 256, "friend%d.connected", i);
-                ret = dict_get_int32 (dict, key, &connected);
-                if (ret)
-                        goto out;
-                if (connected)
-                        connected_str = "Connected";
-                else
-                        connected_str = "Disconnected";
+        keylen = snprintf(key, sizeof(key), "friend%d.connected", i);
+        ret = dict_get_int32n(dict, key, keylen, &connected);
+        if (ret)
+            goto out;
+        if (connected)
+            connected_str = "Connected";
+        else
+            connected_str = "Disconnected";
 
-                cli_out ("%s\t%-*s\t%s ", uuid_buf, hostname_len, hostname_buf,
-                         connected_str);
-                i++;
-        }
+        cli_out("%s\t%-*s\t%s ", uuid_buf, hostname_len, hostname_buf,
+                connected_str);
+        i++;
+    }
 
-        ret = 0;
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 /* function pointer for gf_cli_output_{pool_list,peer_status} */
-typedef int (*cli_friend_output_fn) (dict_t*, int);
+typedef int (*cli_friend_output_fn)(dict_t *, int);
 
-int
-gf_cli_list_friends_cbk (struct rpc_req *req, struct iovec *iov,
-                             int count, void *myframe)
-{
-        gf1_cli_peer_list_rsp      rsp   = {0,};
-        int                        ret   = -1;
-        dict_t                    *dict = NULL;
-        char                       msg[1024] = {0,};
-        char                      *cmd = NULL;
-        cli_friend_output_fn       friend_output_fn;
-        call_frame_t              *frame = NULL;
-        unsigned long              flags = 0;
-
-        frame = myframe;
-        flags = (long)frame->local;
-
-        if (flags == GF_CLI_LIST_POOL_NODES) {
-                cmd = "pool list";
-                friend_output_fn = &gf_cli_output_pool_list;
-        } else {
-                cmd = "peer status";
-                friend_output_fn = &gf_cli_output_peer_status;
+static int
+gf_cli_list_friends_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                        void *myframe)
+{
+    gf1_cli_peer_list_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    dict_t *dict = NULL;
+    char msg[1024] = {
+        0,
+    };
+    const char *cmd = NULL;
+    cli_friend_output_fn friend_output_fn;
+    call_frame_t *frame = NULL;
+    unsigned long flags = 0;
+
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
+
+    GF_ASSERT(myframe);
+
+    frame = myframe;
+
+    flags = (long)frame->local;
+
+    if (flags == GF_CLI_LIST_POOL_NODES) {
+        cmd = "pool list";
+        friend_output_fn = &gf_cli_output_pool_list;
+    } else {
+        cmd = "peer status";
+        friend_output_fn = &gf_cli_output_peer_status;
+    }
+
+    /* 'free' the flags set by gf_cli_list_friends */
+    frame->local = NULL;
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf1_cli_peer_list_rsp);
+    if (ret < 0) {
+        gf_log(frame->this->name, GF_LOG_ERROR, XDR_DECODE_FAIL);
+        // rsp.op_ret   = -1;
+        // rsp.op_errno = EINVAL;
+        goto out;
+    }
+
+    gf_log("cli", GF_LOG_DEBUG, "Received resp to list: %d", rsp.op_ret);
+
+    if (!rsp.op_ret) {
+        if (!rsp.friends.friends_len) {
+            snprintf(msg, sizeof(msg), "%s: No peers present", cmd);
+            if (global_state->mode & GLUSTER_MODE_XML) {
+                ret = cli_xml_output_peer_status(dict, rsp.op_ret, rsp.op_errno,
+                                                 msg);
+                if (ret)
+                    gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+                goto out;
+            }
+            cli_err("%s", msg);
+            ret = 0;
+            goto out;
         }
 
-        /* 'free' the flags set by gf_cli_list_friends */
-        frame->local = NULL;
+        dict = dict_new();
 
-        if (-1 == req->rpc_status) {
-                goto out;
+        if (!dict) {
+            ret = -1;
+            goto out;
         }
 
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf1_cli_peer_list_rsp);
-        if (ret < 0) {
-                gf_log (((call_frame_t *) myframe)->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                //rsp.op_ret   = -1;
-                //rsp.op_errno = EINVAL;
-                goto out;
+        ret = dict_unserialize(rsp.friends.friends_val, rsp.friends.friends_len,
+                               &dict);
+
+        if (ret) {
+            gf_log("", GF_LOG_ERROR, DICT_UNSERIALIZE_FAIL);
+            goto out;
         }
 
-        gf_log ("cli", GF_LOG_DEBUG, "Received resp to list: %d",
-                rsp.op_ret);
+        if (global_state->mode & GLUSTER_MODE_XML) {
+            ret = cli_xml_output_peer_status(dict, rsp.op_ret, rsp.op_errno,
+                                             msg);
+            if (ret)
+                gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+            goto out;
+        }
 
-        ret = rsp.op_ret;
+        ret = dict_get_int32_sizen(dict, "count", &count);
+        if (ret) {
+            goto out;
+        }
 
-        if (!rsp.op_ret) {
-
-                if (!rsp.friends.friends_len) {
-                        snprintf (msg, sizeof (msg),
-                                  "%s: No peers present", cmd);
-                        if (global_state->mode & GLUSTER_MODE_XML) {
-                                ret = cli_xml_output_peer_status (dict,
-                                                                  rsp.op_ret,
-                                                                  rsp.op_errno,
-                                                                  msg);
-                                if (ret)
-                                        gf_log ("cli", GF_LOG_ERROR,
-                                                "Error outputting to xml");
-                                goto out;
-                        }
-                        cli_err ("%s", msg);
-                        ret = 0;
-                        goto out;
-                }
+        ret = friend_output_fn(dict, count);
+        if (ret) {
+            goto out;
+        }
+    } else {
+        if (global_state->mode & GLUSTER_MODE_XML) {
+            ret = cli_xml_output_peer_status(dict, rsp.op_ret, rsp.op_errno,
+                                             NULL);
+            if (ret)
+                gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+        } else {
+            ret = -1;
+        }
+        goto out;
+    }
 
-                dict = dict_new ();
+    ret = 0;
 
-                if (!dict) {
-                        ret = -1;
-                        goto out;
-                }
+out:
+    if (ret)
+        cli_err("%s: failed", cmd);
 
-                ret = dict_unserialize (rsp.friends.friends_val,
-                                        rsp.friends.friends_len,
-                                        &dict);
+    cli_cmd_broadcast_response(ret);
 
-                if (ret) {
-                        gf_log ("", GF_LOG_ERROR,
-                                        "Unable to allocate memory");
-                        goto out;
-                }
+    if (dict)
+        dict_unref(dict);
 
-                if (global_state->mode & GLUSTER_MODE_XML) {
-                        ret = cli_xml_output_peer_status (dict, rsp.op_ret,
-                                                          rsp.op_errno, msg);
-                        if (ret)
-                                gf_log ("cli", GF_LOG_ERROR,
-                                        "Error outputting to xml");
-                        goto out;
-                }
+    if (rsp.friends.friends_val) {
+        free(rsp.friends.friends_val);
+    }
+    return ret;
+}
 
-                ret = dict_get_int32 (dict, "count", &count);
-                if (ret) {
-                        goto out;
-                }
+static int
+gf_cli_get_state_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                     void *myframe)
+{
+    gf_cli_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    dict_t *dict = NULL;
+    char *daemon_name = NULL;
+    char *ofilepath = NULL;
+
+    GF_VALIDATE_OR_GOTO("cli", myframe, out);
+
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,
+               XDR_DECODE_FAIL);
+        goto out;
+    }
+
+    dict = dict_new();
+
+    if (!dict) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len, &dict);
+    if (ret)
+        goto out;
+
+    if (rsp.op_ret) {
+        if (strcmp(rsp.op_errstr, ""))
+            cli_err("Failed to get daemon state: %s", rsp.op_errstr);
+        else
+            cli_err(
+                "Failed to get daemon state. Check glusterd"
+                " log file for more details");
+    } else {
+        ret = dict_get_str_sizen(dict, "daemon", &daemon_name);
+        if (ret)
+            gf_log("cli", GF_LOG_ERROR, "Couldn't get daemon name");
 
-                ret = friend_output_fn (dict, count);
-                if (ret) {
-                        goto out;
-                }
-        } else {
-                if (global_state->mode & GLUSTER_MODE_XML) {
-                        ret = cli_xml_output_peer_status (dict, rsp.op_ret,
-                                                          rsp.op_errno, NULL);
-                        if (ret)
-                                gf_log ("cli", GF_LOG_ERROR,
-                                        "Error outputting to xml");
-                } else {
-                        ret = -1;
-                }
-                goto out;
-        }
+        ret = dict_get_str_sizen(dict, "ofilepath", &ofilepath);
+        if (ret)
+            gf_log("cli", GF_LOG_ERROR, "Couldn't get filepath");
 
+        if (daemon_name && ofilepath)
+            cli_out("%s state dumped to %s", daemon_name, ofilepath);
+    }
 
-        ret = 0;
+    ret = rsp.op_ret;
 
 out:
-        cli_cmd_broadcast_response (ret);
-        if (ret)
-                cli_err ("%s: failed", cmd);
+    if (dict)
+        dict_unref(dict);
 
-        if (dict)
-                dict_destroy (dict);
+    cli_cmd_broadcast_response(ret);
+    gf_free_xdr_cli_rsp(rsp);
 
-        return ret;
+    return ret;
 }
 
-void
-cli_out_options ( char *substr, char *optstr, char *valstr)
+static void
+cli_out_options(char *substr, char *optstr, char *valstr)
 {
-        char                    *ptr1 = NULL;
-        char                    *ptr2 = NULL;
+    char *ptr1 = NULL;
+    char *ptr2 = NULL;
+
+    ptr1 = substr;
+    ptr2 = optstr;
+
+    while (ptr1) {
+        /* Avoiding segmentation fault. */
+        if (!ptr2)
+            return;
+        if (*ptr1 != *ptr2)
+            break;
+        ptr1++;
+        ptr2++;
+    }
+
+    if (*ptr2 == '\0')
+        return;
+    cli_out("%s: %s", ptr2, valstr);
+}
 
-        ptr1 = substr;
-        ptr2 = optstr;
+static int
+_gf_cli_output_volinfo_opts(dict_t *d, char *k, data_t *v, void *tmp)
+{
+    int ret = 0;
+    char *key = NULL;
+    char *ptr = NULL;
+    data_t *value = NULL;
+
+    key = tmp;
+
+    ptr = strstr(k, "option.");
+    if (ptr) {
+        value = v;
+        if (!value) {
+            ret = -1;
+            goto out;
+        }
+        cli_out_options(key, k, v->data);
+    }
+out:
+    return ret;
+}
 
-        while (ptr1)
-        {
-                if (*ptr1 != *ptr2)
-                        break;
-                ptr1++;
-                ptr2++;
-                if (!ptr1)
-                        return;
-                if (!ptr2)
-                        return;
-        }
+static int
+print_brick_details(dict_t *dict, int volcount, int start_index, int end_index,
+                    int replica_count)
+{
+    char key[64] = {
+        0,
+    };
+    int keylen;
+    int index = start_index;
+    int isArbiter = 0;
+    int ret = -1;
+    char *brick = NULL;
+
+    while (index <= end_index) {
+        keylen = snprintf(key, sizeof(key), "volume%d.brick%d", volcount,
+                          index);
+        ret = dict_get_strn(dict, key, keylen, &brick);
+        if (ret)
+            goto out;
+        keylen = snprintf(key, sizeof(key), "volume%d.brick%d.isArbiter",
+                          volcount, index);
+        if (dict_getn(dict, key, keylen))
+            isArbiter = 1;
+        else
+            isArbiter = 0;
 
-        if (*ptr2 == '\0')
-                return;
-        cli_out ("%s: %s",ptr2 , valstr);
+        if (isArbiter)
+            cli_out("Brick%d: %s (arbiter)", index, brick);
+        else
+            cli_out("Brick%d: %s", index, brick);
+        index++;
+    }
+    ret = 0;
+out:
+    return ret;
 }
 
-static int
-_gf_cli_output_volinfo_opts (dict_t *d, char *k,
-                             data_t *v, void *tmp)
-{
-        int     ret   = 0;
-        char   *key   = NULL;
-        char   *ptr   = NULL;
-        data_t *value = NULL;
-
-        key = tmp;
-
-        ptr = strstr (k, "option.");
-        if (ptr) {
-                value = v;
-                if (!value) {
-                        ret = -1;
-                        goto out;
-                }
-                cli_out_options (key, k, v->data);
+static void
+gf_cli_print_number_of_bricks(int type, int brick_count, int dist_count,
+                              int stripe_count, int replica_count,
+                              int disperse_count, int redundancy_count,
+                              int arbiter_count)
+{
+    if (type == GF_CLUSTER_TYPE_NONE) {
+        cli_out("Number of Bricks: %d", brick_count);
+    } else if (type == GF_CLUSTER_TYPE_DISPERSE) {
+        cli_out("Number of Bricks: %d x (%d + %d) = %d",
+                (brick_count / dist_count), disperse_count - redundancy_count,
+                redundancy_count, brick_count);
+    } else {
+        /* For both replicate and stripe, dist_count is
+           good enough */
+        if (arbiter_count == 0) {
+            cli_out("Number of Bricks: %d x %d = %d",
+                    (brick_count / dist_count), dist_count, brick_count);
+        } else {
+            cli_out("Number of Bricks: %d x (%d + %d) = %d",
+                    (brick_count / dist_count), dist_count - arbiter_count,
+                    arbiter_count, brick_count);
         }
-out:
-        return ret;
+    }
 }
 
+static int
+gf_cli_get_volume_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                      void *myframe)
+{
+    int ret = -1;
+    int opt_count = 0;
+    int32_t i = 0;
+    int32_t j = 1;
+    int32_t status = 0;
+    int32_t type = 0;
+    int32_t brick_count = 0;
+    int32_t dist_count = 0;
+    int32_t stripe_count = 0;
+    int32_t replica_count = 0;
+    int32_t disperse_count = 0;
+    int32_t redundancy_count = 0;
+    int32_t arbiter_count = 0;
+    int32_t snap_count = 0;
+    int32_t thin_arbiter_count = 0;
+    int32_t vol_type = 0;
+    int32_t transport = 0;
+    char *volume_id_str = NULL;
+    char *volname = NULL;
+    char *ta_brick = NULL;
+    dict_t *dict = NULL;
+    cli_local_t *local = NULL;
+    char key[64] = {0};
+    int keylen;
+    char err_str[2048] = {0};
+    gf_cli_rsp rsp = {0};
+    char *caps __attribute__((unused)) = NULL;
+    int k __attribute__((unused)) = 0;
+    call_frame_t *frame = NULL;
+
+    GF_ASSERT(myframe);
+
+    if (-1 == req->rpc_status)
+        goto out;
+
+    frame = myframe;
+
+    GF_ASSERT(frame->local);
+
+    local = frame->local;
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(frame->this->name, GF_LOG_ERROR, XDR_DECODE_FAIL);
+        goto out;
+    }
+
+    gf_log("cli", GF_LOG_INFO, "Received resp to get vol: %d", rsp.op_ret);
+
+    if (!rsp.dict.dict_len) {
+        if (global_state->mode & GLUSTER_MODE_XML)
+            goto xml_output;
+        cli_err("No volumes present");
+        ret = 0;
+        goto out;
+    }
+
+    dict = dict_new();
 
-int
-gf_cli_get_volume_cbk (struct rpc_req *req, struct iovec *iov,
-                          int count, void *myframe)
-{
-        int                        ret                  = -1;
-        int                        opt_count            = 0;
-        int32_t                    i                    = 0;
-        int32_t                    j                    = 1;
-        int32_t                    status               = 0;
-        int32_t                    type                 = 0;
-        int32_t                    brick_count          = 0;
-        int32_t                    dist_count           = 0;
-        int32_t                    stripe_count         = 0;
-        int32_t                    replica_count        = 0;
-        int32_t                    disperse_count       = 0;
-        int32_t                    redundancy_count     = 0;
-        int32_t                    vol_type             = 0;
-        int32_t                    transport            = 0;
-        char                      *volume_id_str        = NULL;
-        char                      *brick                = NULL;
-        char                      *volname              = NULL;
-        dict_t                    *dict                 = NULL;
-        cli_local_t               *local                = NULL;
-        char                       key[1024]            = {0};
-        char                       err_str[2048]        = {0};
-        gf_cli_rsp                 rsp                  = {0};
-        char                      *caps                 = NULL;
-        int                        k __attribute__((unused)) = 0;
-
-        if (-1 == req->rpc_status)
-                goto out;
+    if (!dict) {
+        ret = -1;
+        goto out;
+    }
 
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
-        if (ret < 0) {
-                gf_log (((call_frame_t *) myframe)->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                goto out;
-        }
+    ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len, &dict);
 
-        gf_log ("cli", GF_LOG_INFO, "Received resp to get vol: %d",
-                rsp.op_ret);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, DICT_UNSERIALIZE_FAIL);
+        goto out;
+    }
 
-        if (rsp.op_ret) {
-                ret = -1;
-                goto out;
-        }
+    ret = dict_get_int32_sizen(dict, "count", &count);
+    if (ret)
+        goto out;
 
-        if (!rsp.dict.dict_len) {
-                if (global_state->mode & GLUSTER_MODE_XML)
-                        goto xml_output;
-                cli_err ("No volumes present");
+    if (!count) {
+        switch (local->get_vol.flags) {
+            case GF_CLI_GET_NEXT_VOLUME:
+                GF_FREE(local->get_vol.volname);
+                local->get_vol.volname = NULL;
                 ret = 0;
                 goto out;
-        }
-
-        dict = dict_new ();
 
-        if (!dict) {
+            case GF_CLI_GET_VOLUME:
+                snprintf(err_str, sizeof(err_str), "Volume %s does not exist",
+                         local->get_vol.volname);
                 ret = -1;
-                goto out;
+                if (!(global_state->mode & GLUSTER_MODE_XML))
+                    goto out;
         }
+    }
 
-        ret = dict_unserialize (rsp.dict.dict_val,
-                                rsp.dict.dict_len,
-                                &dict);
+    if (rsp.op_ret) {
+        if (global_state->mode & GLUSTER_MODE_XML)
+            goto xml_output;
+        ret = -1;
+        goto out;
+    }
 
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR,
-                        "Unable to allocate memory");
+xml_output:
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        /* For GET_NEXT_VOLUME output is already begun in
+         * and will also end in gf_cli_get_next_volume()
+         */
+        if (local->get_vol.flags == GF_CLI_GET_VOLUME) {
+            ret = cli_xml_output_vol_info_begin(local, rsp.op_ret, rsp.op_errno,
+                                                rsp.op_errstr);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR, XML_ERROR);
                 goto out;
+            }
         }
 
-        ret = dict_get_int32 (dict, "count", &count);
-        if (ret)
+        if (dict) {
+            ret = cli_xml_output_vol_info(local, dict);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR, XML_ERROR);
                 goto out;
+            }
+        }
 
-        local = ((call_frame_t *)myframe)->local;
-
-        if (!count) {
-                switch (local->get_vol.flags) {
-
-                case GF_CLI_GET_NEXT_VOLUME:
-                        GF_FREE (local->get_vol.volname);
-                        local->get_vol.volname = NULL;
-                        ret = 0;
-                        goto out;
-
-                case GF_CLI_GET_VOLUME:
-                        memset (err_str, 0, sizeof (err_str));
-                        snprintf (err_str, sizeof (err_str),
-                                  "Volume %s does not exist",
-                                  local->get_vol.volname);
-                        ret = -1;
-                        if (!(global_state->mode & GLUSTER_MODE_XML))
-                                goto out;
-                }
+        if (local->get_vol.flags == GF_CLI_GET_VOLUME) {
+            ret = cli_xml_output_vol_info_end(local);
+            if (ret)
+                gf_log("cli", GF_LOG_ERROR, XML_ERROR);
         }
+        goto out;
+    }
 
-xml_output:
-        if (global_state->mode & GLUSTER_MODE_XML) {
-                /* For GET_NEXT_VOLUME output is already begun in
-                 * and will also end in gf_cli_get_next_volume()
-                 */
-                if (local->get_vol.flags == GF_CLI_GET_VOLUME) {
-                        ret = cli_xml_output_vol_info_begin
-                                (local, rsp.op_ret, rsp.op_errno,
-                                 rsp.op_errstr);
-                        if (ret) {
-                                gf_log ("cli", GF_LOG_ERROR,
-                                        "Error outputting to xml");
-                                goto out;
-                        }
-                }
+    while (i < count) {
+        cli_out(" ");
+        keylen = snprintf(key, sizeof(key), "volume%d.name", i);
+        ret = dict_get_strn(dict, key, keylen, &volname);
+        if (ret)
+            goto out;
 
-                if (dict) {
-                        ret = cli_xml_output_vol_info (local, dict);
-                        if (ret) {
-                                gf_log ("cli", GF_LOG_ERROR,
-                                        "Error outputting to xml");
-                                goto out;
-                        }
-                }
+        keylen = snprintf(key, sizeof(key), "volume%d.type", i);
+        ret = dict_get_int32n(dict, key, keylen, &type);
+        if (ret)
+            goto out;
 
-                if (local->get_vol.flags == GF_CLI_GET_VOLUME) {
-                        ret = cli_xml_output_vol_info_end (local);
-                        if (ret)
-                                gf_log ("cli", GF_LOG_ERROR,
-                                        "Error outputting to xml");
-                }
-                goto out;
-        }
+        keylen = snprintf(key, sizeof(key), "volume%d.status", i);
+        ret = dict_get_int32n(dict, key, keylen, &status);
+        if (ret)
+            goto out;
 
-        while ( i < count) {
-                cli_out (" ");
-                snprintf (key, 256, "volume%d.name", i);
-                ret = dict_get_str (dict, key, &volname);
-                if (ret)
-                        goto out;
+        keylen = snprintf(key, sizeof(key), "volume%d.brick_count", i);
+        ret = dict_get_int32n(dict, key, keylen, &brick_count);
+        if (ret)
+            goto out;
 
-                snprintf (key, 256, "volume%d.type", i);
-                ret = dict_get_int32 (dict, key, &type);
-                if (ret)
-                        goto out;
+        keylen = snprintf(key, sizeof(key), "volume%d.dist_count", i);
+        ret = dict_get_int32n(dict, key, keylen, &dist_count);
+        if (ret)
+            goto out;
 
-                snprintf (key, 256, "volume%d.status", i);
-                ret = dict_get_int32 (dict, key, &status);
-                if (ret)
-                        goto out;
+        keylen = snprintf(key, sizeof(key), "volume%d.stripe_count", i);
+        ret = dict_get_int32n(dict, key, keylen, &stripe_count);
+        if (ret)
+            goto out;
 
-                snprintf (key, 256, "volume%d.brick_count", i);
-                ret = dict_get_int32 (dict, key, &brick_count);
-                if (ret)
-                        goto out;
+        keylen = snprintf(key, sizeof(key), "volume%d.replica_count", i);
+        ret = dict_get_int32n(dict, key, keylen, &replica_count);
+        if (ret)
+            goto out;
 
-                snprintf (key, 256, "volume%d.dist_count", i);
-                ret = dict_get_int32 (dict, key, &dist_count);
-                if (ret)
-                        goto out;
+        keylen = snprintf(key, sizeof(key), "volume%d.disperse_count", i);
+        ret = dict_get_int32n(dict, key, keylen, &disperse_count);
+        if (ret)
+            goto out;
 
-                snprintf (key, 256, "volume%d.stripe_count", i);
-                ret = dict_get_int32 (dict, key, &stripe_count);
-                if (ret)
-                        goto out;
+        keylen = snprintf(key, sizeof(key), "volume%d.redundancy_count", i);
+        ret = dict_get_int32n(dict, key, keylen, &redundancy_count);
+        if (ret)
+            goto out;
 
-                snprintf (key, 256, "volume%d.replica_count", i);
-                ret = dict_get_int32 (dict, key, &replica_count);
-                if (ret)
-                        goto out;
+        keylen = snprintf(key, sizeof(key), "volume%d.arbiter_count", i);
+        ret = dict_get_int32n(dict, key, keylen, &arbiter_count);
+        if (ret)
+            goto out;
 
-                snprintf (key, 256, "volume%d.disperse_count", i);
-                ret = dict_get_int32 (dict, key, &disperse_count);
-                if (ret)
-                        goto out;
+        keylen = snprintf(key, sizeof(key), "volume%d.transport", i);
+        ret = dict_get_int32n(dict, key, keylen, &transport);
+        if (ret)
+            goto out;
 
-                snprintf (key, 256, "volume%d.redundancy_count", i);
-                ret = dict_get_int32 (dict, key, &redundancy_count);
-                if (ret)
-                        goto out;
+        keylen = snprintf(key, sizeof(key), "volume%d.volume_id", i);
+        ret = dict_get_strn(dict, key, keylen, &volume_id_str);
+        if (ret)
+            goto out;
 
-                snprintf (key, 256, "volume%d.transport", i);
-                ret = dict_get_int32 (dict, key, &transport);
-                if (ret)
-                        goto out;
+        keylen = snprintf(key, sizeof(key), "volume%d.snap_count", i);
+        ret = dict_get_int32n(dict, key, keylen, &snap_count);
+        if (ret)
+            goto out;
 
-                snprintf (key, 256, "volume%d.volume_id", i);
-                ret = dict_get_str (dict, key, &volume_id_str);
-                if (ret)
-                        goto out;
+        keylen = snprintf(key, sizeof(key), "volume%d.thin_arbiter_count", i);
+        ret = dict_get_int32n(dict, key, keylen, &thin_arbiter_count);
+        if (ret)
+            goto out;
 
-                vol_type = type;
+        // Distributed (stripe/replicate/stripe-replica) setups
+        vol_type = get_vol_type(type, dist_count, brick_count);
 
-                // Distributed (stripe/replicate/stripe-replica) setups
-                if ((type > 0) && ( dist_count < brick_count))
-                        vol_type = type + 4;
+        cli_out("Volume Name: %s", volname);
+        cli_out("Type: %s", vol_type_str[vol_type]);
+        cli_out("Volume ID: %s", volume_id_str);
+        cli_out("Status: %s", cli_vol_status_str[status]);
+        cli_out("Snapshot Count: %d", snap_count);
 
-                cli_out ("Volume Name: %s", volname);
-                cli_out ("Type: %s", cli_vol_type_str[vol_type]);
-                cli_out ("Volume ID: %s", volume_id_str);
-                cli_out ("Status: %s", cli_vol_status_str[status]);
+        gf_cli_print_number_of_bricks(
+            type, brick_count, dist_count, stripe_count, replica_count,
+            disperse_count, redundancy_count, arbiter_count);
 
-#ifdef HAVE_BD_XLATOR
-                k = 0;
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "volume%d.xlator%d", i, k);
-                ret = dict_get_str (dict, key, &caps);
-                if (ret)
-                        goto next;
-                do {
-                        j = 0;
-                        cli_out ("Xlator %d: %s", k + 1, caps);
-                        do {
-                                memset (key, 0, sizeof (key));
-                                snprintf (key, sizeof (key),
-                                          "volume%d.xlator%d.caps%d",
-                                          i, k, j++);
-                                ret = dict_get_str (dict, key, &caps);
-                                if (ret)
-                                        break;
-                                cli_out ("Capability %d: %s", j, caps);
-                        } while (1);
-
-                        memset (key, 0, sizeof (key));
-                        snprintf (key, sizeof (key),
-                                  "volume%d.xlator%d", i, ++k);
-                        ret = dict_get_str (dict, key, &caps);
-                        if (ret)
-                                break;
-                } while (1);
-
-next:
-#else
-                caps = 0; /* Avoid compiler warnings when BD not enabled */
-#endif
+        cli_out("Transport-type: %s",
+                ((transport == 0) ? "tcp"
+                                  : (transport == 1) ? "rdma" : "tcp,rdma"));
+        j = 1;
 
-                if (type == GF_CLUSTER_TYPE_STRIPE_REPLICATE) {
-                        cli_out ("Number of Bricks: %d x %d x %d = %d",
-                                 (brick_count / dist_count),
-                                 stripe_count,
-                                 replica_count,
-                                 brick_count);
-                } else if (type == GF_CLUSTER_TYPE_NONE) {
-                        cli_out ("Number of Bricks: %d", brick_count);
-                } else if (type == GF_CLUSTER_TYPE_DISPERSE) {
-                        cli_out ("Number of Bricks: %d x (%d + %d) = %d",
-                                 (brick_count / dist_count),
-                                 disperse_count - redundancy_count,
-                                 redundancy_count, brick_count);
-                } else {
-                        /* For both replicate and stripe, dist_count is
-                           good enough */
-                        cli_out ("Number of Bricks: %d x %d = %d",
-                                 (brick_count / dist_count),
-                                 dist_count, brick_count);
-                }
+        GF_FREE(local->get_vol.volname);
+        local->get_vol.volname = gf_strdup(volname);
 
-                cli_out ("Transport-type: %s",
-                         ((transport == 0)?"tcp":
-                          (transport == 1)?"rdma":
-                          "tcp,rdma"));
-                j = 1;
-
-                GF_FREE (local->get_vol.volname);
-                local->get_vol.volname = gf_strdup (volname);
-
-                if (brick_count)
-                        cli_out ("Bricks:");
-
-                while (j <= brick_count) {
-                        snprintf (key, 1024, "volume%d.brick%d", i, j);
-                        ret = dict_get_str (dict, key, &brick);
-                        if (ret)
-                                goto out;
-
-                        cli_out ("Brick%d: %s", j, brick);
-#ifdef HAVE_BD_XLATOR
-                        snprintf (key, 256, "volume%d.vg%d", i, j);
-                        ret = dict_get_str (dict, key, &caps);
-                        if (!ret)
-                                cli_out ("Brick%d VG: %s", j, caps);
-#endif
-                        j++;
-                }
+        cli_out("Bricks:");
+        ret = print_brick_details(dict, i, j, brick_count, replica_count);
+        if (ret)
+            goto out;
 
-                snprintf (key, 256, "volume%d.opt_count",i);
-                ret = dict_get_int32 (dict, key, &opt_count);
-                if (ret)
-                        goto out;
+        if (thin_arbiter_count) {
+            snprintf(key, sizeof(key), "volume%d.thin_arbiter_brick", i);
+            ret = dict_get_str(dict, key, &ta_brick);
+            if (ret)
+                goto out;
+            cli_out("Thin-arbiter-path: %s", ta_brick);
+        }
 
-                if (!opt_count)
-                        goto out;
+        snprintf(key, sizeof(key), "volume%d.opt_count", i);
+        ret = dict_get_int32(dict, key, &opt_count);
+        if (ret)
+            goto out;
 
-                cli_out ("Options Reconfigured:");
+        if (!opt_count)
+            goto out;
 
-                snprintf (key, 256, "volume%d.option.",i);
+        cli_out("Options Reconfigured:");
 
-                ret = dict_foreach (dict, _gf_cli_output_volinfo_opts, key);
-                if (ret)
-                        goto out;
+        snprintf(key, sizeof(key), "volume%d.option.", i);
 
-                i++;
-        }
+        ret = dict_foreach(dict, _gf_cli_output_volinfo_opts, key);
+        if (ret)
+            goto out;
 
+        i++;
+    }
 
-        ret = 0;
+    ret = 0;
 out:
-        cli_cmd_broadcast_response (ret);
-        if (ret)
-                cli_err ("%s", err_str);
+    if (ret)
+        cli_err("%s", err_str);
 
-        if (dict)
-                dict_destroy (dict);
+    cli_cmd_broadcast_response(ret);
 
-        free (rsp.dict.dict_val);
+    if (dict)
+        dict_unref(dict);
 
-        free (rsp.op_errstr);
+    gf_free_xdr_cli_rsp(rsp);
 
-        gf_log ("cli", GF_LOG_INFO, "Returning: %d", ret);
-        return ret;
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
+    return ret;
 }
 
-int
-gf_cli_create_volume_cbk (struct rpc_req *req, struct iovec *iov,
-                             int count, void *myframe)
+static int
+gf_cli_create_volume_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                         void *myframe)
 {
-        gf_cli_rsp              rsp   = {0,};
-        int                     ret   = -1;
-        cli_local_t             *local = NULL;
-        char                    *volname = NULL;
-        dict_t                  *dict = NULL;
-        dict_t                  *rsp_dict = NULL;
+    gf_cli_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    cli_local_t *local = NULL;
+    char *volname = NULL;
+    dict_t *rsp_dict = NULL;
+    call_frame_t *frame = NULL;
 
-        if (-1 == req->rpc_status) {
-                goto out;
-        }
+    GF_ASSERT(myframe);
 
-        local = ((call_frame_t *) (myframe))->local;
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
 
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
-        if (ret < 0) {
-                gf_log (((call_frame_t *) myframe)->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                goto out;
-        }
+    frame = myframe;
 
-        gf_log ("cli", GF_LOG_INFO, "Received resp to create volume");
+    GF_ASSERT(frame->local);
 
-        dict = local->dict;
+    local = frame->local;
 
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret)
-                goto out;
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(frame->this->name, GF_LOG_ERROR, XDR_DECODE_FAIL);
+        goto out;
+    }
 
-        if (global_state->mode & GLUSTER_MODE_XML) {
-                if (rsp.op_ret == 0) {
-                        rsp_dict = dict_new ();
-                        ret = dict_unserialize (rsp.dict.dict_val,
-                                                rsp.dict.dict_len,
-                                                &rsp_dict);
-                        if (ret) {
-                                gf_log ("cli", GF_LOG_ERROR,
-                                        "Failed rsp_dict unserialization");
-                                goto out;
-                        }
-                }
+    gf_log("cli", GF_LOG_INFO, "Received resp to create volume");
 
-                ret = cli_xml_output_vol_create (rsp_dict, rsp.op_ret,
-                                                 rsp.op_errno, rsp.op_errstr);
-                if (ret)
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Error outputting to xml");
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        if (rsp.op_ret == 0) {
+            rsp_dict = dict_new();
+            ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len,
+                                   &rsp_dict);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR, DICT_UNSERIALIZE_FAIL);
                 goto out;
+            }
         }
 
-        if (rsp.op_ret && strcmp (rsp.op_errstr, ""))
-                cli_err ("volume create: %s: failed: %s", volname,
-                         rsp.op_errstr);
-        else if (rsp.op_ret)
-                cli_err ("volume create: %s: failed", volname);
-        else
-                cli_out ("volume create: %s: success: "
-                         "please start the volume to access data", volname);
-
-        ret = rsp.op_ret;
+        ret = cli_xml_output_vol_create(rsp_dict, rsp.op_ret, rsp.op_errno,
+                                        rsp.op_errstr);
+        if (ret)
+            gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+        goto out;
+    }
+
+    ret = dict_get_str_sizen(local->dict, "volname", &volname);
+    if (ret)
+        goto out;
+
+    if (rsp.op_ret && strcmp(rsp.op_errstr, ""))
+        cli_err("volume create: %s: failed: %s", volname, rsp.op_errstr);
+    else if (rsp.op_ret)
+        cli_err("volume create: %s: failed", volname);
+    else
+        cli_out(
+            "volume create: %s: success: "
+            "please start the volume to access data",
+            volname);
+
+    ret = rsp.op_ret;
 
 out:
-        cli_cmd_broadcast_response (ret);
-        free (rsp.dict.dict_val);
-        free (rsp.op_errstr);
-        return ret;
+    cli_cmd_broadcast_response(ret);
+    gf_free_xdr_cli_rsp(rsp);
+
+    if (rsp_dict)
+        dict_unref(rsp_dict);
+    return ret;
 }
 
-int
-gf_cli_delete_volume_cbk (struct rpc_req *req, struct iovec *iov,
-                             int count, void *myframe)
-{
-        gf_cli_rsp              rsp   = {0,};
-        int                     ret   = -1;
-        cli_local_t             *local = NULL;
-        char                    *volname = NULL;
-        call_frame_t            *frame = NULL;
-        dict_t                  *dict = NULL;
-        dict_t                  *rsp_dict = NULL;
-
-        if (-1 == req->rpc_status) {
-                goto out;
-        }
+static int
+gf_cli_delete_volume_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                         void *myframe)
+{
+    gf_cli_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    cli_local_t *local = NULL;
+    char *volname = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *rsp_dict = NULL;
 
-        frame = myframe;
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
 
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
-        if (ret < 0) {
-                gf_log (frame->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                goto out;
-        }
+    GF_ASSERT(myframe);
+    frame = myframe;
 
-        local = frame->local;
+    GF_ASSERT(frame->local);
 
-        if (local)
-                dict = local->dict;
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret) {
-                gf_log (frame->this->name, GF_LOG_ERROR,
-                        "dict get failed");
-                goto out;
-        }
+    local = frame->local;
 
-        gf_log ("cli", GF_LOG_INFO, "Received resp to delete volume");
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(frame->this->name, GF_LOG_ERROR, XDR_DECODE_FAIL);
+        goto out;
+    }
 
-        if (global_state->mode & GLUSTER_MODE_XML) {
-                if (rsp.op_ret == 0) {
-                        rsp_dict = dict_new ();
-                        ret = dict_unserialize (rsp.dict.dict_val,
-                                                rsp.dict.dict_len,
-                                                &rsp_dict);
-                        if (ret) {
-                                gf_log ("cli", GF_LOG_ERROR,
-                                        "Failed rsp_dict unserialization");
-                                goto out;
-                        }
-                }
+    gf_log("cli", GF_LOG_INFO, "Received resp to delete volume");
 
-                ret = cli_xml_output_generic_volume ("volDelete", rsp_dict,
-                                                     rsp.op_ret, rsp.op_errno,
-                                                     rsp.op_errstr);
-                if (ret)
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Error outputting to xml");
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        if (rsp.op_ret == 0) {
+            rsp_dict = dict_new();
+            ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len,
+                                   &rsp_dict);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR, DICT_UNSERIALIZE_FAIL);
                 goto out;
+            }
         }
 
-        if (rsp.op_ret && strcmp (rsp.op_errstr, ""))
-                cli_err ("volume delete: %s: failed: %s", volname,
-                         rsp.op_errstr);
-        else if (rsp.op_ret)
-                cli_err ("volume delete: %s: failed", volname);
-        else
-                cli_out ("volume delete: %s: success", volname);
+        ret = cli_xml_output_generic_volume("volDelete", rsp_dict, rsp.op_ret,
+                                            rsp.op_errno, rsp.op_errstr);
+        if (ret)
+            gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+        goto out;
+    }
 
-        ret = rsp.op_ret;
+    ret = dict_get_str_sizen(local->dict, "volname", &volname);
+    if (ret) {
+        gf_log(frame->this->name, GF_LOG_ERROR, "dict get failed");
+        goto out;
+    }
+
+    if (rsp.op_ret && strcmp(rsp.op_errstr, ""))
+        cli_err("volume delete: %s: failed: %s", volname, rsp.op_errstr);
+    else if (rsp.op_ret)
+        cli_err("volume delete: %s: failed", volname);
+    else
+        cli_out("volume delete: %s: success", volname);
+
+    ret = rsp.op_ret;
 
 out:
-        cli_cmd_broadcast_response (ret);
-        free (rsp.dict.dict_val);
+    cli_cmd_broadcast_response(ret);
+    gf_free_xdr_cli_rsp(rsp);
 
-        gf_log ("", GF_LOG_INFO, "Returning with %d", ret);
-        return ret;
+    if (rsp_dict)
+        dict_unref(rsp_dict);
+    gf_log("", GF_LOG_DEBUG, RETURNING, ret);
+    return ret;
 }
 
-int
-gf_cli3_1_uuid_get_cbk (struct rpc_req *req, struct iovec *iov,
-                        int count, void *myframe)
+static int
+gf_cli3_1_uuid_get_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                       void *myframe)
 {
-        char                    *uuid_str = NULL;
-        gf_cli_rsp              rsp   = {0,};
-        int                     ret   = -1;
-        cli_local_t             *local = NULL;
-        call_frame_t            *frame = NULL;
-        dict_t                  *dict = NULL;
+    char *uuid_str = NULL;
+    gf_cli_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    cli_local_t *local = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *dict = NULL;
 
-        if (-1 == req->rpc_status)
-                goto out;
+    GF_ASSERT(myframe);
 
-        frame = myframe;
+    if (-1 == req->rpc_status)
+        goto out;
 
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
-        if (ret < 0) {
-                gf_log (frame->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                goto out;
-        }
+    frame = myframe;
 
-        local = frame->local;
-        frame->local = NULL;
+    GF_ASSERT(frame->local);
 
-        gf_log ("cli", GF_LOG_INFO, "Received resp to uuid get");
+    local = frame->local;
 
-        dict = dict_new ();
-        if (!dict) {
-                ret = -1;
-                goto out;
-        }
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(frame->this->name, GF_LOG_ERROR, XDR_DECODE_FAIL);
+        goto out;
+    }
 
-        ret = dict_unserialize (rsp.dict.dict_val, rsp.dict.dict_len,
-                                &dict);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to unserialize "
-                        "response for uuid get");
-                goto out;
-        }
+    frame->local = NULL;
 
-        ret = dict_get_str (dict, "uuid", &uuid_str);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to get uuid "
-                        "from dictionary");
-                goto out;
-        }
+    gf_log("cli", GF_LOG_INFO, "Received resp to uuid get");
 
-        if (global_state->mode & GLUSTER_MODE_XML) {
-                ret = cli_xml_output_dict ("uuidGenerate", dict, rsp.op_ret,
-                                           rsp.op_errno, rsp.op_errstr);
-                if (ret)
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Error outputting to xml");
-                goto out;
-        }
+    dict = dict_new();
+    if (!dict) {
+        ret = -1;
+        goto out;
+    }
 
-        if (rsp.op_ret) {
-                if (strcmp (rsp.op_errstr, "") == 0)
-                        cli_err ("Get uuid was unsuccessful");
-                else
-                        cli_err ("%s", rsp.op_errstr);
+    ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len, &dict);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, DICT_UNSERIALIZE_FAIL);
+        goto out;
+    }
 
-        } else {
-                cli_out ("UUID: %s", uuid_str);
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        ret = cli_xml_output_dict("uuidGenerate", dict, rsp.op_ret,
+                                  rsp.op_errno, rsp.op_errstr);
+        if (ret)
+            gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+        goto out;
+    }
+
+    if (rsp.op_ret) {
+        if (strcmp(rsp.op_errstr, "") == 0)
+            cli_err("Get uuid was unsuccessful");
+        else
+            cli_err("%s", rsp.op_errstr);
 
+    } else {
+        ret = dict_get_str_sizen(dict, "uuid", &uuid_str);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR, "Failed to get uuid from dictionary");
+            goto out;
         }
-        ret = rsp.op_ret;
+        cli_out("UUID: %s", uuid_str);
+    }
+    ret = rsp.op_ret;
 
 out:
-        cli_cmd_broadcast_response (ret);
-        cli_local_wipe (local);
-        if (rsp.dict.dict_val)
-                free (rsp.dict.dict_val);
-        if (dict)
-                dict_unref (dict);
+    cli_cmd_broadcast_response(ret);
+    cli_local_wipe(local);
+    gf_free_xdr_cli_rsp(rsp);
 
-        gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
-        return ret;
+    if (dict)
+        dict_unref(dict);
+
+    gf_log("", GF_LOG_DEBUG, RETURNING, ret);
+    return ret;
 }
 
-int
-gf_cli3_1_uuid_reset_cbk (struct rpc_req *req, struct iovec *iov,
-                             int count, void *myframe)
+static int
+gf_cli3_1_uuid_reset_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                         void *myframe)
 {
-        gf_cli_rsp              rsp   = {0,};
-        int                     ret   = -1;
-        cli_local_t             *local = NULL;
-        call_frame_t            *frame = NULL;
-        dict_t                  *dict = NULL;
+    gf_cli_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    cli_local_t *local = NULL;
+    call_frame_t *frame = NULL;
 
-        if (-1 == req->rpc_status) {
-                goto out;
-        }
+    GF_ASSERT(myframe);
 
-        frame = myframe;
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
 
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
-        if (ret < 0) {
-                gf_log (frame->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                goto out;
-        }
+    frame = myframe;
 
-        local = frame->local;
-        frame->local = NULL;
+    GF_ASSERT(frame->local);
 
-        gf_log ("cli", GF_LOG_INFO, "Received resp to uuid reset");
+    local = frame->local;
 
-        if (global_state->mode & GLUSTER_MODE_XML) {
-                ret = cli_xml_output_dict ("uuidReset", dict, rsp.op_ret,
-                                           rsp.op_errno, rsp.op_errstr);
-                if (ret)
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Error outputting to xml");
-                goto out;
-        }
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(frame->this->name, GF_LOG_ERROR, XDR_DECODE_FAIL);
+        goto out;
+    }
 
-        if (rsp.op_ret && strcmp (rsp.op_errstr, ""))
-                cli_err ("%s", rsp.op_errstr);
-        else
-                cli_out ("resetting the peer uuid has been %s",
-                         (rsp.op_ret) ? "unsuccessful": "successful");
-        ret = rsp.op_ret;
+    frame->local = NULL;
+
+    gf_log("cli", GF_LOG_INFO, "Received resp to uuid reset");
+
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        ret = cli_xml_output_dict("uuidReset", NULL, rsp.op_ret, rsp.op_errno,
+                                  rsp.op_errstr);
+        if (ret)
+            gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+        goto out;
+    }
+
+    if (rsp.op_ret && strcmp(rsp.op_errstr, ""))
+        cli_err("%s", rsp.op_errstr);
+    else
+        cli_out("resetting the peer uuid has been %s",
+                (rsp.op_ret) ? "unsuccessful" : "successful");
+    ret = rsp.op_ret;
 
 out:
-        cli_cmd_broadcast_response (ret);
-        cli_local_wipe (local);
-        if (rsp.dict.dict_val)
-                free (rsp.dict.dict_val);
-        if (dict)
-                dict_unref (dict);
+    cli_cmd_broadcast_response(ret);
+    cli_local_wipe(local);
+    gf_free_xdr_cli_rsp(rsp);
 
-        gf_log ("", GF_LOG_INFO, "Returning with %d", ret);
-        return ret;
+    gf_log("", GF_LOG_DEBUG, RETURNING, ret);
+    return ret;
 }
 
-int
-gf_cli_start_volume_cbk (struct rpc_req *req, struct iovec *iov,
-                             int count, void *myframe)
-{
-        gf_cli_rsp              rsp   = {0,};
-        int                     ret   = -1;
-        cli_local_t             *local = NULL;
-        char                    *volname = NULL;
-        call_frame_t            *frame = NULL;
-        dict_t                  *dict = NULL;
-        dict_t                  *rsp_dict = NULL;
-
-        if (-1 == req->rpc_status) {
-                goto out;
-        }
+static int
+gf_cli_start_volume_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                        void *myframe)
+{
+    gf_cli_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    cli_local_t *local = NULL;
+    char *volname = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *rsp_dict = NULL;
 
-        frame = myframe;
+    GF_ASSERT(myframe);
 
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
-        if (ret < 0) {
-                gf_log (frame->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                goto out;
-        }
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
 
-        if (frame)
-                local = frame->local;
+    frame = myframe;
 
-        if (local)
-                dict = local->dict;
+    GF_ASSERT(frame->local);
 
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret) {
-                gf_log (frame->this->name, GF_LOG_ERROR, "dict get failed");
-                goto out;
-        }
+    local = frame->local;
 
-        gf_log ("cli", GF_LOG_INFO, "Received resp to start volume");
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(frame->this->name, GF_LOG_ERROR, XDR_DECODE_FAIL);
+        goto out;
+    }
 
-        if (global_state->mode & GLUSTER_MODE_XML) {
-                if (rsp.op_ret == 0) {
-                        rsp_dict = dict_new ();
-                        ret = dict_unserialize (rsp.dict.dict_val,
-                                                rsp.dict.dict_len,
-                                                &rsp_dict);
-                        if (ret) {
-                                gf_log ("cli", GF_LOG_ERROR,
-                                        "Failed rsp_dict unserialization");
-                                goto out;
-                        }
-                }
+    gf_log("cli", GF_LOG_INFO, "Received resp to start volume");
 
-                ret = cli_xml_output_generic_volume ("volStart", rsp_dict,
-                                                     rsp.op_ret, rsp.op_errno,
-                                                     rsp.op_errstr);
-                if (ret)
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Error outputting to xml");
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        if (rsp.op_ret == 0) {
+            rsp_dict = dict_new();
+            ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len,
+                                   &rsp_dict);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR, DICT_UNSERIALIZE_FAIL);
                 goto out;
+            }
         }
 
-        if (rsp.op_ret && strcmp (rsp.op_errstr, ""))
-                cli_err ("volume start: %s: failed: %s", volname,
-                         rsp.op_errstr);
-        else if (rsp.op_ret)
-                cli_err ("volume start: %s: failed", volname);
-        else
-                cli_out ("volume start: %s: success", volname);
-
-        ret = rsp.op_ret;
+        ret = cli_xml_output_generic_volume("volStart", rsp_dict, rsp.op_ret,
+                                            rsp.op_errno, rsp.op_errstr);
+        if (ret)
+            gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+        goto out;
+    }
 
-out:
-        cli_cmd_broadcast_response (ret);
-        free (rsp.dict.dict_val);
-        free (rsp.op_errstr);
-        return ret;
-}
+    ret = dict_get_str_sizen(local->dict, "volname", &volname);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "dict get failed");
+        goto out;
+    }
 
-int
-gf_cli_stop_volume_cbk (struct rpc_req *req, struct iovec *iov,
-                             int count, void *myframe)
-{
-        gf_cli_rsp            rsp   = {0,};
-        int                   ret   = -1;
-        cli_local_t           *local = NULL;
-        char                  *volname = NULL;
-        call_frame_t          *frame = NULL;
-        dict_t                *dict = NULL;
-        dict_t                *rsp_dict = NULL;
-
-        if (-1 == req->rpc_status) {
-                goto out;
-        }
+    if (rsp.op_ret && strcmp(rsp.op_errstr, ""))
+        cli_err("volume start: %s: failed: %s", volname, rsp.op_errstr);
+    else if (rsp.op_ret)
+        cli_err("volume start: %s: failed", volname);
+    else
+        cli_out("volume start: %s: success", volname);
 
-        frame = myframe;
+    ret = rsp.op_ret;
 
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
-        if (ret < 0) {
-                gf_log (frame->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                goto out;
-        }
+out:
+    cli_cmd_broadcast_response(ret);
+    gf_free_xdr_cli_rsp(rsp);
 
-        if (frame)
-                local = frame->local;
+    if (rsp_dict)
+        dict_unref(rsp_dict);
+    return ret;
+}
 
-        if (local) {
-                dict = local->dict;
-                ret = dict_get_str (dict, "volname", &volname);
-                if (ret) {
-                        gf_log (frame->this->name, GF_LOG_ERROR,
-                                "Unable to get volname from dict");
-                        goto out;
-                }
-        }
+static int
+gf_cli_stop_volume_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                       void *myframe)
+{
+    gf_cli_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    cli_local_t *local = NULL;
+    char *volname = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *rsp_dict = NULL;
 
-        gf_log ("cli", GF_LOG_INFO, "Received resp to stop volume");
+    GF_ASSERT(myframe);
 
-        if (global_state->mode & GLUSTER_MODE_XML) {
-                if (rsp.op_ret == 0) {
-                        rsp_dict = dict_new ();
-                        ret = dict_unserialize (rsp.dict.dict_val,
-                                                rsp.dict.dict_len,
-                                                &rsp_dict);
-                        if (ret) {
-                                gf_log ("cli", GF_LOG_ERROR,
-                                        "Failed rsp_dict unserialization");
-                                goto out;
-                        }
-                }
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
 
-                ret = cli_xml_output_generic_volume ("volStop", rsp_dict,
-                                                     rsp.op_ret, rsp.op_errno,
-                                                     rsp.op_errstr);
-                if (ret)
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Error outputting to xml");
-                goto out;
-        }
+    frame = myframe;
 
-        if (rsp.op_ret && strcmp (rsp.op_errstr, ""))
-                cli_err ("volume stop: %s: failed: %s", volname, rsp.op_errstr);
-        else if (rsp.op_ret)
-                cli_err ("volume stop: %s: failed", volname);
-        else
-                cli_out ("volume stop: %s: success", volname);
+    GF_ASSERT(frame->local);
 
-        ret = rsp.op_ret;
+    local = frame->local;
 
-out:
-        cli_cmd_broadcast_response (ret);
-        free (rsp.op_errstr);
-        free (rsp.dict.dict_val);
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(frame->this->name, GF_LOG_ERROR, XDR_DECODE_FAIL);
+        goto out;
+    }
 
-        return ret;
-}
+    gf_log("cli", GF_LOG_INFO, "Received resp to stop volume");
 
-int
-gf_cli_print_rebalance_status (dict_t *dict, enum gf_task_types task_type)
-{
-        int                ret          = -1;
-        int                count        = 0;
-        int                i            = 1;
-        char               key[256]     = {0,};
-        gf_defrag_status_t status_rcd   = GF_DEFRAG_STATUS_NOT_STARTED;
-        uint64_t           files        = 0;
-        uint64_t           size         = 0;
-        uint64_t           lookup       = 0;
-        char               *node_name   = NULL;
-        uint64_t           failures     = 0;
-        uint64_t           skipped      = 0;
-        double             elapsed      = 0;
-        char               *status_str  = NULL;
-        char               *size_str    = NULL;
-
-        ret = dict_get_int32 (dict, "count", &count);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "count not set");
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        if (rsp.op_ret == 0) {
+            rsp_dict = dict_new();
+            ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len,
+                                   &rsp_dict);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR, DICT_UNSERIALIZE_FAIL);
                 goto out;
+            }
         }
 
+        ret = cli_xml_output_generic_volume("volStop", rsp_dict, rsp.op_ret,
+                                            rsp.op_errno, rsp.op_errstr);
+        if (ret)
+            gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+        goto out;
+    }
+
+    ret = dict_get_str_sizen(local->dict, "volname", &volname);
+    if (ret) {
+        gf_log(frame->this->name, GF_LOG_ERROR,
+               "Unable to get volname from dict");
+        goto out;
+    }
+
+    if (rsp.op_ret && strcmp(rsp.op_errstr, ""))
+        cli_err("volume stop: %s: failed: %s", volname, rsp.op_errstr);
+    else if (rsp.op_ret)
+        cli_err("volume stop: %s: failed", volname);
+    else
+        cli_out("volume stop: %s: success", volname);
+
+    ret = rsp.op_ret;
 
-        cli_out ("%40s %16s %13s %13s %13s %13s %20s %18s", "Node",
-                 "Rebalanced-files", "size", "scanned", "failures", "skipped",
-                 "status", "run time in secs");
-        cli_out ("%40s %16s %13s %13s %13s %13s %20s %18s", "---------",
-                 "-----------", "-----------", "-----------", "-----------",
-                 "-----------", "------------", "--------------");
-        for (i = 1; i <= count; i++) {
-                /* Reset the variables to prevent carryover of values */
-                node_name = NULL;
-                files = 0;
-                size = 0;
-                lookup = 0;
-                skipped = 0;
-                status_str = NULL;
-                elapsed = 0;
-
-                /* Check if status is NOT_STARTED, and continue early */
-                memset (key, 0, 256);
-                snprintf (key, 256, "status-%d", i);
-                ret = dict_get_int32 (dict, key, (int32_t *)&status_rcd);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_TRACE, "failed to get status");
-                        goto out;
-                }
-                if (GF_DEFRAG_STATUS_NOT_STARTED == status_rcd)
-                        continue;
+out:
+    cli_cmd_broadcast_response(ret);
+    gf_free_xdr_cli_rsp(rsp);
 
+    if (rsp_dict)
+        dict_unref(rsp_dict);
 
-                snprintf (key, 256, "node-name-%d", i);
-                ret = dict_get_str (dict, key, &node_name);
-                if (ret)
-                        gf_log ("cli", GF_LOG_TRACE, "failed to get node-name");
+    return ret;
+}
 
-                memset (key, 0, 256);
-                snprintf (key, 256, "files-%d", i);
-                ret = dict_get_uint64 (dict, key, &files);
-                if (ret)
-                        gf_log ("cli", GF_LOG_TRACE,
-                                "failed to get file count");
+static int
+gf_cli_print_rebalance_status(dict_t *dict, enum gf_task_types task_type)
+{
+    int ret = -1;
+    int count = 0;
+    int i = 1;
+    char key[64] = {
+        0,
+    };
+    int keylen;
+    gf_defrag_status_t status_rcd = GF_DEFRAG_STATUS_NOT_STARTED;
+    uint64_t files = 0;
+    uint64_t size = 0;
+    uint64_t lookup = 0;
+    char *node_name = NULL;
+    uint64_t failures = 0;
+    uint64_t skipped = 0;
+    double elapsed = 0;
+    char *status_str = NULL;
+    char *size_str = NULL;
+    int32_t hrs = 0;
+    uint32_t min = 0;
+    uint32_t sec = 0;
+    gf_boolean_t fix_layout = _gf_false;
+    uint64_t max_time = 0;
+    uint64_t max_elapsed = 0;
+    uint64_t time_left = 0;
+    gf_boolean_t show_estimates = _gf_false;
+
+    ret = dict_get_int32_sizen(dict, "count", &count);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "count not set");
+        goto out;
+    }
+
+    for (i = 1; i <= count; i++) {
+        keylen = snprintf(key, sizeof(key), "status-%d", i);
+        ret = dict_get_int32n(dict, key, keylen, (int32_t *)&status_rcd);
+        /* If information from a node is missing we should skip
+         * the node and try to fetch information of other nodes.
+         * If information is not found for all nodes, we should
+         * error out.
+         */
+        if (!ret)
+            break;
+        if (ret && i == count) {
+            gf_log("cli", GF_LOG_TRACE, "failed to get status");
+            goto out;
+        }
+    }
+
+    /* Fix layout will be sent to all nodes for the volume
+       so every status should be of type
+       GF_DEFRAG_STATUS_LAYOUT_FIX*
+    */
+
+    if ((task_type == GF_TASK_TYPE_REBALANCE) &&
+        (status_rcd >= GF_DEFRAG_STATUS_LAYOUT_FIX_STARTED)) {
+        fix_layout = _gf_true;
+    }
+
+    if (fix_layout) {
+        cli_out("%35s %41s %27s", "Node", "status", "run time in h:m:s");
+        cli_out("%35s %41s %27s", "---------", "-----------", "------------");
+    } else {
+        cli_out("%40s %16s %13s %13s %13s %13s %20s %18s", "Node",
+                "Rebalanced-files", "size", "scanned", "failures", "skipped",
+                "status",
+                "run time in"
+                " h:m:s");
+        cli_out("%40s %16s %13s %13s %13s %13s %20s %18s", "---------",
+                "-----------", "-----------", "-----------", "-----------",
+                "-----------", "------------", "--------------");
+    }
+
+    for (i = 1; i <= count; i++) {
+        /* Reset the variables to prevent carryover of values */
+        node_name = NULL;
+        files = 0;
+        size = 0;
+        lookup = 0;
+        skipped = 0;
+        status_str = NULL;
+        elapsed = 0;
+        time_left = 0;
+
+        /* Check if status is NOT_STARTED, and continue early */
+        keylen = snprintf(key, sizeof(key), "status-%d", i);
+
+        ret = dict_get_int32n(dict, key, keylen, (int32_t *)&status_rcd);
+        if (ret == -ENOENT) {
+            gf_log("cli", GF_LOG_TRACE, "count %d %d", count, i);
+            gf_log("cli", GF_LOG_TRACE, "failed to get status");
+            gf_log("cli", GF_LOG_ERROR, "node down and has failed to set dict");
+            continue;
+            /* skip this node if value not available*/
+        } else if (ret) {
+            gf_log("cli", GF_LOG_TRACE, "count %d %d", count, i);
+            gf_log("cli", GF_LOG_TRACE, "failed to get status");
+            continue;
+            /* skip this node if value not available*/
+        }
+
+        if (GF_DEFRAG_STATUS_NOT_STARTED == status_rcd)
+            continue;
+
+        if (GF_DEFRAG_STATUS_STARTED == status_rcd)
+            show_estimates = _gf_true;
+
+        keylen = snprintf(key, sizeof(key), "node-name-%d", i);
+        ret = dict_get_strn(dict, key, keylen, &node_name);
+        if (ret)
+            gf_log("cli", GF_LOG_TRACE, "failed to get node-name");
 
-                memset (key, 0, 256);
-                snprintf (key, 256, "size-%d", i);
-                ret = dict_get_uint64 (dict, key, &size);
-                if (ret)
-                        gf_log ("cli", GF_LOG_TRACE,
-                                "failed to get size of xfer");
+        snprintf(key, sizeof(key), "files-%d", i);
+        ret = dict_get_uint64(dict, key, &files);
+        if (ret)
+            gf_log("cli", GF_LOG_TRACE, "failed to get file count");
 
-                memset (key, 0, 256);
-                snprintf (key, 256, "lookups-%d", i);
-                ret = dict_get_uint64 (dict, key, &lookup);
-                if (ret)
-                        gf_log ("cli", GF_LOG_TRACE,
-                                "failed to get lookedup file count");
+        snprintf(key, sizeof(key), "size-%d", i);
+        ret = dict_get_uint64(dict, key, &size);
+        if (ret)
+            gf_log("cli", GF_LOG_TRACE, "failed to get size of xfer");
 
-                memset (key, 0, 256);
-                snprintf (key, 256, "failures-%d", i);
-                ret = dict_get_uint64 (dict, key, &failures);
-                if (ret)
-                        gf_log ("cli", GF_LOG_TRACE,
-                                "failed to get failures count");
+        snprintf(key, sizeof(key), "lookups-%d", i);
+        ret = dict_get_uint64(dict, key, &lookup);
+        if (ret)
+            gf_log("cli", GF_LOG_TRACE, "failed to get lookedup file count");
 
-                memset (key, 0, 256);
-                snprintf (key, 256, "skipped-%d", i);
-                ret = dict_get_uint64 (dict, key, &skipped);
-                if (ret)
-                        gf_log ("cli", GF_LOG_TRACE,
-                                "failed to get skipped count");
+        snprintf(key, sizeof(key), "failures-%d", i);
+        ret = dict_get_uint64(dict, key, &failures);
+        if (ret)
+            gf_log("cli", GF_LOG_TRACE, "failed to get failures count");
 
-                /* For remove-brick include skipped count into failure count*/
-                if (task_type != GF_TASK_TYPE_REBALANCE) {
-                        failures += skipped;
-                        skipped = 0;
-                }
+        snprintf(key, sizeof(key), "skipped-%d", i);
+        ret = dict_get_uint64(dict, key, &skipped);
+        if (ret)
+            gf_log("cli", GF_LOG_TRACE, "failed to get skipped count");
 
-                memset (key, 0, 256);
-                snprintf (key, 256, "run-time-%d", i);
-                ret = dict_get_double (dict, key, &elapsed);
-                if (ret)
-                        gf_log ("cli", GF_LOG_TRACE, "failed to get run-time");
-
-                /* Check for array bound */
-                if (status_rcd >= GF_DEFRAG_STATUS_MAX)
-                        status_rcd = GF_DEFRAG_STATUS_MAX;
-
-                status_str = cli_vol_task_status_str[status_rcd];
-                size_str = gf_uint64_2human_readable(size);
-                if (size_str) {
-                        cli_out ("%40s %16"PRIu64 " %13s" " %13"PRIu64 " %13"
-                                 PRIu64" %13"PRIu64 " %20s %18.2f", node_name,
-                                 files, size_str, lookup, failures, skipped,
-                                 status_str, elapsed);
-                } else {
-                        cli_out ("%40s %16"PRIu64 " %13"PRIu64 " %13"PRIu64
-                                 " %13"PRIu64" %13"PRIu64 " %20s %18.2f",
-                                 node_name, files, size, lookup, failures,
-                                 skipped, status_str, elapsed);
-                }
-                GF_FREE(size_str);
+        /* For remove-brick include skipped count into failure count*/
+        if (task_type != GF_TASK_TYPE_REBALANCE) {
+            failures += skipped;
+            skipped = 0;
         }
-out:
-        return ret;
-}
 
-int
-gf_cli_defrag_volume_cbk (struct rpc_req *req, struct iovec *iov,
-                             int count, void *myframe)
-{
-        gf_cli_rsp   rsp          = {0,};
-        cli_local_t  *local       = NULL;
-        char         *volname     = NULL;
-        call_frame_t *frame       = NULL;
-        int          cmd          = 0;
-        int          ret          = -1;
-        dict_t       *dict        = NULL;
-        dict_t       *local_dict  = NULL;
-        char         msg[1024]    = {0,};
-        char         *task_id_str = NULL;
-
-        if (-1 == req->rpc_status) {
-                goto out;
-        }
+        snprintf(key, sizeof(key), "run-time-%d", i);
+        ret = dict_get_double(dict, key, &elapsed);
+        if (ret)
+            gf_log("cli", GF_LOG_TRACE, "failed to get run-time");
 
-        frame = myframe;
+        snprintf(key, sizeof(key), "time-left-%d", i);
+        ret = dict_get_uint64(dict, key, &time_left);
+        if (ret)
+            gf_log("cli", GF_LOG_TRACE, "failed to get time left");
 
-        ret = xdr_to_generic (*iov, &rsp,
-                              (xdrproc_t)xdr_gf_cli_rsp);
-        if (ret < 0) {
-                gf_log (frame->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                goto out;
-        }
+        if (elapsed > max_elapsed)
+            max_elapsed = elapsed;
 
-        if (frame)
-                local = frame->local;
+        if (time_left > max_time)
+            max_time = time_left;
 
-        if (local)
-                local_dict = local->dict;
+        /* Check for array bound */
+        if (status_rcd >= GF_DEFRAG_STATUS_MAX)
+            status_rcd = GF_DEFRAG_STATUS_MAX;
 
-        ret = dict_get_str (local_dict, "volname", &volname);
-        if (ret) {
-                gf_log (frame->this->name, GF_LOG_ERROR,
-                        "Failed to get volname");
-                goto out;
-        }
+        status_str = cli_vol_task_status_str[status_rcd];
+        size_str = gf_uint64_2human_readable(size);
+        hrs = elapsed / 3600;
+        min = ((uint64_t)elapsed % 3600) / 60;
+        sec = ((uint64_t)elapsed % 3600) % 60;
 
-        ret = dict_get_int32 (local_dict, "rebalance-command", (int32_t*)&cmd);
-        if (ret) {
-                gf_log (frame->this->name, GF_LOG_ERROR,
-                        "Failed to get command");
-                goto out;
+        if (fix_layout) {
+            cli_out("%35s %50s %8d:%d:%d", node_name, status_str, hrs, min,
+                    sec);
+        } else {
+            if (size_str) {
+                cli_out("%40s %16" PRIu64
+                        " %13s"
+                        " %13" PRIu64 " %13" PRIu64 " %13" PRIu64
+                        " %20s "
+                        "%8d:%02d:%02d",
+                        node_name, files, size_str, lookup, failures, skipped,
+                        status_str, hrs, min, sec);
+            } else {
+                cli_out("%40s %16" PRIu64 " %13" PRIu64 " %13" PRIu64
+                        " %13" PRIu64 " %13" PRIu64
+                        " %20s"
+                        " %8d:%02d:%02d",
+                        node_name, files, size, lookup, failures, skipped,
+                        status_str, hrs, min, sec);
+            }
+        }
+        GF_FREE(size_str);
+    }
+
+    /* Max time will be non-zero if rebalance is still running */
+    if (max_time) {
+        hrs = max_time / 3600;
+        min = (max_time % 3600) / 60;
+        sec = (max_time % 3600) % 60;
+
+        if (hrs < REBAL_ESTIMATE_SEC_UPPER_LIMIT) {
+            cli_out(
+                "Estimated time left for rebalance to "
+                "complete : %8d:%02d:%02d",
+                hrs, min, sec);
+        } else {
+            cli_out(
+                "Estimated time left for rebalance to "
+                "complete : > 2 months. Please try again "
+                "later.");
+        }
+    } else {
+        /* Rebalance will return 0 if it could not calculate the
+         * estimates or if it is complete.
+         */
+        if (!show_estimates) {
+            goto out;
+        }
+        if (max_elapsed <= REBAL_ESTIMATE_START_TIME) {
+            cli_out(
+                "The estimated time for rebalance to complete "
+                "will be unavailable for the first 10 "
+                "minutes.");
+        } else {
+            cli_out(
+                "Rebalance estimated time unavailable. Please "
+                "try again later.");
         }
+    }
+out:
+    return ret;
+}
 
-        if (rsp.dict.dict_len) {
-                /* Unserialize the dictionary */
-                dict  = dict_new ();
-
-                ret = dict_unserialize (rsp.dict.dict_val,
-                                        rsp.dict.dict_len,
-                                        &dict);
-                if (ret < 0) {
-                        gf_log ("glusterd", GF_LOG_ERROR,
-                                "failed to "
-                                "unserialize req-buffer to dictionary");
-                        goto out;
-                }
+static int
+gf_cli_defrag_volume_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                         void *myframe)
+{
+    gf_cli_rsp rsp = {
+        0,
+    };
+    cli_local_t *local = NULL;
+    char *volname = NULL;
+    call_frame_t *frame = NULL;
+    int cmd = 0;
+    int ret = -1;
+    dict_t *dict = NULL;
+    char msg[1024] = {
+        0,
+    };
+    char *task_id_str = NULL;
+
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
+
+    GF_ASSERT(myframe);
+
+    frame = myframe;
+
+    GF_ASSERT(frame->local);
+
+    local = frame->local;
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(frame->this->name, GF_LOG_ERROR, XDR_DECODE_FAIL);
+        goto out;
+    }
+
+    ret = dict_get_str_sizen(local->dict, "volname", &volname);
+    if (ret) {
+        gf_log(frame->this->name, GF_LOG_ERROR, "Failed to get volname");
+        goto out;
+    }
+
+    ret = dict_get_int32_sizen(local->dict, "rebalance-command",
+                               (int32_t *)&cmd);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to get command");
+        goto out;
+    }
+
+    if (rsp.dict.dict_len) {
+        /* Unserialize the dictionary */
+        dict = dict_new();
+
+        ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len, &dict);
+        if (ret < 0) {
+            gf_log("glusterd", GF_LOG_ERROR, DICT_UNSERIALIZE_FAIL);
+            goto out;
         }
+    }
 
-        if (!((cmd == GF_DEFRAG_CMD_STOP) || (cmd == GF_DEFRAG_CMD_STATUS)) &&
-             !(global_state->mode & GLUSTER_MODE_XML)) {
-                /* All other possibilites are about starting a rebalance */
-                ret = dict_get_str (dict, GF_REBALANCE_TID_KEY, &task_id_str);
-                if (rsp.op_ret && strcmp (rsp.op_errstr, "")) {
-                        snprintf (msg, sizeof (msg), "%s", rsp.op_errstr);
-                } else {
-                        if (!rsp.op_ret) {
-                                snprintf (msg, sizeof (msg),
-                                          "Initiated rebalance on volume %s."
-                                          "\nExecute \"gluster volume rebalance"
-                                          " <volume-name> status\" to check"
-                                          " status.\nID: %s", volname,
-                                          task_id_str);
-                        } else {
-                                snprintf (msg, sizeof (msg),
-                                          "Starting rebalance on volume %s has "
-                                          "been unsuccessful.", volname);
-                        }
-                }
-                goto done;
-        }
-
-        if (cmd == GF_DEFRAG_CMD_STOP) {
-                if (rsp.op_ret == -1) {
-                        if (strcmp (rsp.op_errstr, ""))
-                                snprintf (msg, sizeof (msg),
-                                          "%s", rsp.op_errstr);
-                        else
-                                snprintf (msg, sizeof (msg),
-                                          "rebalance volume %s stop failed",
-                                          volname);
-                        goto done;
-                } else {
-                        snprintf (msg, sizeof (msg),
-                                  "rebalance process may be in the middle of a "
-                                  "file migration.\nThe process will be fully "
-                                  "stopped once the migration of the file is "
-                                  "complete.\nPlease check rebalance process "
-                                  "for completion before doing any further "
-                                  "brick related tasks on the volume.");
-                }
+    if (!((cmd == GF_DEFRAG_CMD_STOP) || (cmd == GF_DEFRAG_CMD_STATUS)) &&
+        !(global_state->mode & GLUSTER_MODE_XML)) {
+        ret = dict_get_str_sizen(dict, GF_REBALANCE_TID_KEY, &task_id_str);
+        if (ret) {
+            gf_log("cli", GF_LOG_WARNING, "failed to get %s from dict",
+                   GF_REBALANCE_TID_KEY);
         }
-        if (cmd == GF_DEFRAG_CMD_STATUS) {
-                if (rsp.op_ret == -1) {
-                        if (strcmp (rsp.op_errstr, ""))
-                                snprintf (msg, sizeof (msg),
-                                          "%s", rsp.op_errstr);
-                        else
-                                snprintf (msg, sizeof (msg),
-                                          "Failed to get the status of "
-                                          "rebalance process");
-                        goto done;
-                }
+        if (rsp.op_ret && strcmp(rsp.op_errstr, "")) {
+            snprintf(msg, sizeof(msg), "%s", rsp.op_errstr);
+        } else {
+            if (!rsp.op_ret) {
+                /* append errstr in the cli msg for successful
+                 * case since unlock failures can be highlighted
+                 * event though rebalance command was successful
+                 */
+                snprintf(msg, sizeof(msg),
+                         "Rebalance on %s has been "
+                         "started successfully. Use "
+                         "rebalance status command to"
+                         " check status of the "
+                         "rebalance process.\nID: %s",
+                         volname, task_id_str);
+            } else {
+                snprintf(msg, sizeof(msg),
+                         "Starting rebalance on volume %s has "
+                         "been unsuccessful.",
+                         volname);
+            }
+        }
+        goto done;
+    }
+
+    if (cmd == GF_DEFRAG_CMD_STOP) {
+        if (rsp.op_ret == -1) {
+            if (strcmp(rsp.op_errstr, ""))
+                snprintf(msg, sizeof(msg), "%s", rsp.op_errstr);
+            else
+                snprintf(msg, sizeof(msg), "rebalance volume %s stop failed",
+                         volname);
+            goto done;
+        } else {
+            /* append errstr in the cli msg for successful case
+             * since unlock failures can be highlighted event though
+             * rebalance command was successful */
+            snprintf(msg, sizeof(msg),
+                     "rebalance process may be in the middle of a "
+                     "file migration.\nThe process will be fully "
+                     "stopped once the migration of the file is "
+                     "complete.\nPlease check rebalance process "
+                     "for completion before doing any further "
+                     "brick related tasks on the volume.\n%s",
+                     rsp.op_errstr);
+        }
+    }
+    if (cmd == GF_DEFRAG_CMD_STATUS) {
+        if (rsp.op_ret == -1) {
+            if (strcmp(rsp.op_errstr, ""))
+                snprintf(msg, sizeof(msg), "%s", rsp.op_errstr);
+            else
+                snprintf(msg, sizeof(msg),
+                         "Failed to get the status of rebalance process");
+            goto done;
+        } else {
+            snprintf(msg, sizeof(msg), "%s", rsp.op_errstr);
         }
+    }
 
-        if (global_state->mode & GLUSTER_MODE_XML) {
-                ret = cli_xml_output_vol_rebalance (cmd, dict, rsp.op_ret,
-                                                    rsp.op_errno,
-                                                    rsp.op_errstr);
-                goto out;
-        }
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        ret = cli_xml_output_vol_rebalance(cmd, dict, rsp.op_ret, rsp.op_errno,
+                                           rsp.op_errstr);
+        goto out;
+    }
 
-        ret = gf_cli_print_rebalance_status (dict, GF_TASK_TYPE_REBALANCE);
-        if (ret)
-                gf_log ("cli", GF_LOG_ERROR,
-                        "Failed to print rebalance status");
+    ret = gf_cli_print_rebalance_status(dict, GF_TASK_TYPE_REBALANCE);
+    if (ret)
+        gf_log("cli", GF_LOG_ERROR, "Failed to print rebalance status");
 
 done:
-        if (global_state->mode & GLUSTER_MODE_XML)
-                cli_xml_output_str ("volRebalance", msg,
-                                    rsp.op_ret, rsp.op_errno,
-                                    rsp.op_errstr);
-        else {
-                if (rsp.op_ret)
-                        cli_err ("volume rebalance: %s: failed: %s", volname,
-                                 msg);
-                else
-                        cli_out ("volume rebalance: %s: success: %s", volname,
-                                 msg);
-        }
-        ret = rsp.op_ret;
+    if (global_state->mode & GLUSTER_MODE_XML)
+        cli_xml_output_str("volRebalance", msg, rsp.op_ret, rsp.op_errno,
+                           rsp.op_errstr);
+    else {
+        if (rsp.op_ret)
+            cli_err("volume rebalance: %s: failed%s%s", volname,
+                    strlen(msg) ? ": " : "", msg);
+        else
+            cli_out("volume rebalance: %s: success%s%s", volname,
+                    strlen(msg) ? ": " : "", msg);
+    }
+    ret = rsp.op_ret;
 
 out:
-        free (rsp.op_errstr); //malloced by xdr
-        free (rsp.dict.dict_val); //malloced by xdr
-        if (dict)
-                dict_unref (dict);
-        cli_cmd_broadcast_response (ret);
-        return ret;
+    gf_free_xdr_cli_rsp(rsp);
+    if (dict)
+        dict_unref(dict);
+    cli_cmd_broadcast_response(ret);
+    return ret;
 }
 
-int
-gf_cli_rename_volume_cbk (struct rpc_req *req, struct iovec *iov,
-                             int count, void *myframe)
+static int
+gf_cli_rename_volume_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                         void *myframe)
 {
-        gf_cli_rsp              rsp   = {0,};
-        int                     ret   = -1;
-        char                    msg[1024] = {0,};
-
-        if (-1 == req->rpc_status) {
-                goto out;
-        }
+    gf_cli_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    char msg[1024] = {
+        0,
+    };
+
+    GF_ASSERT(myframe);
+
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,
+               XDR_DECODE_FAIL);
+        goto out;
+    }
+
+    gf_log("cli", GF_LOG_INFO, "Received resp to probe");
+    snprintf(msg, sizeof(msg), "Rename volume %s",
+             (rsp.op_ret) ? "unsuccessful" : "successful");
+
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        ret = cli_xml_output_str("volRename", msg, rsp.op_ret, rsp.op_errno,
+                                 rsp.op_errstr);
+        if (ret)
+            gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+        goto out;
+    }
 
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
-        if (ret < 0) {
-                gf_log (((call_frame_t *) myframe)->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                goto out;
-        }
+    if (rsp.op_ret)
+        cli_err("volume rename: failed");
+    else
+        cli_out("volume rename: success");
 
+    ret = rsp.op_ret;
 
-        gf_log ("cli", GF_LOG_INFO, "Received resp to probe");
-        snprintf (msg, sizeof (msg), "Rename volume %s",
-                  (rsp.op_ret) ? "unsuccessful": "successful");
+out:
+    cli_cmd_broadcast_response(ret);
+    gf_free_xdr_cli_rsp(rsp);
+    return ret;
+}
 
-        if (global_state->mode & GLUSTER_MODE_XML) {
-                ret = cli_xml_output_str ("volRename", msg, rsp.op_ret,
-                                          rsp.op_errno, rsp.op_errstr);
-                if (ret)
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Error outputting to xml");
-                goto out;
-        }
+static int
+gf_cli_reset_volume_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                        void *myframe)
+{
+    gf_cli_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    char msg[1024] = {
+        0,
+    };
+
+    GF_ASSERT(myframe);
+
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,
+               XDR_DECODE_FAIL);
+        goto out;
+    }
+
+    gf_log("cli", GF_LOG_INFO, "Received resp to reset");
+
+    if (strcmp(rsp.op_errstr, ""))
+        snprintf(msg, sizeof(msg), "%s", rsp.op_errstr);
+    else
+        snprintf(msg, sizeof(msg), "reset volume %s",
+                 (rsp.op_ret) ? "unsuccessful" : "successful");
+
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        ret = cli_xml_output_str("volReset", msg, rsp.op_ret, rsp.op_errno,
+                                 rsp.op_errstr);
+        if (ret)
+            gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+        goto out;
+    }
 
-        if (rsp.op_ret)
-                cli_err ("volume rename: failed");
-        else
-                cli_out ("volume rename: success");
+    if (rsp.op_ret)
+        cli_err("volume reset: failed: %s", msg);
+    else
+        cli_out("volume reset: success: %s", msg);
 
-        ret = rsp.op_ret;
+    ret = rsp.op_ret;
 
 out:
-        cli_cmd_broadcast_response (ret);
-        return ret;
+    cli_cmd_broadcast_response(ret);
+    gf_free_xdr_cli_rsp(rsp);
+    return ret;
 }
 
-int
-gf_cli_reset_volume_cbk (struct rpc_req *req, struct iovec *iov,
-                             int count, void *myframe)
+static int
+gf_cli_ganesha_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                   void *myframe)
 {
-        gf_cli_rsp           rsp   = {0,};
-        int                  ret   = -1;
-        char                 msg[1024] = {0,};
+    gf_cli_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    dict_t *dict = NULL;
 
-        if (-1 == req->rpc_status) {
-                goto out;
-        }
+    GF_ASSERT(myframe);
 
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
-        if (ret < 0) {
-                gf_log (((call_frame_t *) myframe)->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                goto out;
-        }
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
 
-        gf_log ("cli", GF_LOG_INFO, "Received resp to reset");
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,
+               XDR_DECODE_FAIL);
+        goto out;
+    }
 
-        if (strcmp (rsp.op_errstr, ""))
-                snprintf (msg, sizeof (msg), "%s", rsp.op_errstr);
-        else
-                snprintf (msg, sizeof (msg), "reset volume %s",
-                          (rsp.op_ret) ? "unsuccessful": "successful");
+    gf_log("cli", GF_LOG_DEBUG, "Received resp to ganesha");
 
-        if (global_state->mode & GLUSTER_MODE_XML) {
-                ret = cli_xml_output_str ("volReset", msg, rsp.op_ret,
-                                          rsp.op_errno, rsp.op_errstr);
-                if (ret)
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Error outputting to xml");
-                goto out;
-        }
+    dict = dict_new();
 
-        if (rsp.op_ret)
-                cli_err ("volume reset: failed: %s", msg);
+    if (!dict) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len, &dict);
+    if (ret)
+        goto out;
+
+    if (rsp.op_ret) {
+        if (strcmp(rsp.op_errstr, ""))
+            cli_err("nfs-ganesha: failed: %s", rsp.op_errstr);
         else
-                cli_out ("volume reset: success: %s", msg);
+            cli_err("nfs-ganesha: failed");
+    }
 
-        ret = rsp.op_ret;
+    else {
+        cli_out("nfs-ganesha : success ");
+    }
+
+    ret = rsp.op_ret;
 
 out:
-        cli_cmd_broadcast_response (ret);
-        return ret;
+    if (dict)
+        dict_unref(dict);
+    cli_cmd_broadcast_response(ret);
+    return ret;
 }
 
-char *
-is_server_debug_xlator (void *myframe)
+static char *
+is_server_debug_xlator(void *myframe)
 {
-        call_frame_t         *frame        = NULL;
-        cli_local_t          *local        = NULL;
-        char                 **words       = NULL;
-        char                 *key          = NULL;
-        char                 *value        = NULL;
-        char                 *debug_xlator = NULL;
-
-        frame = myframe;
-        local = frame->local;
-        words = (char **)local->words;
-
-        while (*words != NULL) {
-                if (strstr (*words, "trace") == NULL &&
-                    strstr (*words, "error-gen") == NULL) {
-                        words++;
-                        continue;
-                }
-
-                key = *words;
+    call_frame_t *frame = NULL;
+    cli_local_t *local = NULL;
+    char **words = NULL;
+    char *key = NULL;
+    char *value = NULL;
+    char *debug_xlator = NULL;
+
+    frame = myframe;
+    local = frame->local;
+    words = (char **)local->words;
+
+    while (*words != NULL) {
+        if (strstr(*words, "trace") == NULL &&
+            strstr(*words, "error-gen") == NULL) {
+            words++;
+            continue;
+        }
+
+        key = *words;
+        words++;
+        value = *words;
+        if (value == NULL)
+            break;
+        if (strstr(value, "client")) {
+            words++;
+            continue;
+        } else {
+            if (!(strstr(value, "posix") || strstr(value, "acl") ||
+                  strstr(value, "locks") || strstr(value, "io-threads") ||
+                  strstr(value, "marker") || strstr(value, "index"))) {
                 words++;
-                value = *words;
-                if (value == NULL)
-                        break;
-                if (strstr (value, "client")) {
-                        words++;
-                        continue;
-                } else {
-                        if (!(strstr (value, "posix") || strstr (value, "acl")
-                              || strstr (value, "locks") ||
-                              strstr (value, "io-threads") ||
-                              strstr (value, "marker") ||
-                              strstr (value, "index"))) {
-                                words++;
-                                continue;
-                        } else {
-                                debug_xlator = gf_strdup (key);
-                                break;
-                        }
-                }
+                continue;
+            } else {
+                debug_xlator = gf_strdup(key);
+                break;
+            }
         }
+    }
 
-        return debug_xlator;
+    return debug_xlator;
 }
 
-int
-gf_cli_set_volume_cbk (struct rpc_req *req, struct iovec *iov,
-                             int count, void *myframe)
-{
-        gf_cli_rsp           rsp   = {0,};
-        int                  ret   = -1;
-        dict_t               *dict = NULL;
-        char                 *help_str = NULL;
-        char                 msg[1024] = {0,};
-        char                 *debug_xlator = NULL;
-        char                 tmp_str[512] = {0,};
-
-        if (-1 == req->rpc_status) {
-                goto out;
-        }
-
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
-        if (ret < 0) {
-                gf_log (((call_frame_t *) myframe)->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                goto out;
-        }
-
-        gf_log ("cli", GF_LOG_INFO, "Received resp to set");
-
-        dict = dict_new ();
-
-        if (!dict) {
-                ret = -1;
-                goto out;
-        }
-
-        ret = dict_unserialize (rsp.dict.dict_val, rsp.dict.dict_len, &dict);
-
-        /* For brick processes graph change does not happen on the fly.
-         * The process has to be restarted. So this is a check from the
-         * volume set option such that if debug xlators such as trace/errorgen
-         * are provided in the set command, warn the user.
-         */
-        debug_xlator = is_server_debug_xlator (myframe);
-
-        if (dict_get_str (dict, "help-str", &help_str) && !msg[0])
-                snprintf (msg, sizeof (msg), "Set volume %s",
-                          (rsp.op_ret) ? "unsuccessful": "successful");
-        if (rsp.op_ret == 0 && debug_xlator) {
-                snprintf (tmp_str, sizeof (tmp_str), "\n%s translator has been "
-                          "added to the server volume file. Please restart the"
-                          " volume for enabling the translator", debug_xlator);
-        }
-
-        if ((global_state->mode & GLUSTER_MODE_XML) && (help_str == NULL)) {
-                ret = cli_xml_output_str ("volSet", msg, rsp.op_ret,
-                                          rsp.op_errno, rsp.op_errstr);
-                if (ret)
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Error outputting to xml");
-                goto out;
-        }
+static int
+gf_cli_set_volume_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                      void *myframe)
+{
+    gf_cli_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    dict_t *dict = NULL;
+    char *help_str = NULL;
+    char msg[1024] = {
+        0,
+    };
+    char *debug_xlator = NULL;
+    char tmp_str[512] = {
+        0,
+    };
+
+    GF_ASSERT(myframe);
+
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,
+               XDR_DECODE_FAIL);
+        goto out;
+    }
+
+    gf_log("cli", GF_LOG_INFO, "Received resp to set");
+
+    dict = dict_new();
+
+    if (!dict) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len, &dict);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, DICT_UNSERIALIZE_FAIL);
+        goto out;
+    }
+
+    /* For brick processes graph change does not happen on the fly.
+     * The process has to be restarted. So this is a check from the
+     * volume set option such that if debug xlators such as trace/errorgen
+     * are provided in the set command, warn the user.
+     */
+    debug_xlator = is_server_debug_xlator(myframe);
+
+    if (dict_get_str_sizen(dict, "help-str", &help_str) && !msg[0])
+        snprintf(msg, sizeof(msg), "Set volume %s",
+                 (rsp.op_ret) ? "unsuccessful" : "successful");
+    if (rsp.op_ret == 0 && debug_xlator) {
+        snprintf(tmp_str, sizeof(tmp_str),
+                 "\n%s translator has been "
+                 "added to the server volume file. Please restart the"
+                 " volume for enabling the translator",
+                 debug_xlator);
+    }
+
+    if ((global_state->mode & GLUSTER_MODE_XML) && (help_str == NULL)) {
+        ret = cli_xml_output_str("volSet", msg, rsp.op_ret, rsp.op_errno,
+                                 rsp.op_errstr);
+        if (ret)
+            gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+        goto out;
+    }
 
-        if (rsp.op_ret) {
-                if (strcmp (rsp.op_errstr, ""))
-                        cli_err ("volume set: failed: %s", rsp.op_errstr);
-                else
-                        cli_err ("volume set: failed");
+    if (rsp.op_ret) {
+        if (strcmp(rsp.op_errstr, ""))
+            cli_err("volume set: failed: %s", rsp.op_errstr);
+        else
+            cli_err("volume set: failed");
+    } else {
+        if (help_str == NULL) {
+            if (debug_xlator == NULL)
+                cli_out("volume set: success");
+            else
+                cli_out("volume set: success%s", tmp_str);
         } else {
-                if (help_str == NULL) {
-                        if (debug_xlator == NULL)
-                                cli_out ("volume set: success");
-                        else
-                                cli_out ("volume set: success%s", tmp_str);
-                }else {
-                        cli_out ("%s", help_str);
-                }
+            cli_out("%s", help_str);
         }
+    }
 
-        ret = rsp.op_ret;
+    ret = rsp.op_ret;
 
 out:
-        if (dict)
-                dict_unref (dict);
-        GF_FREE (debug_xlator);
-        cli_cmd_broadcast_response (ret);
-        return ret;
+    if (dict)
+        dict_unref(dict);
+    GF_FREE(debug_xlator);
+    cli_cmd_broadcast_response(ret);
+    gf_free_xdr_cli_rsp(rsp);
+    return ret;
 }
 
 int
-gf_cli_add_brick_cbk (struct rpc_req *req, struct iovec *iov,
-                             int count, void *myframe)
+gf_cli_add_brick_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                     void *myframe)
 {
-        gf_cli_rsp                  rsp   = {0,};
-        int                         ret   = -1;
-        char                        msg[1024] = {0,};
-
-        if (-1 == req->rpc_status) {
-                goto out;
-        }
+    gf_cli_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    char msg[1024] = {
+        0,
+    };
+
+    GF_ASSERT(myframe);
+
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,
+               XDR_DECODE_FAIL);
+        goto out;
+    }
+
+    gf_log("cli", GF_LOG_INFO, "Received resp to add brick");
+
+    if (rsp.op_ret && strcmp(rsp.op_errstr, ""))
+        snprintf(msg, sizeof(msg), "%s", rsp.op_errstr);
+    else
+        snprintf(msg, sizeof(msg), "Add Brick %s",
+                 (rsp.op_ret) ? "unsuccessful" : "successful");
+
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        ret = cli_xml_output_str("volAddBrick", msg, rsp.op_ret, rsp.op_errno,
+                                 rsp.op_errstr);
+        if (ret)
+            gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+        goto out;
+    }
 
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
-        if (ret < 0) {
-                gf_log (((call_frame_t *) myframe)->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                goto out;
-        }
+    if (rsp.op_ret)
+        cli_err("volume add-brick: failed: %s", rsp.op_errstr);
+    else
+        cli_out("volume add-brick: success");
+    ret = rsp.op_ret;
 
+out:
+    cli_cmd_broadcast_response(ret);
+    gf_free_xdr_cli_rsp(rsp);
+    return ret;
+}
 
-        gf_log ("cli", GF_LOG_INFO, "Received resp to add brick");
+static int
+gf_cli3_remove_brick_status_cbk(struct rpc_req *req, struct iovec *iov,
+                                int count, void *myframe)
+{
+    gf_cli_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    dict_t *dict = NULL;
+    char msg[1024] = {
+        0,
+    };
+    int32_t command = 0;
+    gf1_op_commands cmd = GF_OP_CMD_NONE;
+    cli_local_t *local = NULL;
+    call_frame_t *frame = NULL;
+    const char *cmd_str;
 
-        if (rsp.op_ret && strcmp (rsp.op_errstr, ""))
-                snprintf (msg, sizeof (msg), "%s", rsp.op_errstr);
-        else
-                snprintf (msg, sizeof (msg), "Add Brick %s",
-                          (rsp.op_ret) ? "unsuccessful": "successful");
+    GF_ASSERT(myframe);
 
-        if (global_state->mode & GLUSTER_MODE_XML) {
-                ret = cli_xml_output_str ("volAddBrick", msg, rsp.op_ret,
-                                          rsp.op_errno, rsp.op_errstr);
-                if (ret)
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Error outputting to xml");
-                goto out;
-        }
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
 
-        if (rsp.op_ret)
-                cli_err ("volume add-brick: failed: %s", rsp.op_errstr);
-        else
-                cli_out ("volume add-brick: success");
-        ret = rsp.op_ret;
+    frame = myframe;
 
-out:
-        cli_cmd_broadcast_response (ret);
-        free (rsp.dict.dict_val);
-        free (rsp.op_errstr);
-        return ret;
-}
+    GF_ASSERT(frame->local);
 
-int
-gf_cli3_remove_brick_status_cbk (struct rpc_req *req, struct iovec *iov,
-                                 int count, void *myframe)
-{
-        gf_cli_rsp               rsp     = {0,};
-        int                      ret     = -1;
-        dict_t                  *dict    = NULL;
-        char                     msg[1024] = {0,};
-        int32_t                  command = 0;
-        gf1_op_commands          cmd = GF_OP_CMD_NONE;
-        cli_local_t             *local = NULL;
-        call_frame_t            *frame = NULL;
-        char                    *cmd_str = "unknown";
-
-        if (-1 == req->rpc_status) {
-                goto out;
-        }
+    local = frame->local;
 
-        frame = myframe;
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(frame->this->name, GF_LOG_ERROR, XDR_DECODE_FAIL);
+        goto out;
+    }
 
-        ret = xdr_to_generic (*iov, &rsp,
-                              (xdrproc_t)xdr_gf_cli_rsp);
-        if (ret < 0) {
-                gf_log (frame->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                goto out;
-        }
+    ret = dict_get_int32_sizen(local->dict, "command", &command);
+    if (ret)
+        goto out;
 
-        if (frame)
-                local = frame->local;
-        ret = dict_get_int32 (local->dict, "command", &command);
-        if (ret)
-                goto out;
-        cmd = command;
+    cmd = command;
 
-        switch (cmd) {
+    switch (cmd) {
         case GF_OP_CMD_STOP:
-                cmd_str = "stop";
-                break;
+            cmd_str = "stop";
+            break;
         case GF_OP_CMD_STATUS:
-                cmd_str = "status";
-                break;
+            cmd_str = "status";
+            break;
         default:
-                break;
-        }
-
-        ret = rsp.op_ret;
-        if (rsp.op_ret == -1) {
-                if (strcmp (rsp.op_errstr, ""))
-                        snprintf (msg, sizeof (msg), "volume remove-brick %s: "
-                                  "failed: %s", cmd_str, rsp.op_errstr);
-                else
-                        snprintf (msg, sizeof (msg), "volume remove-brick %s: "
-                                  "failed", cmd_str);
+            cmd_str = "unknown";
+            break;
+    }
+
+    ret = rsp.op_ret;
+    if (rsp.op_ret == -1) {
+        if (strcmp(rsp.op_errstr, ""))
+            snprintf(msg, sizeof(msg), "volume remove-brick %s: failed: %s",
+                     cmd_str, rsp.op_errstr);
+        else
+            snprintf(msg, sizeof(msg), "volume remove-brick %s: failed",
+                     cmd_str);
 
-                if (global_state->mode & GLUSTER_MODE_XML)
-                        goto xml_output;
+        if (global_state->mode & GLUSTER_MODE_XML)
+            goto xml_output;
 
-                cli_err ("%s", msg);
-                goto out;
-        }
+        cli_err("%s", msg);
+        goto out;
+    }
 
-        if (rsp.dict.dict_len) {
-                /* Unserialize the dictionary */
-                dict  = dict_new ();
+    if (rsp.dict.dict_len) {
+        /* Unserialize the dictionary */
+        dict = dict_new();
 
-                ret = dict_unserialize (rsp.dict.dict_val,
-                                        rsp.dict.dict_len,
-                                        &dict);
-                if (ret < 0) {
-                        strncpy (msg, "failed to unserialize req-buffer to "
-                                 "dictionary", sizeof (msg));
+        ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len, &dict);
+        if (ret < 0) {
+            strncpy(msg, DICT_UNSERIALIZE_FAIL, sizeof(msg));
 
-                        if (global_state->mode & GLUSTER_MODE_XML) {
-                                rsp.op_ret = -1;
-                                goto xml_output;
-                        }
+            if (global_state->mode & GLUSTER_MODE_XML) {
+                rsp.op_ret = -1;
+                goto xml_output;
+            }
 
-                        gf_log ("cli", GF_LOG_ERROR, "%s", msg);
-                        goto out;
-                }
+            gf_log("cli", GF_LOG_ERROR, "%s", msg);
+            goto out;
         }
+    }
 
 xml_output:
-        if (global_state->mode & GLUSTER_MODE_XML) {
-                if (strcmp (rsp.op_errstr, "")) {
-                        ret = cli_xml_output_vol_remove_brick (_gf_true, dict,
-                                                               rsp.op_ret,
-                                                               rsp.op_errno,
-                                                               rsp.op_errstr);
-                } else {
-                        ret = cli_xml_output_vol_remove_brick (_gf_true, dict,
-                                                               rsp.op_ret,
-                                                               rsp.op_errno,
-                                                               msg);
-                }
-                goto out;
-        }
-
-        ret = gf_cli_print_rebalance_status (dict, GF_TASK_TYPE_REMOVE_BRICK);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to print remove-brick "
-                        "rebalance status");
-                goto out;
-        }
-
-        if ((cmd == GF_OP_CMD_STOP) && (rsp.op_ret == 0)) {
-                cli_out ("'remove-brick' process may be in the middle of a "
-                         "file migration.\nThe process will be fully stopped "
-                         "once the migration of the file is complete.\nPlease "
-                         "check remove-brick process for completion before "
-                         "doing any further brick related tasks on the "
-                         "volume.");
-        }
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        if (strcmp(rsp.op_errstr, "")) {
+            ret = cli_xml_output_vol_remove_brick(_gf_true, dict, rsp.op_ret,
+                                                  rsp.op_errno, rsp.op_errstr,
+                                                  "volRemoveBrick");
+        } else {
+            ret = cli_xml_output_vol_remove_brick(_gf_true, dict, rsp.op_ret,
+                                                  rsp.op_errno, msg,
+                                                  "volRemoveBrick");
+        }
+        goto out;
+    }
+
+    ret = gf_cli_print_rebalance_status(dict, GF_TASK_TYPE_REMOVE_BRICK);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR,
+               "Failed to print remove-brick rebalance status");
+        goto out;
+    }
+
+    if ((cmd == GF_OP_CMD_STOP) && (rsp.op_ret == 0)) {
+        cli_out(
+            "'remove-brick' process may be in the middle of a "
+            "file migration.\nThe process will be fully stopped "
+            "once the migration of the file is complete.\nPlease "
+            "check remove-brick process for completion before "
+            "doing any further brick related tasks on the "
+            "volume.");
+    }
 
 out:
-        free (rsp.dict.dict_val); //malloced by xdr
-        if (dict)
-                dict_unref (dict);
-        cli_cmd_broadcast_response (ret);
-        return ret;
+    if (dict)
+        dict_unref(dict);
+    cli_cmd_broadcast_response(ret);
+    gf_free_xdr_cli_rsp(rsp);
+    return ret;
 }
 
-
-int
-gf_cli_remove_brick_cbk (struct rpc_req *req, struct iovec *iov,
-                             int count, void *myframe)
-{
-        gf_cli_rsp                      rsp   = {0,};
-        int                             ret   = -1;
-        char                            msg[1024] = {0,};
-        gf1_op_commands                 cmd = GF_OP_CMD_NONE;
-        char                           *cmd_str = "unknown";
-        cli_local_t                    *local = NULL;
-        call_frame_t                   *frame = NULL;
-        char                           *task_id_str = NULL;
-        dict_t                         *rsp_dict = NULL;
-
-        if (-1 == req->rpc_status) {
-                goto out;
-        }
-
-        frame = myframe;
-        local = frame->local;
-
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
-        if (ret < 0) {
-                gf_log (frame->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                goto out;
-        }
-
-        ret = dict_get_int32 (local->dict, "command", (int32_t *)&cmd);
+static int
+gf_cli_remove_brick_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                        void *myframe)
+{
+    gf_cli_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    char msg[1024] = {
+        0,
+    };
+    gf1_op_commands cmd = GF_OP_CMD_NONE;
+    char *cmd_str = "unknown";
+    cli_local_t *local = NULL;
+    call_frame_t *frame = NULL;
+    char *task_id_str = NULL;
+    dict_t *rsp_dict = NULL;
+
+    GF_ASSERT(myframe);
+
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
+
+    frame = myframe;
+
+    GF_ASSERT(frame->local);
+
+    local = frame->local;
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(frame->this->name, GF_LOG_ERROR, XDR_DECODE_FAIL);
+        goto out;
+    }
+
+    ret = dict_get_int32_sizen(local->dict, "command", (int32_t *)&cmd);
+    if (ret) {
+        gf_log("", GF_LOG_ERROR, "failed to get command");
+        goto out;
+    }
+
+    if (rsp.dict.dict_len) {
+        rsp_dict = dict_new();
+        if (!rsp_dict) {
+            ret = -1;
+            goto out;
+        }
+
+        ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len, &rsp_dict);
         if (ret) {
-                 gf_log ("", GF_LOG_ERROR, "failed to get command");
-                 goto out;
-        }
-
-        if (rsp.dict.dict_len) {
-                rsp_dict = dict_new ();
-                if (!rsp_dict) {
-                        ret = -1;
-                        goto out;
-                }
-
-                ret = dict_unserialize (rsp.dict.dict_val, rsp.dict.dict_len,
-                                        &rsp_dict);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Failed to unserialize rsp_dict");
-                        goto out;
-                }
+            gf_log("cli", GF_LOG_ERROR, DICT_UNSERIALIZE_FAIL);
+            goto out;
         }
+    }
 
-        switch (cmd) {
+    switch (cmd) {
+        case GF_OP_CMD_DETACH_START:
         case GF_OP_CMD_START:
-                cmd_str = "start";
-
-                ret = dict_get_str (rsp_dict, GF_REMOVE_BRICK_TID_KEY, &task_id_str);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "remove-brick-id is not present in dict");
-                }
-                break;
+            cmd_str = "start";
+
+            ret = dict_get_str_sizen(rsp_dict, GF_REMOVE_BRICK_TID_KEY,
+                                     &task_id_str);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR,
+                       "remove-brick-id is not present in dict");
+            }
+            break;
         case GF_OP_CMD_COMMIT:
-                cmd_str = "commit";
-                break;
+            cmd_str = "commit";
+            break;
         case GF_OP_CMD_COMMIT_FORCE:
-                cmd_str = "commit force";
-                break;
+            cmd_str = "commit force";
+            break;
         default:
-                cmd_str = "unknown";
-                break;
-        }
+            cmd_str = "unknown";
+            break;
+    }
+
+    gf_log("cli", GF_LOG_INFO, "Received resp to remove brick");
+
+    if (rsp.op_ret && strcmp(rsp.op_errstr, ""))
+        snprintf(msg, sizeof(msg), "%s", rsp.op_errstr);
+    else
+        snprintf(msg, sizeof(msg), "Remove Brick %s %s", cmd_str,
+                 (rsp.op_ret) ? "unsuccessful" : "successful");
+
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        ret = cli_xml_output_vol_remove_brick(_gf_false, rsp_dict, rsp.op_ret,
+                                              rsp.op_errno, msg,
+                                              "volRemoveBrick");
+        if (ret)
+            gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+        goto out;
+    }
+
+    if (rsp.op_ret) {
+        cli_err("volume remove-brick %s: failed: %s", cmd_str, msg);
+    } else {
+        cli_out("volume remove-brick %s: success", cmd_str);
+        if (GF_OP_CMD_START == cmd && task_id_str != NULL)
+            cli_out("ID: %s", task_id_str);
+        if (GF_OP_CMD_COMMIT == cmd)
+            cli_out(
+                "Check the removed bricks to ensure all files "
+                "are migrated.\nIf files with data are "
+                "found on the brick path, copy them via a "
+                "gluster mount point before re-purposing the "
+                "removed brick. ");
+    }
+
+    ret = rsp.op_ret;
 
-        gf_log ("cli", GF_LOG_INFO, "Received resp to remove brick");
+out:
+    cli_cmd_broadcast_response(ret);
+    gf_free_xdr_cli_rsp(rsp);
 
-        if (rsp.op_ret && strcmp (rsp.op_errstr, ""))
-                snprintf (msg, sizeof (msg), "%s", rsp.op_errstr);
-        else
-                snprintf (msg, sizeof (msg), "Remove Brick %s %s", cmd_str,
-                          (rsp.op_ret) ? "unsuccessful": "successful");
+    if (rsp_dict)
+        dict_unref(rsp_dict);
 
-        if (global_state->mode & GLUSTER_MODE_XML) {
-                ret = cli_xml_output_vol_remove_brick (_gf_false, rsp_dict,
-                                                       rsp.op_ret, rsp.op_errno,
-                                                       msg);
-                if (ret)
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Error outputting to xml");
-                goto out;
-        }
+    return ret;
+}
 
-        if (rsp.op_ret) {
-                cli_err ("volume remove-brick %s: failed: %s", cmd_str,
-                         msg);
-        } else {
-                cli_out ("volume remove-brick %s: success", cmd_str);
-                if (GF_OP_CMD_START == cmd && task_id_str != NULL)
-                        cli_out ("ID: %s", task_id_str);
-        }
+static int
+gf_cli_reset_brick_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                       void *myframe)
+{
+    gf_cli_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    cli_local_t *local = NULL;
+    call_frame_t *frame = NULL;
+    const char *rb_operation_str = NULL;
+    dict_t *rsp_dict = NULL;
+    char msg[1024] = {
+        0,
+    };
+    char *reset_op = NULL;
+
+    GF_ASSERT(myframe);
+
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
+
+    frame = myframe;
+
+    GF_ASSERT(frame->local);
+
+    local = frame->local;
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(frame->this->name, GF_LOG_ERROR, XDR_DECODE_FAIL);
+        goto out;
+    }
+
+    ret = dict_get_str_sizen(local->dict, "operation", &reset_op);
+    if (ret) {
+        gf_log(frame->this->name, GF_LOG_ERROR, "dict_get on operation failed");
+        goto out;
+    }
+
+    if (strcmp(reset_op, "GF_RESET_OP_START") &&
+        strcmp(reset_op, "GF_RESET_OP_COMMIT") &&
+        strcmp(reset_op, "GF_RESET_OP_COMMIT_FORCE")) {
+        ret = -1;
+        goto out;
+    }
+
+    if (rsp.dict.dict_len) {
+        /* Unserialize the dictionary */
+        rsp_dict = dict_new();
+
+        ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len, &rsp_dict);
+        if (ret < 0) {
+            gf_log(frame->this->name, GF_LOG_ERROR, DICT_UNSERIALIZE_FAIL);
+            goto out;
+        }
+    }
+
+    if (rsp.op_ret && (strcmp(rsp.op_errstr, ""))) {
+        rb_operation_str = rsp.op_errstr;
+    } else {
+        if (!strcmp(reset_op, "GF_RESET_OP_START")) {
+            if (rsp.op_ret)
+                rb_operation_str = "reset-brick start operation failed";
+            else
+                rb_operation_str = "reset-brick start operation successful";
+        } else if (!strcmp(reset_op, "GF_RESET_OP_COMMIT")) {
+            if (rsp.op_ret)
+                rb_operation_str = "reset-brick commit operation failed";
+            else
+                rb_operation_str = "reset-brick commit operation successful";
+        } else if (!strcmp(reset_op, "GF_RESET_OP_COMMIT_FORCE")) {
+            if (rsp.op_ret)
+                rb_operation_str = "reset-brick commit force operation failed";
+            else
+                rb_operation_str =
+                    "reset-brick commit force operation successful";
+        }
+    }
+
+    gf_log("cli", GF_LOG_INFO, "Received resp to reset brick");
+    snprintf(msg, sizeof(msg), "%s",
+             rb_operation_str ? rb_operation_str : "Unknown operation");
+
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        ret = cli_xml_output_vol_replace_brick(rsp_dict, rsp.op_ret,
+                                               rsp.op_errno, msg);
+        if (ret)
+            gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+        goto out;
+    }
 
-        ret = rsp.op_ret;
+    if (rsp.op_ret)
+        cli_err("volume reset-brick: failed: %s", msg);
+    else
+        cli_out("volume reset-brick: success: %s", msg);
+    ret = rsp.op_ret;
 
 out:
-        cli_cmd_broadcast_response (ret);
-        free (rsp.dict.dict_val);
-        free (rsp.op_errstr);
-
-        return ret;
-}
-
-
+    if (frame)
+        frame->local = NULL;
 
-int
-gf_cli_replace_brick_cbk (struct rpc_req *req, struct iovec *iov,
-                             int count, void *myframe)
-{
-        gf_cli_rsp                       rsp              = {0,};
-        int                              ret              = -1;
-        cli_local_t                     *local            = NULL;
-        call_frame_t                    *frame            = NULL;
-        dict_t                          *dict             = NULL;
-        char                            *src_brick        = NULL;
-        char                            *dst_brick        = NULL;
-        char                            *status_reply     = NULL;
-        gf1_cli_replace_op               replace_op       = 0;
-        char                            *rb_operation_str = NULL;
-        dict_t                          *rsp_dict         = NULL;
-        char                             msg[1024]        = {0,};
-        char                            *task_id_str      = NULL;
-
-        if (-1 == req->rpc_status) {
-                goto out;
-        }
+    if (local)
+        cli_local_wipe(local);
 
-        frame = (call_frame_t *) myframe;
+    cli_cmd_broadcast_response(ret);
+    gf_free_xdr_cli_rsp(rsp);
+    if (rsp_dict)
+        dict_unref(rsp_dict);
 
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    return ret;
+}
+static int
+gf_cli_replace_brick_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                         void *myframe)
+{
+    gf_cli_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    cli_local_t *local = NULL;
+    call_frame_t *frame = NULL;
+    const char *rb_operation_str = NULL;
+    dict_t *rsp_dict = NULL;
+    char msg[1024] = {
+        0,
+    };
+    char *replace_op = NULL;
+
+    GF_ASSERT(myframe);
+
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
+
+    frame = myframe;
+
+    GF_ASSERT(frame->local);
+
+    local = frame->local;
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(frame->this->name, GF_LOG_ERROR, XDR_DECODE_FAIL);
+        goto out;
+    }
+
+    ret = dict_get_str_sizen(local->dict, "operation", &replace_op);
+    if (ret) {
+        gf_log(frame->this->name, GF_LOG_ERROR, "dict_get on operation failed");
+        goto out;
+    }
+
+    if (rsp.dict.dict_len) {
+        /* Unserialize the dictionary */
+        rsp_dict = dict_new();
+
+        ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len, &rsp_dict);
         if (ret < 0) {
-                gf_log (frame->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                goto out;
-        }
-
-        local = frame->local;
-        GF_ASSERT (local);
-        dict = local->dict;
-
-        ret = dict_get_int32 (dict, "operation", (int32_t *)&replace_op);
-        if (ret) {
-                gf_log ("", GF_LOG_DEBUG,
-                        "dict_get on operation failed");
-                goto out;
-        }
-
-        if (rsp.dict.dict_len) {
-                /* Unserialize the dictionary */
-                rsp_dict  = dict_new ();
-
-                ret = dict_unserialize (rsp.dict.dict_val,
-                                rsp.dict.dict_len,
-                                &rsp_dict);
-                if (ret < 0) {
-                        gf_log ("glusterd", GF_LOG_ERROR,
-                                        "failed to "
-                                        "unserialize rsp buffer to dictionary");
-                        goto out;
-                }
+            gf_log(frame->this->name, GF_LOG_ERROR, DICT_UNSERIALIZE_FAIL);
+            goto out;
         }
+    }
 
-        switch (replace_op) {
-        case GF_REPLACE_OP_START:
-                if (rsp.op_ret) {
-                        rb_operation_str = gf_strdup ("replace-brick failed to"
-                                                      " start");
-                } else {
-                        ret = dict_get_str (rsp_dict, GF_REPLACE_BRICK_TID_KEY,
-                                            &task_id_str);
-                        if (ret) {
-                                gf_log ("cli", GF_LOG_ERROR, "Failed to get "
-                                        "\"replace-brick-id\" from dict");
-                                goto out;
-                        }
-                        ret = gf_asprintf (&rb_operation_str,
-                                           "replace-brick started successfully"
-                                           "\nID: %s", task_id_str);
-                        if (ret < 0)
-                                goto out;
-                }
-                break;
-
-        case GF_REPLACE_OP_STATUS:
-
-                if (rsp.op_ret || ret) {
-                        rb_operation_str = gf_strdup ("replace-brick status "
-                                                      "unknown");
-                } else {
-                        ret = dict_get_str (rsp_dict, "status-reply",
-                                            &status_reply);
-                        if (ret) {
-                                gf_log (frame->this->name, GF_LOG_ERROR, "failed to"
-                                        "get status");
-                                goto out;
-                        }
-
-                        rb_operation_str = gf_strdup (status_reply);
-                }
-
-                break;
-
-        case GF_REPLACE_OP_PAUSE:
-                if (rsp.op_ret)
-                        rb_operation_str = gf_strdup ("replace-brick pause "
-                                                      "failed");
-                else
-                        rb_operation_str = gf_strdup ("replace-brick paused "
-                                                      "successfully");
-                break;
-
-        case GF_REPLACE_OP_ABORT:
-                if (rsp.op_ret)
-                        rb_operation_str = gf_strdup ("replace-brick abort "
-                                                      "failed");
-                else
-                        rb_operation_str = gf_strdup ("replace-brick aborted "
-                                                      "successfully");
-                break;
-
-        case GF_REPLACE_OP_COMMIT:
-        case GF_REPLACE_OP_COMMIT_FORCE:
-                ret = dict_get_str (dict, "src-brick", &src_brick);
-                if (ret) {
-                        gf_log ("", GF_LOG_DEBUG,
-                                "dict_get on src-brick failed");
-                        goto out;
-                }
-
-                ret = dict_get_str (dict, "dst-brick", &dst_brick);
-                if (ret) {
-                        gf_log ("", GF_LOG_DEBUG,
-                                "dict_get on dst-brick failed");
-                        goto out;
-                }
-
+    if (!strcmp(replace_op, "GF_REPLACE_OP_COMMIT_FORCE")) {
+        if (rsp.op_ret || ret)
+            rb_operation_str = "replace-brick commit force operation failed";
+        else
+            rb_operation_str =
+                "replace-brick commit force operation successful";
+    } else {
+        gf_log(frame->this->name, GF_LOG_DEBUG, "Unknown operation");
+    }
+
+    if (rsp.op_ret && (strcmp(rsp.op_errstr, ""))) {
+        rb_operation_str = rsp.op_errstr;
+    }
+
+    gf_log("cli", GF_LOG_INFO, "Received resp to replace brick");
+    snprintf(msg, sizeof(msg), "%s",
+             rb_operation_str ? rb_operation_str : "Unknown operation");
+
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        ret = cli_xml_output_vol_replace_brick(rsp_dict, rsp.op_ret,
+                                               rsp.op_errno, msg);
+        if (ret)
+            gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+        goto out;
+    }
 
-                if (rsp.op_ret || ret)
-                        rb_operation_str = gf_strdup ("replace-brick commit "
-                                                      "failed");
-                else
-                        rb_operation_str = gf_strdup ("replace-brick commit "
-                                                      "successful");
+    if (rsp.op_ret)
+        cli_err("volume replace-brick: failed: %s", msg);
+    else
+        cli_out("volume replace-brick: success: %s", msg);
+    ret = rsp.op_ret;
 
-                break;
+out:
+    if (frame)
+        frame->local = NULL;
 
-        default:
-                gf_log ("", GF_LOG_DEBUG,
-                        "Unknown operation");
-                break;
-        }
+    if (local)
+        cli_local_wipe(local);
 
-        if (rsp.op_ret && (strcmp (rsp.op_errstr, ""))) {
-                rb_operation_str = gf_strdup (rsp.op_errstr);
-        }
+    cli_cmd_broadcast_response(ret);
+    gf_free_xdr_cli_rsp(rsp);
+    if (rsp_dict)
+        dict_unref(rsp_dict);
 
-        gf_log ("cli", GF_LOG_INFO, "Received resp to replace brick");
-        snprintf (msg, sizeof (msg), "%s",
-                  rb_operation_str ? rb_operation_str : "Unknown operation");
+    return ret;
+}
 
-        if (global_state->mode & GLUSTER_MODE_XML) {
-                ret = cli_xml_output_vol_replace_brick (replace_op, rsp_dict,
-                                                        rsp.op_ret,
-                                                        rsp.op_errno, msg);
-                if (ret)
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Error outputting to xml");
-                goto out;
-        }
+static int
+gf_cli_log_rotate_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                      void *myframe)
+{
+    gf_cli_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    char msg[1024] = {
+        0,
+    };
+
+    GF_ASSERT(myframe);
+
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,
+               XDR_DECODE_FAIL);
+        goto out;
+    }
+
+    gf_log("cli", GF_LOG_DEBUG, "Received resp to log rotate");
+
+    if (rsp.op_ret && strcmp(rsp.op_errstr, ""))
+        snprintf(msg, sizeof(msg), "%s", rsp.op_errstr);
+    else
+        snprintf(msg, sizeof(msg), "log rotate %s",
+                 (rsp.op_ret) ? "unsuccessful" : "successful");
+
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        ret = cli_xml_output_str("volLogRotate", msg, rsp.op_ret, rsp.op_errno,
+                                 rsp.op_errstr);
+        if (ret)
+            gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+        goto out;
+    }
 
-        if (rsp.op_ret)
-                cli_err ("volume replace-brick: failed: %s", msg);
-        else
-                cli_out ("volume replace-brick: success: %s", msg);
-        ret = rsp.op_ret;
+    if (rsp.op_ret)
+        cli_err("volume log-rotate: failed: %s", msg);
+    else
+        cli_out("volume log-rotate: success");
+    ret = rsp.op_ret;
 
 out:
-        if (frame)
-                frame->local = NULL;
+    cli_cmd_broadcast_response(ret);
+    gf_free_xdr_cli_rsp(rsp);
 
-        if (local) {
-                dict_unref (local->dict);
-                cli_local_wipe (local);
-        }
+    return ret;
+}
 
-        if (rb_operation_str)
-                GF_FREE (rb_operation_str);
+static int
+gf_cli_sync_volume_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                       void *myframe)
+{
+    gf_cli_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    char msg[1024] = {
+        0,
+    };
+
+    GF_ASSERT(myframe);
+
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,
+               XDR_DECODE_FAIL);
+        goto out;
+    }
+
+    gf_log("cli", GF_LOG_DEBUG, "Received resp to sync");
+
+    if (rsp.op_ret && strcmp(rsp.op_errstr, ""))
+        snprintf(msg, sizeof(msg), "volume sync: failed: %s", rsp.op_errstr);
+    else
+        snprintf(msg, sizeof(msg), "volume sync: %s",
+                 (rsp.op_ret) ? "failed" : "success");
+
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        ret = cli_xml_output_str("volSync", msg, rsp.op_ret, rsp.op_errno,
+                                 rsp.op_errstr);
+        if (ret)
+            gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+        goto out;
+    }
 
-        cli_cmd_broadcast_response (ret);
-        free (rsp.dict.dict_val);
-        if (rsp_dict)
-                dict_unref (rsp_dict);
+    if (rsp.op_ret)
+        cli_err("%s", msg);
+    else
+        cli_out("%s", msg);
+    ret = rsp.op_ret;
 
-        return ret;
+out:
+    cli_cmd_broadcast_response(ret);
+    gf_free_xdr_cli_rsp(rsp);
+    return ret;
 }
 
-
 static int
-gf_cli_log_rotate_cbk (struct rpc_req *req, struct iovec *iov,
-                          int count, void *myframe)
+print_quota_list_usage_output(cli_local_t *local, char *path, int64_t avail,
+                              char *sl_str, quota_limits_t *limits,
+                              quota_meta_t *used_space, gf_boolean_t sl,
+                              gf_boolean_t hl, double sl_num,
+                              gf_boolean_t limit_set)
 {
-        gf_cli_rsp             rsp   = {0,};
-        int                    ret   = -1;
-        char                   msg[1024] = {0,};
-
-        if (-1 == req->rpc_status) {
-                goto out;
-        }
-
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
-        if (ret < 0) {
-                gf_log (((call_frame_t *) myframe)->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                goto out;
+    int32_t ret = -1;
+    char *used_str = NULL;
+    char *avail_str = NULL;
+    char *hl_str = NULL;
+    char *sl_val = NULL;
+
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        ret = cli_quota_xml_output(local, path, limits->hl, sl_str, sl_num,
+                                   used_space->size, avail, sl ? "Yes" : "No",
+                                   hl ? "Yes" : "No", limit_set);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Failed to output in xml format for quota list command");
         }
+        goto out;
+    }
 
-        gf_log ("cli", GF_LOG_DEBUG, "Received resp to log rotate");
+    used_str = gf_uint64_2human_readable(used_space->size);
 
-        if (rsp.op_ret && strcmp (rsp.op_errstr, ""))
-                snprintf (msg, sizeof (msg), "%s", rsp.op_errstr);
-        else
-                snprintf (msg, sizeof (msg), "log rotate %s",
-                          (rsp.op_ret) ? "unsuccessful": "successful");
+    if (limit_set) {
+        hl_str = gf_uint64_2human_readable(limits->hl);
+        sl_val = gf_uint64_2human_readable(sl_num);
 
-        if (global_state->mode & GLUSTER_MODE_XML) {
-                ret = cli_xml_output_str ("volLogRotate", msg, rsp.op_ret,
-                                          rsp.op_errno, rsp.op_errstr);
-                if (ret)
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Error outputting to xml");
-                goto out;
+        if (!used_str) {
+            cli_out("%-40s %7s %7s(%s) %8" PRIu64 "%9" PRIu64
+                    ""
+                    "%15s %18s",
+                    path, hl_str, sl_str, sl_val, used_space->size, avail,
+                    sl ? "Yes" : "No", hl ? "Yes" : "No");
+        } else {
+            avail_str = gf_uint64_2human_readable(avail);
+            cli_out("%-40s %7s %7s(%s) %8s %7s %15s %20s", path, hl_str, sl_str,
+                    sl_val, used_str, avail_str, sl ? "Yes" : "No",
+                    hl ? "Yes" : "No");
         }
+    } else {
+        cli_out("%-36s %10s %10s %14s %9s %15s %18s", path, "N/A", "N/A",
+                used_str, "N/A", "N/A", "N/A");
+    }
 
-        if (rsp.op_ret)
-                cli_err ("volume log-rotate: failed: %s", msg);
-        else
-                cli_out ("volume log-rotate: success");
-        ret = rsp.op_ret;
-
+    ret = 0;
 out:
-        cli_cmd_broadcast_response (ret);
-        free (rsp.dict.dict_val);
+    GF_FREE(hl_str);
+    GF_FREE(used_str);
+    GF_FREE(avail_str);
+    GF_FREE(sl_val);
 
-        return ret;
+    return ret;
 }
 
 static int
-gf_cli_sync_volume_cbk (struct rpc_req *req, struct iovec *iov,
-                           int count, void *myframe)
+print_quota_list_object_output(cli_local_t *local, char *path, int64_t avail,
+                               char *sl_str, quota_limits_t *limits,
+                               quota_meta_t *used_space, gf_boolean_t sl,
+                               gf_boolean_t hl, double sl_num,
+                               gf_boolean_t limit_set)
 {
-        gf_cli_rsp                     rsp   = {0,};
-        int                            ret   = -1;
-        char                           msg[1024] = {0,};
+    int32_t ret = -1;
+    int64_t sl_val = sl_num;
 
-        if (-1 == req->rpc_status) {
-                goto out;
-        }
-
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
-        if (ret < 0) {
-                gf_log (((call_frame_t *) myframe)->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                goto out;
-        }
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        ret = cli_quota_object_xml_output(local, path, sl_str, sl_val, limits,
+                                          used_space, avail, sl ? "Yes" : "No",
+                                          hl ? "Yes" : "No", limit_set);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Failed to output in xml format for quota list command");
+        }
+        goto out;
+    }
+
+    if (limit_set) {
+        cli_out("%-40s %9" PRIu64 " %9s(%" PRId64 ") %10" PRIu64
+                ""
+                "%10" PRIu64 " %11" PRIu64 " %15s %20s",
+                path, limits->hl, sl_str, sl_val, used_space->file_count,
+                used_space->dir_count, avail, sl ? "Yes" : "No",
+                hl ? "Yes" : "No");
+    } else {
+        cli_out("%-40s %9s %9s %10" PRIu64 " %10" PRIu64 " %11s %15s %20s",
+                path, "N/A", "N/A", used_space->file_count,
+                used_space->dir_count, "N/A", "N/A", "N/A");
+    }
+    ret = 0;
 
-        gf_log ("cli", GF_LOG_DEBUG, "Received resp to sync");
+out:
 
-        if (rsp.op_ret && strcmp (rsp.op_errstr, ""))
-                snprintf (msg, sizeof (msg), "volume sync: failed: %s",
-                          rsp.op_errstr);
-        else
-                snprintf (msg, sizeof (msg), "volume sync: %s",
-                          (rsp.op_ret) ? "failed": "success");
+    return ret;
+}
 
-        if (global_state->mode & GLUSTER_MODE_XML) {
-                ret = cli_xml_output_str ("volSync", msg, rsp.op_ret,
-                                          rsp.op_errno, rsp.op_errstr);
-                if (ret)
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Error outputting to xml");
+static int
+print_quota_list_output(cli_local_t *local, char *path, char *default_sl,
+                        quota_limits_t *limits, quota_meta_t *used_space,
+                        int type, gf_boolean_t limit_set)
+{
+    int64_t avail = 0;
+    char percent_str[20] = {0};
+    char *sl_final = NULL;
+    int ret = -1;
+    double sl_num = 0;
+    gf_boolean_t sl = _gf_false;
+    gf_boolean_t hl = _gf_false;
+    int64_t used_size = 0;
+
+    GF_ASSERT(local);
+    GF_ASSERT(path);
+
+    if (limit_set) {
+        if (limits->sl < 0) {
+            ret = gf_string2percent(default_sl, &sl_num);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR,
+                       "could not convert default soft limit to percent");
+                goto out;
+            }
+            sl_num = (sl_num * limits->hl) / 100;
+            sl_final = default_sl;
+        } else {
+            sl_num = (limits->sl * limits->hl) / 100;
+            ret = snprintf(percent_str, sizeof(percent_str), "%" PRIu64 "%%",
+                           limits->sl);
+            if (ret < 0)
                 goto out;
+            sl_final = percent_str;
         }
-
-        if (rsp.op_ret)
-                cli_err ("%s", msg);
+        if (type == GF_QUOTA_OPTION_TYPE_LIST)
+            used_size = used_space->size;
         else
-                cli_out ("%s", msg);
-        ret = rsp.op_ret;
-
+            used_size = used_space->file_count + used_space->dir_count;
+
+        if (limits->hl > used_size) {
+            avail = limits->hl - used_size;
+            hl = _gf_false;
+            if (used_size > sl_num)
+                sl = _gf_true;
+            else
+                sl = _gf_false;
+        } else {
+            avail = 0;
+            hl = sl = _gf_true;
+        }
+    }
+
+    if (type == GF_QUOTA_OPTION_TYPE_LIST)
+        ret = print_quota_list_usage_output(local, path, avail, sl_final,
+                                            limits, used_space, sl, hl, sl_num,
+                                            limit_set);
+    else
+        ret = print_quota_list_object_output(local, path, avail, sl_final,
+                                             limits, used_space, sl, hl, sl_num,
+                                             limit_set);
 out:
-        cli_cmd_broadcast_response (ret);
-        return ret;
+    return ret;
 }
 
 static int
-print_quota_list_output (char *mountdir, char *default_sl, char *path)
-{
-        int64_t used_space       = 0;
-        int64_t avail            = 0;
-        char    *used_str         = NULL;
-        char    *avail_str        = NULL;
-        int     ret               = -1;
-        char    *sl_final         = NULL;
-        char    *hl_str           = NULL;
-        double  sl_num           = 0;
-        gf_boolean_t sl          = _gf_false;
-        gf_boolean_t hl          = _gf_false;
-        char percent_str[20]     = {0};
-
-        struct quota_limit {
-                int64_t hl;
-                int64_t sl;
-        } __attribute__ ((__packed__)) existing_limits;
-
-        ret = sys_lgetxattr (mountdir, "trusted.glusterfs.quota.limit-set",
-                             (void *)&existing_limits,
-                             sizeof (existing_limits));
-        if (ret < 0) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to get the xattr "
-                        "trusted.glusterfs.quota.limit-set on %s. Reason : %s",
-                        mountdir, strerror (errno));
-                switch (errno) {
+print_quota_list_from_mountdir(cli_local_t *local, char *mountdir,
+                               char *default_sl, char *path, int type)
+{
+    int ret = -1;
+    ssize_t xattr_size = 0;
+    quota_limits_t limits = {
+        0,
+    };
+    quota_meta_t used_space = {
+        0,
+    };
+    char *key = NULL;
+    gf_boolean_t limit_set = _gf_true;
+
+    GF_ASSERT(local);
+    GF_ASSERT(mountdir);
+    GF_ASSERT(path);
+
+    if (type == GF_QUOTA_OPTION_TYPE_LIST)
+        key = QUOTA_LIMIT_KEY;
+    else
+        key = QUOTA_LIMIT_OBJECTS_KEY;
+
+    ret = sys_lgetxattr(mountdir, key, (void *)&limits, sizeof(limits));
+    if (ret < 0) {
+        gf_log("cli", GF_LOG_ERROR,
+               "Failed to get the xattr %s on %s. Reason : %s", key, mountdir,
+               strerror(errno));
+
+        switch (errno) {
 #if defined(ENODATA)
-                case ENODATA:
+            case ENODATA:
 #endif
 #if defined(ENOATTR) && (ENOATTR != ENODATA)
-                case ENOATTR:
+            case ENOATTR:
 #endif
-                        cli_err ("%-40s %s", path, "Limit not set");
-                        break;
-                default:
-                        cli_err ("%-40s %s", path, strerror (errno));
-                        break;
-                }
-
-                goto out;
-        }
-
-        existing_limits.hl = ntoh64 (existing_limits.hl);
-        existing_limits.sl = ntoh64 (existing_limits.sl);
+                /* If it's an ENOATTR, quota/inode-quota is
+                 * configured(limit is set at least for one directory).
+                 * The user is trying to issue 'list/list-objects'
+                 * command for a directory on which quota limit is
+                 * not set and we are showing the used-space in case
+                 * of list-usage and showing (dir_count, file_count)
+                 * in case of list-objects. Other labels are
+                 * shown "N/A".
+                 */
 
-        hl_str = gf_uint64_2human_readable (existing_limits.hl);
+                limit_set = _gf_false;
+                goto enoattr;
+                break;
 
-        if (existing_limits.sl < 0) {
-                ret = gf_string2percent (default_sl, &sl_num);
-                sl_num = (sl_num * existing_limits.hl) / 100;
-                sl_final = default_sl;
-        } else {
-                sl_num = (existing_limits.sl * existing_limits.hl) / 100;
-                snprintf (percent_str, sizeof (percent_str), "%"PRIu64"%%",
-                          existing_limits.sl);
-                sl_final = percent_str;
+            default:
+                cli_err("%-40s %s", path, strerror(errno));
+                break;
         }
 
-        ret = sys_lgetxattr (mountdir, "trusted.glusterfs.quota.size",
-                             &used_space, sizeof (used_space));
+        goto out;
+    }
 
-        if (ret < 0) {
-                cli_out ("%-40s %7s %9s %11s %7s %15s %20s",
-                         path, hl_str, sl_final,
-                         "N/A", "N/A", "N/A", "N/A");
-        } else {
-                used_space = ntoh64 (used_space);
-
-                used_str = gf_uint64_2human_readable (used_space);
-
-                if (existing_limits.hl > used_space) {
-                        avail = existing_limits.hl - used_space;
-                        hl = _gf_false;
-                        if (used_space > sl_num)
-                                sl = _gf_true;
-                        else
-                                sl = _gf_false;
-                } else {
-                        avail = 0;
-                        hl = sl = _gf_true;
-                }
+    limits.hl = ntoh64(limits.hl);
+    limits.sl = ntoh64(limits.sl);
 
-                avail_str = gf_uint64_2human_readable (avail);
-                if (used_str == NULL) {
-                        cli_out ("%-40s %7s %9s %11"PRIu64
-                                 "%9"PRIu64" %15s %18s", path, hl_str,
-                                  sl_final, used_space, avail, sl? "Yes" : "No",
-                                  hl? "Yes" : "No");
-                } else {
-                        cli_out ("%-40s %7s %9s %11s %7s %15s %20s", path, hl_str,
-                                 sl_final, used_str, avail_str, sl? "Yes" : "No",
-                                 hl? "Yes" : "No");
-                }
-        }
+enoattr:
+    xattr_size = sys_lgetxattr(mountdir, QUOTA_SIZE_KEY, NULL, 0);
+    if (xattr_size < (sizeof(int64_t) * 2) &&
+        type == GF_QUOTA_OPTION_TYPE_LIST_OBJECTS) {
+        ret = -1;
 
+        /* This can happen when glusterfs is upgraded from 3.6 to 3.7
+         * and the xattr healing is not completed.
+         */
+    } else if (xattr_size > (sizeof(int64_t) * 2)) {
+        ret = sys_lgetxattr(mountdir, QUOTA_SIZE_KEY, &used_space,
+                            sizeof(used_space));
+    } else if (xattr_size > 0) {
+        /* This is for compatibility.
+         * Older version had only file usage
+         */
+        ret = sys_lgetxattr(mountdir, QUOTA_SIZE_KEY, &(used_space.size),
+                            sizeof(used_space.size));
+        used_space.file_count = 0;
+        used_space.dir_count = 0;
+    } else {
+        ret = -1;
+    }
+
+    if (ret < 0) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to get quota size on path %s: %s",
+               mountdir, strerror(errno));
+        print_quota_list_empty(path, type);
+        goto out;
+    }
+
+    used_space.size = ntoh64(used_space.size);
+    used_space.file_count = ntoh64(used_space.file_count);
+    used_space.dir_count = ntoh64(used_space.dir_count);
+
+    ret = print_quota_list_output(local, path, default_sl, &limits, &used_space,
+                                  type, limit_set);
 out:
-        GF_FREE (used_str);
-        GF_FREE (avail_str);
-        GF_FREE (hl_str);
-        return ret;
+    return ret;
 }
 
-int
-gf_cli_print_limit_list_from_dict (char *volname, dict_t *dict,
-                                   char *default_sl, int count, char *op_errstr)
+static int
+gluster_remove_auxiliary_mount(char *volname)
 {
-        int  ret               = -1;
-        int  i                 = 0;
-        char key[1024]         = {0,};
-        char mountdir[PATH_MAX] = {0,};
-        char *path              = NULL;
+    int ret = -1;
+    char mountdir[PATH_MAX] = {
+        0,
+    };
+    xlator_t *this = NULL;
 
-        if (!dict|| count <= 0)
-                goto out;
+    this = THIS;
+    GF_ASSERT(this);
 
-        /* Need to check if any quota limits are set on the volume before trying
-         * to list them
-         */
-        if (!_limits_set_on_volume (volname)) {
-                ret = 0;
-                cli_out ("quota: No quota configured on volume %s", volname);
-                goto out;
-        }
+    GLUSTERD_GET_QUOTA_LIST_MOUNT_PATH(mountdir, volname, "/");
+    ret = gf_umount_lazy(this->name, mountdir, 1);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "umount on %s failed, reason : %s",
+               mountdir, strerror(errno));
+    }
 
-        /* Check if the mount is online before doing any listing */
-        if (!_quota_aux_mount_online (volname)) {
-                ret = -1;
-                goto out;
-        }
+    return ret;
+}
 
-        cli_out ("                  Path                   Hard-limit "
-                 "Soft-limit   Used  Available  Soft-limit exceeded?"
-                 "  Hard-limit exceeded?");
-        cli_out ("--------------------------------------------------------"
-                 "--------------------------------------------------------"
-                 "-----------");
+static int
+gf_cli_print_limit_list_from_dict(cli_local_t *local, char *volname,
+                                  dict_t *dict, char *default_sl, int count,
+                                  int op_ret, int op_errno, char *op_errstr)
+{
+    int ret = -1;
+    int i = 0;
+    char key[32] = {
+        0,
+    };
+    int keylen;
+    char mountdir[PATH_MAX] = {
+        0,
+    };
+    char *path = NULL;
+    int type = -1;
+
+    if (!dict || count <= 0)
+        goto out;
+
+    ret = dict_get_int32_sizen(dict, "type", &type);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to get quota type");
+        goto out;
+    }
+
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        ret = cli_xml_output_vol_quota_limit_list_begin(local, op_ret, op_errno,
+                                                        op_errstr);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+            goto out;
+        }
+    } else {
+        print_quota_list_header(type);
+    }
 
-        while (count--) {
-                snprintf (key, sizeof (key), "path%d", i++);
+    while (count--) {
+        keylen = snprintf(key, sizeof(key), "path%d", i++);
 
-                ret = dict_get_str (dict, key, &path);
-                if (ret < 0) {
-                        gf_log ("cli", GF_LOG_DEBUG, "Path not present in limit"
-                                " list");
-                        continue;
-                }
+        ret = dict_get_strn(dict, key, keylen, &path);
+        if (ret < 0) {
+            gf_log("cli", GF_LOG_DEBUG, "Path not present in limit list");
+            continue;
+        }
 
-                ret = gf_canonicalize_path (path);
-                if (ret)
-                        goto out;
-                GLUSTERD_GET_QUOTA_AUX_MOUNT_PATH (mountdir, volname, path);
-                ret = print_quota_list_output (mountdir, default_sl, path);
+        ret = gf_canonicalize_path(path);
+        if (ret)
+            goto out;
+        GLUSTERD_GET_QUOTA_LIST_MOUNT_PATH(mountdir, volname, path);
+        ret = print_quota_list_from_mountdir(local, mountdir, default_sl, path,
+                                             type);
+    }
 
-        }
 out:
-        return ret;
+    return ret;
 }
 
-int
-print_quota_list_from_quotad (call_frame_t *frame, dict_t *rsp_dict)
-{
-        int64_t used_space    = 0;
-        int64_t avail         = 0;
-        int64_t *limit         = NULL;
-        char    *used_str      = NULL;
-        char    *avail_str     = NULL;
-        char    *hl_str        = NULL;
-        char    *sl_final      = NULL;
-        char    *path          = NULL;
-        char    *default_sl    = NULL;
-        int     ret            = -1;
-        cli_local_t *local     = NULL;
-        dict_t *gd_rsp_dict    = NULL;
-        double sl_num          = 0;
-        gf_boolean_t sl        = _gf_false;
-        gf_boolean_t hl        = _gf_false;
-        char  percent_str[20]  = {0,};
-
-        local = frame->local;
-        gd_rsp_dict = local->dict;
-
-        struct quota_limit {
-                int64_t hl;
-                int64_t sl;
-        } __attribute__ ((__packed__)) *existing_limits = NULL;
-
-        ret = dict_get_str (rsp_dict, GET_ANCESTRY_PATH_KEY, &path);
+static int
+print_quota_list_from_quotad(call_frame_t *frame, dict_t *rsp_dict)
+{
+    char *path = NULL;
+    char *default_sl = NULL;
+    int ret = -1;
+    cli_local_t *local = NULL;
+    dict_t *gd_rsp_dict = NULL;
+    quota_meta_t used_space = {
+        0,
+    };
+    quota_limits_t limits = {
+        0,
+    };
+    quota_limits_t *size_limits = NULL;
+    int32_t type = 0;
+    int32_t success_count = 0;
+
+    GF_ASSERT(frame);
+
+    local = frame->local;
+    gd_rsp_dict = local->dict;
+
+    ret = dict_get_int32_sizen(rsp_dict, "type", &type);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to get type");
+        goto out;
+    }
+
+    ret = dict_get_str_sizen(rsp_dict, GET_ANCESTRY_PATH_KEY, &path);
+    if (ret) {
+        gf_log("cli", GF_LOG_WARNING, "path key is not present in dict");
+        goto out;
+    }
+
+    ret = dict_get_str_sizen(gd_rsp_dict, "default-soft-limit", &default_sl);
+    if (ret) {
+        gf_log(frame->this->name, GF_LOG_ERROR,
+               "failed to get default soft limit");
+        goto out;
+    }
+
+    if (type == GF_QUOTA_OPTION_TYPE_LIST) {
+        ret = dict_get_bin(rsp_dict, QUOTA_LIMIT_KEY, (void **)&size_limits);
         if (ret) {
-                gf_log ("cli", GF_LOG_WARNING, "path key is not present "
-                        "in dict");
-                goto out;
+            gf_log("cli", GF_LOG_WARNING, "limit key not present in dict on %s",
+                   path);
+            goto out;
         }
-
-        ret = dict_get_bin (rsp_dict, QUOTA_LIMIT_KEY, (void**)&limit);
+    } else {
+        ret = dict_get_bin(rsp_dict, QUOTA_LIMIT_OBJECTS_KEY,
+                           (void **)&size_limits);
         if (ret) {
-                gf_log ("cli", GF_LOG_WARNING,
-                        "limit key not present in dict");
-                goto out;
-        }
-
-        ret = dict_get_str (gd_rsp_dict, "default-soft-limit", &default_sl);
-        if (ret) {
-                gf_log (frame->this->name, GF_LOG_ERROR, "failed to "
-                        "get default soft limit");
+            gf_log("cli", GF_LOG_WARNING,
+                   "object limit key not present in dict on %s", path);
+            goto out;
+        }
+    }
+
+    limits.hl = ntoh64(size_limits->hl);
+    limits.sl = ntoh64(size_limits->sl);
+
+    if (type == GF_QUOTA_OPTION_TYPE_LIST)
+        ret = quota_dict_get_meta(rsp_dict, QUOTA_SIZE_KEY,
+                                  SLEN(QUOTA_SIZE_KEY), &used_space);
+    else
+        ret = quota_dict_get_inode_meta(rsp_dict, QUOTA_SIZE_KEY,
+                                        SLEN(QUOTA_SIZE_KEY), &used_space);
+
+    if (ret < 0) {
+        gf_log("cli", GF_LOG_WARNING, "size key not present in dict");
+        print_quota_list_empty(path, type);
+        goto out;
+    }
+
+    LOCK(&local->lock);
+    {
+        ret = dict_get_int32_sizen(gd_rsp_dict, "quota-list-success-count",
+                                   &success_count);
+        if (ret)
+            success_count = 0;
+
+        ret = dict_set_int32_sizen(gd_rsp_dict, "quota-list-success-count",
+                                   success_count + 1);
+    }
+    UNLOCK(&local->lock);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR,
+               "Failed to set quota-list-success-count in dict");
+        goto out;
+    }
+
+    if (success_count == 0) {
+        if (!(global_state->mode & GLUSTER_MODE_XML)) {
+            print_quota_list_header(type);
+        } else {
+            ret = cli_xml_output_vol_quota_limit_list_begin(local, 0, 0, NULL);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR, XML_ERROR);
                 goto out;
+            }
         }
-        existing_limits = (struct quota_limit *)limit;
-        existing_limits->hl = ntoh64 (existing_limits->hl);
-        existing_limits->sl = ntoh64 (existing_limits->sl);
+    }
 
-        hl_str = gf_uint64_2human_readable (existing_limits->hl);
+    ret = print_quota_list_output(local, path, default_sl, &limits, &used_space,
+                                  type, _gf_true);
+out:
+    return ret;
+}
 
-        if (existing_limits->sl < 0) {
-                ret = gf_string2percent (default_sl, &sl_num);
-                sl_num = (sl_num * existing_limits->hl) / 100;
-                sl_final = default_sl;
-        } else {
-                sl_num = (existing_limits->sl * existing_limits->hl) / 100;
-                snprintf (percent_str, sizeof (percent_str), "%"PRIu64"%%",
-                          existing_limits->sl);
-                sl_final =  percent_str;
-        }
+static void *
+cli_cmd_broadcast_response_detached(void *opaque)
+{
+    int32_t ret = 0;
 
-        ret = dict_get_bin (rsp_dict, QUOTA_SIZE_KEY, (void**)&limit);
-        if (ret < 0) {
-                gf_log ("cli", GF_LOG_WARNING,
-                        "size key not present in dict");
-                cli_out ("%-40s %7s %9s %11s %7s %15s %20s", path, hl_str,
-                         sl_final, "N/A", "N/A", "N/A", "N/A");
-        } else {
-                used_space = *limit;
-                used_space = ntoh64 (used_space);
-                used_str = gf_uint64_2human_readable (used_space);
-
-                if (existing_limits->hl > used_space) {
-                        avail = existing_limits->hl - used_space;
-                        hl = _gf_false;
-                        if (used_space > sl_num)
-                                sl = _gf_true;
-                        else
-                                sl = _gf_false;
-                } else {
-                        avail = 0;
-                        hl = sl = _gf_true;
-                }
-                avail_str = gf_uint64_2human_readable (avail);
-                if (used_str == NULL)
-                        cli_out ("%-40s %7s %9s %11"PRIu64
-                                 "%9"PRIu64" %15s %20s", path, hl_str,
-                                 sl_final, used_space, avail, sl? "Yes" : "No",
-                                 hl? "Yes" : "No");
-                else
-                        cli_out ("%-40s %7s %9s %11s %7s %15s %20s", path,
-                                 hl_str, sl_final, used_str, avail_str,
-                                 sl? "Yes" : "No", hl? "Yes" : "No");
-        }
+    ret = (intptr_t)opaque;
+    cli_cmd_broadcast_response(ret);
 
-        ret = 0;
-out:
-        GF_FREE (used_str);
-        GF_FREE (avail_str);
-        GF_FREE (hl_str);
-        return ret;
+    return NULL;
 }
 
-int
-cli_quotad_getlimit_cbk (struct rpc_req *req, struct iovec *iov,
-                          int count, void *myframe)
+static int32_t
+cli_quota_compare_path(struct list_head *list1, struct list_head *list2)
 {
-    //TODO: we need to gather the path, hard-limit, soft-limit and used space
-        gf_cli_rsp         rsp         = {0,};
-        int                ret         = -1;
-        dict_t            *dict        = NULL;
-        call_frame_t      *frame       = NULL;
+    struct list_node *node1 = NULL;
+    struct list_node *node2 = NULL;
+    dict_t *dict1 = NULL;
+    dict_t *dict2 = NULL;
+    char *path1 = NULL;
+    char *path2 = NULL;
+    int ret = 0;
 
-        if (-1 == req->rpc_status) {
-                goto out;
-        }
+    node1 = list_entry(list1, struct list_node, list);
+    node2 = list_entry(list2, struct list_node, list);
+
+    dict1 = node1->ptr;
+    dict2 = node2->ptr;
+
+    ret = dict_get_str_sizen(dict1, GET_ANCESTRY_PATH_KEY, &path1);
+    if (ret < 0)
+        return 0;
 
-        frame = myframe;
+    ret = dict_get_str_sizen(dict2, GET_ANCESTRY_PATH_KEY, &path2);
+    if (ret < 0)
+        return 0;
 
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    return strcmp(path1, path2);
+}
+
+static int
+cli_quotad_getlimit_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                        void *myframe)
+{
+    /*TODO: we need to gather the path, hard-limit, soft-limit and used space*/
+    gf_cli_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    dict_t *dict = NULL;
+    struct list_node *node = NULL;
+    struct list_node *tmpnode = NULL;
+    call_frame_t *frame = NULL;
+    cli_local_t *local = NULL;
+    int32_t list_count = 0;
+    pthread_t th_id = {
+        0,
+    };
+    int32_t max_count = 0;
+
+    GF_ASSERT(myframe);
+
+    frame = myframe;
+
+    GF_ASSERT(frame->local);
+
+    local = frame->local;
+
+    LOCK(&local->lock);
+    {
+        ret = dict_get_int32_sizen(local->dict, "quota-list-count",
+                                   &list_count);
+        if (ret)
+            list_count = 0;
+
+        list_count++;
+        ret = dict_set_int32_sizen(local->dict, "quota-list-count", list_count);
+    }
+    UNLOCK(&local->lock);
+
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to set quota-list-count in dict");
+        goto out;
+    }
+
+    if (-1 == req->rpc_status) {
+        if (list_count == 0)
+            cli_err(
+                "Connection failed. Please check if quota "
+                "daemon is operational.");
+        ret = -1;
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(frame->this->name, GF_LOG_ERROR, XDR_DECODE_FAIL);
+        goto out;
+    }
+
+    if (rsp.op_ret) {
+        ret = -1;
+        if (strcmp(rsp.op_errstr, ""))
+            cli_err("quota command failed : %s", rsp.op_errstr);
+        else
+            cli_err("quota command : failed");
+        goto out;
+    }
+
+    if (rsp.dict.dict_len) {
+        /* Unserialize the dictionary */
+        dict = dict_new();
+
+        ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len, &dict);
         if (ret < 0) {
-                gf_log (frame->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                goto out;
+            gf_log("cli", GF_LOG_ERROR, DICT_UNSERIALIZE_FAIL);
+            goto out;
         }
 
-        if (rsp.op_ret) {
-                ret = -1;
-                if (strcmp (rsp.op_errstr, ""))
-                        cli_err ("quota command failed : %s", rsp.op_errstr);
-                else
-                        cli_err ("quota command : failed");
-                goto out;
+        node = list_node_add_order(dict, &local->dict_list,
+                                   cli_quota_compare_path);
+        if (node == NULL) {
+            gf_log("cli", GF_LOG_ERROR, "failed to add node to the list");
+            dict_unref(dict);
+            ret = -1;
+            goto out;
         }
+    }
 
-        if (rsp.dict.dict_len) {
-                /* Unserialize the dictionary */
-                dict  = dict_new ();
+    ret = dict_get_int32_sizen(local->dict, "max_count", &max_count);
+    if (ret < 0) {
+        gf_log("cli", GF_LOG_ERROR, "failed to get max_count");
+        goto out;
+    }
 
-                ret = dict_unserialize (rsp.dict.dict_val,
-                                        rsp.dict.dict_len,
-                                        &dict);
-                if (ret < 0) {
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "failed to "
-                                "unserialize req-buffer to dictionary");
-                        goto out;
-                }
-                print_quota_list_from_quotad (frame, dict);
+    if (list_count == max_count) {
+        list_for_each_entry_safe(node, tmpnode, &local->dict_list, list)
+        {
+            dict = node->ptr;
+            print_quota_list_from_quotad(frame, dict);
+            list_node_del(node);
+            dict_unref(dict);
         }
+    }
 
 out:
-        cli_cmd_broadcast_response (ret);
-        if (dict)
-                dict_unref (dict);
+    /* Bad Fix: CLI holds the lock to process a command.
+     * When processing quota list command, below sequence of steps executed
+     * in the same thread and causing deadlock
+     *
+     * 1) CLI holds the lock
+     * 2) Send rpc_clnt_submit request to quotad for quota usage
+     * 3) If quotad is down, rpc_clnt_submit invokes cbk function with error
+     * 4) cbk function cli_quotad_getlimit_cbk invokes
+     *    cli_cmd_broadcast_response which tries to hold lock to broadcast
+     *    the results and hangs, because same thread has already holding
+     *    the lock
+     *
+     * Broadcasting response in a separate thread which is not a
+     * good fix. This needs to be re-visted with better solution
+     */
+    if (ret == -1) {
+        ret = pthread_create(&th_id, NULL, cli_cmd_broadcast_response_detached,
+                             (void *)-1);
+        if (ret)
+            gf_log("cli", GF_LOG_ERROR, "pthread_create failed: %s",
+                   strerror(errno));
+    } else {
+        cli_cmd_broadcast_response(ret);
+    }
+    gf_free_xdr_cli_rsp(rsp);
 
-        free (rsp.dict.dict_val);
-        return ret;
+    return ret;
 }
 
-int
-cli_quotad_getlimit (call_frame_t *frame, xlator_t *this, void *data)
+static int
+cli_quotad_getlimit(call_frame_t *frame, xlator_t *this, void *data)
 {
-        gf_cli_req          req = {{0,}};
-        int                 ret = 0;
-        dict_t             *dict = NULL;
+    gf_cli_req req = {{
+        0,
+    }};
+    int ret = 0;
+    dict_t *dict = NULL;
 
-        if (!frame || !this ||  !data) {
-                ret = -1;
-                goto out;
-        }
+    if (!frame || !this || !data) {
+        ret = -1;
+        goto out;
+    }
 
-        dict = data;
-        ret = dict_allocate_and_serialize (dict, &req.dict.dict_val,
-                                           &req.dict.dict_len);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to serialize the data");
+    dict = data;
+    ret = add_cli_cmd_timeout_to_dict(dict);
 
-                goto out;
-        }
-
-        ret = cli_cmd_submit (global_quotad_rpc, &req, frame, &cli_quotad_clnt,
-                              GF_AGGREGATOR_GETLIMIT, NULL,
-                              this, cli_quotad_getlimit_cbk,
-                              (xdrproc_t) xdr_gf_cli_req);
+    ret = dict_allocate_and_serialize(dict, &req.dict.dict_val,
+                                      &req.dict.dict_len);
+    if (ret < 0) {
+        gf_log(this->name, GF_LOG_ERROR, DICT_SERIALIZE_FAIL);
 
-out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+        goto out;
+    }
 
+    ret = cli_cmd_submit(global_quotad_rpc, &req, frame, &cli_quotad_clnt,
+                         GF_AGGREGATOR_GETLIMIT, NULL, this,
+                         cli_quotad_getlimit_cbk, (xdrproc_t)xdr_gf_cli_req);
 
+out:
+    GF_FREE(req.dict.dict_val);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
+    return ret;
 }
 
-void
-gf_cli_quota_list (char *volname, dict_t *dict, int count, char *op_errstr,
-                   char *default_sl)
+static void
+gf_cli_quota_list(cli_local_t *local, char *volname, dict_t *dict,
+                  char *default_sl, int count, int op_ret, int op_errno,
+                  char *op_errstr)
 {
-        GF_VALIDATE_OR_GOTO ("cli", volname, out);
+    if (!cli_cmd_connected())
+        goto out;
 
-        if (!connected)
-                goto out;
+    if (count > 0) {
+        GF_VALIDATE_OR_GOTO("cli", volname, out);
 
-        if (count > 0)
-                gf_cli_print_limit_list_from_dict (volname, dict, default_sl,
-                                                   count, op_errstr);
+        gf_cli_print_limit_list_from_dict(local, volname, dict, default_sl,
+                                          count, op_ret, op_errno, op_errstr);
+    }
 out:
-        return;
+    return;
 }
 
-int
-gf_cli_quota_cbk (struct rpc_req *req, struct iovec *iov,
-                     int count, void *myframe)
-{
-        gf_cli_rsp         rsp         = {0,};
-        int                ret         = -1;
-        dict_t            *dict        = NULL;
-        char              *volname     = NULL;
-        int32_t            type        = 0;
-        call_frame_t      *frame       = NULL;
-        char              *default_sl  = NULL;
-        char              *limit_list  = NULL;
-        cli_local_t       *local       = NULL;
-        dict_t            *aggr        = NULL;
-        char              *default_sl_dup  = NULL;
-        int32_t            entry_count      = 0;
-        if (-1 == req->rpc_status) {
-                goto out;
+static int
+gf_cli_quota_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                 void *myframe)
+{
+    gf_cli_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    dict_t *dict = NULL;
+    char *volname = NULL;
+    int32_t type = 0;
+    call_frame_t *frame = NULL;
+    char *default_sl = NULL;
+    cli_local_t *local = NULL;
+    char *default_sl_dup = NULL;
+    int32_t entry_count = 0;
+
+    GF_ASSERT(myframe);
+
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
+
+    frame = myframe;
+
+    GF_ASSERT(frame->local);
+
+    local = frame->local;
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(frame->this->name, GF_LOG_ERROR, XDR_DECODE_FAIL);
+        goto out;
+    }
+
+    if (rsp.op_ret) {
+        ret = -1;
+        if (global_state->mode & GLUSTER_MODE_XML)
+            goto xml_output;
+
+        if (strcmp(rsp.op_errstr, "")) {
+            cli_err("quota command failed : %s", rsp.op_errstr);
+            if (rsp.op_ret == -ENOENT)
+                cli_err("please enter the path relative to the volume");
+        } else {
+            cli_err("quota command : failed");
         }
 
-        frame = myframe;
-        local = frame->local;
-        aggr  = local->dict;
+        goto out;
+    }
+
+    if (rsp.dict.dict_len) {
+        /* Unserialize the dictionary */
+        dict = dict_new();
 
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+        ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len, &dict);
         if (ret < 0) {
-                gf_log (frame->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                goto out;
+            gf_log("cli", GF_LOG_ERROR, DICT_UNSERIALIZE_FAIL);
+            goto out;
         }
+    }
 
-        if (rsp.op_ret) {
-                ret = -1;
-                if (global_state->mode & GLUSTER_MODE_XML)
-                        goto xml_output;
+    gf_log("cli", GF_LOG_DEBUG, "Received resp to quota command");
 
-                if (strcmp (rsp.op_errstr, ""))
-                        cli_err ("quota command failed : %s", rsp.op_errstr);
-                else
-                        cli_err ("quota command : failed");
+    ret = dict_get_str_sizen(dict, "default-soft-limit", &default_sl);
+    if (ret)
+        gf_log(frame->this->name, GF_LOG_TRACE,
+               "failed to get default soft limit");
 
-                goto out;
+    // default-soft-limit is part of rsp_dict only iff we sent
+    // GLUSTER_CLI_QUOTA with type being GF_QUOTA_OPTION_TYPE_LIST
+    if (default_sl) {
+        default_sl_dup = gf_strdup(default_sl);
+        if (!default_sl_dup) {
+            ret = -1;
+            goto out;
         }
-
-        if (rsp.dict.dict_len) {
-                /* Unserialize the dictionary */
-                dict  = dict_new ();
-
-                ret = dict_unserialize (rsp.dict.dict_val,
-                                        rsp.dict.dict_len,
-                                        &dict);
-                if (ret < 0) {
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "failed to "
-                                "unserialize req-buffer to dictionary");
-                        goto out;
-                }
+        ret = dict_set_dynstr_sizen(local->dict, "default-soft-limit",
+                                    default_sl_dup);
+        if (ret) {
+            gf_log(frame->this->name, GF_LOG_TRACE,
+                   "failed to set default soft limit");
+            GF_FREE(default_sl_dup);
         }
+    }
 
-        gf_log ("cli", GF_LOG_DEBUG, "Received resp to quota command");
+    ret = dict_get_str_sizen(dict, "volname", &volname);
+    if (ret)
+        gf_log(frame->this->name, GF_LOG_ERROR, "failed to get volname");
 
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret)
-                gf_log (frame->this->name, GF_LOG_ERROR,
-                        "failed to get volname");
+    ret = dict_get_int32_sizen(dict, "type", &type);
+    if (ret)
+        gf_log(frame->this->name, GF_LOG_TRACE, "failed to get type");
 
-        ret = dict_get_str (dict, "default-soft-limit", &default_sl);
-        if (ret)
-                gf_log (frame->this->name, GF_LOG_TRACE, "failed to get "
-                        "default soft limit");
+    ret = dict_get_int32_sizen(dict, "count", &entry_count);
+    if (ret)
+        gf_log(frame->this->name, GF_LOG_TRACE, "failed to get count");
 
-        // default-soft-limit is part of rsp_dict only iff we sent
-        // GLUSTER_CLI_QUOTA with type being GF_QUOTA_OPTION_TYPE_LIST
-        if (default_sl) {
-                default_sl_dup = gf_strdup (default_sl);
-                if (!default_sl_dup) {
-                        ret = -1;
-                        goto out;
-                }
-                ret = dict_set_dynstr (aggr, "default-soft-limit",
-                                       default_sl_dup);
-                if (ret) {
-                        gf_log (frame->this->name, GF_LOG_TRACE,
-                                "failed to set default soft limit");
-                        GF_FREE (default_sl_dup);
-                }
+    if ((type == GF_QUOTA_OPTION_TYPE_LIST) ||
+        (type == GF_QUOTA_OPTION_TYPE_LIST_OBJECTS)) {
+        gf_cli_quota_list(local, volname, dict, default_sl, entry_count,
+                          rsp.op_ret, rsp.op_errno, rsp.op_errstr);
+
+        if (global_state->mode & GLUSTER_MODE_XML) {
+            ret = cli_xml_output_vol_quota_limit_list_end(local);
+            if (ret < 0) {
+                ret = -1;
+                gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+            }
+            goto out;
         }
+    }
 
-        ret = dict_get_int32 (dict, "type", &type);
-        if (ret)
-                gf_log (frame->this->name, GF_LOG_TRACE,
-                        "failed to get type");
+xml_output:
 
-        ret = dict_get_int32 (dict, "count", &entry_count);
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        ret = cli_xml_output_str("volQuota", NULL, rsp.op_ret, rsp.op_errno,
+                                 rsp.op_errstr);
         if (ret)
-                gf_log (frame->this->name, GF_LOG_TRACE, "failed to get count");
+            gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+        goto out;
+    }
 
-        if (type == GF_QUOTA_OPTION_TYPE_LIST) {
-                if (global_state->mode & GLUSTER_MODE_XML) {
-                        ret = cli_xml_output_vol_quota_limit_list
-                                (volname, limit_list, rsp.op_ret,
-                                 rsp.op_errno, rsp.op_errstr);
-                        if (ret)
-                                gf_log ("cli", GF_LOG_ERROR,
-                                        "Error outputting to xml");
-                        goto out;
-                }
-
-                gf_cli_quota_list (volname, dict, entry_count, rsp.op_errstr,
-                                   default_sl);
-        }
+    if (!rsp.op_ret && type != GF_QUOTA_OPTION_TYPE_LIST &&
+        type != GF_QUOTA_OPTION_TYPE_LIST_OBJECTS)
+        cli_out("volume quota : success");
 
-xml_output:
-        if (global_state->mode & GLUSTER_MODE_XML) {
-                ret = cli_xml_output_str ("volQuota", NULL, rsp.op_ret,
-                                          rsp.op_errno, rsp.op_errstr);
-                if (ret)
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Error outputting to xml");
-                goto out;
-        }
+    ret = rsp.op_ret;
+out:
 
-        if (!rsp.op_ret && type != GF_QUOTA_OPTION_TYPE_LIST)
-                cli_out ("volume quota : success");
+    if ((type == GF_QUOTA_OPTION_TYPE_LIST) ||
+        (type == GF_QUOTA_OPTION_TYPE_LIST_OBJECTS)) {
+        gluster_remove_auxiliary_mount(volname);
+    }
 
-        ret = rsp.op_ret;
-out:
-        cli_cmd_broadcast_response (ret);
-        if (dict)
-                dict_unref (dict);
+    cli_cmd_broadcast_response(ret);
+    if (dict)
+        dict_unref(dict);
 
-        free (rsp.dict.dict_val);
+    gf_free_xdr_cli_rsp(rsp);
 
-        return ret;
+    return ret;
 }
 
-int
-gf_cli_getspec_cbk (struct rpc_req *req, struct iovec *iov,
-                       int count, void *myframe)
+static int
+gf_cli_getspec_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                   void *myframe)
 {
-        gf_getspec_rsp          rsp   = {0,};
-        int                     ret   = -1;
-        char                   *spec  = NULL;
-        call_frame_t           *frame = NULL;
-
-        if (-1 == req->rpc_status) {
-                goto out;
-        }
-
-        frame = myframe;
-
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_getspec_rsp);
-        if (ret < 0) {
-                gf_log (frame->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                goto out;
-        }
-
-        if (rsp.op_ret == -1) {
-                gf_log (frame->this->name, GF_LOG_ERROR,
-                        "getspec failed");
-                goto out;
-        }
-
-        gf_log ("cli", GF_LOG_INFO, "Received resp to getspec");
-
-        spec = GF_MALLOC (rsp.op_ret + 1, cli_mt_char);
-        if (!spec) {
-                gf_log("", GF_LOG_ERROR, "out of memory");
-                goto out;
-        }
-        memcpy (spec, rsp.spec, rsp.op_ret);
-        spec[rsp.op_ret] = '\0';
-        cli_out ("%s", spec);
-        GF_FREE (spec);
-
-        ret = 0;
+    gf_getspec_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    char *spec = NULL;
+
+    GF_ASSERT(myframe);
+
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_getspec_rsp);
+    if (ret < 0) {
+        gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,
+               XDR_DECODE_FAIL);
+        goto out;
+    }
+
+    if (rsp.op_ret == -1) {
+        gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,
+               "getspec failed");
+        ret = -1;
+        goto out;
+    }
+
+    gf_log("cli", GF_LOG_INFO, "Received resp to getspec");
+
+    spec = GF_MALLOC(rsp.op_ret + 1, cli_mt_char);
+    if (!spec) {
+        gf_log("", GF_LOG_ERROR, "out of memory");
+        ret = -1;
+        goto out;
+    }
+    memcpy(spec, rsp.spec, rsp.op_ret);
+    spec[rsp.op_ret] = '\0';
+    cli_out("%s", spec);
+    GF_FREE(spec);
+
+    ret = 0;
 
 out:
-        cli_cmd_broadcast_response (ret);
-        return ret;
+    cli_cmd_broadcast_response(ret);
+    gf_free_xdr_getspec_rsp(rsp);
+    return ret;
 }
 
-int
-gf_cli_pmap_b2p_cbk (struct rpc_req *req, struct iovec *iov,
-                        int count, void *myframe)
+static int
+gf_cli_pmap_b2p_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                    void *myframe)
 {
-        pmap_port_by_brick_rsp rsp = {0,};
-        int                     ret   = -1;
-        char                   *spec  = NULL;
-        call_frame_t           *frame = NULL;
+    pmap_port_by_brick_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    char *spec = NULL;
 
-        if (-1 == req->rpc_status) {
-                goto out;
-        }
+    GF_ASSERT(myframe);
 
-        frame = myframe;
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
 
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_pmap_port_by_brick_rsp);
-        if (ret < 0) {
-                gf_log (frame->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                goto out;
-        }
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_pmap_port_by_brick_rsp);
+    if (ret < 0) {
+        gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,
+               XDR_DECODE_FAIL);
+        goto out;
+    }
 
-        if (rsp.op_ret == -1) {
-                gf_log (frame->this->name, GF_LOG_ERROR,
-                        "pump_b2p failed");
-                goto out;
-        }
+    if (rsp.op_ret == -1) {
+        gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,
+               "pump_b2p failed");
+        ret = -1;
+        goto out;
+    }
 
-        gf_log ("cli", GF_LOG_INFO, "Received resp to pmap b2p");
+    gf_log("cli", GF_LOG_INFO, "Received resp to pmap b2p");
 
-        cli_out ("%d", rsp.port);
-        GF_FREE (spec);
+    cli_out("%d", rsp.port);
+    GF_FREE(spec);
 
-        ret = rsp.op_ret;
+    ret = rsp.op_ret;
 
 out:
-        cli_cmd_broadcast_response (ret);
-        return ret;
+    cli_cmd_broadcast_response(ret);
+    return ret;
 }
 
-
-int32_t
-gf_cli_probe (call_frame_t *frame, xlator_t *this,
-                 void *data)
+static int32_t
+gf_cli_probe(call_frame_t *frame, xlator_t *this, void *data)
 {
-        gf_cli_req         req      = {{0,},};
-        int                ret      = 0;
-        dict_t            *dict     = NULL;
-        int                port     = 0;
-
-        if (!frame || !this ||  !data) {
-                ret = -1;
-                goto out;
-        }
-
-        dict = data;
-
-        ret = dict_get_int32 (dict, "port", &port);
-        if (ret) {
-                ret = dict_set_int32 (dict, "port", CLI_GLUSTERD_PORT);
-                if (ret)
-                        goto out;
-        }
+    gf_cli_req req = {
+        {
+            0,
+        },
+    };
+    int ret = 0;
+    dict_t *dict = NULL;
+    int port = 0;
+
+    if (!frame || !this || !data) {
+        ret = -1;
+        goto out;
+    }
+
+    dict = data;
+
+    ret = dict_get_int32_sizen(dict, "port", &port);
+    if (ret) {
+        ret = dict_set_int32_sizen(dict, "port", CLI_GLUSTERD_PORT);
+        if (ret)
+            goto out;
+    }
 
-        ret = cli_to_glusterd (&req, frame, gf_cli_probe_cbk,
-                               (xdrproc_t) xdr_gf_cli_req, dict,
-                               GLUSTER_CLI_PROBE, this, cli_rpc_prog, NULL);
+    ret = cli_to_glusterd(&req, frame, gf_cli_probe_cbk,
+                          (xdrproc_t)xdr_gf_cli_req, dict, GLUSTER_CLI_PROBE,
+                          this, cli_rpc_prog, NULL);
 
 out:
-        GF_FREE (req.dict.dict_val);
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    GF_FREE(req.dict.dict_val);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
 
-        return ret;
+    return ret;
 }
 
-int32_t
-gf_cli_deprobe (call_frame_t *frame, xlator_t *this,
-                   void *data)
+static int32_t
+gf_cli_deprobe(call_frame_t *frame, xlator_t *this, void *data)
 {
-        gf_cli_req           req      = {{0,},};
-        int                  ret      = 0;
-        dict_t              *dict     = NULL;
-        int                  port     = 0;
-        int                  flags    = 0;
-
-        if (!frame || !this ||  !data) {
-                ret = -1;
-                goto out;
-        }
-
-        dict = data;
-        ret = dict_get_int32 (dict, "port", &port);
-        if (ret) {
-                ret = dict_set_int32 (dict, "port", CLI_GLUSTERD_PORT);
-                if (ret)
-                        goto out;
-        }
+    gf_cli_req req = {
+        {
+            0,
+        },
+    };
+    int ret = 0;
+    dict_t *dict = NULL;
+    int port = 0;
+    int flags = 0;
+
+    if (!frame || !this || !data) {
+        ret = -1;
+        goto out;
+    }
+
+    dict = data;
+    ret = dict_get_int32_sizen(dict, "port", &port);
+    if (ret) {
+        ret = dict_set_int32_sizen(dict, "port", CLI_GLUSTERD_PORT);
+        if (ret)
+            goto out;
+    }
 
-        ret = dict_get_int32 (dict, "flags", &flags);
-        if (ret) {
-                ret = dict_set_int32 (dict, "flags", 0);
-                if (ret)
-                        goto out;
-        }
+    ret = dict_get_int32_sizen(dict, "flags", &flags);
+    if (ret) {
+        ret = dict_set_int32_sizen(dict, "flags", 0);
+        if (ret)
+            goto out;
+    }
 
-        ret = cli_to_glusterd (&req, frame, gf_cli_deprobe_cbk,
-                               (xdrproc_t)xdr_gf_cli_req, dict,
-                              GLUSTER_CLI_DEPROBE, this, cli_rpc_prog, NULL);
+    ret = cli_to_glusterd(&req, frame, gf_cli_deprobe_cbk,
+                          (xdrproc_t)xdr_gf_cli_req, dict, GLUSTER_CLI_DEPROBE,
+                          this, cli_rpc_prog, NULL);
 
 out:
-        GF_FREE (req.dict.dict_val);
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    GF_FREE(req.dict.dict_val);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
 
-        return ret;
+    return ret;
 }
 
-int32_t
-gf_cli_list_friends (call_frame_t *frame, xlator_t *this,
-                     void *data)
+static int32_t
+gf_cli_list_friends(call_frame_t *frame, xlator_t *this, void *data)
 {
-        gf1_cli_peer_list_req   req = {0,};
-        int                     ret = 0;
-        unsigned long           flags = 0;
+    gf1_cli_peer_list_req req = {
+        0,
+    };
+    int ret = 0;
+    unsigned long flags = 0;
+
+    if (!frame || !this) {
+        ret = -1;
+        goto out;
+    }
+
+    GF_ASSERT(frame->local == NULL);
+
+    flags = (long)data;
+    req.flags = flags;
+    frame->local = (void *)flags;
+    ret = cli_cmd_submit(
+        NULL, &req, frame, cli_rpc_prog, GLUSTER_CLI_LIST_FRIENDS, NULL, this,
+        gf_cli_list_friends_cbk, (xdrproc_t)xdr_gf1_cli_peer_list_req);
 
-        if (!frame || !this) {
-                ret = -1;
-                goto out;
-        }
+out:
+    if (ret && frame) {
+        /*
+         * If everything goes fine, gf_cli_list_friends_cbk()
+         * [invoked through cli_cmd_submit()]resets the
+         * frame->local to NULL. In case cli_cmd_submit()
+         * fails in between, RESET frame->local here.
+         */
+        frame->local = NULL;
+    }
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
+    return ret;
+}
 
-        GF_ASSERT (frame->local == NULL);
+static int32_t
+gf_cli_get_state(call_frame_t *frame, xlator_t *this, void *data)
+{
+    gf_cli_req req = {
+        {
+            0,
+        },
+    };
+    int ret = 0;
+    dict_t *dict = NULL;
 
-        flags = (long)data;
-        req.flags = flags;
-        frame->local = (void*)flags;
-        ret = cli_cmd_submit (NULL, &req, frame, cli_rpc_prog,
-                              GLUSTER_CLI_LIST_FRIENDS, NULL,
-                              this, gf_cli_list_friends_cbk,
-                              (xdrproc_t) xdr_gf1_cli_peer_list_req);
+    dict = data;
 
-out:
-        if (ret) {
-                /*
-                 * If everything goes fine, gf_cli_list_friends_cbk()
-                 * [invoked through cli_cmd_submit()]resets the
-                 * frame->local to NULL. In case cli_cmd_submit()
-                 * fails in between, RESET frame->local here.
-                 */
-                frame->local = NULL;
-        }
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    ret = cli_to_glusterd(&req, frame, gf_cli_get_state_cbk,
+                          (xdrproc_t)xdr_gf_cli_req, dict,
+                          GLUSTER_CLI_GET_STATE, this, cli_rpc_prog, NULL);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
+    GF_FREE(req.dict.dict_val);
+
+    return ret;
 }
 
-int32_t
-gf_cli_get_next_volume (call_frame_t *frame, xlator_t *this,
-                           void *data)
+static int32_t
+gf_cli_get_next_volume(call_frame_t *frame, xlator_t *this, void *data)
 {
+    int ret = 0;
+    cli_cmd_volume_get_ctx_t *ctx = NULL;
+    cli_local_t *local = NULL;
 
-        int                             ret = 0;
-        cli_cmd_volume_get_ctx_t        *ctx = NULL;
-        cli_local_t                     *local = NULL;
+    if (!frame || !this || !data) {
+        ret = -1;
+        goto out;
+    }
 
-        if (!frame || !this || !data) {
-                ret = -1;
-                goto out;
-        }
-
-        ctx = data;
-        local = frame->local;
+    ctx = data;
+    local = frame->local;
 
-        if (global_state->mode & GLUSTER_MODE_XML) {
-                ret = cli_xml_output_vol_info_begin (local, 0, 0, "");
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Error outputting to xml");
-                        goto out;
-                }
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        ret = cli_xml_output_vol_info_begin(local, 0, 0, "");
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+            goto out;
         }
+    }
 
-        ret = gf_cli_get_volume (frame, this, data);
+    ret = gf_cli_get_volume(frame, this, data);
 
+    if (!local || !local->get_vol.volname) {
+        if ((global_state->mode & GLUSTER_MODE_XML))
+            goto end_xml;
 
-        if (!local || !local->get_vol.volname) {
-                if ((global_state->mode & GLUSTER_MODE_XML))
-                        goto end_xml;
-
-                cli_err ("No volumes present");
-                goto out;
-        }
+        cli_err("No volumes present");
+        goto out;
+    }
 
+    ctx->volname = local->get_vol.volname;
 
+    while (ctx->volname) {
+        ret = gf_cli_get_volume(frame, this, ctx);
+        if (ret)
+            goto out;
         ctx->volname = local->get_vol.volname;
-
-        while (ctx->volname) {
-                ret = gf_cli_get_volume (frame, this, ctx);
-                if (ret)
-                        goto out;
-                ctx->volname = local->get_vol.volname;
-        }
+    }
 
 end_xml:
-        if (global_state->mode & GLUSTER_MODE_XML) {
-                ret = cli_xml_output_vol_info_end (local);
-                if (ret)
-                        gf_log ("cli", GF_LOG_ERROR, "Error outputting to xml");
-        }
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        ret = cli_xml_output_vol_info_end(local);
+        if (ret)
+            gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+    }
 
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
+    return ret;
 }
 
-int32_t
-gf_cli_get_volume (call_frame_t *frame, xlator_t *this,
-                      void *data)
+static int32_t
+gf_cli_get_volume(call_frame_t *frame, xlator_t *this, void *data)
 {
-        gf_cli_req                      req = {{0,}};
-        int                             ret = 0;
-        cli_cmd_volume_get_ctx_t        *ctx = NULL;
-        dict_t                          *dict = NULL;
-        int32_t                         flags = 0;
-
-        if (!frame || !this || !data) {
-                ret = -1;
-                goto out;
-        }
-
-        ctx = data;
-
-        dict = dict_new ();
-        if (!dict)
-                goto out;
-
-        if (ctx->volname) {
-                ret = dict_set_str (dict, "volname", ctx->volname);
-                if (ret)
-                        goto out;
-        }
-
-        flags = ctx->flags;
-        ret = dict_set_int32 (dict, "flags", flags);
-        if (ret) {
-                gf_log (frame->this->name, GF_LOG_ERROR, "failed to set flags");
-                goto out;
-        }
-
-        ret = dict_allocate_and_serialize (dict, &req.dict.dict_val,
-                                           &req.dict.dict_len);
-
-        ret = cli_cmd_submit (NULL, &req, frame, cli_rpc_prog,
-                              GLUSTER_CLI_GET_VOLUME, NULL,
-                              this, gf_cli_get_volume_cbk,
-                              (xdrproc_t) xdr_gf_cli_req);
+    gf_cli_req req = {{
+        0,
+    }};
+    int ret = 0;
+    cli_cmd_volume_get_ctx_t *ctx = NULL;
+    dict_t *dict = NULL;
+    int32_t flags = 0;
+
+    if (!this || !data) {
+        ret = -1;
+        goto out;
+    }
+
+    ctx = data;
+
+    dict = dict_new();
+    if (!dict) {
+        gf_log(THIS->name, GF_LOG_ERROR, "Failed to create the dict");
+        ret = -1;
+        goto out;
+    }
+
+    if (ctx->volname) {
+        ret = dict_set_str_sizen(dict, "volname", ctx->volname);
+        if (ret)
+            goto out;
+    }
+
+    flags = ctx->flags;
+    ret = dict_set_int32_sizen(dict, "flags", flags);
+    if (ret) {
+        gf_log(frame->this->name, GF_LOG_ERROR, "failed to set flags");
+        goto out;
+    }
+
+    ret = dict_allocate_and_serialize(dict, &req.dict.dict_val,
+                                      &req.dict.dict_len);
+    if (ret) {
+        gf_log(frame->this->name, GF_LOG_ERROR, DICT_SERIALIZE_FAIL);
+        goto out;
+    }
+
+    ret = cli_cmd_submit(NULL, &req, frame, cli_rpc_prog,
+                         GLUSTER_CLI_GET_VOLUME, NULL, this,
+                         gf_cli_get_volume_cbk, (xdrproc_t)xdr_gf_cli_req);
 
 out:
-        if (dict)
-                dict_unref (dict);
+    if (dict)
+        dict_unref(dict);
 
-        GF_FREE (req.dict.dict_val);
+    GF_FREE(req.dict.dict_val);
 
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
+    return ret;
 }
 
-int32_t
-gf_cli3_1_uuid_get (call_frame_t *frame, xlator_t *this,
-                      void *data)
+static int32_t
+gf_cli3_1_uuid_get(call_frame_t *frame, xlator_t *this, void *data)
 {
-        gf_cli_req                      req = {{0,}};
-        int                             ret = 0;
-        dict_t                          *dict = NULL;
+    gf_cli_req req = {{
+        0,
+    }};
+    int ret = 0;
+    dict_t *dict = NULL;
 
-        if (!frame || !this || !data) {
-                ret = -1;
-                goto out;
-        }
-
-        dict = data;
-        ret = cli_to_glusterd (&req, frame, gf_cli3_1_uuid_get_cbk,
-                               (xdrproc_t)xdr_gf_cli_req, dict,
-                               GLUSTER_CLI_UUID_GET, this, cli_rpc_prog,
-                               NULL);
-out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    dict = data;
+    ret = cli_to_glusterd(&req, frame, gf_cli3_1_uuid_get_cbk,
+                          (xdrproc_t)xdr_gf_cli_req, dict, GLUSTER_CLI_UUID_GET,
+                          this, cli_rpc_prog, NULL);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
+    GF_FREE(req.dict.dict_val);
+    return ret;
 }
 
-int32_t
-gf_cli3_1_uuid_reset (call_frame_t *frame, xlator_t *this,
-                      void *data)
+static int32_t
+gf_cli3_1_uuid_reset(call_frame_t *frame, xlator_t *this, void *data)
 {
-        gf_cli_req                      req = {{0,}};
-        int                             ret = 0;
-        dict_t                          *dict = NULL;
+    gf_cli_req req = {{
+        0,
+    }};
+    int ret = 0;
+    dict_t *dict = NULL;
 
-        if (!frame || !this || !data) {
-                ret = -1;
-                goto out;
-        }
-
-        dict = data;
-        ret = cli_to_glusterd (&req, frame, gf_cli3_1_uuid_reset_cbk,
-                               (xdrproc_t)xdr_gf_cli_req, dict,
-                               GLUSTER_CLI_UUID_RESET, this, cli_rpc_prog,
-                               NULL);
-out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    dict = data;
+    ret = cli_to_glusterd(&req, frame, gf_cli3_1_uuid_reset_cbk,
+                          (xdrproc_t)xdr_gf_cli_req, dict,
+                          GLUSTER_CLI_UUID_RESET, this, cli_rpc_prog, NULL);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
+    GF_FREE(req.dict.dict_val);
+    return ret;
 }
 
-int32_t
-gf_cli_create_volume (call_frame_t *frame, xlator_t *this,
-                         void *data)
+static int32_t
+gf_cli_create_volume(call_frame_t *frame, xlator_t *this, void *data)
 {
-        gf_cli_req              req = {{0,}};
-        int                     ret = 0;
-        dict_t                  *dict = NULL;
-
-        if (!frame || !this ||  !data) {
-                ret = -1;
-                goto out;
-        }
+    gf_cli_req req = {{
+        0,
+    }};
+    int ret = 0;
+    dict_t *dict = NULL;
 
-        dict = data;
+    dict = data;
 
-        ret = cli_to_glusterd (&req, frame, gf_cli_create_volume_cbk,
-                               (xdrproc_t) xdr_gf_cli_req, dict,
-                               GLUSTER_CLI_CREATE_VOLUME, this, cli_rpc_prog,
-                               NULL);
+    ret = cli_to_glusterd(&req, frame, gf_cli_create_volume_cbk,
+                          (xdrproc_t)xdr_gf_cli_req, dict,
+                          GLUSTER_CLI_CREATE_VOLUME, this, cli_rpc_prog, NULL);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
 
-out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    GF_FREE(req.dict.dict_val);
 
-        GF_FREE (req.dict.dict_val);
-
-        return ret;
+    return ret;
 }
 
-int32_t
-gf_cli_delete_volume (call_frame_t *frame, xlator_t *this,
-                         void *data)
+static int32_t
+gf_cli_delete_volume(call_frame_t *frame, xlator_t *this, void *data)
 {
-        gf_cli_req              req = {{0,}};
-        int                     ret = 0;
-        dict_t                  *dict = NULL;
-
-        if (!frame || !this ||  !data) {
-                ret = -1;
-                goto out;
-        }
-
-        dict = data;
+    gf_cli_req req = {{
+        0,
+    }};
+    int ret = 0;
+    dict_t *dict = NULL;
 
-        ret = cli_to_glusterd (&req, frame, gf_cli_delete_volume_cbk,
-                               (xdrproc_t) xdr_gf_cli_req, dict,
-                               GLUSTER_CLI_DELETE_VOLUME, this, cli_rpc_prog,
-                               NULL);
+    dict = data;
 
-out:
-        GF_FREE (req.dict.dict_val);
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    ret = cli_to_glusterd(&req, frame, gf_cli_delete_volume_cbk,
+                          (xdrproc_t)xdr_gf_cli_req, dict,
+                          GLUSTER_CLI_DELETE_VOLUME, this, cli_rpc_prog, NULL);
+    GF_FREE(req.dict.dict_val);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
 
-        return ret;
+    return ret;
 }
 
-int32_t
-gf_cli_start_volume (call_frame_t *frame, xlator_t *this,
-                         void *data)
+static int32_t
+gf_cli_start_volume(call_frame_t *frame, xlator_t *this, void *data)
 {
-        gf_cli_req              req = {{0,}};
-        int                     ret = 0;
-        dict_t                  *dict = NULL;
+    gf_cli_req req = {{
+        0,
+    }};
+    int ret = 0;
+    dict_t *dict = NULL;
 
-        if (!frame || !this ||  !data) {
-                ret = -1;
-                goto out;
-        }
+    dict = data;
 
-        dict = data;
+    ret = cli_to_glusterd(&req, frame, gf_cli_start_volume_cbk,
+                          (xdrproc_t)xdr_gf_cli_req, dict,
+                          GLUSTER_CLI_START_VOLUME, this, cli_rpc_prog, NULL);
 
-        ret = cli_to_glusterd (&req, frame, gf_cli_start_volume_cbk,
-                               (xdrproc_t) xdr_gf_cli_req, dict,
-                               GLUSTER_CLI_START_VOLUME, this, cli_rpc_prog,
-                               NULL);
-
-out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
+    GF_FREE(req.dict.dict_val);
 
-        return ret;
+    return ret;
 }
 
-int32_t
-gf_cli_stop_volume (call_frame_t *frame, xlator_t *this,
-                         void *data)
+static int32_t
+gf_cli_stop_volume(call_frame_t *frame, xlator_t *this, void *data)
 {
-        gf_cli_req             req = {{0,}};
-        int                    ret = 0;
-        dict_t                 *dict = data;
-
-        if (!frame || !this ||  !data) {
-                ret = -1;
-                goto out;
-        }
+    gf_cli_req req = {{
+        0,
+    }};
+    int ret = 0;
+    dict_t *dict = data;
 
-        dict = data;
+    dict = data;
 
-        ret = cli_to_glusterd (&req, frame, gf_cli_stop_volume_cbk,
-                               (xdrproc_t) xdr_gf_cli_req, dict,
-                               GLUSTER_CLI_STOP_VOLUME, this, cli_rpc_prog,
-                               NULL);
+    ret = cli_to_glusterd(&req, frame, gf_cli_stop_volume_cbk,
+                          (xdrproc_t)xdr_gf_cli_req, dict,
+                          GLUSTER_CLI_STOP_VOLUME, this, cli_rpc_prog, NULL);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
+    GF_FREE(req.dict.dict_val);
 
-out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-
-        return ret;
+    return ret;
 }
 
-int32_t
-gf_cli_defrag_volume (call_frame_t *frame, xlator_t *this,
-                         void *data)
+static int32_t
+gf_cli_defrag_volume(call_frame_t *frame, xlator_t *this, void *data)
 {
-        gf_cli_req              req     =  {{0,}};
-        int                     ret     = 0;
-        dict_t                 *dict    = NULL;
+    gf_cli_req req = {{
+        0,
+    }};
+    int ret = 0;
+    dict_t *dict = NULL;
 
-        if (!frame || !this ||  !data) {
-                ret = -1;
-                goto out;
-        }
-
-        dict = data;
-
-        ret = cli_to_glusterd (&req, frame, gf_cli_defrag_volume_cbk,
-                               (xdrproc_t) xdr_gf_cli_req, dict,
-                               GLUSTER_CLI_DEFRAG_VOLUME, this, cli_rpc_prog,
-                               NULL);
+    dict = data;
 
-out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    ret = cli_to_glusterd(&req, frame, gf_cli_defrag_volume_cbk,
+                          (xdrproc_t)xdr_gf_cli_req, dict,
+                          GLUSTER_CLI_DEFRAG_VOLUME, this, cli_rpc_prog, NULL);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
+    GF_FREE(req.dict.dict_val);
 
-        return ret;
+    return ret;
 }
 
-int32_t
-gf_cli_rename_volume (call_frame_t *frame, xlator_t *this,
-                         void *data)
+static int32_t
+gf_cli_rename_volume(call_frame_t *frame, xlator_t *this, void *data)
 {
-        gf_cli_req              req = {{0,}};
-        int                     ret = 0;
-        dict_t                  *dict = NULL;
+    gf_cli_req req = {{
+        0,
+    }};
+    int ret = 0;
+    dict_t *dict = NULL;
 
-        if (!frame || !this ||  !data) {
-                ret = -1;
-                goto out;
-        }
+    if (!frame || !this || !data) {
+        ret = -1;
+        goto out;
+    }
 
-        dict = data;
+    dict = data;
 
-        ret = dict_allocate_and_serialize (dict, &req.dict.dict_val,
-                                           &req.dict.dict_len);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to serialize the data");
-
-                goto out;
-        }
+    ret = dict_allocate_and_serialize(dict, &req.dict.dict_val,
+                                      &req.dict.dict_len);
+    if (ret < 0) {
+        gf_log(this->name, GF_LOG_ERROR, DICT_SERIALIZE_FAIL);
 
+        goto out;
+    }
 
-        ret = cli_cmd_submit (NULL, &req, frame, cli_rpc_prog,
-                              GLUSTER_CLI_RENAME_VOLUME, NULL,
-                              this, gf_cli_rename_volume_cbk,
-                              (xdrproc_t) xdr_gf_cli_req);
+    ret = cli_cmd_submit(NULL, &req, frame, cli_rpc_prog,
+                         GLUSTER_CLI_RENAME_VOLUME, NULL, this,
+                         gf_cli_rename_volume_cbk, (xdrproc_t)xdr_gf_cli_req);
 
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    GF_FREE(req.dict.dict_val);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
 
-        return ret;
+    return ret;
 }
 
-int32_t
-gf_cli_reset_volume (call_frame_t *frame, xlator_t *this,
-                        void *data)
+static int32_t
+gf_cli_reset_volume(call_frame_t *frame, xlator_t *this, void *data)
 {
-        gf_cli_req              req =  {{0,}};
-        int                     ret = 0;
-        dict_t                  *dict = NULL;
+    gf_cli_req req = {{
+        0,
+    }};
+    int ret = 0;
+    dict_t *dict = NULL;
 
-        if (!frame || !this ||  !data) {
-                ret = -1;
-                goto out;
-        }
-
-        dict = data;
-
-        ret = cli_to_glusterd (&req, frame, gf_cli_reset_volume_cbk,
-                               (xdrproc_t) xdr_gf_cli_req, dict,
-                               GLUSTER_CLI_RESET_VOLUME, this, cli_rpc_prog,
-                               NULL);
+    dict = data;
 
-out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    ret = cli_to_glusterd(&req, frame, gf_cli_reset_volume_cbk,
+                          (xdrproc_t)xdr_gf_cli_req, dict,
+                          GLUSTER_CLI_RESET_VOLUME, this, cli_rpc_prog, NULL);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
+    GF_FREE(req.dict.dict_val);
+    return ret;
 }
 
-int32_t
-gf_cli_set_volume (call_frame_t *frame, xlator_t *this,
-                         void *data)
+static int32_t
+gf_cli_ganesha(call_frame_t *frame, xlator_t *this, void *data)
 {
-        gf_cli_req              req =  {{0,}};
-        int                     ret = 0;
-        dict_t                  *dict = NULL;
-
-        if (!frame || !this ||  !data) {
-                ret = -1;
-                goto out;
-        }
+    gf_cli_req req = {{
+        0,
+    }};
+    int ret = 0;
+    dict_t *dict = NULL;
 
-        dict = data;
+    dict = data;
 
-        ret = cli_to_glusterd (&req, frame, gf_cli_set_volume_cbk,
-                               (xdrproc_t) xdr_gf_cli_req, dict,
-                               GLUSTER_CLI_SET_VOLUME, this, cli_rpc_prog,
-                               NULL);
+    ret = cli_to_glusterd(&req, frame, gf_cli_ganesha_cbk,
+                          (xdrproc_t)xdr_gf_cli_req, dict, GLUSTER_CLI_GANESHA,
+                          this, cli_rpc_prog, NULL);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
 
-out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-
-        return ret;
+    return ret;
 }
 
-int32_t
-gf_cli_add_brick (call_frame_t *frame, xlator_t *this,
-                         void *data)
+static int32_t
+gf_cli_set_volume(call_frame_t *frame, xlator_t *this, void *data)
 {
-        gf_cli_req              req =  {{0,}};
-        int                     ret = 0;
-        dict_t                  *dict = NULL;
-        char                    *volname = NULL;
-        int32_t                 count = 0;
+    gf_cli_req req = {{
+        0,
+    }};
+    int ret = 0;
+    dict_t *dict = NULL;
 
-        if (!frame || !this ||  !data) {
-                ret = -1;
-                goto out;
-        }
-
-        dict = data;
-
-        ret = dict_get_str (dict, "volname", &volname);
-
-        if (ret)
-                goto out;
-
-        ret = dict_get_int32 (dict, "count", &count);
-        if (ret)
-                goto out;
-
-        ret = cli_to_glusterd (&req, frame, gf_cli_add_brick_cbk,
-                               (xdrproc_t) xdr_gf_cli_req, dict,
-                               GLUSTER_CLI_ADD_BRICK, this, cli_rpc_prog, NULL);
-
-out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    dict = data;
 
-        GF_FREE (req.dict.dict_val);
+    ret = cli_to_glusterd(&req, frame, gf_cli_set_volume_cbk,
+                          (xdrproc_t)xdr_gf_cli_req, dict,
+                          GLUSTER_CLI_SET_VOLUME, this, cli_rpc_prog, NULL);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
+    GF_FREE(req.dict.dict_val);
 
-        return ret;
+    return ret;
 }
 
 int32_t
-gf_cli_remove_brick (call_frame_t *frame, xlator_t *this,
-                         void *data)
-{
-        gf_cli_req                req =  {{0,}};;
-        gf_cli_req                status_req =  {{0,}};;
-        int                       ret = 0;
-        dict_t                   *dict = NULL;
-        int32_t                   command = 0;
-        char                     *volname = NULL;
-        dict_t                   *req_dict = NULL;
-        int32_t                   cmd = 0;
-
-        if (!frame || !this ||  !data) {
-                ret = -1;
-                goto out;
-        }
-
-        dict = data;
-
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret)
-                goto out;
-
-        ret = dict_get_int32 (dict, "command", &command);
-        if (ret)
-                goto out;
+gf_cli_add_brick(call_frame_t *frame, xlator_t *this, void *data)
+{
+    gf_cli_req req = {{
+        0,
+    }};
+    int ret = 0;
+    dict_t *dict = NULL;
 
-        if ((command != GF_OP_CMD_STATUS) &&
-            (command != GF_OP_CMD_STOP)) {
+    dict = data;
 
+    ret = cli_to_glusterd(&req, frame, gf_cli_add_brick_cbk,
+                          (xdrproc_t)xdr_gf_cli_req, dict,
+                          GLUSTER_CLI_ADD_BRICK, this, cli_rpc_prog, NULL);
 
-                ret = cli_to_glusterd (&req, frame, gf_cli_remove_brick_cbk,
-                                       (xdrproc_t) xdr_gf_cli_req, dict,
-                                       GLUSTER_CLI_REMOVE_BRICK, this,
-                                       cli_rpc_prog, NULL);
-        } else {
-                /* Need rebalance status to be sent :-) */
-                req_dict = dict_new ();
-                if (!req_dict) {
-                        ret = -1;
-                        goto out;
-                }
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
 
-                ret = dict_set_str (req_dict, "volname", volname);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Failed to set dict");
-                        goto out;
-                }
+    GF_FREE(req.dict.dict_val);
 
-                if (command == GF_OP_CMD_STATUS)
-                        cmd |= GF_DEFRAG_CMD_STATUS;
-                else
-                        cmd |= GF_DEFRAG_CMD_STOP;
+    return ret;
+}
 
-                ret = dict_set_int32 (req_dict, "rebalance-command", (int32_t) cmd);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Failed to set dict");
-                        goto out;
-                }
+static int32_t
+gf_cli_remove_brick(call_frame_t *frame, xlator_t *this, void *data)
+{
+    gf_cli_req req = {{
+        0,
+    }};
+    ;
+    gf_cli_req status_req = {{
+        0,
+    }};
+    ;
+    int ret = 0;
+    dict_t *dict = NULL;
+    int32_t command = 0;
+    int32_t cmd = 0;
+
+    if (!frame || !this) {
+        ret = -1;
+        goto out;
+    }
+
+    dict = data;
+
+    ret = dict_get_int32_sizen(dict, "command", &command);
+    if (ret)
+        goto out;
+
+    if ((command != GF_OP_CMD_STATUS) && (command != GF_OP_CMD_STOP)) {
+        ret = cli_to_glusterd(
+            &req, frame, gf_cli_remove_brick_cbk, (xdrproc_t)xdr_gf_cli_req,
+            dict, GLUSTER_CLI_REMOVE_BRICK, this, cli_rpc_prog, NULL);
+    } else {
+        /* Need rebalance status to be sent :-) */
+        if (command == GF_OP_CMD_STATUS)
+            cmd |= GF_DEFRAG_CMD_STATUS;
+        else
+            cmd |= GF_DEFRAG_CMD_STOP;
 
-                ret = cli_to_glusterd (&status_req, frame,
-                                       gf_cli3_remove_brick_status_cbk,
-                                       (xdrproc_t) xdr_gf_cli_req, req_dict,
-                                       GLUSTER_CLI_DEFRAG_VOLUME, this,
-                                       cli_rpc_prog, NULL);
+        ret = dict_set_int32_sizen(dict, "rebalance-command", (int32_t)cmd);
+        if (ret) {
+            gf_log(this->name, GF_LOG_ERROR, "Failed to set dict");
+            goto out;
+        }
 
-                }
+        ret = cli_to_glusterd(
+            &status_req, frame, gf_cli3_remove_brick_status_cbk,
+            (xdrproc_t)xdr_gf_cli_req, dict, GLUSTER_CLI_DEFRAG_VOLUME, this,
+            cli_rpc_prog, NULL);
+    }
 
 out:
-        if (req_dict)
-                dict_unref (req_dict);
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
 
-        GF_FREE (req.dict.dict_val);
+    GF_FREE(req.dict.dict_val);
 
-        GF_FREE (status_req.dict.dict_val);
+    GF_FREE(status_req.dict.dict_val);
 
-        return ret;
+    return ret;
 }
 
-int32_t
-gf_cli_replace_brick (call_frame_t *frame, xlator_t *this,
-                         void *data)
-{
-        gf_cli_req                  req        =  {{0,}};
-        int                         ret        = 0;
-        dict_t                     *dict       = NULL;
-        char                       *src_brick  = NULL;
-        char                       *dst_brick  = NULL;
-        char                       *volname    = NULL;
-        int32_t                     op         = 0;
-
-        if (!frame || !this ||  !data) {
-                ret = -1;
-                goto out;
-        }
-
-        dict = data;
-
-        ret = dict_get_int32 (dict, "operation", &op);
-        if (ret) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "dict_get on operation failed");
-                goto out;
-        }
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "dict_get on volname failed");
-                goto out;
-        }
-
-        ret = dict_get_str (dict, "src-brick", &src_brick);
-        if (ret) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "dict_get on src-brick failed");
-                goto out;
-        }
-
-        ret = dict_get_str (dict, "dst-brick", &dst_brick);
+static int32_t
+gf_cli_reset_brick(call_frame_t *frame, xlator_t *this, void *data)
+{
+    gf_cli_req req = {{
+        0,
+    }};
+    int ret = 0;
+    dict_t *dict = NULL;
+    char *dst_brick = NULL;
+    char *op = NULL;
+
+    if (!frame || !this || !data) {
+        ret = -1;
+        goto out;
+    }
+
+    dict = data;
+
+    ret = dict_get_str_sizen(dict, "operation", &op);
+    if (ret) {
+        gf_log(this->name, GF_LOG_DEBUG, "dict_get on operation failed");
+        goto out;
+    }
+
+    if (!strcmp(op, "GF_RESET_OP_COMMIT") ||
+        !strcmp(op, "GF_RESET_OP_COMMIT_FORCE")) {
+        ret = dict_get_str_sizen(dict, "dst-brick", &dst_brick);
         if (ret) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "dict_get on dst-brick failed");
-                goto out;
+            gf_log(this->name, GF_LOG_DEBUG, "dict_get on dst-brick failed");
+            goto out;
         }
+    }
 
-        gf_log (this->name, GF_LOG_DEBUG,
-                "Received command replace-brick %s with "
-                "%s with operation=%d", src_brick,
-                dst_brick, op);
-
-        ret = cli_to_glusterd (&req, frame, gf_cli_replace_brick_cbk,
-                               (xdrproc_t) xdr_gf_cli_req, dict,
-                               GLUSTER_CLI_REPLACE_BRICK, this, cli_rpc_prog,
-                               NULL);
+    ret = cli_to_glusterd(&req, frame, gf_cli_reset_brick_cbk,
+                          (xdrproc_t)xdr_gf_cli_req, dict,
+                          GLUSTER_CLI_RESET_BRICK, this, cli_rpc_prog, NULL);
 
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
 
-        GF_FREE (req.dict.dict_val);
+    GF_FREE(req.dict.dict_val);
 
-        return ret;
+    return ret;
 }
 
-
-int32_t
-gf_cli_log_rotate (call_frame_t *frame, xlator_t *this,
-                      void *data)
+static int32_t
+gf_cli_replace_brick(call_frame_t *frame, xlator_t *this, void *data)
 {
-        gf_cli_req                req = {{0,}};
-        int                       ret = 0;
-        dict_t                   *dict = NULL;
-
-        if (!frame || !this ||  !data) {
-                ret = -1;
-                goto out;
-        }
-
-        dict = data;
-
-        ret = cli_to_glusterd (&req, frame, gf_cli_log_rotate_cbk,
-                               (xdrproc_t) xdr_gf_cli_req, dict,
-                               GLUSTER_CLI_LOG_ROTATE, this, cli_rpc_prog,
-                               NULL);
+    gf_cli_req req = {{
+        0,
+    }};
+    int ret = 0;
+    dict_t *dict = NULL;
+    int32_t op = 0;
+
+    if (!frame || !this || !data) {
+        ret = -1;
+        goto out;
+    }
+
+    dict = data;
+
+    ret = dict_get_int32_sizen(dict, "operation", &op);
+    if (ret) {
+        gf_log(this->name, GF_LOG_DEBUG, "dict_get on operation failed");
+        goto out;
+    }
+
+    ret = cli_to_glusterd(&req, frame, gf_cli_replace_brick_cbk,
+                          (xdrproc_t)xdr_gf_cli_req, dict,
+                          GLUSTER_CLI_REPLACE_BRICK, this, cli_rpc_prog, NULL);
 
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
 
-        GF_FREE (req.dict.dict_val);
-        return ret;
+    GF_FREE(req.dict.dict_val);
+
+    return ret;
 }
 
-int32_t
-gf_cli_sync_volume (call_frame_t *frame, xlator_t *this,
-                       void *data)
+static int32_t
+gf_cli_log_rotate(call_frame_t *frame, xlator_t *this, void *data)
 {
-        int               ret = 0;
-        gf_cli_req        req = {{0,}};
-        dict_t            *dict = NULL;
-
-        if (!frame || !this || !data) {
-                ret = -1;
-                goto out;
-        }
+    gf_cli_req req = {{
+        0,
+    }};
+    int ret = 0;
+    dict_t *dict = NULL;
 
-        dict = data;
+    dict = data;
 
-        ret = cli_to_glusterd (&req, frame, gf_cli_sync_volume_cbk,
-                               (xdrproc_t) xdr_gf_cli_req, dict,
-                               GLUSTER_CLI_SYNC_VOLUME, this, cli_rpc_prog,
-                               NULL);
+    ret = cli_to_glusterd(&req, frame, gf_cli_log_rotate_cbk,
+                          (xdrproc_t)xdr_gf_cli_req, dict,
+                          GLUSTER_CLI_LOG_ROTATE, this, cli_rpc_prog, NULL);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
 
-out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        GF_FREE (req.dict.dict_val);
-
-        return ret;
+    GF_FREE(req.dict.dict_val);
+    return ret;
 }
 
-int32_t
-gf_cli_getspec (call_frame_t *frame, xlator_t *this,
-                         void *data)
+static int32_t
+gf_cli_sync_volume(call_frame_t *frame, xlator_t *this, void *data)
 {
-        gf_getspec_req          req = {0,};
-        int                     ret = 0;
-        dict_t                  *dict = NULL;
-        dict_t                  *op_dict = NULL;
-
-        if (!frame || !this ||  !data) {
-                ret = -1;
-                goto out;
-        }
+    int ret = 0;
+    gf_cli_req req = {{
+        0,
+    }};
+    dict_t *dict = NULL;
 
-        dict = data;
+    dict = data;
 
-        ret = dict_get_str (dict, "volid", &req.key);
-        if (ret)
-                goto out;
+    ret = cli_to_glusterd(&req, frame, gf_cli_sync_volume_cbk,
+                          (xdrproc_t)xdr_gf_cli_req, dict,
+                          GLUSTER_CLI_SYNC_VOLUME, this, cli_rpc_prog, NULL);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
+    GF_FREE(req.dict.dict_val);
 
-        op_dict = dict_new ();
-        if (!dict) {
-                ret = -1;
-                goto out;
-        }
-
-        // Set the supported min and max op-versions, so glusterd can make a
-        // decision
-        ret = dict_set_int32 (op_dict, "min-op-version", GD_OP_VERSION_MIN);
-        if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "Failed to set min-op-version"
-                        " in request dict");
-                goto out;
-        }
-
-        ret = dict_set_int32 (op_dict, "max-op-version", GD_OP_VERSION_MAX);
-        if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "Failed to set max-op-version"
-                        " in request dict");
-                goto out;
-        }
-
-        ret = dict_allocate_and_serialize (op_dict, &req.xdata.xdata_val,
-                                           &req.xdata.xdata_len);
-        if (ret < 0) {
-                gf_log (THIS->name, GF_LOG_ERROR,
-                        "Failed to serialize dictionary");
-                goto out;
-        }
+    return ret;
+}
 
-        ret = cli_cmd_submit (NULL, &req, frame, &cli_handshake_prog,
-                              GF_HNDSK_GETSPEC, NULL,
-                              this, gf_cli_getspec_cbk,
-                              (xdrproc_t) xdr_gf_getspec_req);
+static int32_t
+gf_cli_getspec(call_frame_t *frame, xlator_t *this, void *data)
+{
+    gf_getspec_req req = {
+        0,
+    };
+    int ret = 0;
+    dict_t *dict = NULL;
+    dict_t *op_dict = NULL;
+
+    if (!frame || !this) {
+        ret = -1;
+        goto out;
+    }
+
+    dict = data;
+
+    ret = dict_get_str_sizen(dict, "volid", &req.key);
+    if (ret)
+        goto out;
+
+    op_dict = dict_new();
+    if (!op_dict) {
+        ret = -1;
+        goto out;
+    }
+
+    // Set the supported min and max op-versions, so glusterd can make a
+    // decision
+    ret = dict_set_int32_sizen(op_dict, "min-op-version", GD_OP_VERSION_MIN);
+    if (ret) {
+        gf_log(THIS->name, GF_LOG_ERROR,
+               "Failed to set min-op-version in request dict");
+        goto out;
+    }
+
+    ret = dict_set_int32_sizen(op_dict, "max-op-version", GD_OP_VERSION_MAX);
+    if (ret) {
+        gf_log(THIS->name, GF_LOG_ERROR,
+               "Failed to set max-op-version in request dict");
+        goto out;
+    }
+
+    ret = dict_allocate_and_serialize(op_dict, &req.xdata.xdata_val,
+                                      &req.xdata.xdata_len);
+    if (ret < 0) {
+        gf_log(THIS->name, GF_LOG_ERROR, DICT_SERIALIZE_FAIL);
+        goto out;
+    }
+
+    ret = cli_cmd_submit(NULL, &req, frame, &cli_handshake_prog,
+                         GF_HNDSK_GETSPEC, NULL, this, gf_cli_getspec_cbk,
+                         (xdrproc_t)xdr_gf_getspec_req);
 
 out:
-        if (op_dict) {
-                dict_unref(op_dict);
-        }
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    if (op_dict) {
+        dict_unref(op_dict);
+    }
+    GF_FREE(req.xdata.xdata_val);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
 
-        return ret;
+    return ret;
 }
 
-int32_t
-gf_cli_quota (call_frame_t *frame, xlator_t *this,
-                 void *data)
+static int32_t
+gf_cli_quota(call_frame_t *frame, xlator_t *this, void *data)
 {
-        gf_cli_req          req = {{0,}};
-        int                 ret = 0;
-        dict_t             *dict = NULL;
+    gf_cli_req req = {{
+        0,
+    }};
+    int ret = 0;
+    dict_t *dict = NULL;
 
-        if (!frame || !this ||  !data) {
-                ret = -1;
-                goto out;
-        }
-
-        dict = data;
-
-        ret = cli_to_glusterd (&req, frame, gf_cli_quota_cbk,
-                               (xdrproc_t) xdr_gf_cli_req, dict,
-                               GLUSTER_CLI_QUOTA, this, cli_rpc_prog, NULL);
-
-out:
-        GF_FREE (req.dict.dict_val);
+    dict = data;
 
-        return ret;
+    ret = cli_to_glusterd(&req, frame, gf_cli_quota_cbk,
+                          (xdrproc_t)xdr_gf_cli_req, dict, GLUSTER_CLI_QUOTA,
+                          this, cli_rpc_prog, NULL);
+    GF_FREE(req.dict.dict_val);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
+    return ret;
 }
 
-int32_t
-gf_cli_pmap_b2p (call_frame_t *frame, xlator_t *this, void *data)
+static int32_t
+gf_cli_pmap_b2p(call_frame_t *frame, xlator_t *this, void *data)
 {
-        pmap_port_by_brick_req  req = {0,};
-        int                     ret = 0;
-        dict_t                  *dict = NULL;
+    pmap_port_by_brick_req req = {
+        0,
+    };
+    int ret = 0;
+    dict_t *dict = NULL;
 
-        if (!frame || !this ||  !data) {
-                ret = -1;
-                goto out;
-        }
+    if (!frame || !this) {
+        ret = -1;
+        goto out;
+    }
 
-        dict = data;
+    dict = data;
 
-        ret = dict_get_str (dict, "brick", &req.brick);
-        if (ret)
-                goto out;
+    ret = dict_get_str_sizen(dict, "brick", &req.brick);
+    if (ret)
+        goto out;
 
-        ret = cli_cmd_submit (NULL, &req, frame, &cli_pmap_prog,
-                              GF_PMAP_PORTBYBRICK, NULL,
-                              this, gf_cli_pmap_b2p_cbk,
-                              (xdrproc_t) xdr_pmap_port_by_brick_req);
+    ret = cli_cmd_submit(NULL, &req, frame, &cli_pmap_prog, GF_PMAP_PORTBYBRICK,
+                         NULL, this, gf_cli_pmap_b2p_cbk,
+                         (xdrproc_t)xdr_pmap_port_by_brick_req);
 
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
 
-        return ret;
+    return ret;
 }
 
 static int
-gf_cli_fsm_log_cbk (struct rpc_req *req, struct iovec *iov,
-                       int count, void *myframe)
-{
-        gf1_cli_fsm_log_rsp        rsp   = {0,};
-        int                        ret   = -1;
-        dict_t                     *dict = NULL;
-        int                        tr_count = 0;
-        char                       key[256] = {0};
-        int                        i = 0;
-        char                       *old_state = NULL;
-        char                       *new_state = NULL;
-        char                       *event = NULL;
-        char                       *time = NULL;
-
-        if (-1 == req->rpc_status) {
-                goto out;
-        }
-
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf1_cli_fsm_log_rsp);
-        if (ret < 0) {
-                gf_log (((call_frame_t *) myframe)->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                goto out;
-        }
-
-        if (rsp.op_ret) {
-                if (strcmp (rsp.op_errstr, ""))
-                        cli_err ("%s", rsp.op_errstr);
-                cli_err ("fsm log unsuccessful");
-                ret = rsp.op_ret;
-                goto out;
-        }
-
-        dict = dict_new ();
-        if (!dict) {
-                ret = -1;
-                goto out;
-        }
-
-        ret = dict_unserialize (rsp.fsm_log.fsm_log_val,
-                                rsp.fsm_log.fsm_log_len,
-                                &dict);
-
-        if (ret) {
-                cli_err ("bad response");
-                goto out;
-        }
-
-        ret = dict_get_int32 (dict, "count", &tr_count);
-        if (tr_count)
-                cli_out("number of transitions: %d", tr_count);
-        else
-                cli_err("No transitions");
-        for (i = 0; i < tr_count; i++) {
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "log%d-old-state", i);
-                ret = dict_get_str (dict, key, &old_state);
-                if (ret)
-                        goto out;
+gf_cli_fsm_log_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                   void *myframe)
+{
+    gf1_cli_fsm_log_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    dict_t *dict = NULL;
+    int tr_count = 0;
+    char key[64] = {0};
+    int keylen;
+    int i = 0;
+    char *old_state = NULL;
+    char *new_state = NULL;
+    char *event = NULL;
+    char *time = NULL;
+
+    GF_ASSERT(myframe);
+
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf1_cli_fsm_log_rsp);
+    if (ret < 0) {
+        gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,
+               XDR_DECODE_FAIL);
+        goto out;
+    }
+
+    if (rsp.op_ret) {
+        if (strcmp(rsp.op_errstr, ""))
+            cli_err("%s", rsp.op_errstr);
+        cli_err("fsm log unsuccessful");
+        ret = rsp.op_ret;
+        goto out;
+    }
+
+    dict = dict_new();
+    if (!dict) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_unserialize(rsp.fsm_log.fsm_log_val, rsp.fsm_log.fsm_log_len,
+                           &dict);
+
+    if (ret) {
+        cli_err(DICT_UNSERIALIZE_FAIL);
+        goto out;
+    }
+
+    ret = dict_get_int32_sizen(dict, "count", &tr_count);
+    if (!ret && tr_count)
+        cli_out("number of transitions: %d", tr_count);
+    else
+        cli_err("No transitions");
+    for (i = 0; i < tr_count; i++) {
+        keylen = snprintf(key, sizeof(key), "log%d-old-state", i);
+        ret = dict_get_strn(dict, key, keylen, &old_state);
+        if (ret)
+            goto out;
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "log%d-event", i);
-                ret = dict_get_str (dict, key, &event);
-                if (ret)
-                        goto out;
+        keylen = snprintf(key, sizeof(key), "log%d-event", i);
+        ret = dict_get_strn(dict, key, keylen, &event);
+        if (ret)
+            goto out;
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "log%d-new-state", i);
-                ret = dict_get_str (dict, key, &new_state);
-                if (ret)
-                        goto out;
+        keylen = snprintf(key, sizeof(key), "log%d-new-state", i);
+        ret = dict_get_strn(dict, key, keylen, &new_state);
+        if (ret)
+            goto out;
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "log%d-time", i);
-                ret = dict_get_str (dict, key, &time);
-                if (ret)
-                        goto out;
-                cli_out ("Old State: [%s]\n"
-                         "New State: [%s]\n"
-                         "Event    : [%s]\n"
-                         "timestamp: [%s]\n", old_state, new_state, event, time);
-        }
+        keylen = snprintf(key, sizeof(key), "log%d-time", i);
+        ret = dict_get_strn(dict, key, keylen, &time);
+        if (ret)
+            goto out;
+        cli_out(
+            "Old State: [%s]\n"
+            "New State: [%s]\n"
+            "Event    : [%s]\n"
+            "timestamp: [%s]\n",
+            old_state, new_state, event, time);
+    }
 
-        ret = rsp.op_ret;
+    ret = rsp.op_ret;
 
 out:
-        cli_cmd_broadcast_response (ret);
-        return ret;
+    cli_cmd_broadcast_response(ret);
+    if (dict) {
+        dict_unref(dict);
+    }
+    gf_free_xdr_fsm_log_rsp(rsp);
+
+    return ret;
 }
 
-int32_t
-gf_cli_fsm_log (call_frame_t *frame, xlator_t *this, void *data)
+static int32_t
+gf_cli_fsm_log(call_frame_t *frame, xlator_t *this, void *data)
 {
-        int                        ret = -1;
-        gf1_cli_fsm_log_req        req = {0,};
-
-        GF_ASSERT (frame);
-        GF_ASSERT (this);
-        GF_ASSERT (data);
-
-        if (!frame || !this || !data)
-                goto out;
-        req.name = data;
-        ret = cli_cmd_submit (NULL, &req, frame, cli_rpc_prog,
-                              GLUSTER_CLI_FSM_LOG, NULL,
-                              this, gf_cli_fsm_log_cbk,
-                              (xdrproc_t) xdr_gf1_cli_fsm_log_req);
+    int ret = -1;
+    gf1_cli_fsm_log_req req = {
+        0,
+    };
+
+    GF_ASSERT(frame);
+    GF_ASSERT(this);
+    GF_ASSERT(data);
+
+    if (!frame || !this || !data)
+        goto out;
+    req.name = data;
+    ret = cli_cmd_submit(NULL, &req, frame, cli_rpc_prog, GLUSTER_CLI_FSM_LOG,
+                         NULL, this, gf_cli_fsm_log_cbk,
+                         (xdrproc_t)xdr_gf1_cli_fsm_log_req);
 
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
 
-        return ret;
+    return ret;
 }
 
-int
-gf_cli_gsync_config_command (dict_t *dict)
-{
-        runner_t runner     = {0,};
-        char *subop         = NULL;
-        char *gwd           = NULL;
-        char *slave         = NULL;
-        char *confpath      = NULL;
-        char *master        = NULL;
-        char *op_name       = NULL;
-        int   ret           = -1;
-        char  conf_path[PATH_MAX] = "";
-
-        if (dict_get_str (dict, "subop", &subop) != 0)
-                return -1;
-
-        if (strcmp (subop, "get") != 0 && strcmp (subop, "get-all") != 0) {
-                cli_out (GEOREP" config updated successfully");
-                return 0;
-        }
-
-        if (dict_get_str (dict, "glusterd_workdir", &gwd) != 0 ||
-            dict_get_str (dict, "slave", &slave) != 0)
-                return -1;
-
-        if (dict_get_str (dict, "master", &master) != 0)
-                master = NULL;
-        if (dict_get_str (dict, "op_name", &op_name) != 0)
-                op_name = NULL;
-
-        ret = dict_get_str (dict, "conf_path", &confpath);
-        if (!confpath) {
-                ret = snprintf (conf_path, sizeof(conf_path) - 1,
-                                "%s/"GEOREP"/gsyncd_template.conf", gwd);
-                conf_path[ret] = '\0';
-                confpath = conf_path;
-        }
-
-        runinit (&runner);
-        runner_add_args (&runner, GSYNCD_PREFIX"/gsyncd", "-c", NULL);
-        runner_argprintf (&runner, "%s", confpath);
-        runner_argprintf (&runner, "--iprefix=%s", DATADIR);
-        if (master)
-                runner_argprintf (&runner, ":%s", master);
-        runner_add_arg (&runner, slave);
-        runner_argprintf (&runner, "--config-%s", subop);
-        if (op_name)
-                runner_add_arg (&runner, op_name);
-
-        return runner_run (&runner);
-}
-
-char*
-get_struct_variable (int mem_num, gf_gsync_status_t *sts_val)
-{
-        switch (mem_num) {
-        case 0:  return (sts_val->node);
-        case 1:  return (sts_val->master);
-        case 2:  return (sts_val->brick);
-        case 3:  return (sts_val->slave_node);
-        case 4:  return (sts_val->worker_status);
-        case 5:  return (sts_val->checkpoint_status);
-        case 6:  return (sts_val->crawl_status);
-        case 7:  return (sts_val->files_syncd);
-        case 8:  return (sts_val->files_remaining);
-        case 9:  return (sts_val->bytes_remaining);
-        case 10: return (sts_val->purges_remaining);
-        case 11: return (sts_val->total_files_skipped);
-        case 12: return (sts_val->brick_host_uuid);
-        case 13: return (sts_val->slavekey);
-        case 14: return (sts_val->session_slave);
-        default:
-                 goto out;
-        }
-
-out:
-        return NULL;
+static int
+gf_cli_gsync_config_command(dict_t *dict)
+{
+    runner_t runner = {
+        0,
+    };
+    char *subop = NULL;
+    char *gwd = NULL;
+    char *slave = NULL;
+    char *confpath = NULL;
+    char *master = NULL;
+    char *op_name = NULL;
+    int ret = -1;
+    char conf_path[PATH_MAX] = "";
+
+    if (dict_get_str_sizen(dict, "subop", &subop) != 0)
+        return -1;
+
+    if (strcmp(subop, "get") != 0 && strcmp(subop, "get-all") != 0) {
+        cli_out(GEOREP " config updated successfully");
+        return 0;
+    }
+
+    if (dict_get_str_sizen(dict, "glusterd_workdir", &gwd) != 0 ||
+        dict_get_str_sizen(dict, "slave", &slave) != 0)
+        return -1;
+
+    if (dict_get_str_sizen(dict, "master", &master) != 0)
+        master = NULL;
+    if (dict_get_str_sizen(dict, "op_name", &op_name) != 0)
+        op_name = NULL;
+
+    ret = dict_get_str_sizen(dict, "conf_path", &confpath);
+    if (ret || !confpath) {
+        ret = snprintf(conf_path, sizeof(conf_path) - 1,
+                       "%s/" GEOREP "/gsyncd_template.conf", gwd);
+        conf_path[ret] = '\0';
+        confpath = conf_path;
+    }
+
+    runinit(&runner);
+    runner_add_args(&runner, GSYNCD_PREFIX "/gsyncd", "-c", NULL);
+    runner_argprintf(&runner, "%s", confpath);
+    runner_argprintf(&runner, "--iprefix=%s", DATADIR);
+    if (master)
+        runner_argprintf(&runner, ":%s", master);
+    runner_add_arg(&runner, slave);
+    runner_argprintf(&runner, "--config-%s", subop);
+    if (op_name)
+        runner_add_arg(&runner, op_name);
+
+    return runner_run(&runner);
 }
 
-int
-gf_cli_print_status (char **title_values,
-                     gf_gsync_status_t **sts_vals,
-                     int *spacing, int gsync_count,
-                     int number_of_fields, int is_detail)
-{
-        int     i                        = 0;
-        int     j                        = 0;
-        int     ret                      = 0;
-        int     status_fields            = 6; /* Indexed at 0 */
-        int     total_spacing            = 0;
-        char  **output_values            = NULL;
-        char   *tmp                      = NULL;
-        char   *hyphens                  = NULL;
-
-        /* calculating spacing for hyphens */
-        for (i = 0; i < number_of_fields; i++) {
-                /* Suppressing detail output for status */
-                if ((!is_detail) && (i > status_fields)) {
-                       /* Suppressing detailed output for
-                        * status */
-                        continue;
-                }
-                spacing[i] += 3; /* Adding extra space to
-                                    distinguish between fields */
-                total_spacing += spacing[i];
-        }
-        total_spacing += 4; /* For the spacing between the fields */
-
-        /* char pointers for each field */
-        output_values = GF_CALLOC (number_of_fields, sizeof (char *),
-                                   gf_common_mt_char);
-        if (!output_values) {
-                ret = -1;
-                goto out;
-        }
-        for (i = 0; i < number_of_fields; i++) {
-                output_values[i] = GF_CALLOC (spacing[i] + 1, sizeof (char),
-                                              gf_common_mt_char);
-                if (!output_values[i]) {
-                        ret = -1;
-                        goto out;
-                }
-        }
-
-        hyphens = GF_CALLOC (total_spacing + 1, sizeof (char),
-                             gf_common_mt_char);
-        if (!hyphens) {
+static int
+gf_cli_print_status(char **title_values, gf_gsync_status_t **sts_vals,
+                    int *spacing, int gsync_count, int number_of_fields,
+                    int is_detail)
+{
+    int i = 0;
+    int j = 0;
+    int ret = 0;
+    int status_fields = 8; /* Indexed at 0 */
+    int total_spacing = 0;
+    char **output_values = NULL;
+    char *tmp = NULL;
+    char *hyphens = NULL;
+
+    /* calculating spacing for hyphens */
+    for (i = 0; i < number_of_fields; i++) {
+        /* Suppressing detail output for status */
+        if ((!is_detail) && (i > status_fields)) {
+            /* Suppressing detailed output for
+             * status */
+            continue;
+        }
+        spacing[i] += 3; /* Adding extra space to
+                            distinguish between fields */
+        total_spacing += spacing[i];
+    }
+    total_spacing += 4; /* For the spacing between the fields */
+
+    /* char pointers for each field */
+    output_values = GF_MALLOC(number_of_fields * sizeof(char *),
+                              gf_common_mt_char);
+    if (!output_values) {
+        ret = -1;
+        goto out;
+    }
+    for (i = 0; i < number_of_fields; i++) {
+        output_values[i] = GF_CALLOC(spacing[i] + 1, sizeof(char),
+                                     gf_common_mt_char);
+        if (!output_values[i]) {
+            ret = -1;
+            goto out;
+        }
+    }
+
+    cli_out(" ");
+
+    /* setting the title "NODE", "MASTER", etc. from title_values[]
+       and printing the same */
+    for (j = 0; j < number_of_fields; j++) {
+        if ((!is_detail) && (j > status_fields)) {
+            /* Suppressing detailed output for
+             * status */
+            output_values[j][0] = '\0';
+            continue;
+        }
+        memset(output_values[j], ' ', spacing[j]);
+        memcpy(output_values[j], title_values[j], strlen(title_values[j]));
+        output_values[j][spacing[j]] = '\0';
+    }
+    cli_out("%s %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s", output_values[0],
+            output_values[1], output_values[2], output_values[3],
+            output_values[4], output_values[5], output_values[6],
+            output_values[7], output_values[8], output_values[9],
+            output_values[10], output_values[11], output_values[12],
+            output_values[13], output_values[14], output_values[15]);
+
+    hyphens = GF_MALLOC((total_spacing + 1) * sizeof(char), gf_common_mt_char);
+    if (!hyphens) {
+        ret = -1;
+        goto out;
+    }
+
+    /* setting and printing the hyphens */
+    memset(hyphens, '-', total_spacing);
+    hyphens[total_spacing] = '\0';
+    cli_out("%s", hyphens);
+
+    for (i = 0; i < gsync_count; i++) {
+        for (j = 0; j < number_of_fields; j++) {
+            if ((!is_detail) && (j > status_fields)) {
+                /* Suppressing detailed output for
+                 * status */
+                output_values[j][0] = '\0';
+                continue;
+            }
+            tmp = get_struct_variable(j, sts_vals[i]);
+            if (!tmp) {
+                gf_log("", GF_LOG_ERROR, "struct member empty.");
                 ret = -1;
                 goto out;
+            }
+            memset(output_values[j], ' ', spacing[j]);
+            memcpy(output_values[j], tmp, strlen(tmp));
+            output_values[j][spacing[j]] = '\0';
         }
 
-        cli_out (" ");
-
-        /* setting the title "NODE", "MASTER", etc. from title_values[]
-           and printing the same */
-        for (j = 0; j < number_of_fields; j++) {
-                if ((!is_detail) && (j > status_fields)) {
-                       /* Suppressing detailed output for
-                        * status */
-                       output_values[j][0] = '\0';
-                       continue;
-                }
-                memset (output_values[j], ' ', spacing[j]);
-                memcpy (output_values[j], title_values[j],
-                        strlen(title_values[j]));
-                output_values[j][spacing[j]] = '\0';
-        }
-        cli_out ("%s %s %s %s %s %s %s %s %s %s %s %s",
-                 output_values[0], output_values[1],
-                 output_values[2], output_values[3],
-                 output_values[4], output_values[5],
-                 output_values[6], output_values[7],
-                 output_values[8], output_values[9],
-                 output_values[10], output_values[11]);
-
-        /* setting and printing the hyphens */
-        memset (hyphens, '-', total_spacing);
-        hyphens[total_spacing] = '\0';
-        cli_out ("%s", hyphens);
-
-        for (i = 0; i < gsync_count; i++) {
-                for (j = 0; j < number_of_fields; j++) {
-                        if ((!is_detail) && (j > status_fields)) {
-                                /* Suppressing detailed output for
-                                 * status */
-                                output_values[j][0] = '\0';
-                                continue;
-                        }
-                        tmp = get_struct_variable(j, sts_vals[i]);
-                        if (!tmp) {
-                                gf_log ("", GF_LOG_ERROR,
-                                        "struct member empty.");
-                                ret = -1;
-                                goto out;
-                        }
-                        memset (output_values[j], ' ', spacing[j]);
-                        memcpy (output_values[j], tmp, strlen (tmp));
-                        output_values[j][spacing[j]] = '\0';
-                }
-
-                cli_out ("%s %s %s %s %s %s %s %s %s %s %s %s",
-                         output_values[0], output_values[1],
-                         output_values[2], output_values[3],
-                         output_values[4], output_values[5],
-                         output_values[6], output_values[7],
-                         output_values[8], output_values[9],
-                         output_values[10], output_values[11]);
-        }
+        cli_out("%s %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s",
+                output_values[0], output_values[1], output_values[2],
+                output_values[3], output_values[4], output_values[5],
+                output_values[6], output_values[7], output_values[8],
+                output_values[9], output_values[10], output_values[11],
+                output_values[12], output_values[13], output_values[14],
+                output_values[15]);
+    }
 
 out:
-        if (output_values) {
-                for (i = 0; i < number_of_fields; i++) {
-                        if (output_values[i])
-                                GF_FREE (output_values[i]);
-                }
-                GF_FREE (output_values);
+    if (output_values) {
+        for (i = 0; i < number_of_fields; i++) {
+            if (output_values[i])
+                GF_FREE(output_values[i]);
         }
+        GF_FREE(output_values);
+    }
 
-        if (hyphens)
-                GF_FREE (hyphens);
+    if (hyphens)
+        GF_FREE(hyphens);
 
-        return ret;
+    return ret;
 }
 
 int
-gf_cli_read_status_data (dict_t *dict,
-                         gf_gsync_status_t **sts_vals,
-                         int *spacing, int gsync_count,
-                         int number_of_fields)
-{
-        char   *tmp                    = NULL;
-        char    sts_val_name[PATH_MAX] = "";
-        int     ret                    = 0;
-        int     i                      = 0;
-        int     j                      = 0;
-
-        /* Storing per node status info in each object */
-        for (i = 0; i < gsync_count; i++) {
-                snprintf (sts_val_name, sizeof(sts_val_name), "status_value%d", i);
-
-                /* Fetching the values from dict, and calculating
-                   the max length for each field */
-                ret = dict_get_bin (dict, sts_val_name, (void **)&(sts_vals[i]));
-                if (ret)
-                        goto out;
-
-                for (j = 0; j < number_of_fields; j++) {
-                        tmp = get_struct_variable(j, sts_vals[i]);
-                        if (!tmp) {
-                                gf_log ("", GF_LOG_ERROR,
-                                        "struct member empty.");
-                                ret = -1;
-                                goto out;
-                        }
-                        if (strlen (tmp) > spacing[j])
-                                spacing[j] = strlen (tmp);
-                }
-        }
+gf_gsync_status_t_comparator(const void *p, const void *q)
+{
+    char *slavekey1 = NULL;
+    char *slavekey2 = NULL;
 
-out:
-        return ret;
-}
+    slavekey1 = get_struct_variable(20, (*(gf_gsync_status_t **)p));
+    slavekey2 = get_struct_variable(20, (*(gf_gsync_status_t **)q));
+    if (!slavekey1 || !slavekey2) {
+        gf_log("cli", GF_LOG_ERROR, "struct member empty.");
+        return 0;
+    }
 
-int
-gf_cli_gsync_status_output (dict_t *dict, gf_boolean_t is_detail)
-{
-        int                     gsync_count    = 0;
-        int                     i              = 0;
-        int                     ret            = 0;
-        int                     spacing[13]    = {0};
-        int                     num_of_fields  = 12;
-        char                    errmsg[1024]   = "";
-        char                   *master         = NULL;
-        char                   *slave          = NULL;
-        char                   *title_values[] = {"MASTER NODE", "MASTER VOL",
-                                                  "MASTER BRICK", "SLAVE",
-                                                  "STATUS", "CHECKPOINT STATUS",
-                                                  "CRAWL STATUS", "FILES SYNCD",
-                                                  "FILES PENDING", "BYTES PENDING",
-                                                  "DELETES PENDING", "FILES SKIPPED"};
-        gf_gsync_status_t     **sts_vals       = NULL;
-
-        /* Checks if any session is active or not */
-        ret = dict_get_int32 (dict, "gsync-count", &gsync_count);
-        if (ret) {
-                ret = dict_get_str (dict, "master", &master);
-
-                ret = dict_get_str (dict, "slave", &slave);
-
-                if (master) {
-                        if (slave)
-                                snprintf (errmsg, sizeof(errmsg), "No active "
-                                          "geo-replication sessions between %s"
-                                          " and %s", master, slave);
-                        else
-                                snprintf (errmsg, sizeof(errmsg), "No active "
-                                          "geo-replication sessions for %s",
-                                          master);
-                } else
-                        snprintf (errmsg, sizeof(errmsg), "No active "
-                                  "geo-replication sessions");
-
-                gf_log ("cli", GF_LOG_INFO, "%s", errmsg);
-                cli_out ("%s", errmsg);
-                ret = 0;
-                goto out;
-        }
+    return strcmp(slavekey1, slavekey2);
+}
 
-        for (i = 0; i < num_of_fields; i++)
-                spacing[i] = strlen(title_values[i]);
+static int
+gf_cli_read_status_data(dict_t *dict, gf_gsync_status_t **sts_vals,
+                        int *spacing, int gsync_count, int number_of_fields)
+{
+    char *tmp = NULL;
+    char sts_val_name[PATH_MAX] = "";
+    int ret = 0;
+    int i = 0;
+    int j = 0;
+
+    /* Storing per node status info in each object */
+    for (i = 0; i < gsync_count; i++) {
+        snprintf(sts_val_name, sizeof(sts_val_name), "status_value%d", i);
+
+        /* Fetching the values from dict, and calculating
+           the max length for each field */
+        ret = dict_get_bin(dict, sts_val_name, (void **)&(sts_vals[i]));
+        if (ret)
+            goto out;
 
-        /* gsync_count = number of nodes reporting output.
-           each sts_val object will store output of each
-           node */
-        sts_vals = GF_CALLOC (gsync_count, sizeof (gf_gsync_status_t *),
-                              gf_common_mt_char);
-        if (!sts_vals) {
+        for (j = 0; j < number_of_fields; j++) {
+            tmp = get_struct_variable(j, sts_vals[i]);
+            if (!tmp) {
+                gf_log("", GF_LOG_ERROR, "struct member empty.");
                 ret = -1;
                 goto out;
+            }
+            if (strlen(tmp) > spacing[j])
+                spacing[j] = strlen(tmp);
         }
-        for (i = 0; i < gsync_count; i++) {
-                sts_vals[i] = GF_CALLOC (1, sizeof (gf_gsync_status_t),
-                                         gf_common_mt_char);
-                if (!sts_vals[i]) {
-                        ret = -1;
-                        goto out;
-                }
-        }
+    }
 
-        ret = gf_cli_read_status_data (dict, sts_vals, spacing,
-                                       gsync_count, num_of_fields);
-        if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to read status data");
-                goto out;
-        }
+    /* Sort based on Session Slave */
+    qsort(sts_vals, gsync_count, sizeof(gf_gsync_status_t *),
+          gf_gsync_status_t_comparator);
 
-        ret = gf_cli_print_status (title_values, sts_vals, spacing, gsync_count,
-                                   num_of_fields, is_detail);
-        if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to print status output");
-                goto out;
-        }
+out:
+    return ret;
+}
+
+static int
+gf_cli_gsync_status_output(dict_t *dict, gf_boolean_t is_detail)
+{
+    int gsync_count = 0;
+    int i = 0;
+    int ret = 0;
+    int spacing[16] = {0};
+    int num_of_fields = 16;
+    char errmsg[1024] = "";
+    char *master = NULL;
+    char *slave = NULL;
+    static char *title_values[] = {"MASTER NODE",
+                                   "MASTER VOL",
+                                   "MASTER BRICK",
+                                   "SLAVE USER",
+                                   "SLAVE",
+                                   "SLAVE NODE",
+                                   "STATUS",
+                                   "CRAWL STATUS",
+                                   "LAST_SYNCED",
+                                   "ENTRY",
+                                   "DATA",
+                                   "META",
+                                   "FAILURES",
+                                   "CHECKPOINT TIME",
+                                   "CHECKPOINT COMPLETED",
+                                   "CHECKPOINT COMPLETION TIME"};
+    gf_gsync_status_t **sts_vals = NULL;
+
+    /* Checks if any session is active or not */
+    ret = dict_get_int32_sizen(dict, "gsync-count", &gsync_count);
+    if (ret) {
+        ret = dict_get_str_sizen(dict, "master", &master);
+
+        ret = dict_get_str_sizen(dict, "slave", &slave);
+
+        if (master) {
+            if (slave)
+                snprintf(errmsg, sizeof(errmsg),
+                         "No active geo-replication sessions between %s"
+                         " and %s",
+                         master, slave);
+            else
+                snprintf(errmsg, sizeof(errmsg),
+                         "No active geo-replication sessions for %s", master);
+        } else
+            snprintf(errmsg, sizeof(errmsg),
+                     "No active geo-replication sessions");
+
+        gf_log("cli", GF_LOG_INFO, "%s", errmsg);
+        cli_out("%s", errmsg);
+        ret = -1;
+        goto out;
+    }
+
+    for (i = 0; i < num_of_fields; i++)
+        spacing[i] = strlen(title_values[i]);
+
+    /* gsync_count = number of nodes reporting output.
+       each sts_val object will store output of each
+       node */
+    sts_vals = GF_MALLOC(gsync_count * sizeof(gf_gsync_status_t *),
+                         gf_common_mt_char);
+    if (!sts_vals) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = gf_cli_read_status_data(dict, sts_vals, spacing, gsync_count,
+                                  num_of_fields);
+    if (ret) {
+        gf_log("", GF_LOG_ERROR, "Unable to read status data");
+        goto out;
+    }
+
+    ret = gf_cli_print_status(title_values, sts_vals, spacing, gsync_count,
+                              num_of_fields, is_detail);
+    if (ret) {
+        gf_log("", GF_LOG_ERROR, "Unable to print status output");
+        goto out;
+    }
 
 out:
-        if (sts_vals)
-                GF_FREE (sts_vals);
+    if (sts_vals)
+        GF_FREE(sts_vals);
 
-        return ret;
+    return ret;
 }
 
 static int32_t
-write_contents_to_common_pem_file (dict_t *dict, int output_count)
-{
-        char            *workdir                   = NULL;
-        char             common_pem_file[PATH_MAX] = "";
-        char            *output                    = NULL;
-        char             output_name[PATH_MAX]     = "";
-        int              bytes_written             = 0;
-        int              fd                        = -1;
-        int              ret                       = -1;
-        int              i                         = -1;
-
-        ret = dict_get_str (dict, "glusterd_workdir", &workdir);
-        if (ret || !workdir) {
-                gf_log ("", GF_LOG_ERROR, "Unable to fetch workdir");
+write_contents_to_common_pem_file(dict_t *dict, int output_count)
+{
+    char *workdir = NULL;
+    char common_pem_file[PATH_MAX] = "";
+    char *output = NULL;
+    char output_name[32] = "";
+    int bytes_written = 0;
+    int fd = -1;
+    int ret = -1;
+    int i = -1;
+
+    ret = dict_get_str_sizen(dict, "glusterd_workdir", &workdir);
+    if (ret || !workdir) {
+        gf_log("", GF_LOG_ERROR, "Unable to fetch workdir");
+        ret = -1;
+        goto out;
+    }
+
+    snprintf(common_pem_file, sizeof(common_pem_file),
+             "%s/geo-replication/common_secret.pem.pub", workdir);
+
+    sys_unlink(common_pem_file);
+
+    fd = open(common_pem_file, O_WRONLY | O_CREAT, 0600);
+    if (fd == -1) {
+        gf_log("", GF_LOG_ERROR, "Failed to open %s Error : %s",
+               common_pem_file, strerror(errno));
+        ret = -1;
+        goto out;
+    }
+
+    for (i = 1; i <= output_count; i++) {
+        ret = snprintf(output_name, sizeof(output_name), "output_%d", i);
+        ret = dict_get_strn(dict, output_name, ret, &output);
+        if (ret) {
+            gf_log("", GF_LOG_ERROR, "Failed to get %s.", output_name);
+            cli_out("Unable to fetch output.");
+        }
+        if (output) {
+            bytes_written = sys_write(fd, output, strlen(output));
+            if (bytes_written != strlen(output)) {
+                gf_log("", GF_LOG_ERROR, "Failed to write to %s",
+                       common_pem_file);
                 ret = -1;
                 goto out;
-        }
-
-        snprintf (common_pem_file, sizeof(common_pem_file),
-                  "%s/geo-replication/common_secret.pem.pub",
-                  workdir);
-
-        unlink (common_pem_file);
-
-        fd = open (common_pem_file, O_WRONLY | O_CREAT, 0600);
-        if (fd == -1) {
-                gf_log ("", GF_LOG_ERROR, "Failed to open %s"
-                        " Error : %s", common_pem_file,
-                        strerror (errno));
+            }
+            /* Adding the new line character */
+            bytes_written = sys_write(fd, "\n", 1);
+            if (bytes_written != 1) {
+                gf_log("", GF_LOG_ERROR, "Failed to add new line char");
                 ret = -1;
                 goto out;
+            }
+            output = NULL;
         }
+    }
 
-        for (i = 1; i <= output_count; i++) {
-                memset (output_name, '\0', sizeof (output_name));
-                snprintf (output_name, sizeof (output_name),
-                          "output_%d", i);
-                ret = dict_get_str (dict, output_name, &output);
-                if (ret) {
-                        gf_log ("", GF_LOG_ERROR, "Failed to get %s.",
-                                output_name);
-                        cli_out ("Unable to fetch output.");
-                }
-                if (output) {
-                        bytes_written = write (fd, output, strlen(output));
-                        if (bytes_written != strlen(output)) {
-                                gf_log ("", GF_LOG_ERROR, "Failed to write "
-                                        "to %s", common_pem_file);
-                                ret = -1;
-                                goto out;
-                        }
-                        /* Adding the new line character */
-                        bytes_written = write (fd, "\n", strlen("\n"));
-                        if (bytes_written != strlen("\n")) {
-                                gf_log ("", GF_LOG_ERROR,
-                                        "Failed to add new line char");
-                                ret = -1;
-                                goto out;
-                        }
-                        output = NULL;
-                }
-        }
-
-        cli_out ("Common secret pub file present at %s", common_pem_file);
-        ret = 0;
+    cli_out("Common secret pub file present at %s", common_pem_file);
+    ret = 0;
 out:
-        if (fd)
-                close (fd);
+    if (fd >= 0)
+        sys_close(fd);
 
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_log("", GF_LOG_DEBUG, RETURNING, ret);
+    return ret;
 }
 
-int
-gf_cli_sys_exec_cbk (struct rpc_req *req, struct iovec *iov,
-                     int count, void *myframe)
-{
-        int                     ret     = -1;
-        int                     output_count     = -1;
-        int                     i     = -1;
-        char                   *output  = NULL;
-        char                   *command = NULL;
-        char                    output_name[PATH_MAX] = "";
-        gf_cli_rsp              rsp     = {0, };
-        dict_t                  *dict   = NULL;
-        call_frame_t            *frame  = NULL;
-
-        if (req->rpc_status == -1) {
-                ret = -1;
-                goto out;
-        }
-
-        frame = myframe;
-
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
-        if (ret < 0) {
-                gf_log (frame->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                goto out;
-       }
-
-        dict = dict_new ();
-
-        if (!dict) {
-                ret = -1;
-                goto out;
-        }
-
-        ret = dict_unserialize (rsp.dict.dict_val, rsp.dict.dict_len, &dict);
+static int
+gf_cli_sys_exec_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                    void *myframe)
+{
+    int ret = -1;
+    int output_count = -1;
+    int i = -1;
+    char *output = NULL;
+    char *command = NULL;
+    char output_name[32] = "";
+    gf_cli_rsp rsp = {
+        0,
+    };
+    dict_t *dict = NULL;
+
+    GF_ASSERT(myframe);
+
+    if (req->rpc_status == -1) {
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,
+               XDR_DECODE_FAIL);
+        goto out;
+    }
+
+    dict = dict_new();
+
+    if (!dict) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len, &dict);
+
+    if (ret)
+        goto out;
+
+    if (rsp.op_ret) {
+        cli_err("%s", rsp.op_errstr ? rsp.op_errstr : "Command failed.");
+        ret = rsp.op_ret;
+        goto out;
+    }
 
-        if (ret)
-                goto out;
+    ret = dict_get_int32_sizen(dict, "output_count", &output_count);
+    if (ret) {
+        cli_out("Command executed successfully.");
+        ret = 0;
+        goto out;
+    }
 
-        if (rsp.op_ret) {
-                cli_err ("%s", rsp.op_errstr ? rsp.op_errstr :
-                         "Command failed.");
-                ret = rsp.op_ret;
-                goto out;
-        }
+    ret = dict_get_str_sizen(dict, "command", &command);
+    if (ret) {
+        gf_log("", GF_LOG_ERROR, "Unable to get command from dict");
+        goto out;
+    }
 
-        ret = dict_get_int32 (dict, "output_count", &output_count);
-        if (ret) {
-                cli_out ("Command executed successfully.");
-                ret = 0;
-                goto out;
-        }
+    if (!strcmp(command, "gsec_create")) {
+        ret = write_contents_to_common_pem_file(dict, output_count);
+        if (!ret)
+            goto out;
+    }
 
-        ret = dict_get_str (dict, "command", &command);
+    for (i = 1; i <= output_count; i++) {
+        ret = snprintf(output_name, sizeof(output_name), "output_%d", i);
+        ret = dict_get_strn(dict, output_name, ret, &output);
         if (ret) {
-                gf_log ("", GF_LOG_ERROR,
-                        "Unable to get command from dict");
-                goto out;
+            gf_log("", GF_LOG_ERROR, "Failed to get %s.", output_name);
+            cli_out("Unable to fetch output.");
         }
-
-        if (!strcmp (command, "gsec_create")) {
-                ret = write_contents_to_common_pem_file (dict, output_count);
-                if (!ret)
-                        goto out;
-        }
-
-        for (i = 1; i <= output_count; i++) {
-                memset (output_name, '\0', sizeof (output_name));
-                snprintf (output_name, sizeof (output_name),
-                          "output_%d", i);
-                ret = dict_get_str (dict, output_name, &output);
-                if (ret) {
-                        gf_log ("", GF_LOG_ERROR, "Failed to get %s.",
-                                output_name);
-                        cli_out ("Unable to fetch output.");
-                }
-                if (output) {
-                        cli_out ("%s", output);
-                        output = NULL;
-                }
+        if (output) {
+            cli_out("%s", output);
+            output = NULL;
         }
+    }
 
-        ret = 0;
+    ret = 0;
 out:
-        if (dict)
-                dict_unref (dict);
-        cli_cmd_broadcast_response (ret);
-
-        free (rsp.dict.dict_val);
-
-        return ret;
+    if (dict)
+        dict_unref(dict);
+    cli_cmd_broadcast_response(ret);
+    gf_free_xdr_cli_rsp(rsp);
+    return ret;
 }
 
-int
-gf_cli_copy_file_cbk (struct rpc_req *req, struct iovec *iov,
-                      int count, void *myframe)
+static int
+gf_cli_copy_file_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                     void *myframe)
 {
-        int                     ret     = -1;
-        gf_cli_rsp              rsp     = {0, };
-        dict_t                  *dict   = NULL;
-        call_frame_t            *frame  = NULL;
+    int ret = -1;
+    gf_cli_rsp rsp = {
+        0,
+    };
+    dict_t *dict = NULL;
 
-        if (req->rpc_status == -1) {
-                ret = -1;
-                goto out;
-        }
+    GF_ASSERT(myframe);
 
-        frame = myframe;
+    if (req->rpc_status == -1) {
+        goto out;
+    }
 
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
-        if (ret < 0) {
-                gf_log (frame->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                goto out;
-       }
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,
+               XDR_DECODE_FAIL);
+        goto out;
+    }
 
-        dict = dict_new ();
+    dict = dict_new();
 
-        if (!dict) {
-                ret = -1;
-                goto out;
-        }
+    if (!dict) {
+        ret = -1;
+        goto out;
+    }
 
-        ret = dict_unserialize (rsp.dict.dict_val, rsp.dict.dict_len, &dict);
+    ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len, &dict);
 
-       if (ret)
-                goto out;
+    if (ret)
+        goto out;
 
-        if (rsp.op_ret) {
-                cli_err ("%s", rsp.op_errstr ? rsp.op_errstr :
-                         "Copy unsuccessful");
-                ret = rsp.op_ret;
-                goto out;
-        }
+    if (rsp.op_ret) {
+        cli_err("%s", rsp.op_errstr ? rsp.op_errstr : "Copy unsuccessful");
+        ret = rsp.op_ret;
+        goto out;
+    }
 
-        cli_out ("Successfully copied file.");
+    cli_out("Successfully copied file.");
 
 out:
-        if (dict)
-                dict_unref (dict);
-        cli_cmd_broadcast_response (ret);
-
-        free (rsp.dict.dict_val);
-
-        return ret;
+    if (dict)
+        dict_unref(dict);
+    cli_cmd_broadcast_response(ret);
+    gf_free_xdr_cli_rsp(rsp);
+    return ret;
 }
 
-int
-gf_cli_gsync_set_cbk (struct rpc_req *req, struct iovec *iov,
-                         int count, void *myframe)
+static int
+gf_cli_gsync_set_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                     void *myframe)
 {
-        int                     ret     = -1;
-        gf_cli_rsp              rsp     = {0, };
-        dict_t                  *dict   = NULL;
-        char                    *gsync_status = NULL;
-        char                    *master = NULL;
-        char                    *slave  = NULL;
-        int32_t                  type   = 0;
-        call_frame_t            *frame  = NULL;
-        gf_boolean_t             status_detail = _gf_false;
-
-
-        if (req->rpc_status == -1) {
-                ret = -1;
-                goto out;
-        }
-
-        frame = myframe;
-
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
-        if (ret < 0) {
-                gf_log (frame->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                goto out;
-        }
-
-        dict = dict_new ();
-
-        if (!dict) {
-                ret = -1;
-                goto out;
-        }
-
-        ret = dict_unserialize (rsp.dict.dict_val, rsp.dict.dict_len, &dict);
-
+    int ret = -1;
+    gf_cli_rsp rsp = {
+        0,
+    };
+    dict_t *dict = NULL;
+    char *gsync_status = NULL;
+    char *master = NULL;
+    char *slave = NULL;
+    int32_t type = 0;
+    gf_boolean_t status_detail = _gf_false;
+
+    GF_ASSERT(myframe);
+
+    if (req->rpc_status == -1) {
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,
+               XDR_DECODE_FAIL);
+        goto out;
+    }
+
+    dict = dict_new();
+
+    if (!dict) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len, &dict);
+
+    if (ret)
+        goto out;
+
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        ret = cli_xml_output_vol_gsync(dict, rsp.op_ret, rsp.op_errno,
+                                       rsp.op_errstr);
         if (ret)
-                goto out;
-
-        if (global_state->mode & GLUSTER_MODE_XML) {
-                ret = cli_xml_output_vol_gsync (dict, rsp.op_ret, rsp.op_errno,
-                                                rsp.op_errstr);
-                if (ret)
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Error outputting to xml");
-                goto out;
-        }
-
-        if (rsp.op_ret) {
-                cli_err ("%s", rsp.op_errstr ? rsp.op_errstr :
-                         GEOREP" command unsuccessful");
-                ret = rsp.op_ret;
-                goto out;
-        }
-
-        ret = dict_get_str (dict, "gsync-status", &gsync_status);
-        if (!ret)
-                cli_out ("%s", gsync_status);
-        else
-                ret = 0;
-
-        ret = dict_get_int32 (dict, "type", &type);
-        if (ret) {
-                gf_log (frame->this->name, GF_LOG_ERROR, "failed to get type");
-                goto out;
-        }
-
-        switch (type) {
-                case GF_GSYNC_OPTION_TYPE_START:
-                case GF_GSYNC_OPTION_TYPE_STOP:
-                        if (dict_get_str (dict, "master", &master) != 0)
-                                master = "???";
-                        if (dict_get_str (dict, "slave", &slave) != 0)
-                                slave = "???";
-
-                        cli_out ("%s " GEOREP " session between %s & %s"
-                                 " has been successful",
-                                 type == GF_GSYNC_OPTION_TYPE_START ?
-                                  "Starting" : "Stopping",
-                                 master, slave);
-                break;
+            gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+        goto out;
+    }
 
-                case GF_GSYNC_OPTION_TYPE_PAUSE:
-                case GF_GSYNC_OPTION_TYPE_RESUME:
-                        if (dict_get_str (dict, "master", &master) != 0)
-                                master = "???";
-                        if (dict_get_str (dict, "slave", &slave) != 0)
-                                slave = "???";
-
-                        cli_out ("%s " GEOREP " session between %s & %s"
-                                 " has been successful",
-                                 type == GF_GSYNC_OPTION_TYPE_PAUSE ?
-                                  "Pausing" : "Resuming",
-                                 master, slave);
-                break;
-
-                case GF_GSYNC_OPTION_TYPE_CONFIG:
-                        ret = gf_cli_gsync_config_command (dict);
-                break;
-
-                case GF_GSYNC_OPTION_TYPE_STATUS:
-                        status_detail = dict_get_str_boolean (dict,
-                                                              "status-detail",
-                                                              _gf_false);
-                        if (status_detail)
-                                ret = gf_cli_gsync_status_output (dict, status_detail);
-                        else
-                                ret = gf_cli_gsync_status_output (dict, status_detail);
-                break;
-
-                case GF_GSYNC_OPTION_TYPE_DELETE:
-                        if (dict_get_str (dict, "master", &master) != 0)
-                                master = "???";
-                        if (dict_get_str (dict, "slave", &slave) != 0)
-                                slave = "???";
-                        cli_out ("Deleting " GEOREP " session between %s & %s"
-                                 " has been successful", master, slave);
-                break;
-
-                case GF_GSYNC_OPTION_TYPE_CREATE:
-                        if (dict_get_str (dict, "master", &master) != 0)
-                                master = "???";
-                        if (dict_get_str (dict, "slave", &slave) != 0)
-                                slave = "???";
-                        cli_out ("Creating " GEOREP " session between %s & %s"
-                                 " has been successful", master, slave);
-                break;
+    if (rsp.op_ret) {
+        cli_err("%s",
+                rsp.op_errstr ? rsp.op_errstr : GEOREP " command unsuccessful");
+        ret = rsp.op_ret;
+        goto out;
+    }
+
+    ret = dict_get_str_sizen(dict, "gsync-status", &gsync_status);
+    if (!ret)
+        cli_out("%s", gsync_status);
+
+    ret = dict_get_int32_sizen(dict, "type", &type);
+    if (ret) {
+        gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,
+               "failed to get type");
+        goto out;
+    }
+
+    switch (type) {
+        case GF_GSYNC_OPTION_TYPE_START:
+        case GF_GSYNC_OPTION_TYPE_STOP:
+            if (dict_get_str_sizen(dict, "master", &master) != 0)
+                master = "???";
+            if (dict_get_str_sizen(dict, "slave", &slave) != 0)
+                slave = "???";
+
+            cli_out(
+                "%s " GEOREP " session between %s & %s has been successful",
+                type == GF_GSYNC_OPTION_TYPE_START ? "Starting" : "Stopping",
+                master, slave);
+            break;
+
+        case GF_GSYNC_OPTION_TYPE_PAUSE:
+        case GF_GSYNC_OPTION_TYPE_RESUME:
+            if (dict_get_str_sizen(dict, "master", &master) != 0)
+                master = "???";
+            if (dict_get_str_sizen(dict, "slave", &slave) != 0)
+                slave = "???";
+
+            cli_out("%s " GEOREP " session between %s & %s has been successful",
+                    type == GF_GSYNC_OPTION_TYPE_PAUSE ? "Pausing" : "Resuming",
+                    master, slave);
+            break;
+
+        case GF_GSYNC_OPTION_TYPE_CONFIG:
+            ret = gf_cli_gsync_config_command(dict);
+            break;
+
+        case GF_GSYNC_OPTION_TYPE_STATUS:
+            status_detail = dict_get_str_boolean(dict, "status-detail",
+                                                 _gf_false);
+            ret = gf_cli_gsync_status_output(dict, status_detail);
+            break;
+
+        case GF_GSYNC_OPTION_TYPE_DELETE:
+            if (dict_get_str_sizen(dict, "master", &master) != 0)
+                master = "???";
+            if (dict_get_str_sizen(dict, "slave", &slave) != 0)
+                slave = "???";
+            cli_out("Deleting " GEOREP
+                    " session between %s & %s has been successful",
+                    master, slave);
+            break;
+
+        case GF_GSYNC_OPTION_TYPE_CREATE:
+            if (dict_get_str_sizen(dict, "master", &master) != 0)
+                master = "???";
+            if (dict_get_str_sizen(dict, "slave", &slave) != 0)
+                slave = "???";
+            cli_out("Creating " GEOREP
+                    " session between %s & %s has been successful",
+                    master, slave);
+            break;
 
-                default:
-                        cli_out (GEOREP" command executed successfully");
-        }
+        default:
+            cli_out(GEOREP " command executed successfully");
+    }
 
 out:
-        if (dict)
-                dict_unref (dict);
-        cli_cmd_broadcast_response (ret);
-
-        free (rsp.dict.dict_val);
-
-        return ret;
+    if (dict)
+        dict_unref(dict);
+    cli_cmd_broadcast_response(ret);
+    gf_free_xdr_cli_rsp(rsp);
+    return ret;
 }
 
-int32_t
-gf_cli_sys_exec (call_frame_t *frame, xlator_t *this, void *data)
+static int32_t
+gf_cli_sys_exec(call_frame_t *frame, xlator_t *this, void *data)
 {
-        int                      ret    = 0;
-        dict_t                  *dict   = NULL;
-        gf_cli_req               req = {{0,}};
-
+    int ret = 0;
+    dict_t *dict = NULL;
+    gf_cli_req req = {{
+        0,
+    }};
+
+    dict = data;
+
+    ret = cli_to_glusterd(&req, frame, gf_cli_sys_exec_cbk,
+                          (xdrproc_t)xdr_gf_cli_req, dict, GLUSTER_CLI_SYS_EXEC,
+                          this, cli_rpc_prog, NULL);
+    if (ret)
         if (!frame || !this || !data) {
-                ret = -1;
-                gf_log ("cli", GF_LOG_ERROR, "Invalid data");
-                goto out;
+            ret = -1;
+            gf_log("cli", GF_LOG_ERROR, "Invalid data");
         }
 
-        dict = data;
-
-        ret = cli_to_glusterd (&req, frame, gf_cli_sys_exec_cbk,
-                               (xdrproc_t) xdr_gf_cli_req, dict,
-                               GLUSTER_CLI_SYS_EXEC, this, cli_rpc_prog,
-                               NULL);
-out:
-        GF_FREE (req.dict.dict_val);
-        return ret;
+    GF_FREE(req.dict.dict_val);
+    return ret;
 }
 
-int32_t
-gf_cli_copy_file (call_frame_t *frame, xlator_t *this, void *data)
+static int32_t
+gf_cli_copy_file(call_frame_t *frame, xlator_t *this, void *data)
 {
-        int                      ret    = 0;
-        dict_t                  *dict   = NULL;
-        gf_cli_req               req = {{0,}};
-
+    int ret = 0;
+    dict_t *dict = NULL;
+    gf_cli_req req = {{
+        0,
+    }};
+
+    dict = data;
+
+    ret = cli_to_glusterd(&req, frame, gf_cli_copy_file_cbk,
+                          (xdrproc_t)xdr_gf_cli_req, dict,
+                          GLUSTER_CLI_COPY_FILE, this, cli_rpc_prog, NULL);
+    if (ret)
         if (!frame || !this || !data) {
-                ret = -1;
-                gf_log ("cli", GF_LOG_ERROR, "Invalid data");
-                goto out;
+            ret = -1;
+            gf_log("cli", GF_LOG_ERROR, "Invalid data");
         }
 
-        dict = data;
-
-        ret = cli_to_glusterd (&req, frame, gf_cli_copy_file_cbk,
-                               (xdrproc_t) xdr_gf_cli_req, dict,
-                               GLUSTER_CLI_COPY_FILE, this, cli_rpc_prog,
-                               NULL);
-out:
-        GF_FREE (req.dict.dict_val);
-        return ret;
+    GF_FREE(req.dict.dict_val);
+    return ret;
 }
 
-int32_t
-gf_cli_gsync_set (call_frame_t *frame, xlator_t *this,
-                     void *data)
+static int32_t
+gf_cli_gsync_set(call_frame_t *frame, xlator_t *this, void *data)
 {
-        int                      ret    = 0;
-        dict_t                  *dict   = NULL;
-        gf_cli_req               req = {{0,}};
+    int ret = 0;
+    dict_t *dict = NULL;
+    gf_cli_req req = {{
+        0,
+    }};
 
-        if (!frame || !this || !data) {
-                ret = -1;
-                goto out;
-        }
+    dict = data;
 
-        dict = data;
+    ret = cli_to_glusterd(&req, frame, gf_cli_gsync_set_cbk,
+                          (xdrproc_t)xdr_gf_cli_req, dict,
+                          GLUSTER_CLI_GSYNC_SET, this, cli_rpc_prog, NULL);
+    GF_FREE(req.dict.dict_val);
 
-        ret = cli_to_glusterd (&req, frame, gf_cli_gsync_set_cbk,
-                               (xdrproc_t) xdr_gf_cli_req, dict,
-                               GLUSTER_CLI_GSYNC_SET, this, cli_rpc_prog,
-                               NULL);
-
-out:
-        GF_FREE (req.dict.dict_val);
-
-        return ret;
+    return ret;
 }
 
-
 int
-cli_profile_info_percentage_cmp (void *a, void *b)
+cli_profile_info_percentage_cmp(void *a, void *b)
 {
-        cli_profile_info_t *ia = NULL;
-        cli_profile_info_t *ib = NULL;
-        int                ret = 0;
+    cli_profile_info_t *ia = NULL;
+    cli_profile_info_t *ib = NULL;
 
-        ia = a;
-        ib = b;
-        if (ia->percentage_avg_latency < ib->percentage_avg_latency)
-                ret = -1;
-        else if (ia->percentage_avg_latency > ib->percentage_avg_latency)
-                ret = 1;
-        else
-                ret = 0;
-        return ret;
-}
+    ia = a;
+    ib = b;
+    if (ia->percentage_avg_latency < ib->percentage_avg_latency)
+        return -1;
+    else if (ia->percentage_avg_latency > ib->percentage_avg_latency)
+        return 1;
 
+    return 0;
+}
 
-void
-cmd_profile_volume_brick_out (dict_t *dict, int count, int interval)
-{
-        char                    key[256] = {0};
-        int                     i = 0;
-        uint64_t                sec = 0;
-        uint64_t                r_count = 0;
-        uint64_t                w_count = 0;
-        uint64_t                rb_counts[32] = {0};
-        uint64_t                wb_counts[32] = {0};
-        cli_profile_info_t      profile_info[GF_FOP_MAXVALUE] = {{0}};
-        char                    output[128] = {0};
-        int                     per_line = 0;
-        char                    read_blocks[128] = {0};
-        char                    write_blocks[128] = {0};
-        int                     index = 0;
-        int                     is_header_printed = 0;
-        int                     ret = 0;
-        double                  total_percentage_latency = 0;
-
-        for (i = 0; i < 32; i++) {
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%d-%d-read-%d", count,
-                          interval, (1 << i));
-                ret = dict_get_uint64 (dict, key, &rb_counts[i]);
-        }
-
-        for (i = 0; i < 32; i++) {
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%d-%d-write-%d", count, interval,
-                          (1<<i));
-                ret = dict_get_uint64 (dict, key, &wb_counts[i]);
+static void
+cmd_profile_volume_brick_out(dict_t *dict, int count, int interval)
+{
+    char key[256] = {0};
+    int i = 0;
+    uint64_t sec = 0;
+    uint64_t r_count = 0;
+    uint64_t w_count = 0;
+    uint64_t rb_counts[32] = {0};
+    uint64_t wb_counts[32] = {0};
+    cli_profile_info_t profile_info[GF_FOP_MAXVALUE] = {{0}};
+    cli_profile_info_t upcall_info[GF_UPCALL_FLAGS_MAXVALUE] = {
+        {0},
+    };
+    char output[128] = {0};
+    int per_line = 0;
+    char read_blocks[128] = {0};
+    char write_blocks[128] = {0};
+    int index = 0;
+    int is_header_printed = 0;
+    int ret = 0;
+    double total_percentage_latency = 0;
+
+    for (i = 0; i < 32; i++) {
+        snprintf(key, sizeof(key), "%d-%d-read-%" PRIu32, count, interval,
+                 (1U << i));
+        ret = dict_get_uint64(dict, key, &rb_counts[i]);
+        if (ret) {
+            gf_log("cli", GF_LOG_DEBUG, "failed to get %s from dict", key);
         }
+    }
 
-        for (i = 0; i < GF_FOP_MAXVALUE; i++) {
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%d-%d-%d-hits", count,
-                          interval, i);
-                ret = dict_get_uint64 (dict, key, &profile_info[i].fop_hits);
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%d-%d-%d-avglatency", count,
-                          interval, i);
-                ret = dict_get_double (dict, key, &profile_info[i].avg_latency);
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%d-%d-%d-minlatency", count,
-                          interval, i);
-                ret = dict_get_double (dict, key, &profile_info[i].min_latency);
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%d-%d-%d-maxlatency", count,
-                          interval, i);
-                ret = dict_get_double (dict, key, &profile_info[i].max_latency);
-                profile_info[i].fop_name = (char *)gf_fop_list[i];
-
-                total_percentage_latency +=
-                       (profile_info[i].fop_hits * profile_info[i].avg_latency);
-        }
-        if (total_percentage_latency) {
-                for (i = 0; i < GF_FOP_MAXVALUE; i++) {
-                        profile_info[i].percentage_avg_latency = 100 * (
-                     (profile_info[i].avg_latency* profile_info[i].fop_hits) /
-                                total_percentage_latency);
-                }
-                gf_array_insertionsort (profile_info, 1, GF_FOP_MAXVALUE - 1,
-                                    sizeof (cli_profile_info_t),
-                                    cli_profile_info_percentage_cmp);
+    for (i = 0; i < 32; i++) {
+        snprintf(key, sizeof(key), "%d-%d-write-%" PRIu32, count, interval,
+                 (1U << i));
+        ret = dict_get_uint64(dict, key, &wb_counts[i]);
+        if (ret) {
+            gf_log("cli", GF_LOG_DEBUG, "failed to get %s from dict", key);
         }
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%d-%d-duration", count, interval);
-        ret = dict_get_uint64 (dict, key, &sec);
+    }
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%d-%d-total-read", count, interval);
-        ret = dict_get_uint64 (dict, key, &r_count);
+    for (i = 0; i < GF_UPCALL_FLAGS_MAXVALUE; i++) {
+        snprintf(key, sizeof(key), "%d-%d-%d-upcall-hits", count, interval, i);
+        ret = dict_get_uint64(dict, key, &upcall_info[i].fop_hits);
+        if (ret) {
+            gf_log("cli", GF_LOG_DEBUG, "failed to get %s from dict", key);
+        }
+        upcall_info[i].fop_name = (char *)gf_upcall_list[i];
+    }
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%d-%d-total-write", count, interval);
-        ret = dict_get_uint64 (dict, key, &w_count);
+    for (i = 0; i < GF_FOP_MAXVALUE; i++) {
+        snprintf(key, sizeof(key), "%d-%d-%d-hits", count, interval, i);
+        ret = dict_get_uint64(dict, key, &profile_info[i].fop_hits);
+        if (ret) {
+            gf_log("cli", GF_LOG_DEBUG, "failed to get %s from dict", key);
+        }
 
-        if (ret == 0) {
+        snprintf(key, sizeof(key), "%d-%d-%d-avglatency", count, interval, i);
+        ret = dict_get_double(dict, key, &profile_info[i].avg_latency);
+        if (ret) {
+            gf_log("cli", GF_LOG_DEBUG, "failed to get %s from dict", key);
         }
 
-        if (interval == -1)
-                cli_out ("Cumulative Stats:");
-        else
-                cli_out ("Interval %d Stats:", interval);
-        snprintf (output, sizeof (output), "%14s", "Block Size:");
-        snprintf (read_blocks, sizeof (read_blocks), "%14s", "No. of Reads:");
-        snprintf (write_blocks, sizeof (write_blocks), "%14s", "No. of Writes:");
-        index = 14;
-        for (i = 0; i < 32; i++) {
-                if ((rb_counts[i] == 0) && (wb_counts[i] == 0))
-                        continue;
-                per_line++;
-                snprintf (output+index, sizeof (output)-index, "%19db+ ", (1<<i));
-                if (rb_counts[i]) {
-                        snprintf (read_blocks+index, sizeof (read_blocks)-index,
-                                  "%21"PRId64" ", rb_counts[i]);
-                } else {
-                        snprintf (read_blocks+index, sizeof (read_blocks)-index,
-                                  "%21s ", "0");
-                }
-                if (wb_counts[i]) {
-                        snprintf (write_blocks+index, sizeof (write_blocks)-index,
-                                  "%21"PRId64" ", wb_counts[i]);
-                } else {
-                        snprintf (write_blocks+index, sizeof (write_blocks)-index,
-                                  "%21s ", "0");
-                }
-                index += 22;
-                if (per_line == 3) {
-                        cli_out ("%s", output);
-                        cli_out ("%s", read_blocks);
-                        cli_out ("%s", write_blocks);
-                        cli_out (" ");
-                        per_line = 0;
-                        memset (output, 0, sizeof (output));
-                        memset (read_blocks, 0, sizeof (read_blocks));
-                        memset (write_blocks, 0, sizeof (write_blocks));
-                        snprintf (output, sizeof (output), "%14s", "Block Size:");
-                        snprintf (read_blocks, sizeof (read_blocks), "%14s",
-                                  "No. of Reads:");
-                        snprintf (write_blocks, sizeof (write_blocks), "%14s",
-                                  "No. of Writes:");
-                        index = 14;
-                }
+        snprintf(key, sizeof(key), "%d-%d-%d-minlatency", count, interval, i);
+        ret = dict_get_double(dict, key, &profile_info[i].min_latency);
+        if (ret) {
+            gf_log("cli", GF_LOG_DEBUG, "failed to get %s from dict", key);
         }
 
-        if (per_line != 0) {
-                cli_out ("%s", output);
-                cli_out ("%s", read_blocks);
-                cli_out ("%s", write_blocks);
+        snprintf(key, sizeof(key), "%d-%d-%d-maxlatency", count, interval, i);
+        ret = dict_get_double(dict, key, &profile_info[i].max_latency);
+        if (ret) {
+            gf_log("cli", GF_LOG_DEBUG, "failed to get %s from dict", key);
         }
+        profile_info[i].fop_name = (char *)gf_fop_list[i];
+
+        total_percentage_latency += (profile_info[i].fop_hits *
+                                     profile_info[i].avg_latency);
+    }
+    if (total_percentage_latency) {
         for (i = 0; i < GF_FOP_MAXVALUE; i++) {
-                if (profile_info[i].fop_hits == 0)
-                        continue;
-                if (is_header_printed == 0) {
-                        cli_out ("%10s %13s %13s %13s %14s %11s", "%-latency",
-                                 "Avg-latency", "Min-Latency", "Max-Latency",
-                                 "No. of calls", "Fop");
-                        cli_out ("%10s %13s %13s %13s %14s %11s", "---------",
-                                 "-----------", "-----------", "-----------",
-                                 "------------", "----");
-                        is_header_printed = 1;
-                }
-                if (profile_info[i].fop_hits) {
-                        cli_out ("%10.2lf %10.2lf us %10.2lf us %10.2lf us"
-                                 " %14"PRId64" %11s",
-                                 profile_info[i].percentage_avg_latency,
-                                 profile_info[i].avg_latency,
-                                 profile_info[i].min_latency,
-                                 profile_info[i].max_latency,
-                                 profile_info[i].fop_hits,
-                                 profile_info[i].fop_name);
-                }
+            profile_info[i]
+                .percentage_avg_latency = 100 * ((profile_info[i].avg_latency *
+                                                  profile_info[i].fop_hits) /
+                                                 total_percentage_latency);
+        }
+        gf_array_insertionsort(profile_info, 1, GF_FOP_MAXVALUE - 1,
+                               sizeof(cli_profile_info_t),
+                               cli_profile_info_percentage_cmp);
+    }
+    snprintf(key, sizeof(key), "%d-%d-duration", count, interval);
+    ret = dict_get_uint64(dict, key, &sec);
+    if (ret) {
+        gf_log("cli", GF_LOG_DEBUG, "failed to get %s from dict", key);
+    }
+
+    snprintf(key, sizeof(key), "%d-%d-total-read", count, interval);
+    ret = dict_get_uint64(dict, key, &r_count);
+    if (ret) {
+        gf_log("cli", GF_LOG_DEBUG, "failed to get %s from dict", key);
+    }
+
+    snprintf(key, sizeof(key), "%d-%d-total-write", count, interval);
+    ret = dict_get_uint64(dict, key, &w_count);
+    if (ret) {
+        gf_log("cli", GF_LOG_DEBUG, "failed to get %s from dict", key);
+    }
+
+    if (interval == -1)
+        cli_out("Cumulative Stats:");
+    else
+        cli_out("Interval %d Stats:", interval);
+    snprintf(output, sizeof(output), "%14s", "Block Size:");
+    snprintf(read_blocks, sizeof(read_blocks), "%14s", "No. of Reads:");
+    snprintf(write_blocks, sizeof(write_blocks), "%14s", "No. of Writes:");
+    index = 14;
+    for (i = 0; i < 32; i++) {
+        if ((rb_counts[i] == 0) && (wb_counts[i] == 0))
+            continue;
+        per_line++;
+        snprintf(output + index, sizeof(output) - index, "%19" PRIu32 "b+ ",
+                 (1U << i));
+        if (rb_counts[i]) {
+            snprintf(read_blocks + index, sizeof(read_blocks) - index,
+                     "%21" PRId64 " ", rb_counts[i]);
+        } else {
+            snprintf(read_blocks + index, sizeof(read_blocks) - index, "%21s ",
+                     "0");
         }
-        cli_out (" ");
-        cli_out ("%12s: %"PRId64" seconds", "Duration", sec);
-        cli_out ("%12s: %"PRId64" bytes", "Data Read", r_count);
-        cli_out ("%12s: %"PRId64" bytes", "Data Written", w_count);
-        cli_out (" ");
+        if (wb_counts[i]) {
+            snprintf(write_blocks + index, sizeof(write_blocks) - index,
+                     "%21" PRId64 " ", wb_counts[i]);
+        } else {
+            snprintf(write_blocks + index, sizeof(write_blocks) - index,
+                     "%21s ", "0");
+        }
+        index += 22;
+        if (per_line == 3) {
+            cli_out("%s", output);
+            cli_out("%s", read_blocks);
+            cli_out("%s", write_blocks);
+            cli_out(" ");
+            per_line = 0;
+            snprintf(output, sizeof(output), "%14s", "Block Size:");
+            snprintf(read_blocks, sizeof(read_blocks), "%14s", "No. of Reads:");
+            snprintf(write_blocks, sizeof(write_blocks), "%14s",
+                     "No. of Writes:");
+            index = 14;
+        }
+    }
+
+    if (per_line != 0) {
+        cli_out("%s", output);
+        cli_out("%s", read_blocks);
+        cli_out("%s", write_blocks);
+    }
+    for (i = 0; i < GF_FOP_MAXVALUE; i++) {
+        if (profile_info[i].fop_hits == 0)
+            continue;
+        if (is_header_printed == 0) {
+            cli_out("%10s %13s %13s %13s %14s %11s", "%-latency", "Avg-latency",
+                    "Min-Latency", "Max-Latency", "No. of calls", "Fop");
+            cli_out("%10s %13s %13s %13s %14s %11s", "---------", "-----------",
+                    "-----------", "-----------", "------------", "----");
+            is_header_printed = 1;
+        }
+        if (profile_info[i].fop_hits) {
+            cli_out(
+                "%10.2lf %10.2lf us %10.2lf us %10.2lf us"
+                " %14" PRId64 " %11s",
+                profile_info[i].percentage_avg_latency,
+                profile_info[i].avg_latency, profile_info[i].min_latency,
+                profile_info[i].max_latency, profile_info[i].fop_hits,
+                profile_info[i].fop_name);
+        }
+    }
+
+    for (i = 0; i < GF_UPCALL_FLAGS_MAXVALUE; i++) {
+        if (upcall_info[i].fop_hits == 0)
+            continue;
+        if (upcall_info[i].fop_hits) {
+            cli_out(
+                "%10.2lf %10.2lf us %10.2lf us %10.2lf us"
+                " %14" PRId64 " %11s",
+                upcall_info[i].percentage_avg_latency,
+                upcall_info[i].avg_latency, upcall_info[i].min_latency,
+                upcall_info[i].max_latency, upcall_info[i].fop_hits,
+                upcall_info[i].fop_name);
+        }
+    }
+
+    cli_out(" ");
+    cli_out("%12s: %" PRId64 " seconds", "Duration", sec);
+    cli_out("%12s: %" PRId64 " bytes", "Data Read", r_count);
+    cli_out("%12s: %" PRId64 " bytes", "Data Written", w_count);
+    cli_out(" ");
 }
 
-int32_t
-gf_cli_profile_volume_cbk (struct rpc_req *req, struct iovec *iov,
-                              int count, void *myframe)
-{
-        gf_cli_rsp                        rsp   = {0,};
-        int                               ret   = -1;
-        dict_t                            *dict = NULL;
-        gf1_cli_stats_op                  op = GF_CLI_STATS_NONE;
-        char                              key[256] = {0};
-        int                               interval = 0;
-        int                               i = 1;
-        int32_t                           brick_count = 0;
-        char                              *volname = NULL;
-        char                              *brick = NULL;
-        char                              str[1024] = {0,};
-        int                               stats_cleared = 0;
-        gf1_cli_info_op                   info_op = GF_CLI_INFO_NONE;
-
-        if (-1 == req->rpc_status) {
-                goto out;
+static int32_t
+gf_cli_profile_volume_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                          void *myframe)
+{
+    gf_cli_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    dict_t *dict = NULL;
+    gf1_cli_stats_op op = GF_CLI_STATS_NONE;
+    char key[64] = {0};
+    int len;
+    int interval = 0;
+    int i = 1;
+    int32_t brick_count = 0;
+    char *volname = NULL;
+    char *brick = NULL;
+    char str[1024] = {
+        0,
+    };
+    int stats_cleared = 0;
+    gf1_cli_info_op info_op = GF_CLI_INFO_NONE;
+
+    GF_ASSERT(myframe);
+
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
+
+    gf_log("cli", GF_LOG_DEBUG, "Received resp to profile");
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,
+               XDR_DECODE_FAIL);
+        goto out;
+    }
+
+    dict = dict_new();
+
+    if (!dict) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len, &dict);
+
+    if (ret) {
+        gf_log("", GF_LOG_ERROR, DICT_UNSERIALIZE_FAIL);
+        goto out;
+    }
+
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        ret = cli_xml_output_vol_profile(dict, rsp.op_ret, rsp.op_errno,
+                                         rsp.op_errstr);
+        if (ret)
+            gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+        goto out;
+    }
+
+    ret = dict_get_int32_sizen(dict, "op", (int32_t *)&op);
+    if (ret)
+        goto out;
+
+    ret = dict_get_str_sizen(dict, "volname", &volname);
+    if (ret)
+        goto out;
+
+    if (rsp.op_ret && strcmp(rsp.op_errstr, "")) {
+        cli_err("%s", rsp.op_errstr);
+    } else {
+        switch (op) {
+            case GF_CLI_STATS_START:
+                cli_out("Starting volume profile on %s has been %s ", volname,
+                        (rsp.op_ret) ? "unsuccessful" : "successful");
+                break;
+            case GF_CLI_STATS_STOP:
+                cli_out("Stopping volume profile on %s has been %s ", volname,
+                        (rsp.op_ret) ? "unsuccessful" : "successful");
+                break;
+            case GF_CLI_STATS_INFO:
+                break;
+            default:
+                cli_out("volume profile on %s has been %s ", volname,
+                        (rsp.op_ret) ? "unsuccessful" : "successful");
+                break;
         }
+    }
 
-        gf_log ("cli", GF_LOG_DEBUG, "Received resp to profile");
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
-        if (ret < 0) {
-                gf_log (((call_frame_t *) myframe)->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                goto out;
-        }
+    if (rsp.op_ret) {
+        ret = rsp.op_ret;
+        goto out;
+    }
 
-        dict = dict_new ();
+    if (GF_CLI_STATS_INFO != op) {
+        ret = 0;
+        goto out;
+    }
 
-        if (!dict) {
-                ret = -1;
-                goto out;
-        }
+    ret = dict_get_int32_sizen(dict, "count", &brick_count);
+    if (ret)
+        goto out;
 
-        ret = dict_unserialize (rsp.dict.dict_val,
-                                rsp.dict.dict_len,
-                                &dict);
+    if (!brick_count) {
+        cli_err("All bricks of volume %s are down.", volname);
+        goto out;
+    }
 
-        if (ret) {
-                gf_log ("", GF_LOG_ERROR,
-                                "Unable to allocate memory");
-                goto out;
-        } else {
-                dict->extra_stdfree = rsp.dict.dict_val;
-        }
+    ret = dict_get_int32_sizen(dict, "info-op", (int32_t *)&info_op);
+    if (ret)
+        goto out;
 
-        if (global_state->mode & GLUSTER_MODE_XML) {
-                ret = cli_xml_output_vol_profile (dict, rsp.op_ret,
-                                                  rsp.op_errno,
-                                                  rsp.op_errstr);
-                if (ret)
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Error outputting to xml");
-                goto out;
+    while (i <= brick_count) {
+        len = snprintf(key, sizeof(key), "%d-brick", i);
+        ret = dict_get_strn(dict, key, len, &brick);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR, "Couldn't get brick name");
+            goto out;
         }
 
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret)
-                goto out;
+        ret = dict_get_str_boolean(dict, "nfs", _gf_false);
 
-        ret = dict_get_int32 (dict, "op", (int32_t*)&op);
         if (ret)
-                goto out;
-
-        if (rsp.op_ret && strcmp (rsp.op_errstr, "")) {
-                cli_err ("%s", rsp.op_errstr);
+            len = snprintf(str, sizeof(str), "NFS Server : %s", brick);
+        else
+            len = snprintf(str, sizeof(str), "Brick: %s", brick);
+        cli_out("%s", str);
+        memset(str, '-', len);
+        cli_out("%s", str);
+
+        if (GF_CLI_INFO_CLEAR == info_op) {
+            len = snprintf(key, sizeof(key), "%d-stats-cleared", i);
+            ret = dict_get_int32n(dict, key, len, &stats_cleared);
+            if (ret)
+                goto out;
+            cli_out(stats_cleared ? "Cleared stats."
+                                  : "Failed to clear stats.");
         } else {
-                switch (op) {
-                case GF_CLI_STATS_START:
-                        cli_out ("Starting volume profile on %s has been %s ",
-                                 volname,
-                                 (rsp.op_ret) ? "unsuccessful": "successful");
-                        break;
-                case GF_CLI_STATS_STOP:
-                        cli_out ("Stopping volume profile on %s has been %s ",
-                                 volname,
-                                 (rsp.op_ret) ? "unsuccessful": "successful");
-                        break;
-                case GF_CLI_STATS_INFO:
-                        break;
-                default:
-                        cli_out ("volume profile on %s has been %s ",
-                                 volname,
-                                 (rsp.op_ret) ? "unsuccessful": "successful");
-                        break;
-                }
-        }
-
-        if (rsp.op_ret) {
-                ret = rsp.op_ret;
-                goto out;
-        }
+            len = snprintf(key, sizeof(key), "%d-cumulative", i);
+            ret = dict_get_int32n(dict, key, len, &interval);
+            if (ret == 0)
+                cmd_profile_volume_brick_out(dict, i, interval);
 
-        if (GF_CLI_STATS_INFO != op) {
-                ret = 0;
-                goto out;
+            len = snprintf(key, sizeof(key), "%d-interval", i);
+            ret = dict_get_int32n(dict, key, len, &interval);
+            if (ret == 0)
+                cmd_profile_volume_brick_out(dict, i, interval);
         }
+        i++;
+    }
+    ret = rsp.op_ret;
 
-        ret = dict_get_int32 (dict, "info-op", (int32_t*)&info_op);
-        if (ret)
-                goto out;
-
-        ret = dict_get_int32 (dict, "count", &brick_count);
-        if (ret)
-                goto out;
+out:
+    if (dict)
+        dict_unref(dict);
+    cli_cmd_broadcast_response(ret);
+    gf_free_xdr_cli_rsp(rsp);
+    return ret;
+}
 
-        if (!brick_count) {
-                cli_err ("All bricks of volume %s are down.", volname);
-                goto out;
-        }
+static int32_t
+gf_cli_profile_volume(call_frame_t *frame, xlator_t *this, void *data)
+{
+    int ret = -1;
+    gf_cli_req req = {{
+        0,
+    }};
+    dict_t *dict = NULL;
 
-        while (i <= brick_count) {
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%d-brick", i);
-                ret = dict_get_str (dict, key, &brick);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Couldn't get brick name");
-                        goto out;
-                }
+    GF_ASSERT(frame);
+    GF_ASSERT(this);
+    GF_ASSERT(data);
 
-                ret = dict_get_str_boolean (dict, "nfs", _gf_false);
+    dict = data;
 
-                if (ret)
-                        snprintf (str, sizeof (str), "NFS Server : %s", brick);
-                else
-                        snprintf (str, sizeof (str), "Brick: %s", brick);
-                cli_out ("%s", str);
-                memset (str, '-', strlen (str));
-                cli_out ("%s", str);
-
-                if (GF_CLI_INFO_CLEAR == info_op) {
-                        snprintf (key, sizeof (key), "%d-stats-cleared", i);
-                        ret = dict_get_int32 (dict, key, &stats_cleared);
-                        if (ret)
-                                goto out;
-                        cli_out (stats_cleared ? "Cleared stats." :
-                                                 "Failed to clear stats.");
-                } else {
-                        snprintf (key, sizeof (key), "%d-cumulative", i);
-                        ret = dict_get_int32 (dict, key, &interval);
-                        if (ret == 0)
-                                cmd_profile_volume_brick_out (dict, i,
-                                                              interval);
-
-                        snprintf (key, sizeof (key), "%d-interval", i);
-                        ret = dict_get_int32 (dict, key, &interval);
-                        if (ret == 0)
-                                cmd_profile_volume_brick_out (dict, i,
-                                                              interval);
-                }
-                i++;
-        }
-        ret = rsp.op_ret;
+    ret = cli_to_glusterd(&req, frame, gf_cli_profile_volume_cbk,
+                          (xdrproc_t)xdr_gf_cli_req, dict,
+                          GLUSTER_CLI_PROFILE_VOLUME, this, cli_rpc_prog, NULL);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
 
-out:
-        if (dict)
-                dict_unref (dict);
-        free (rsp.op_errstr);
-        cli_cmd_broadcast_response (ret);
-        return ret;
+    GF_FREE(req.dict.dict_val);
+    return ret;
 }
 
-int32_t
-gf_cli_profile_volume (call_frame_t *frame, xlator_t *this, void *data)
+static int32_t
+gf_cli_top_volume_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                      void *myframe)
 {
-        int                        ret   = -1;
-        gf_cli_req                 req   = {{0,}};
-        dict_t                     *dict = NULL;
+    gf_cli_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    dict_t *dict = NULL;
+    gf1_cli_stats_op op = GF_CLI_STATS_NONE;
+    char key[256] = {0};
+    int keylen;
+    int i = 0;
+    int32_t brick_count = 0;
+    char brick[1024];
+    int32_t members = 0;
+    char *filename;
+    char *bricks;
+    uint64_t value = 0;
+    int32_t j = 0;
+    gf1_cli_top_op top_op = GF_CLI_TOP_NONE;
+    uint64_t nr_open = 0;
+    uint64_t max_nr_open = 0;
+    double throughput = 0;
+    double time = 0;
+    int32_t time_sec = 0;
+    long int time_usec = 0;
+    char timestr[GF_TIMESTR_SIZE] = {
+        0,
+    };
+    char *openfd_str = NULL;
+    gf_boolean_t nfs = _gf_false;
+    gf_boolean_t clear_stats = _gf_false;
+    int stats_cleared = 0;
+
+    GF_ASSERT(myframe);
+
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
+
+    gf_log("cli", GF_LOG_DEBUG, "Received resp to top");
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,
+               XDR_DECODE_FAIL);
+        goto out;
+    }
+
+    if (rsp.op_ret) {
+        if (strcmp(rsp.op_errstr, ""))
+            cli_err("%s", rsp.op_errstr);
+        cli_err("volume top unsuccessful");
+        ret = rsp.op_ret;
+        goto out;
+    }
 
-        GF_ASSERT (frame);
-        GF_ASSERT (this);
-        GF_ASSERT (data);
+    dict = dict_new();
 
-        if (!frame || !this || !data)
-                goto out;
-        dict = data;
+    if (!dict) {
+        ret = -1;
+        goto out;
+    }
 
-        ret = cli_to_glusterd (&req, frame, gf_cli_profile_volume_cbk,
-                               (xdrproc_t) xdr_gf_cli_req, dict,
-                               GLUSTER_CLI_PROFILE_VOLUME, this, cli_rpc_prog,
-                               NULL);
+    ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len, &dict);
 
-out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    if (ret) {
+        gf_log("", GF_LOG_ERROR, DICT_UNSERIALIZE_FAIL);
+        goto out;
+    }
 
-        GF_FREE (req.dict.dict_val);
-        return ret;
-}
+    ret = dict_get_int32_sizen(dict, "op", (int32_t *)&op);
 
-int32_t
-gf_cli_top_volume_cbk (struct rpc_req *req, struct iovec *iov,
-                              int count, void *myframe)
-{
-        gf_cli_rsp                        rsp   = {0,};
-        int                               ret   = -1;
-        dict_t                           *dict = NULL;
-        gf1_cli_stats_op                  op = GF_CLI_STATS_NONE;
-        char                              key[256] = {0};
-        int                               i = 0;
-        int32_t                           brick_count = 0;
-        char                              brick[1024];
-        int32_t                           members = 0;
-        char                             *filename;
-        char                             *bricks;
-        uint64_t                          value = 0;
-        int32_t                           j = 0;
-        gf1_cli_top_op                    top_op = GF_CLI_TOP_NONE;
-        uint64_t                          nr_open = 0;
-        uint64_t                          max_nr_open = 0;
-        double                            throughput = 0;
-        double                            time = 0;
-        int32_t                           time_sec = 0;
-        long int                          time_usec = 0;
-        char                              timestr[256] = {0, };
-        char                             *openfd_str = NULL;
-        gf_boolean_t                      nfs = _gf_false;
-        gf_boolean_t                      clear_stats = _gf_false;
-        int                               stats_cleared = 0;
-
-        if (-1 == req->rpc_status) {
-                goto out;
-        }
+    if (op != GF_CLI_STATS_TOP) {
+        ret = 0;
+        goto out;
+    }
 
-        gf_log ("cli", GF_LOG_DEBUG, "Received resp to top");
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
-        if (ret < 0) {
-                gf_log (((call_frame_t *) myframe)->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                goto out;
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        ret = cli_xml_output_vol_top(dict, rsp.op_ret, rsp.op_errno,
+                                     rsp.op_errstr);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR, XML_ERROR);
         }
+        goto out;
+    }
 
-        if (rsp.op_ret) {
-                if (strcmp (rsp.op_errstr, ""))
-                        cli_err ("%s", rsp.op_errstr);
-                cli_err ("volume top unsuccessful");
-                ret = rsp.op_ret;
-                goto out;
-        }
+    ret = dict_get_int32_sizen(dict, "count", &brick_count);
+    if (ret)
+        goto out;
+    keylen = snprintf(key, sizeof(key), "%d-top-op", 1);
+    ret = dict_get_int32n(dict, key, keylen, (int32_t *)&top_op);
+    if (ret)
+        goto out;
 
-        dict = dict_new ();
+    clear_stats = dict_get_str_boolean(dict, "clear-stats", _gf_false);
 
-        if (!dict) {
-                ret = -1;
-                goto out;
-        }
+    while (i < brick_count) {
+        i++;
+        keylen = snprintf(brick, sizeof(brick), "%d-brick", i);
+        ret = dict_get_strn(dict, brick, keylen, &bricks);
+        if (ret)
+            goto out;
 
-        ret = dict_unserialize (rsp.dict.dict_val,
-                                rsp.dict.dict_len,
-                                &dict);
+        nfs = dict_get_str_boolean(dict, "nfs", _gf_false);
 
-        if (ret) {
-                gf_log ("", GF_LOG_ERROR,
-                                "Unable to allocate memory");
+        if (clear_stats) {
+            keylen = snprintf(key, sizeof(key), "%d-stats-cleared", i);
+            ret = dict_get_int32n(dict, key, keylen, &stats_cleared);
+            if (ret)
                 goto out;
+            cli_out(stats_cleared ? "Cleared stats for %s %s"
+                                  : "Failed to clear stats for %s %s",
+                    nfs ? "NFS server on" : "brick", bricks);
+            continue;
         }
 
-        ret = dict_get_int32 (dict, "op", (int32_t*)&op);
+        if (nfs)
+            cli_out("NFS Server : %s", bricks);
+        else
+            cli_out("Brick: %s", bricks);
 
-        if (op != GF_CLI_STATS_TOP) {
-                ret = 0;
-                goto out;
-        }
+        keylen = snprintf(key, sizeof(key), "%d-members", i);
+        ret = dict_get_int32n(dict, key, keylen, &members);
 
-        if (global_state->mode & GLUSTER_MODE_XML) {
-                ret = cli_xml_output_vol_top (dict, rsp.op_ret,
-                                              rsp.op_errno,
-                                              rsp.op_errstr);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Error outputting to xml");
+        switch (top_op) {
+            case GF_CLI_TOP_OPEN:
+                snprintf(key, sizeof(key), "%d-current-open", i);
+                ret = dict_get_uint64(dict, key, &nr_open);
+                if (ret)
+                    break;
+                snprintf(key, sizeof(key), "%d-max-open", i);
+                ret = dict_get_uint64(dict, key, &max_nr_open);
+                if (ret)
+                    goto out;
+                keylen = snprintf(key, sizeof(key), "%d-max-openfd-time", i);
+                ret = dict_get_strn(dict, key, keylen, &openfd_str);
+                if (ret)
+                    goto out;
+                cli_out("Current open fds: %" PRIu64 ", Max open fds: %" PRIu64
+                        ", Max openfd time: %s",
+                        nr_open, max_nr_open, openfd_str);
+            case GF_CLI_TOP_READ:
+            case GF_CLI_TOP_WRITE:
+            case GF_CLI_TOP_OPENDIR:
+            case GF_CLI_TOP_READDIR:
+                if (!members) {
+                    continue;
+                }
+                cli_out("Count\t\tfilename\n=======================");
+                break;
+            case GF_CLI_TOP_READ_PERF:
+            case GF_CLI_TOP_WRITE_PERF:
+                snprintf(key, sizeof(key), "%d-throughput", i);
+                ret = dict_get_double(dict, key, &throughput);
+                if (!ret) {
+                    snprintf(key, sizeof(key), "%d-time", i);
+                    ret = dict_get_double(dict, key, &time);
                 }
+                if (!ret)
+                    cli_out("Throughput %.2f MBps time %.4f secs", throughput,
+                            time / 1e6);
+
+                if (!members) {
+                    continue;
+                }
+                cli_out("%*s %-*s %-*s", VOL_TOP_PERF_SPEED_WIDTH, "MBps",
+                        VOL_TOP_PERF_FILENAME_DEF_WIDTH, "Filename",
+                        VOL_TOP_PERF_TIME_WIDTH, "Time");
+                cli_out("%*s %-*s %-*s", VOL_TOP_PERF_SPEED_WIDTH,
+                        "====", VOL_TOP_PERF_FILENAME_DEF_WIDTH,
+                        "========", VOL_TOP_PERF_TIME_WIDTH, "====");
+                break;
+            default:
                 goto out;
         }
 
-        ret = dict_get_int32 (dict, "count", &brick_count);
-        if (ret)
-                goto out;
-        snprintf (key, sizeof (key), "%d-top-op", 1);
-        ret = dict_get_int32 (dict, key, (int32_t*)&top_op);
-        if (ret)
-                goto out;
-
-        clear_stats = dict_get_str_boolean (dict, "clear-stats", _gf_false);
-
-        while (i < brick_count) {
-                i++;
-                snprintf (brick, sizeof (brick), "%d-brick", i);
-                ret = dict_get_str (dict, brick, &bricks);
+        for (j = 1; j <= members; j++) {
+            keylen = snprintf(key, sizeof(key), "%d-filename-%d", i, j);
+            ret = dict_get_strn(dict, key, keylen, &filename);
+            if (ret)
+                break;
+            snprintf(key, sizeof(key), "%d-value-%d", i, j);
+            ret = dict_get_uint64(dict, key, &value);
+            if (ret)
+                goto out;
+            if (top_op == GF_CLI_TOP_READ_PERF ||
+                top_op == GF_CLI_TOP_WRITE_PERF) {
+                keylen = snprintf(key, sizeof(key), "%d-time-sec-%d", i, j);
+                ret = dict_get_int32n(dict, key, keylen, (int32_t *)&time_sec);
                 if (ret)
-                        goto out;
-
-                nfs = dict_get_str_boolean (dict, "nfs", _gf_false);
-
-                if (clear_stats) {
-                        memset (key, 0, sizeof (key));
-                        snprintf (key, sizeof (key), "%d-stats-cleared", i);
-                        ret = dict_get_int32 (dict, key, &stats_cleared);
-                        if (ret)
-                                goto out;
-                        cli_out (stats_cleared ? "Cleared stats for %s %s" :
-                                 "Failed to clear stats for %s %s",
-                                 nfs ? "NFS server on" : "brick", bricks);
-                        continue;
-                }
-
-                if (nfs)
-                        cli_out ("NFS Server : %s", bricks);
+                    goto out;
+                keylen = snprintf(key, sizeof(key), "%d-time-usec-%d", i, j);
+                ret = dict_get_int32n(dict, key, keylen, (int32_t *)&time_usec);
+                if (ret)
+                    goto out;
+                gf_time_fmt(timestr, sizeof timestr, time_sec, gf_timefmt_FT);
+                snprintf(timestr + strlen(timestr),
+                         sizeof timestr - strlen(timestr), ".%ld", time_usec);
+                if (strlen(filename) < VOL_TOP_PERF_FILENAME_DEF_WIDTH)
+                    cli_out("%*" PRIu64 " %-*s %-*s", VOL_TOP_PERF_SPEED_WIDTH,
+                            value, VOL_TOP_PERF_FILENAME_DEF_WIDTH, filename,
+                            VOL_TOP_PERF_TIME_WIDTH, timestr);
                 else
-                        cli_out ("Brick: %s", bricks);
-
-                snprintf(key, sizeof (key), "%d-members", i);
-                ret = dict_get_int32 (dict, key, &members);
-
-                switch (top_op) {
-                case GF_CLI_TOP_OPEN:
-                        snprintf (key, sizeof (key), "%d-current-open", i);
-                        ret = dict_get_uint64 (dict, key, &nr_open);
-                        if (ret)
-                                break;
-                        snprintf (key, sizeof (key), "%d-max-open", i);
-                        ret = dict_get_uint64 (dict, key, &max_nr_open);
-                        if (ret)
-                                goto out;
-                        snprintf (key, sizeof (key), "%d-max-openfd-time", i);
-                        ret = dict_get_str (dict, key, &openfd_str);
-                        if (ret)
-                                goto out;
-                        cli_out ("Current open fds: %"PRIu64", Max open"
-                                " fds: %"PRIu64", Max openfd time: %s", nr_open,
-                                 max_nr_open, openfd_str);
-                case GF_CLI_TOP_READ:
-                case GF_CLI_TOP_WRITE:
-                case GF_CLI_TOP_OPENDIR:
-                case GF_CLI_TOP_READDIR:
-                        if (!members) {
-                                continue;
-                        }
-                        cli_out ("Count\t\tfilename\n=======================");
-                        break;
-                case GF_CLI_TOP_READ_PERF:
-                case GF_CLI_TOP_WRITE_PERF:
-                        snprintf (key, sizeof (key), "%d-throughput", i);
-                        ret = dict_get_double (dict, key, &throughput);
-                        if (!ret) {
-                                snprintf (key, sizeof (key), "%d-time", i);
-                                ret = dict_get_double (dict, key, &time);
-                        }
-                        if (!ret)
-                                cli_out ("Throughput %.2f MBps time %.4f secs", throughput,
-                                          time / 1e6);
-
-                        if (!members) {
-                                continue;
-                        }
-                        cli_out ("%*s %-*s %-*s",
-                                 VOL_TOP_PERF_SPEED_WIDTH, "MBps",
-                                 VOL_TOP_PERF_FILENAME_DEF_WIDTH, "Filename",
-                                 VOL_TOP_PERF_TIME_WIDTH, "Time");
-                        cli_out ("%*s %-*s %-*s",
-                                 VOL_TOP_PERF_SPEED_WIDTH, "====",
-                                 VOL_TOP_PERF_FILENAME_DEF_WIDTH, "========",
-                                 VOL_TOP_PERF_TIME_WIDTH, "====");
-                        break;
-                default:
-                        goto out;
-                }
-
-                for (j = 1; j <= members; j++) {
-                        snprintf (key, sizeof (key), "%d-filename-%d", i, j);
-                        ret = dict_get_str (dict, key, &filename);
-                        if (ret)
-                                break;
-                        snprintf (key, sizeof (key), "%d-value-%d", i, j);
-                        ret = dict_get_uint64 (dict, key, &value);
-                        if (ret)
-                                goto out;
-                        if ( top_op == GF_CLI_TOP_READ_PERF ||
-                                top_op == GF_CLI_TOP_WRITE_PERF) {
-                                snprintf (key, sizeof (key), "%d-time-sec-%d", i, j);
-                                ret = dict_get_int32 (dict, key, (int32_t *)&time_sec);
-                                if (ret)
-                                        goto out;
-                                snprintf (key, sizeof (key), "%d-time-usec-%d", i, j);
-                                ret = dict_get_int32 (dict, key, (int32_t *)&time_usec);
-                                if (ret)
-                                        goto out;
-                                gf_time_fmt (timestr, sizeof timestr,
-                                             time_sec, gf_timefmt_FT);
-                                snprintf (timestr + strlen (timestr), sizeof timestr - strlen (timestr),
-                                  ".%ld", time_usec);
-                                if (strlen (filename) < VOL_TOP_PERF_FILENAME_DEF_WIDTH)
-                                        cli_out ("%*"PRIu64" %-*s %-*s",
-                                                 VOL_TOP_PERF_SPEED_WIDTH,
-                                                 value,
-                                                 VOL_TOP_PERF_FILENAME_DEF_WIDTH,
-                                                 filename,
-                                                 VOL_TOP_PERF_TIME_WIDTH,
-                                                 timestr);
-                                else
-                                        cli_out ("%*"PRIu64" ...%-*s %-*s",
-                                                 VOL_TOP_PERF_SPEED_WIDTH,
-                                                 value,
-                                                 VOL_TOP_PERF_FILENAME_ALT_WIDTH ,
-                                                 filename + strlen (filename) -
-                                                 VOL_TOP_PERF_FILENAME_ALT_WIDTH,
-                                                 VOL_TOP_PERF_TIME_WIDTH,
-                                                 timestr);
-                        } else {
-                                cli_out ("%"PRIu64"\t\t%s", value, filename);
-                        }
-                }
-        }
-        ret = rsp.op_ret;
+                    cli_out("%*" PRIu64 " ...%-*s %-*s",
+                            VOL_TOP_PERF_SPEED_WIDTH, value,
+                            VOL_TOP_PERF_FILENAME_ALT_WIDTH,
+                            filename + strlen(filename) -
+                                VOL_TOP_PERF_FILENAME_ALT_WIDTH,
+                            VOL_TOP_PERF_TIME_WIDTH, timestr);
+            } else {
+                cli_out("%" PRIu64 "\t\t%s", value, filename);
+            }
+        }
+    }
+    ret = rsp.op_ret;
 
 out:
-        cli_cmd_broadcast_response (ret);
+    cli_cmd_broadcast_response(ret);
 
-        if (dict)
-                dict_unref (dict);
+    if (dict)
+        dict_unref(dict);
 
-        free (rsp.dict.dict_val);
-        return ret;
+    gf_free_xdr_cli_rsp(rsp);
+    return ret;
 }
 
-int32_t
-gf_cli_top_volume (call_frame_t *frame, xlator_t *this, void *data)
+static int32_t
+gf_cli_top_volume(call_frame_t *frame, xlator_t *this, void *data)
 {
-        int                        ret   = -1;
-        gf_cli_req                 req   = {{0,}};
-        dict_t                     *dict = NULL;
-
-        GF_ASSERT (frame);
-        GF_ASSERT (this);
-        GF_ASSERT (data);
+    int ret = -1;
+    gf_cli_req req = {{
+        0,
+    }};
+    dict_t *dict = NULL;
 
-        if (!frame || !this || !data)
-                goto out;
-        dict = data;
+    GF_ASSERT(frame);
+    GF_ASSERT(this);
+    GF_ASSERT(data);
 
-        ret = cli_to_glusterd (&req, frame, gf_cli_top_volume_cbk,
-                               (xdrproc_t) xdr_gf_cli_req, dict,
-                               GLUSTER_CLI_PROFILE_VOLUME, this, cli_rpc_prog,
-                               NULL);
+    dict = data;
 
-out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        GF_FREE (req.dict.dict_val);
-        return ret;
+    ret = cli_to_glusterd(&req, frame, gf_cli_top_volume_cbk,
+                          (xdrproc_t)xdr_gf_cli_req, dict,
+                          GLUSTER_CLI_PROFILE_VOLUME, this, cli_rpc_prog, NULL);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
+    GF_FREE(req.dict.dict_val);
+    return ret;
 }
 
-
-int
-gf_cli_getwd_cbk (struct rpc_req *req, struct iovec *iov,
-                       int count, void *myframe)
+static int
+gf_cli_getwd_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                 void *myframe)
 {
-        gf1_cli_getwd_rsp rsp   = {0,};
-        int               ret   = -1;
-
-        if (-1 == req->rpc_status) {
-                goto out;
-        }
-
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf1_cli_getwd_rsp);
-        if (ret < 0) {
-                gf_log (((call_frame_t *) myframe)->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                goto out;
-        }
-
-        if (rsp.op_ret == -1) {
-                cli_err ("getwd failed");
-                ret = rsp.op_ret;
-                goto out;
-        }
+    gf1_cli_getwd_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+
+    GF_ASSERT(myframe);
+
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf1_cli_getwd_rsp);
+    if (ret < 0) {
+        gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,
+               XDR_DECODE_FAIL);
+        goto out;
+    }
+
+    if (rsp.op_ret == -1) {
+        cli_err("getwd failed");
+        ret = rsp.op_ret;
+        goto out;
+    }
 
-        gf_log ("cli", GF_LOG_INFO, "Received resp to getwd");
+    gf_log("cli", GF_LOG_INFO, "Received resp to getwd");
 
-        cli_out ("%s", rsp.wd);
+    cli_out("%s", rsp.wd);
 
-        ret = 0;
+    ret = 0;
 
 out:
-        cli_cmd_broadcast_response (ret);
-        return ret;
+    cli_cmd_broadcast_response(ret);
+    if (rsp.wd) {
+        free(rsp.wd);
+    }
+
+    return ret;
 }
 
-int32_t
-gf_cli_getwd (call_frame_t *frame, xlator_t *this, void *data)
+static int32_t
+gf_cli_getwd(call_frame_t *frame, xlator_t *this, void *data)
 {
-        int                      ret = -1;
-        gf1_cli_getwd_req        req = {0,};
+    int ret = -1;
+    gf1_cli_getwd_req req = {
+        0,
+    };
 
-        GF_ASSERT (frame);
-        GF_ASSERT (this);
+    GF_ASSERT(frame);
+    GF_ASSERT(this);
 
-        if (!frame || !this)
-                goto out;
+    if (!frame || !this)
+        goto out;
 
-        ret = cli_cmd_submit (NULL, &req, frame, cli_rpc_prog,
-                              GLUSTER_CLI_GETWD, NULL,
-                              this, gf_cli_getwd_cbk,
-                              (xdrproc_t) xdr_gf1_cli_getwd_req);
+    ret = cli_cmd_submit(NULL, &req, frame, cli_rpc_prog, GLUSTER_CLI_GETWD,
+                         NULL, this, gf_cli_getwd_cbk,
+                         (xdrproc_t)xdr_gf1_cli_getwd_req);
 
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
 
-        return ret;
+    return ret;
 }
 
-void
-cli_print_volume_status_mempool (dict_t *dict, char *prefix)
-{
-        int             ret = -1;
-        int32_t         mempool_count = 0;
-        char            *name = NULL;
-        int32_t         hotcount = 0;
-        int32_t         coldcount = 0;
-        uint64_t        paddedsizeof = 0;
-        uint64_t        alloccount = 0;
-        int32_t         maxalloc = 0;
-        uint64_t        pool_misses = 0;
-        int32_t         maxstdalloc = 0;
-        char            key[1024] = {0,};
-        int             i = 0;
-
-        GF_ASSERT (dict);
-        GF_ASSERT (prefix);
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.mempool-count",prefix);
-        ret = dict_get_int32 (dict, key, &mempool_count);
+static void
+cli_print_volume_status_mempool(dict_t *dict, char *prefix)
+{
+    int ret = -1;
+    int32_t mempool_count = 0;
+    char *name = NULL;
+    int32_t hotcount = 0;
+    int32_t coldcount = 0;
+    uint64_t paddedsizeof = 0;
+    uint64_t alloccount = 0;
+    int32_t maxalloc = 0;
+    uint64_t pool_misses = 0;
+    int32_t maxstdalloc = 0;
+    char key[128] = {
+        /* prefix is really small 'brick%d' really */
+        0,
+    };
+    int keylen;
+    int i = 0;
+
+    GF_ASSERT(dict);
+    GF_ASSERT(prefix);
+
+    keylen = snprintf(key, sizeof(key), "%s.mempool-count", prefix);
+    ret = dict_get_int32n(dict, key, keylen, &mempool_count);
+    if (ret)
+        goto out;
+
+    cli_out("Mempool Stats\n-------------");
+    cli_out("%-30s %9s %9s %12s %10s %8s %8s %12s", "Name", "HotCount",
+            "ColdCount", "PaddedSizeof", "AllocCount", "MaxAlloc", "Misses",
+            "Max-StdAlloc");
+    cli_out("%-30s %9s %9s %12s %10s %8s %8s %12s", "----", "--------",
+            "---------", "------------", "----------", "--------", "--------",
+            "------------");
+
+    for (i = 0; i < mempool_count; i++) {
+        keylen = snprintf(key, sizeof(key), "%s.pool%d.name", prefix, i);
+        ret = dict_get_strn(dict, key, keylen, &name);
         if (ret)
-                goto out;
-
-        cli_out ("Mempool Stats\n-------------");
-        cli_out ("%-30s %9s %9s %12s %10s %8s %8s %12s", "Name", "HotCount",
-                 "ColdCount", "PaddedSizeof", "AllocCount", "MaxAlloc",
-                 "Misses", "Max-StdAlloc");
-        cli_out ("%-30s %9s %9s %12s %10s %8s %8s %12s", "----", "--------",
-                 "---------", "------------", "----------",
-                 "--------", "--------", "------------");
-
-        for (i = 0; i < mempool_count; i++) {
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%s.pool%d.name", prefix, i);
-                ret = dict_get_str (dict, key, &name);
-                if (ret)
-                        goto out;
+            goto out;
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%s.pool%d.hotcount", prefix, i);
-                ret = dict_get_int32 (dict, key, &hotcount);
-                if (ret)
-                        goto out;
+        keylen = snprintf(key, sizeof(key), "%s.pool%d.hotcount", prefix, i);
+        ret = dict_get_int32n(dict, key, keylen, &hotcount);
+        if (ret)
+            goto out;
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%s.pool%d.coldcount", prefix, i);
-                ret = dict_get_int32 (dict, key, &coldcount);
-                if (ret)
-                        goto out;
+        keylen = snprintf(key, sizeof(key), "%s.pool%d.coldcount", prefix, i);
+        ret = dict_get_int32n(dict, key, keylen, &coldcount);
+        if (ret)
+            goto out;
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%s.pool%d.paddedsizeof",
-                          prefix, i);
-                ret = dict_get_uint64 (dict, key, &paddedsizeof);
-                if (ret)
-                        goto out;
+        snprintf(key, sizeof(key), "%s.pool%d.paddedsizeof", prefix, i);
+        ret = dict_get_uint64(dict, key, &paddedsizeof);
+        if (ret)
+            goto out;
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%s.pool%d.alloccount", prefix, i);
-                ret = dict_get_uint64 (dict, key, &alloccount);
-                if (ret)
-                        goto out;
+        snprintf(key, sizeof(key), "%s.pool%d.alloccount", prefix, i);
+        ret = dict_get_uint64(dict, key, &alloccount);
+        if (ret)
+            goto out;
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%s.pool%d.max_alloc", prefix, i);
-                ret = dict_get_int32 (dict, key, &maxalloc);
-                if (ret)
-                        goto out;
+        keylen = snprintf(key, sizeof(key), "%s.pool%d.max_alloc", prefix, i);
+        ret = dict_get_int32n(dict, key, keylen, &maxalloc);
+        if (ret)
+            goto out;
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%s.pool%d.max-stdalloc", prefix, i);
-                ret = dict_get_int32 (dict, key, &maxstdalloc);
-                if (ret)
-                        goto out;
+        keylen = snprintf(key, sizeof(key), "%s.pool%d.max-stdalloc", prefix,
+                          i);
+        ret = dict_get_int32n(dict, key, keylen, &maxstdalloc);
+        if (ret)
+            goto out;
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%s.pool%d.pool-misses", prefix, i);
-                ret = dict_get_uint64 (dict, key, &pool_misses);
-                if (ret)
-                        goto out;
+        snprintf(key, sizeof(key), "%s.pool%d.pool-misses", prefix, i);
+        ret = dict_get_uint64(dict, key, &pool_misses);
+        if (ret)
+            goto out;
 
-                cli_out ("%-30s %9d %9d %12"PRIu64" %10"PRIu64" %8d %8"PRIu64
-                         " %12d", name, hotcount, coldcount, paddedsizeof,
-                         alloccount, maxalloc, pool_misses, maxstdalloc);
-        }
+        cli_out("%-30s %9d %9d %12" PRIu64 " %10" PRIu64 " %8d %8" PRIu64
+                " %12d",
+                name, hotcount, coldcount, paddedsizeof, alloccount, maxalloc,
+                pool_misses, maxstdalloc);
+    }
 
 out:
-        return;
-
+    return;
 }
 
-void
-cli_print_volume_status_mem (dict_t *dict, gf_boolean_t notbrick)
+static void
+cli_print_volume_status_mem(dict_t *dict, gf_boolean_t notbrick)
 {
-        int             ret = -1;
-        char            *volname = NULL;
-        char            *hostname = NULL;
-        char            *path = NULL;
-        int             online = -1;
-        char            key[1024] = {0,};
-        int             brick_index_max = -1;
-        int             other_count = 0;
-        int             index_max = 0;
-        int             val = 0;
-        int             i = 0;
-
-        GF_ASSERT (dict);
-
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret)
-                goto out;
-        cli_out ("Memory status for volume : %s", volname);
-
-        ret = dict_get_int32 (dict, "brick-index-max", &brick_index_max);
+    int ret = -1;
+    char *volname = NULL;
+    char *hostname = NULL;
+    char *path = NULL;
+    int online = -1;
+    char key[64] = {
+        0,
+    };
+    int brick_index_max = -1;
+    int other_count = 0;
+    int index_max = 0;
+    int val = 0;
+    int i = 0;
+
+    GF_ASSERT(dict);
+
+    ret = dict_get_str_sizen(dict, "volname", &volname);
+    if (ret)
+        goto out;
+    cli_out("Memory status for volume : %s", volname);
+
+    ret = dict_get_int32_sizen(dict, "brick-index-max", &brick_index_max);
+    if (ret)
+        goto out;
+    ret = dict_get_int32_sizen(dict, "other-count", &other_count);
+    if (ret)
+        goto out;
+
+    index_max = brick_index_max + other_count;
+
+    for (i = 0; i <= index_max; i++) {
+        cli_out("----------------------------------------------");
+
+        snprintf(key, sizeof(key), "brick%d.hostname", i);
+        ret = dict_get_str(dict, key, &hostname);
         if (ret)
-                goto out;
-        ret = dict_get_int32 (dict, "other-count", &other_count);
+            continue;
+        snprintf(key, sizeof(key), "brick%d.path", i);
+        ret = dict_get_str(dict, key, &path);
         if (ret)
-                goto out;
-
-        index_max = brick_index_max + other_count;
-
-        for (i = 0; i <= index_max; i++) {
-                cli_out ("----------------------------------------------");
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "brick%d.hostname", i);
-                ret = dict_get_str (dict, key, &hostname);
-                if (ret)
-                        continue;
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "brick%d.path", i);
-                ret = dict_get_str (dict, key, &path);
-                if (ret)
-                        continue;
-                if (notbrick)
-                        cli_out ("%s : %s", hostname, path);
-                else
-                        cli_out ("Brick : %s:%s", hostname, path);
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "brick%d.status", i);
-                ret = dict_get_int32 (dict, key, &online);
-                if (ret)
-                        goto out;
-                if (!online) {
-                        if (notbrick)
-                                cli_out ("%s is offline", hostname);
-                        else
-                                cli_out ("Brick is offline");
-                        continue;
-                }
+            continue;
+        if (notbrick)
+            cli_out("%s : %s", hostname, path);
+        else
+            cli_out("Brick : %s:%s", hostname, path);
 
-                cli_out ("Mallinfo\n--------");
+        snprintf(key, sizeof(key), "brick%d.status", i);
+        ret = dict_get_int32(dict, key, &online);
+        if (ret)
+            goto out;
+        if (!online) {
+            if (notbrick)
+                cli_out("%s is offline", hostname);
+            else
+                cli_out("Brick is offline");
+            continue;
+        }
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "brick%d.mallinfo.arena", i);
-                ret = dict_get_int32 (dict, key, &val);
-                if (ret)
-                        goto out;
-                cli_out ("%-8s : %d","Arena", val);
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "brick%d.mallinfo.ordblks", i);
-                ret = dict_get_int32 (dict, key, &val);
-                if(ret)
-                        goto out;
-                cli_out ("%-8s : %d","Ordblks", val);
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "brick%d.mallinfo.smblks", i);
-                ret = dict_get_int32 (dict, key, &val);
-                if(ret)
-                        goto out;
-                cli_out ("%-8s : %d","Smblks", val);
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "brick%d.mallinfo.hblks", i);
-                ret = dict_get_int32 (dict, key, &val);
-                if(ret)
-                        goto out;
-                cli_out ("%-8s : %d", "Hblks", val);
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "brick%d.mallinfo.hblkhd", i);
-                ret = dict_get_int32 (dict, key, &val);
-                if (ret)
-                        goto out;
-                cli_out ("%-8s : %d", "Hblkhd", val);
+        cli_out("Mallinfo\n--------");
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "brick%d.mallinfo.usmblks", i);
-                ret = dict_get_int32 (dict, key, &val);
-                if (ret)
-                        goto out;
-                cli_out ("%-8s : %d", "Usmblks", val);
+        snprintf(key, sizeof(key), "brick%d.mallinfo.arena", i);
+        ret = dict_get_int32(dict, key, &val);
+        if (ret)
+            goto out;
+        cli_out("%-8s : %d", "Arena", val);
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "brick%d.mallinfo.fsmblks", i);
-                ret = dict_get_int32 (dict, key, &val);
-                if (ret)
-                        goto out;
-                cli_out ("%-8s : %d", "Fsmblks", val);
+        snprintf(key, sizeof(key), "brick%d.mallinfo.ordblks", i);
+        ret = dict_get_int32(dict, key, &val);
+        if (ret)
+            goto out;
+        cli_out("%-8s : %d", "Ordblks", val);
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "brick%d.mallinfo.uordblks", i);
-                ret = dict_get_int32 (dict, key, &val);
-                if (ret)
-                        goto out;
-                cli_out ("%-8s : %d", "Uordblks", val);
+        snprintf(key, sizeof(key), "brick%d.mallinfo.smblks", i);
+        ret = dict_get_int32(dict, key, &val);
+        if (ret)
+            goto out;
+        cli_out("%-8s : %d", "Smblks", val);
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "brick%d.mallinfo.fordblks", i);
-                ret = dict_get_int32 (dict, key, &val);
-                if (ret)
-                        goto out;
-                cli_out ("%-8s : %d", "Fordblks", val);
+        snprintf(key, sizeof(key), "brick%d.mallinfo.hblks", i);
+        ret = dict_get_int32(dict, key, &val);
+        if (ret)
+            goto out;
+        cli_out("%-8s : %d", "Hblks", val);
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "brick%d.mallinfo.keepcost", i);
-                ret = dict_get_int32 (dict, key, &val);
-                if (ret)
-                        goto out;
-                cli_out ("%-8s : %d", "Keepcost", val);
+        snprintf(key, sizeof(key), "brick%d.mallinfo.hblkhd", i);
+        ret = dict_get_int32(dict, key, &val);
+        if (ret)
+            goto out;
+        cli_out("%-8s : %d", "Hblkhd", val);
 
-                cli_out (" ");
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "brick%d", i);
-                cli_print_volume_status_mempool (dict, key);
-        }
-out:
-        cli_out ("----------------------------------------------\n");
-        return;
-}
+        snprintf(key, sizeof(key), "brick%d.mallinfo.usmblks", i);
+        ret = dict_get_int32(dict, key, &val);
+        if (ret)
+            goto out;
+        cli_out("%-8s : %d", "Usmblks", val);
 
-void
-cli_print_volume_status_clients (dict_t *dict, gf_boolean_t notbrick)
-{
-        int             ret = -1;
-        char            *volname = NULL;
-        int             brick_index_max = -1;
-        int             other_count = 0;
-        int             index_max = 0;
-        char            *hostname = NULL;
-        char            *path = NULL;
-        int             online = -1;
-        int             client_count = 0;
-        char            *clientname = NULL;
-        uint64_t        bytesread = 0;
-        uint64_t        byteswrite = 0;
-        char            key[1024] = {0,};
-        int             i = 0;
-        int             j = 0;
-
-        GF_ASSERT (dict);
-
-        ret = dict_get_str (dict, "volname", &volname);
+        snprintf(key, sizeof(key), "brick%d.mallinfo.fsmblks", i);
+        ret = dict_get_int32(dict, key, &val);
         if (ret)
-                goto out;
-        cli_out ("Client connections for volume %s", volname);
+            goto out;
+        cli_out("%-8s : %d", "Fsmblks", val);
 
-        ret = dict_get_int32 (dict, "brick-index-max", &brick_index_max);
+        snprintf(key, sizeof(key), "brick%d.mallinfo.uordblks", i);
+        ret = dict_get_int32(dict, key, &val);
         if (ret)
-                goto out;
-        ret = dict_get_int32 (dict, "other-count", &other_count);
+            goto out;
+        cli_out("%-8s : %d", "Uordblks", val);
+
+        snprintf(key, sizeof(key), "brick%d.mallinfo.fordblks", i);
+        ret = dict_get_int32(dict, key, &val);
         if (ret)
-                goto out;
+            goto out;
+        cli_out("%-8s : %d", "Fordblks", val);
 
-        index_max = brick_index_max + other_count;
+        snprintf(key, sizeof(key), "brick%d.mallinfo.keepcost", i);
+        ret = dict_get_int32(dict, key, &val);
+        if (ret)
+            goto out;
+        cli_out("%-8s : %d", "Keepcost", val);
 
-        for (i = 0; i <= index_max; i++) {
-                cli_out ("----------------------------------------------");
+        cli_out(" ");
+        snprintf(key, sizeof(key), "brick%d", i);
+        cli_print_volume_status_mempool(dict, key);
+    }
+out:
+    cli_out("----------------------------------------------\n");
+    return;
+}
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "brick%d.hostname", i);
-                ret = dict_get_str (dict, key, &hostname);
+static void
+cli_print_volume_status_client_list(dict_t *dict, gf_boolean_t notbrick)
+{
+    int ret = -1;
+    char *volname = NULL;
+    int client_count = 0;
+    int current_count = 0;
+    char key[64] = {
+        0,
+    };
+    int i = 0;
+    int total = 0;
+    char *name = NULL;
+    gf_boolean_t is_fuse_done = _gf_false;
+    gf_boolean_t is_gfapi_done = _gf_false;
+    gf_boolean_t is_rebalance_done = _gf_false;
+    gf_boolean_t is_glustershd_done = _gf_false;
+    gf_boolean_t is_quotad_done = _gf_false;
+    gf_boolean_t is_snapd_done = _gf_false;
+
+    GF_ASSERT(dict);
+
+    ret = dict_get_str_sizen(dict, "volname", &volname);
+    if (ret)
+        goto out;
+    cli_out("Client connections for volume %s", volname);
+
+    ret = dict_get_int32_sizen(dict, "client-count", &client_count);
+    if (ret)
+        goto out;
+
+    cli_out("%-48s %15s", "Name", "count");
+    cli_out("%-48s %15s", "-----", "------");
+    for (i = 0; i < client_count; i++) {
+        name = NULL;
+        snprintf(key, sizeof(key), "client%d.name", i);
+        ret = dict_get_str(dict, key, &name);
+
+        if (!strncmp(name, "fuse", 4)) {
+            if (!is_fuse_done) {
+                is_fuse_done = _gf_true;
+                ret = dict_get_int32_sizen(dict, "fuse-count", &current_count);
                 if (ret)
-                        goto out;
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "brick%d.path", i);
-                ret = dict_get_str (dict, key, &path);
+                    goto out;
+                total = total + current_count;
+                goto print;
+            }
+            continue;
+        } else if (!strncmp(name, "gfapi", 5)) {
+            if (!is_gfapi_done) {
+                is_gfapi_done = _gf_true;
+                ret = dict_get_int32_sizen(dict, "gfapi-count", &current_count);
                 if (ret)
-                        goto out;
-
-                if (notbrick)
-                        cli_out ("%s : %s", hostname, path);
-                else
-                        cli_out ("Brick : %s:%s", hostname, path);
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "brick%d.status", i);
-                ret = dict_get_int32 (dict, key, &online);
+                    goto out;
+                total = total + current_count;
+                goto print;
+            }
+            continue;
+        } else if (!strcmp(name, "rebalance")) {
+            if (!is_rebalance_done) {
+                is_rebalance_done = _gf_true;
+                ret = dict_get_int32_sizen(dict, "rebalance-count",
+                                           &current_count);
                 if (ret)
-                        goto out;
-                if (!online) {
-                        if (notbrick)
-                                cli_out ("%s is offline", hostname);
-                        else
-                                cli_out ("Brick is offline");
-                        continue;
-                }
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "brick%d.clientcount", i);
-                ret = dict_get_int32 (dict, key, &client_count);
+                    goto out;
+                total = total + current_count;
+                goto print;
+            }
+            continue;
+        } else if (!strcmp(name, "glustershd")) {
+            if (!is_glustershd_done) {
+                is_glustershd_done = _gf_true;
+                ret = dict_get_int32_sizen(dict, "glustershd-count",
+                                           &current_count);
                 if (ret)
-                        goto out;
-
-                cli_out ("Clients connected : %d", client_count);
-                if (client_count == 0)
-                        continue;
-
-                cli_out ("%-48s %15s %15s", "Hostname", "BytesRead",
-                         "BytesWritten");
-                cli_out ("%-48s %15s %15s", "--------", "---------",
-                         "------------");
-                for (j =0; j < client_count; j++) {
-                        memset (key, 0, sizeof (key));
-                        snprintf (key, sizeof (key),
-                                  "brick%d.client%d.hostname", i, j);
-                        ret = dict_get_str (dict, key, &clientname);
-                        if (ret)
-                                goto out;
-
-                        memset (key, 0, sizeof (key));
-                        snprintf (key, sizeof (key),
-                                  "brick%d.client%d.bytesread", i, j);
-                        ret = dict_get_uint64 (dict, key, &bytesread);
-                        if (ret)
-                                goto out;
-
-                        memset (key, 0, sizeof (key));
-                        snprintf (key, sizeof (key),
-                                 "brick%d.client%d.byteswrite", i, j);
-                        ret = dict_get_uint64 (dict, key, &byteswrite);
-                        if (ret)
-                                goto out;
-
-                        cli_out ("%-48s %15"PRIu64" %15"PRIu64,
-                                 clientname, bytesread, byteswrite);
-                }
+                    goto out;
+                total = total + current_count;
+                goto print;
+            }
+            continue;
+        } else if (!strcmp(name, "quotad")) {
+            if (!is_quotad_done) {
+                is_quotad_done = _gf_true;
+                ret = dict_get_int32_sizen(dict, "quotad-count",
+                                           &current_count);
+                if (ret)
+                    goto out;
+                total = total + current_count;
+                goto print;
+            }
+            continue;
+        } else if (!strcmp(name, "snapd")) {
+            if (!is_snapd_done) {
+                is_snapd_done = _gf_true;
+                ret = dict_get_int32_sizen(dict, "snapd-count", &current_count);
+                if (ret)
+                    goto out;
+                total = total + current_count;
+                goto print;
+            }
+            continue;
         }
+
+    print:
+        cli_out("%-48s %15d", name, current_count);
+    }
 out:
-        cli_out ("----------------------------------------------\n");
-        return;
+    cli_out("\ntotal clients for volume %s : %d ", volname, total);
+    cli_out(
+        "-----------------------------------------------------------------\n");
+    return;
 }
 
-void
-cli_print_volume_status_inode_entry (dict_t *dict, char *prefix)
+static void
+cli_print_volume_status_clients(dict_t *dict, gf_boolean_t notbrick)
 {
-        int             ret = -1;
-        char            key[1024] = {0,};
-        char            *gfid = NULL;
-        uint64_t        nlookup = 0;
-        uint32_t        ref = 0;
-        int             ia_type = 0;
-        char            inode_type;
-
-        GF_ASSERT (dict);
-        GF_ASSERT (prefix);
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.gfid", prefix);
-        ret = dict_get_str (dict, key, &gfid);
-        if (ret)
-                goto out;
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.nlookup", prefix);
-        ret = dict_get_uint64 (dict, key, &nlookup);
-        if (ret)
-                goto out;
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.ref", prefix);
-        ret = dict_get_uint32 (dict, key, &ref);
-        if (ret)
-                goto out;
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.ia_type", prefix);
-        ret = dict_get_int32 (dict, key, &ia_type);
-        if (ret)
-                goto out;
+    int ret = -1;
+    char *volname = NULL;
+    int brick_index_max = -1;
+    int other_count = 0;
+    int index_max = 0;
+    char *hostname = NULL;
+    char *path = NULL;
+    int online = -1;
+    int client_count = 0;
+    char *clientname = NULL;
+    uint64_t bytesread = 0;
+    uint64_t byteswrite = 0;
+    uint32_t opversion = 0;
+    char key[128] = {
+        0,
+    };
+    int i = 0;
+    int j = 0;
+
+    GF_ASSERT(dict);
+
+    ret = dict_get_str_sizen(dict, "volname", &volname);
+    if (ret)
+        goto out;
+    cli_out("Client connections for volume %s", volname);
+
+    ret = dict_get_int32_sizen(dict, "brick-index-max", &brick_index_max);
+    if (ret)
+        goto out;
+    ret = dict_get_int32_sizen(dict, "other-count", &other_count);
+    if (ret)
+        goto out;
+
+    index_max = brick_index_max + other_count;
+
+    for (i = 0; i <= index_max; i++) {
+        hostname = NULL;
+        path = NULL;
+        online = -1;
+        client_count = 0;
+        clientname = NULL;
+        bytesread = 0;
+        byteswrite = 0;
+
+        cli_out("----------------------------------------------");
+
+        snprintf(key, sizeof(key), "brick%d.hostname", i);
+        ret = dict_get_str(dict, key, &hostname);
+
+        snprintf(key, sizeof(key), "brick%d.path", i);
+        ret = dict_get_str(dict, key, &path);
+
+        if (hostname && path) {
+            if (notbrick)
+                cli_out("%s : %s", hostname, path);
+            else
+                cli_out("Brick : %s:%s", hostname, path);
+        }
+        snprintf(key, sizeof(key), "brick%d.status", i);
+        ret = dict_get_int32(dict, key, &online);
+        if (!online) {
+            if (notbrick)
+                cli_out("%s is offline", hostname);
+            else
+                cli_out("Brick is offline");
+            continue;
+        }
+
+        snprintf(key, sizeof(key), "brick%d.clientcount", i);
+        ret = dict_get_int32(dict, key, &client_count);
+
+        if (hostname && path)
+            cli_out("Clients connected : %d", client_count);
+        if (client_count == 0)
+            continue;
+
+        cli_out("%-48s %15s %15s %15s", "Hostname", "BytesRead", "BytesWritten",
+                "OpVersion");
+        cli_out("%-48s %15s %15s %15s", "--------", "---------", "------------",
+                "---------");
+        for (j = 0; j < client_count; j++) {
+            snprintf(key, sizeof(key), "brick%d.client%d.hostname", i, j);
+            ret = dict_get_str(dict, key, &clientname);
+
+            snprintf(key, sizeof(key), "brick%d.client%d.bytesread", i, j);
+            ret = dict_get_uint64(dict, key, &bytesread);
+
+            snprintf(key, sizeof(key), "brick%d.client%d.byteswrite", i, j);
+            ret = dict_get_uint64(dict, key, &byteswrite);
+
+            snprintf(key, sizeof(key), "brick%d.client%d.opversion", i, j);
+            ret = dict_get_uint32(dict, key, &opversion);
+
+            cli_out("%-48s %15" PRIu64 " %15" PRIu64 " %15" PRIu32, clientname,
+                    bytesread, byteswrite, opversion);
+        }
+    }
+out:
+    cli_out("----------------------------------------------\n");
+    return;
+}
 
-        switch (ia_type) {
+#ifdef DEBUG /* this function is only used in debug */
+static void
+cli_print_volume_status_inode_entry(dict_t *dict, char *prefix)
+{
+    int ret = -1;
+    char key[1024] = {
+        0,
+    };
+    char *gfid = NULL;
+    uint64_t nlookup = 0;
+    uint32_t ref = 0;
+    int ia_type = 0;
+    char inode_type;
+
+    GF_ASSERT(dict);
+    GF_ASSERT(prefix);
+
+    snprintf(key, sizeof(key), "%s.gfid", prefix);
+    ret = dict_get_str(dict, key, &gfid);
+    if (ret)
+        goto out;
+
+    snprintf(key, sizeof(key), "%s.nlookup", prefix);
+    ret = dict_get_uint64(dict, key, &nlookup);
+    if (ret)
+        goto out;
+
+    snprintf(key, sizeof(key), "%s.ref", prefix);
+    ret = dict_get_uint32(dict, key, &ref);
+    if (ret)
+        goto out;
+
+    snprintf(key, sizeof(key), "%s.ia_type", prefix);
+    ret = dict_get_int32(dict, key, &ia_type);
+    if (ret)
+        goto out;
+
+    switch (ia_type) {
         case IA_IFREG:
-                inode_type = 'R';
-                break;
+            inode_type = 'R';
+            break;
         case IA_IFDIR:
-                inode_type = 'D';
-                break;
+            inode_type = 'D';
+            break;
         case IA_IFLNK:
-                inode_type = 'L';
-                break;
+            inode_type = 'L';
+            break;
         case IA_IFBLK:
-                inode_type = 'B';
-                break;
+            inode_type = 'B';
+            break;
         case IA_IFCHR:
-                inode_type = 'C';
-                break;
+            inode_type = 'C';
+            break;
         case IA_IFIFO:
-                inode_type = 'F';
-                break;
+            inode_type = 'F';
+            break;
         case IA_IFSOCK:
-                inode_type = 'S';
-                break;
+            inode_type = 'S';
+            break;
         default:
-                inode_type = 'I';
-                break;
-        }
+            inode_type = 'I';
+            break;
+    }
 
-        cli_out ("%-40s %14"PRIu64" %14"PRIu32" %9c",
-                 gfid, nlookup, ref, inode_type);
+    cli_out("%-40s %14" PRIu64 " %14" PRIu32 " %9c", gfid, nlookup, ref,
+            inode_type);
 
 out:
-        return;
-
+    return;
 }
+#endif
 
-void
-cli_print_volume_status_itables (dict_t *dict, char *prefix)
+static void
+cli_print_volume_status_itables(dict_t *dict, char *prefix)
 {
-        int             ret = -1;
-        char            key[1024] = {0,};
-        uint32_t        active_size = 0;
-        uint32_t        lru_size = 0;
-        uint32_t        purge_size = 0;
-        int             i =0;
-
-        GF_ASSERT (dict);
-        GF_ASSERT (prefix);
+    int ret = -1;
+    char key[1024] = {
+        0,
+    };
+    uint32_t active_size = 0;
+    uint32_t lru_size = 0;
+    uint32_t purge_size = 0;
+    uint32_t lru_limit = 0;
+#ifdef DEBUG
+    int i = 0;
+#endif
+    GF_ASSERT(dict);
+    GF_ASSERT(prefix);
+
+    snprintf(key, sizeof(key), "%s.lru_limit", prefix);
+    ret = dict_get_uint32(dict, key, &lru_limit);
+    if (ret)
+        goto out;
+    cli_out("LRU limit     : %u", lru_limit);
+
+    snprintf(key, sizeof(key), "%s.active_size", prefix);
+    ret = dict_get_uint32(dict, key, &active_size);
+    if (ret)
+        goto out;
+
+#ifdef DEBUG
+    if (active_size != 0) {
+        cli_out("Active inodes:");
+        cli_out("%-40s %14s %14s %9s", "GFID", "Lookups", "Ref", "IA type");
+        cli_out("%-40s %14s %14s %9s", "----", "-------", "---", "-------");
+    }
+    for (i = 0; i < active_size; i++) {
+        snprintf(key, sizeof(key), "%s.active%d", prefix, i);
+        cli_print_volume_status_inode_entry(dict, key);
+    }
+    cli_out(" ");
+#else
+    cli_out("Active Inodes : %u", active_size);
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.active_size", prefix);
-        ret = dict_get_uint32 (dict, key, &active_size);
-        if (ret)
-                goto out;
-        if (active_size != 0) {
-                cli_out ("Active inodes:");
-                cli_out ("%-40s %14s %14s %9s", "GFID", "Lookups", "Ref",
-                         "IA type");
-                cli_out ("%-40s %14s %14s %9s", "----", "-------", "---",
-                         "-------");
-        }
-        for (i = 0; i < active_size; i++) {
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%s.active%d", prefix, i);
-                cli_print_volume_status_inode_entry (dict, key);
-        }
-        cli_out (" ");
+#endif
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.lru_size", prefix);
-        ret = dict_get_uint32 (dict, key, &lru_size);
-        if (ret)
-                goto out;
-        if (lru_size != 0) {
-                cli_out ("LRU inodes:");
-                cli_out ("%-40s %14s %14s %9s", "GFID", "Lookups", "Ref",
-                         "IA type");
-                cli_out ("%-40s %14s %14s %9s", "----", "-------", "---",
-                         "-------");
-        }
-        for (i = 0; i < lru_size; i++) {
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%s.lru%d", prefix, i);
-                cli_print_volume_status_inode_entry (dict, key);
-        }
-        cli_out (" ");
+    snprintf(key, sizeof(key), "%s.lru_size", prefix);
+    ret = dict_get_uint32(dict, key, &lru_size);
+    if (ret)
+        goto out;
+
+#ifdef DEBUG
+    if (lru_size != 0) {
+        cli_out("LRU inodes:");
+        cli_out("%-40s %14s %14s %9s", "GFID", "Lookups", "Ref", "IA type");
+        cli_out("%-40s %14s %14s %9s", "----", "-------", "---", "-------");
+    }
+    for (i = 0; i < lru_size; i++) {
+        snprintf(key, sizeof(key), "%s.lru%d", prefix, i);
+        cli_print_volume_status_inode_entry(dict, key);
+    }
+    cli_out(" ");
+#else
+    cli_out("LRU Inodes    : %u", lru_size);
+#endif
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.purge_size", prefix);
-        ret = dict_get_uint32 (dict, key, &purge_size);
-        if (ret)
-                goto out;
-        if (purge_size != 0) {
-                cli_out ("Purged inodes:");
-                cli_out ("%-40s %14s %14s %9s", "GFID", "Lookups", "Ref",
-                         "IA type");
-                cli_out ("%-40s %14s %14s %9s", "----", "-------", "---",
-                         "-------");
-        }
-        for (i = 0; i < purge_size; i++) {
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%s.purge%d", prefix, i);
-                cli_print_volume_status_inode_entry (dict, key);
-        }
+    snprintf(key, sizeof(key), "%s.purge_size", prefix);
+    ret = dict_get_uint32(dict, key, &purge_size);
+    if (ret)
+        goto out;
+#ifdef DEBUG
+    if (purge_size != 0) {
+        cli_out("Purged inodes:");
+        cli_out("%-40s %14s %14s %9s", "GFID", "Lookups", "Ref", "IA type");
+        cli_out("%-40s %14s %14s %9s", "----", "-------", "---", "-------");
+    }
+    for (i = 0; i < purge_size; i++) {
+        snprintf(key, sizeof(key), "%s.purge%d", prefix, i);
+        cli_print_volume_status_inode_entry(dict, key);
+    }
+#else
+    cli_out("Purge Inodes  : %u", purge_size);
+#endif
 
 out:
-        return;
+    return;
 }
 
-void
-cli_print_volume_status_inode (dict_t *dict, gf_boolean_t notbrick)
+static void
+cli_print_volume_status_inode(dict_t *dict, gf_boolean_t notbrick)
 {
-        int             ret = -1;
-        char            *volname = NULL;
-        int             brick_index_max = -1;
-        int             other_count = 0;
-        int             index_max = 0;
-        char            *hostname = NULL;
-        char            *path = NULL;
-        int             online = -1;
-        int             conn_count = 0;
-        char            key[1024] = {0,};
-        int             i = 0;
-        int             j = 0;
-
-        GF_ASSERT (dict);
-
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret)
-                goto out;
-        cli_out ("Inode tables for volume %s", volname);
-
-        ret = dict_get_int32 (dict, "brick-index-max", &brick_index_max);
+    int ret = -1;
+    char *volname = NULL;
+    int brick_index_max = -1;
+    int other_count = 0;
+    int index_max = 0;
+    char *hostname = NULL;
+    char *path = NULL;
+    int online = -1;
+    int conn_count = 0;
+    char key[64] = {
+        0,
+    };
+    int i = 0;
+    int j = 0;
+
+    GF_ASSERT(dict);
+
+    ret = dict_get_str_sizen(dict, "volname", &volname);
+    if (ret)
+        goto out;
+    cli_out("Inode tables for volume %s", volname);
+
+    ret = dict_get_int32_sizen(dict, "brick-index-max", &brick_index_max);
+    if (ret)
+        goto out;
+    ret = dict_get_int32_sizen(dict, "other-count", &other_count);
+    if (ret)
+        goto out;
+
+    index_max = brick_index_max + other_count;
+
+    for (i = 0; i <= index_max; i++) {
+        cli_out("----------------------------------------------");
+
+        snprintf(key, sizeof(key), "brick%d.hostname", i);
+        ret = dict_get_str(dict, key, &hostname);
         if (ret)
-                goto out;
-        ret = dict_get_int32 (dict, "other-count", &other_count);
+            goto out;
+        snprintf(key, sizeof(key), "brick%d.path", i);
+        ret = dict_get_str(dict, key, &path);
         if (ret)
-                goto out;
-
-        index_max = brick_index_max + other_count;
-
-        for ( i = 0; i <= index_max; i++) {
-                cli_out ("----------------------------------------------");
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "brick%d.hostname", i);
-                ret = dict_get_str (dict, key, &hostname);
-                if (ret)
-                        goto out;
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "brick%d.path", i);
-                ret = dict_get_str (dict, key, &path);
-                if (ret)
-                        goto out;
-                if (notbrick)
-                        cli_out ("%s : %s", hostname, path);
-                else
-                        cli_out ("Brick : %s:%s", hostname, path);
+            goto out;
+        if (notbrick)
+            cli_out("%s : %s", hostname, path);
+        else
+            cli_out("Brick : %s:%s", hostname, path);
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "brick%d.status", i);
-                ret = dict_get_int32 (dict, key, &online);
-                if (ret)
-                        goto out;
-                if (!online) {
-                        if (notbrick)
-                                cli_out ("%s is offline", hostname);
-                        else
-                                cli_out ("Brick is offline");
-                        continue;
-                }
+        snprintf(key, sizeof(key), "brick%d.status", i);
+        ret = dict_get_int32(dict, key, &online);
+        if (ret)
+            goto out;
+        if (!online) {
+            if (notbrick)
+                cli_out("%s is offline", hostname);
+            else
+                cli_out("Brick is offline");
+            continue;
+        }
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "brick%d.conncount", i);
-                ret = dict_get_int32 (dict, key, &conn_count);
-                if (ret)
-                        goto out;
+        snprintf(key, sizeof(key), "brick%d.conncount", i);
+        ret = dict_get_int32(dict, key, &conn_count);
+        if (ret)
+            goto out;
 
-                for (j = 0; j < conn_count; j++) {
-                        if (conn_count > 1)
-                                cli_out ("Connection %d:", j+1);
+        for (j = 0; j < conn_count; j++) {
+            if (conn_count > 1)
+                cli_out("Connection %d:", j + 1);
 
-                        memset (key, 0, sizeof (key));
-                        snprintf (key, sizeof (key), "brick%d.conn%d.itable",
-                                  i, j);
-                        cli_print_volume_status_itables (dict, key);
-                        cli_out (" ");
-                }
+            snprintf(key, sizeof(key), "brick%d.conn%d.itable", i, j);
+            cli_print_volume_status_itables(dict, key);
+            cli_out(" ");
         }
+    }
 out:
-        cli_out ("----------------------------------------------");
-        return;
+    cli_out("----------------------------------------------");
+    return;
 }
 
 void
-cli_print_volume_status_fdtable (dict_t *dict, char *prefix)
-{
-        int             ret = -1;
-        char            key[1024] = {0,};
-        int             refcount = 0;
-        uint32_t        maxfds = 0;
-        int             firstfree = 0;
-        int             openfds = 0;
-        int             fd_pid = 0;
-        int             fd_refcount = 0;
-        int             fd_flags = 0;
-        int             i = 0;
-
-        GF_ASSERT (dict);
-        GF_ASSERT (prefix);
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.refcount", prefix);
-        ret = dict_get_int32 (dict, key, &refcount);
+cli_print_volume_status_fdtable(dict_t *dict, char *prefix)
+{
+    int ret = -1;
+    char key[256] = {
+        0,
+    };
+    int refcount = 0;
+    uint32_t maxfds = 0;
+    int firstfree = 0;
+    int openfds = 0;
+    int fd_pid = 0;
+    int fd_refcount = 0;
+    int fd_flags = 0;
+    int i = 0;
+
+    GF_ASSERT(dict);
+    GF_ASSERT(prefix);
+
+    snprintf(key, sizeof(key), "%s.refcount", prefix);
+    ret = dict_get_int32(dict, key, &refcount);
+    if (ret)
+        goto out;
+
+    snprintf(key, sizeof(key), "%s.maxfds", prefix);
+    ret = dict_get_uint32(dict, key, &maxfds);
+    if (ret)
+        goto out;
+
+    snprintf(key, sizeof(key), "%s.firstfree", prefix);
+    ret = dict_get_int32(dict, key, &firstfree);
+    if (ret)
+        goto out;
+
+    cli_out("RefCount = %d  MaxFDs = %d  FirstFree = %d", refcount, maxfds,
+            firstfree);
+
+    snprintf(key, sizeof(key), "%s.openfds", prefix);
+    ret = dict_get_int32(dict, key, &openfds);
+    if (ret)
+        goto out;
+    if (0 == openfds) {
+        cli_err("No open fds");
+        goto out;
+    }
+
+    cli_out("%-19s %-19s %-19s %-19s", "FD Entry", "PID", "RefCount", "Flags");
+    cli_out("%-19s %-19s %-19s %-19s", "--------", "---", "--------", "-----");
+
+    for (i = 0; i < maxfds; i++) {
+        snprintf(key, sizeof(key), "%s.fdentry%d.pid", prefix, i);
+        ret = dict_get_int32(dict, key, &fd_pid);
         if (ret)
-                goto out;
+            continue;
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.maxfds", prefix);
-        ret = dict_get_uint32 (dict, key, &maxfds);
+        snprintf(key, sizeof(key), "%s.fdentry%d.refcount", prefix, i);
+        ret = dict_get_int32(dict, key, &fd_refcount);
         if (ret)
-                goto out;
+            continue;
 
-        memset (key, 0 ,sizeof (key));
-        snprintf (key, sizeof (key), "%s.firstfree", prefix);
-        ret = dict_get_int32 (dict, key, &firstfree);
+        snprintf(key, sizeof(key), "%s.fdentry%d.flags", prefix, i);
+        ret = dict_get_int32(dict, key, &fd_flags);
         if (ret)
-                goto out;
+            continue;
+
+        cli_out("%-19d %-19d %-19d %-19d", i, fd_pid, fd_refcount, fd_flags);
+    }
 
-        cli_out ("RefCount = %d  MaxFDs = %d  FirstFree = %d",
-                 refcount, maxfds, firstfree);
+out:
+    return;
+}
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.openfds", prefix);
-        ret = dict_get_int32 (dict, key, &openfds);
+static void
+cli_print_volume_status_fd(dict_t *dict, gf_boolean_t notbrick)
+{
+    int ret = -1;
+    char *volname = NULL;
+    int brick_index_max = -1;
+    int other_count = 0;
+    int index_max = 0;
+    char *hostname = NULL;
+    char *path = NULL;
+    int online = -1;
+    int conn_count = 0;
+    char key[64] = {
+        0,
+    };
+    int i = 0;
+    int j = 0;
+
+    GF_ASSERT(dict);
+
+    ret = dict_get_str_sizen(dict, "volname", &volname);
+    if (ret)
+        goto out;
+    cli_out("FD tables for volume %s", volname);
+
+    ret = dict_get_int32_sizen(dict, "brick-index-max", &brick_index_max);
+    if (ret)
+        goto out;
+    ret = dict_get_int32_sizen(dict, "other-count", &other_count);
+    if (ret)
+        goto out;
+
+    index_max = brick_index_max + other_count;
+
+    for (i = 0; i <= index_max; i++) {
+        cli_out("----------------------------------------------");
+
+        snprintf(key, sizeof(key), "brick%d.hostname", i);
+        ret = dict_get_str(dict, key, &hostname);
         if (ret)
-                goto out;
-        if (0 == openfds) {
-                cli_err ("No open fds");
-                goto out;
-        }
+            goto out;
+        snprintf(key, sizeof(key), "brick%d.path", i);
+        ret = dict_get_str(dict, key, &path);
+        if (ret)
+            goto out;
 
-        cli_out ("%-19s %-19s %-19s %-19s", "FD Entry", "PID",
-                 "RefCount", "Flags");
-        cli_out ("%-19s %-19s %-19s %-19s", "--------", "---",
-                 "--------", "-----");
+        if (notbrick)
+            cli_out("%s : %s", hostname, path);
+        else
+            cli_out("Brick : %s:%s", hostname, path);
 
-        for (i = 0; i < maxfds ; i++) {
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%s.fdentry%d.pid", prefix, i);
-                ret = dict_get_int32 (dict, key, &fd_pid);
-                if (ret)
-                        continue;
+        snprintf(key, sizeof(key), "brick%d.status", i);
+        ret = dict_get_int32(dict, key, &online);
+        if (ret)
+            goto out;
+        if (!online) {
+            if (notbrick)
+                cli_out("%s is offline", hostname);
+            else
+                cli_out("Brick is offline");
+            continue;
+        }
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%s.fdentry%d.refcount",
-                          prefix, i);
-                ret = dict_get_int32 (dict, key, &fd_refcount);
-                if (ret)
-                        continue;
+        snprintf(key, sizeof(key), "brick%d.conncount", i);
+        ret = dict_get_int32(dict, key, &conn_count);
+        if (ret)
+            goto out;
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%s.fdentry%d.flags", prefix, i);
-                ret = dict_get_int32 (dict, key, &fd_flags);
-                if (ret)
-                        continue;
+        for (j = 0; j < conn_count; j++) {
+            cli_out("Connection %d:", j + 1);
 
-                cli_out ("%-19d %-19d %-19d %-19d", i, fd_pid, fd_refcount,
-                         fd_flags);
+            snprintf(key, sizeof(key), "brick%d.conn%d.fdtable", i, j);
+            cli_print_volume_status_fdtable(dict, key);
+            cli_out(" ");
         }
-
+    }
 out:
-        return;
+    cli_out("----------------------------------------------");
+    return;
 }
 
-void
-cli_print_volume_status_fd (dict_t *dict, gf_boolean_t notbrick)
+static void
+cli_print_volume_status_call_frame(dict_t *dict, char *prefix)
 {
-        int             ret = -1;
-        char            *volname = NULL;
-        int             brick_index_max = -1;
-        int             other_count = 0;
-        int             index_max = 0;
-        char            *hostname = NULL;
-        char            *path = NULL;
-        int             online = -1;
-        int             conn_count = 0;
-        char            key[1024] = {0,};
-        int             i = 0;
-        int             j = 0;
+    int ret = -1;
+    char key[1024] = {
+        0,
+    };
+    int ref_count = 0;
+    char *translator = 0;
+    int complete = 0;
+    char *parent = NULL;
+    char *wind_from = NULL;
+    char *wind_to = NULL;
+    char *unwind_from = NULL;
+    char *unwind_to = NULL;
+
+    if (!dict || !prefix)
+        return;
 
-        GF_ASSERT (dict);
+    snprintf(key, sizeof(key), "%s.refcount", prefix);
+    ret = dict_get_int32(dict, key, &ref_count);
+    if (ret)
+        return;
 
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret)
-                goto out;
-        cli_out ("FD tables for volume %s", volname);
+    snprintf(key, sizeof(key), "%s.translator", prefix);
+    ret = dict_get_str(dict, key, &translator);
+    if (ret)
+        return;
 
-        ret = dict_get_int32 (dict, "brick-index-max", &brick_index_max);
-        if (ret)
-                goto out;
-        ret = dict_get_int32 (dict, "other-count", &other_count);
-        if (ret)
-                goto out;
+    snprintf(key, sizeof(key), "%s.complete", prefix);
+    ret = dict_get_int32(dict, key, &complete);
+    if (ret)
+        return;
 
-        index_max = brick_index_max + other_count;
+    cli_out("  Ref Count   = %d", ref_count);
+    cli_out("  Translator  = %s", translator);
+    cli_out("  Completed   = %s", (complete ? "Yes" : "No"));
 
-        for (i = 0; i <= index_max; i++) {
-                cli_out ("----------------------------------------------");
+    snprintf(key, sizeof(key), "%s.parent", prefix);
+    ret = dict_get_str(dict, key, &parent);
+    if (!ret)
+        cli_out("  Parent      = %s", parent);
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "brick%d.hostname", i);
-                ret = dict_get_str (dict, key, &hostname);
-                if (ret)
-                        goto out;
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "brick%d.path", i);
-                ret = dict_get_str (dict, key, &path);
-                if (ret)
-                        goto out;
+    snprintf(key, sizeof(key), "%s.windfrom", prefix);
+    ret = dict_get_str(dict, key, &wind_from);
+    if (!ret)
+        cli_out("  Wind From   = %s", wind_from);
 
-                if (notbrick)
-                        cli_out ("%s : %s", hostname, path);
-                else
-                        cli_out ("Brick : %s:%s", hostname, path);
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "brick%d.status", i);
-                ret = dict_get_int32 (dict, key, &online);
-                if (ret)
-                        goto out;
-                if (!online) {
-                        if (notbrick)
-                                cli_out ("%s is offline", hostname);
-                        else
-                                cli_out ("Brick is offline");
-                        continue;
-                }
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "brick%d.conncount", i);
-                ret = dict_get_int32 (dict, key, &conn_count);
-                if (ret)
-                        goto out;
+    snprintf(key, sizeof(key), "%s.windto", prefix);
+    ret = dict_get_str(dict, key, &wind_to);
+    if (!ret)
+        cli_out("  Wind To     = %s", wind_to);
 
-                for (j = 0; j < conn_count; j++) {
-                        cli_out ("Connection %d:", j+1);
+    snprintf(key, sizeof(key), "%s.unwindfrom", prefix);
+    ret = dict_get_str(dict, key, &unwind_from);
+    if (!ret)
+        cli_out("  Unwind From = %s", unwind_from);
 
-                        memset (key, 0, sizeof (key));
-                        snprintf (key, sizeof (key), "brick%d.conn%d.fdtable",
-                                  i, j);
-                        cli_print_volume_status_fdtable (dict, key);
-                        cli_out (" ");
-                }
-        }
-out:
-        cli_out ("----------------------------------------------");
-        return;
+    snprintf(key, sizeof(key), "%s.unwindto", prefix);
+    ret = dict_get_str(dict, key, &unwind_to);
+    if (!ret)
+        cli_out("  Unwind To   = %s", unwind_to);
 }
 
-void
-cli_print_volume_status_call_frame (dict_t *dict, char *prefix)
+static void
+cli_print_volume_status_call_stack(dict_t *dict, char *prefix)
 {
-        int             ret = -1;
-        char            key[1024] = {0,};
-        int             ref_count = 0;
-        char            *translator = 0;
-        int             complete = 0;
-        char            *parent = NULL;
-        char            *wind_from = NULL;
-        char            *wind_to = NULL;
-        char            *unwind_from = NULL;
-        char            *unwind_to = NULL;
+    int ret = -1;
+    char key[256] = {
+        0,
+    };
+    int uid = 0;
+    int gid = 0;
+    int pid = 0;
+    uint64_t unique = 0;
+    // char            *op = NULL;
+    int count = 0;
+    int i = 0;
+
+    if (!prefix)
+        return;
 
-        if (!dict || !prefix)
-                return;
+    snprintf(key, sizeof(key), "%s.uid", prefix);
+    ret = dict_get_int32(dict, key, &uid);
+    if (ret)
+        return;
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.refcount", prefix);
-        ret = dict_get_int32 (dict, key, &ref_count);
-        if (ret)
-                return;
+    snprintf(key, sizeof(key), "%s.gid", prefix);
+    ret = dict_get_int32(dict, key, &gid);
+    if (ret)
+        return;
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.translator", prefix);
-        ret = dict_get_str (dict, key, &translator);
-        if (ret)
-                return;
+    snprintf(key, sizeof(key), "%s.pid", prefix);
+    ret = dict_get_int32(dict, key, &pid);
+    if (ret)
+        return;
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.complete", prefix);
-        ret = dict_get_int32 (dict, key, &complete);
-        if (ret)
-                return;
+    snprintf(key, sizeof(key), "%s.unique", prefix);
+    ret = dict_get_uint64(dict, key, &unique);
+    if (ret)
+        return;
 
-        cli_out ("  Ref Count   = %d", ref_count);
-        cli_out ("  Translator  = %s", translator);
-        cli_out ("  Completed   = %s", (complete ? "Yes" : "No"));
+    /*
+    snprintf (key, sizeof (key), "%s.op", prefix);
+    ret = dict_get_str (dict, key, &op);
+    if (ret)
+            return;
+    */
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.parent", prefix);
-        ret = dict_get_str (dict, key, &parent);
-        if (!ret)
-                cli_out ("  Parent      = %s", parent);
+    snprintf(key, sizeof(key), "%s.count", prefix);
+    ret = dict_get_int32(dict, key, &count);
+    if (ret)
+        return;
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.windfrom", prefix);
-        ret = dict_get_str (dict, key, &wind_from);
-        if (!ret)
-                cli_out ("  Wind From   = %s", wind_from);
+    cli_out(" UID    : %d", uid);
+    cli_out(" GID    : %d", gid);
+    cli_out(" PID    : %d", pid);
+    cli_out(" Unique : %" PRIu64, unique);
+    // cli_out ("\tOp     : %s", op);
+    cli_out(" Frames : %d", count);
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.windto", prefix);
-        ret = dict_get_str (dict, key, &wind_to);
-        if (!ret)
-                cli_out ("  Wind To     = %s", wind_to);
+    for (i = 0; i < count; i++) {
+        cli_out(" Frame %d", i + 1);
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.unwindfrom", prefix);
-        ret = dict_get_str (dict, key, &unwind_from);
-        if (!ret)
-                cli_out ("  Unwind From = %s", unwind_from);
+        snprintf(key, sizeof(key), "%s.frame%d", prefix, i);
+        cli_print_volume_status_call_frame(dict, key);
+    }
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.unwindto", prefix);
-        ret = dict_get_str (dict, key, &unwind_to);
-        if (!ret)
-                cli_out ("  Unwind To   = %s", unwind_to);
+    cli_out(" ");
 }
 
-void
-cli_print_volume_status_call_stack (dict_t *dict, char *prefix)
+static void
+cli_print_volume_status_callpool(dict_t *dict, gf_boolean_t notbrick)
 {
-        int             ret = -1;
-        char            key[1024] = {0,};
-        int             uid = 0;
-        int             gid = 0;
-        int             pid = 0;
-        uint64_t        unique = 0;
-        //char            *op = NULL;
-        int             count = 0;
-        int             i = 0;
-
-        if (!dict || !prefix)
-                return;
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.uid", prefix);
-        ret = dict_get_int32 (dict, key, &uid);
+    int ret = -1;
+    char *volname = NULL;
+    int brick_index_max = -1;
+    int other_count = 0;
+    int index_max = 0;
+    char *hostname = NULL;
+    char *path = NULL;
+    int online = -1;
+    int call_count = 0;
+    char key[64] = {
+        0,
+    };
+    int i = 0;
+    int j = 0;
+
+    GF_ASSERT(dict);
+
+    ret = dict_get_str_sizen(dict, "volname", &volname);
+    if (ret)
+        goto out;
+    cli_out("Pending calls for volume %s", volname);
+
+    ret = dict_get_int32_sizen(dict, "brick-index-max", &brick_index_max);
+    if (ret)
+        goto out;
+    ret = dict_get_int32_sizen(dict, "other-count", &other_count);
+    if (ret)
+        goto out;
+
+    index_max = brick_index_max + other_count;
+
+    for (i = 0; i <= index_max; i++) {
+        cli_out("----------------------------------------------");
+
+        snprintf(key, sizeof(key), "brick%d.hostname", i);
+        ret = dict_get_str(dict, key, &hostname);
         if (ret)
-                return;
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.gid", prefix);
-        ret = dict_get_int32 (dict, key, &gid);
+            goto out;
+        snprintf(key, sizeof(key), "brick%d.path", i);
+        ret = dict_get_str(dict, key, &path);
         if (ret)
-                return;
+            goto out;
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.pid", prefix);
-        ret = dict_get_int32 (dict, key, &pid);
-        if (ret)
-                return;
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.unique", prefix);
-        ret = dict_get_uint64 (dict, key, &unique);
-        if (ret)
-                return;
+        if (notbrick)
+            cli_out("%s : %s", hostname, path);
+        else
+            cli_out("Brick : %s:%s", hostname, path);
 
-        /*
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.op", prefix);
-        ret = dict_get_str (dict, key, &op);
+        snprintf(key, sizeof(key), "brick%d.status", i);
+        ret = dict_get_int32(dict, key, &online);
         if (ret)
-                return;
-        */
-
+            goto out;
+        if (!online) {
+            if (notbrick)
+                cli_out("%s is offline", hostname);
+            else
+                cli_out("Brick is offline");
+            continue;
+        }
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.count", prefix);
-        ret = dict_get_int32 (dict, key, &count);
+        snprintf(key, sizeof(key), "brick%d.callpool.count", i);
+        ret = dict_get_int32(dict, key, &call_count);
         if (ret)
-                return;
+            goto out;
+        cli_out("Pending calls: %d", call_count);
 
-        cli_out (" UID    : %d", uid);
-        cli_out (" GID    : %d", gid);
-        cli_out (" PID    : %d", pid);
-        cli_out (" Unique : %"PRIu64, unique);
-        //cli_out ("\tOp     : %s", op);
-        cli_out (" Frames : %d", count);
+        if (0 == call_count)
+            continue;
 
-        for (i = 0; i < count; i++) {
-                cli_out (" Frame %d", i+1);
+        for (j = 0; j < call_count; j++) {
+            cli_out("Call Stack%d", j + 1);
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%s.frame%d", prefix, i);
-                cli_print_volume_status_call_frame (dict, key);
+            snprintf(key, sizeof(key), "brick%d.callpool.stack%d", i, j);
+            cli_print_volume_status_call_stack(dict, key);
         }
+    }
 
-        cli_out (" ");
+out:
+    cli_out("----------------------------------------------");
+    return;
 }
 
-void
-cli_print_volume_status_callpool (dict_t *dict, gf_boolean_t notbrick)
+static void
+cli_print_volume_status_tasks(dict_t *dict)
 {
-        int             ret = -1;
-        char            *volname = NULL;
-        int             brick_index_max = -1;
-        int             other_count = 0;
-        int             index_max = 0;
-        char            *hostname = NULL;
-        char            *path = NULL;
-        int             online = -1;
-        int             call_count = 0;
-        char            key[1024] = {0,};
-        int             i = 0;
-        int             j = 0;
+    int ret = -1;
+    int i = 0;
+    int j = 0;
+    int count = 0;
+    int task_count = 0;
+    int status = 0;
+    char *op = NULL;
+    char *task_id_str = NULL;
+    char *volname = NULL;
+    char key[64] = {
+        0,
+    };
+    char task[32] = {
+        0,
+    };
+    char *brick = NULL;
+
+    ret = dict_get_int32_sizen(dict, "tasks", &task_count);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to get tasks count");
+        return;
+    }
 
-        GF_ASSERT (dict);
+    ret = dict_get_str_sizen(dict, "volname", &volname);
+    if (ret)
+        goto out;
 
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret)
-                goto out;
-        cli_out ("Pending calls for volume %s", volname);
+    cli_out("Task Status of Volume %s", volname);
+    cli_print_line(CLI_BRICK_STATUS_LINE_LEN);
 
-        ret = dict_get_int32 (dict, "brick-index-max", &brick_index_max);
-        if (ret)
-                goto out;
-        ret = dict_get_int32 (dict, "other-count", &other_count);
+    if (task_count == 0) {
+        cli_out("There are no active volume tasks");
+        cli_out(" ");
+        return;
+    }
+
+    for (i = 0; i < task_count; i++) {
+        snprintf(key, sizeof(key), "task%d.type", i);
+        ret = dict_get_str(dict, key, &op);
         if (ret)
-                goto out;
+            return;
+        cli_out("%-20s : %-20s", "Task", op);
 
-        index_max = brick_index_max + other_count;
+        snprintf(key, sizeof(key), "task%d.id", i);
+        ret = dict_get_str(dict, key, &task_id_str);
+        if (ret)
+            return;
+        cli_out("%-20s : %-20s", "ID", task_id_str);
 
-        for (i = 0; i <= index_max; i++) {
-                cli_out ("----------------------------------------------");
+        snprintf(key, sizeof(key), "task%d.status", i);
+        ret = dict_get_int32(dict, key, &status);
+        if (ret)
+            return;
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "brick%d.hostname", i);
-                ret = dict_get_str (dict, key, &hostname);
-                if (ret)
-                        goto out;
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "brick%d.path", i);
-                ret = dict_get_str (dict, key, &path);
-                if (ret)
-                        goto out;
+        snprintf(task, sizeof(task), "task%d", i);
 
-                if (notbrick)
-                        cli_out ("%s : %s", hostname, path);
-                else
-                        cli_out ("Brick : %s:%s", hostname, path);
+        if (!strcmp(op, "Remove brick")) {
+            snprintf(key, sizeof(key), "%s.count", task);
+            ret = dict_get_int32(dict, key, &count);
+            if (ret)
+                goto out;
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "brick%d.status", i);
-                ret = dict_get_int32 (dict, key, &online);
-                if (ret)
-                        goto out;
-                if (!online) {
-                        if (notbrick)
-                                cli_out ("%s is offline", hostname);
-                        else
-                                cli_out ("Brick is offline");
-                        continue;
-                }
+            cli_out("%-20s", "Removed bricks:");
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "brick%d.callpool.count", i);
-                ret = dict_get_int32 (dict, key, &call_count);
+            for (j = 1; j <= count; j++) {
+                snprintf(key, sizeof(key), "%s.brick%d", task, j);
+                ret = dict_get_str(dict, key, &brick);
                 if (ret)
-                        goto out;
-                cli_out ("Pending calls: %d", call_count);
+                    goto out;
 
-                if (0 == call_count)
-                        continue;
-
-                for (j = 0; j < call_count; j++) {
-                        cli_out ("Call Stack%d", j+1);
-
-                        memset (key, 0, sizeof (key));
-                        snprintf (key, sizeof (key),
-                                  "brick%d.callpool.stack%d", i, j);
-                        cli_print_volume_status_call_stack (dict, key);
-                }
+                cli_out("%-20s", brick);
+            }
         }
+        cli_out("%-20s : %-20s", "Status", cli_vol_task_status_str[status]);
+        cli_out(" ");
+    }
 
 out:
-        cli_out ("----------------------------------------------");
-        return;
+    return;
 }
 
-static void
-cli_print_volume_status_tasks (dict_t *dict)
-{
-        int             ret         = -1;
-        int             i           = 0;
-        int             j           = 0;
-        int             count       = 0;
-        int             task_count  = 0;
-        int             status      = 0;
-        char           *op          = NULL;
-        char           *task_id_str = NULL;
-        char           *volname     = NULL;
-        char            key[1024]   = {0,};
-        char            task[1024]  = {0,};
-        char           *brick       = NULL;
-        char           *src_brick   = NULL;
-        char           *dest_brick  = NULL;
-
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret)
-                goto out;
-
-        ret = dict_get_int32 (dict, "tasks", &task_count);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to get tasks count");
-                return;
+static int
+gf_cli_status_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                  void *myframe)
+{
+    int ret = -1;
+    int brick_index_max = -1;
+    int other_count = 0;
+    int index_max = 0;
+    int i = 0;
+    int type = -1;
+    int hot_brick_count = -1;
+    int pid = -1;
+    uint32_t cmd = 0;
+    gf_boolean_t notbrick = _gf_false;
+    char key[64] = {
+        0,
+    };
+    char *hostname = NULL;
+    char *path = NULL;
+    char *volname = NULL;
+    dict_t *dict = NULL;
+    gf_cli_rsp rsp = {
+        0,
+    };
+    cli_volume_status_t status = {0};
+    cli_local_t *local = NULL;
+    gf_boolean_t wipe_local = _gf_false;
+    char msg[1024] = {
+        0,
+    };
+
+    GF_ASSERT(myframe);
+
+    if (req->rpc_status == -1)
+        goto out;
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,
+               XDR_DECODE_FAIL);
+        goto out;
+    }
+
+    gf_log("cli", GF_LOG_DEBUG, "Received response to status cmd");
+
+    local = ((call_frame_t *)myframe)->local;
+    if (!local) {
+        local = cli_local_get();
+        if (!local) {
+            ret = -1;
+            gf_log("cli", GF_LOG_ERROR, "Failed to get local");
+            goto out;
         }
+        wipe_local = _gf_true;
+    }
 
-        cli_out ("Task Status of Volume %s", volname);
-        cli_print_line (CLI_BRICK_STATUS_LINE_LEN);
+    if (rsp.op_ret) {
+        if (strcmp(rsp.op_errstr, ""))
+            snprintf(msg, sizeof(msg), "%s", rsp.op_errstr);
+        else
+            snprintf(msg, sizeof(msg),
+                     "Unable to obtain volume status information.");
 
-        if (task_count == 0) {
-                cli_out ("There are no active volume tasks");
-                cli_out (" ");
-                return;
+        if (global_state->mode & GLUSTER_MODE_XML) {
+            if (!local->all)
+                cli_xml_output_str("volStatus", msg, rsp.op_ret, rsp.op_errno,
+                                   rsp.op_errstr);
+            ret = 0;
+            goto out;
         }
 
-        for (i = 0; i < task_count; i++) {
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "task%d.type", i);
-                ret = dict_get_str(dict, key, &op);
-                if (ret)
-                        return;
-                cli_out ("%-20s : %-20s", "Task", op);
+        cli_err("%s", msg);
+        if (local && local->all) {
+            ret = 0;
+            cli_out(" ");
+        } else
+            ret = -1;
+
+        goto out;
+    }
+
+    dict = dict_new();
+    if (!dict) {
+        gf_log(THIS->name, GF_LOG_ERROR, "Failed to create the dict");
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len, &dict);
+    if (ret)
+        goto out;
+
+    ret = dict_get_uint32(dict, "cmd", &cmd);
+    if (ret)
+        goto out;
+
+    if ((cmd & GF_CLI_STATUS_ALL)) {
+        if (local && local->dict) {
+            dict_ref(dict);
+            ret = dict_set_static_ptr(local->dict, "rsp-dict", dict);
+            ret = 0;
+        } else {
+            gf_log("cli", GF_LOG_ERROR, "local not found");
+            ret = -1;
+        }
+        goto out;
+    }
+
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        if (!local->all) {
+            ret = cli_xml_output_vol_status_begin(local, rsp.op_ret,
+                                                  rsp.op_errno, rsp.op_errstr);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+                goto xml_end;
+            }
+        }
+        if (cmd & GF_CLI_STATUS_TASKS) {
+            ret = cli_xml_output_vol_status_tasks_detail(local, dict);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR, "Error outputting to xml");
+                goto xml_end;
+            }
+        } else {
+            ret = cli_xml_output_vol_status(local, dict);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+                goto xml_end;
+            }
+        }
+
+    xml_end:
+        if (!local->all) {
+            ret = cli_xml_output_vol_status_end(local);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+            }
+        }
+        goto out;
+    }
+
+    if ((cmd & GF_CLI_STATUS_NFS) || (cmd & GF_CLI_STATUS_SHD) ||
+        (cmd & GF_CLI_STATUS_QUOTAD) || (cmd & GF_CLI_STATUS_SNAPD) ||
+        (cmd & GF_CLI_STATUS_BITD) || (cmd & GF_CLI_STATUS_SCRUB))
+        notbrick = _gf_true;
+
+    switch (cmd & GF_CLI_STATUS_MASK) {
+        case GF_CLI_STATUS_MEM:
+            cli_print_volume_status_mem(dict, notbrick);
+            goto cont;
+            break;
+        case GF_CLI_STATUS_CLIENTS:
+            cli_print_volume_status_clients(dict, notbrick);
+            goto cont;
+            break;
+        case GF_CLI_STATUS_CLIENT_LIST:
+            cli_print_volume_status_client_list(dict, notbrick);
+            goto cont;
+            break;
+        case GF_CLI_STATUS_INODE:
+            cli_print_volume_status_inode(dict, notbrick);
+            goto cont;
+            break;
+        case GF_CLI_STATUS_FD:
+            cli_print_volume_status_fd(dict, notbrick);
+            goto cont;
+            break;
+        case GF_CLI_STATUS_CALLPOOL:
+            cli_print_volume_status_callpool(dict, notbrick);
+            goto cont;
+            break;
+        case GF_CLI_STATUS_TASKS:
+            cli_print_volume_status_tasks(dict);
+            goto cont;
+            break;
+        default:
+            break;
+    }
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "task%d.id", i);
-                ret = dict_get_str (dict, key, &task_id_str);
-                if (ret)
-                        return;
-                cli_out ("%-20s : %-20s", "ID", task_id_str);
+    ret = dict_get_str_sizen(dict, "volname", &volname);
+    if (ret)
+        goto out;
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "task%d.status", i);
-                ret = dict_get_int32 (dict, key, &status);
-                if (ret)
-                        return;
-
-                snprintf (task, sizeof (task), "task%d", i);
-
-                /*
-                   Replace brick only has two states - In progress and Complete
-                   Ref: xlators/mgmt/glusterd/src/glusterd-replace-brick.c
-                */
-
-                if (!strcmp (op, "Replace brick")) {
-                        if (status)
-                                status = GF_DEFRAG_STATUS_COMPLETE;
-                        else
-                                status = GF_DEFRAG_STATUS_STARTED;
-
-                        memset (key, 0, sizeof (key));
-                        snprintf (key, sizeof (key), "%s.src-brick", task);
-                        ret = dict_get_str (dict, key, &src_brick);
-                        if (ret)
-                                goto out;
-
-                        cli_out ("%-20s : %-20s", "Source Brick", src_brick);
-
-                        memset (key, 0, sizeof (key));
-                        snprintf (key, sizeof (key), "%s.dst-brick", task);
-                        ret = dict_get_str (dict, key, &dest_brick);
-                        if (ret)
-                                goto out;
-
-                        cli_out ("%-20s : %-20s", "Destination Brick",
-                                 dest_brick);
-
-                } else if (!strcmp (op, "Remove brick")) {
-                        memset (key, 0, sizeof (key));
-                        snprintf (key, sizeof (key), "%s.count", task);
-                        ret = dict_get_int32 (dict, key, &count);
-                        if (ret)
-                                goto out;
-
-                        cli_out ("%-20s", "Removed bricks:");
-
-                        for (j = 1; j <= count; j++) {
-                                memset (key, 0, sizeof (key));
-                                snprintf (key, sizeof (key),"%s.brick%d",
-                                          task, j);
-                                ret = dict_get_str (dict, key, &brick);
-                                if (ret)
-                                        goto out;
-
-                                cli_out ("%-20s", brick);
-                        }
-                }
-                cli_out ("%-20s : %-20s", "Status",
-                         cli_vol_task_status_str[status]);
-                cli_out (" ");
-        }
+    ret = dict_get_int32_sizen(dict, "brick-index-max", &brick_index_max);
+    if (ret)
+        goto out;
 
-out:
-        return;
-}
+    ret = dict_get_int32_sizen(dict, "other-count", &other_count);
+    if (ret)
+        goto out;
 
-static int
-gf_cli_status_cbk (struct rpc_req *req, struct iovec *iov,
-                      int count, void *myframe)
-{
-        int                             ret             = -1;
-        int                             brick_index_max = -1;
-        int                             other_count     = 0;
-        int                             index_max       = 0;
-        int                             i               = 0;
-        int                             pid             = -1;
-        uint32_t                        cmd             = 0;
-        gf_boolean_t                    notbrick        = _gf_false;
-        char                            key[1024]       = {0,};
-        char                           *hostname        = NULL;
-        char                           *path            = NULL;
-        char                           *volname         = NULL;
-        dict_t                         *dict            = NULL;
-        gf_cli_rsp                      rsp             = {0,};
-        cli_volume_status_t             status          = {0};
-        cli_local_t                    *local           = NULL;
-        gf_boolean_t                    wipe_local      = _gf_false;
-        char                            msg[1024]       = {0,};
-
-        if (req->rpc_status == -1)
-                goto out;
+    index_max = brick_index_max + other_count;
 

path: root/NEWS

Gluster moves to Gerrit.