2672 files changed, 640435 insertions, 394737 deletions

diff --git a/.clang-format b/.clang-format
new file mode 100644
index 00000000000..84c2efe3fad
--- /dev/null
+++ b/.clang-format
@@ -0,0 +1,107 @@
+---
+Language:        Cpp
+# BasedOnStyle:  Chromium
+AccessModifierOffset: -1
+AlignAfterOpenBracket: Align
+AlignConsecutiveAssignments: false
+AlignConsecutiveDeclarations: false
+AlignEscapedNewlines: Right
+AlignOperands:   true
+AlignTrailingComments: true
+AllowAllParametersOfDeclarationOnNextLine: false
+AllowShortBlocksOnASingleLine: false
+AllowShortCaseLabelsOnASingleLine: false
+AllowShortFunctionsOnASingleLine: Inline
+AllowShortIfStatementsOnASingleLine: false
+AllowShortLoopsOnASingleLine: false
+AlwaysBreakAfterDefinitionReturnType: All
+AlwaysBreakAfterReturnType: All
+AlwaysBreakBeforeMultilineStrings: true
+AlwaysBreakTemplateDeclarations: true
+BinPackArguments: true
+BinPackParameters: true
+BraceWrapping:
+  AfterClass:      false
+  AfterControlStatement: false
+  AfterEnum:       false
+  AfterFunction:   true
+  AfterNamespace:  false
+  AfterObjCDeclaration: false
+  AfterStruct:     false
+  AfterUnion:      false
+  BeforeCatch:     false
+  BeforeElse:      false
+  IndentBraces:    false
+  SplitEmptyFunction: true
+  SplitEmptyRecord: true
+  SplitEmptyNamespace: true
+BreakBeforeBinaryOperators: None
+BreakBeforeBraces: Linux
+BreakBeforeInheritanceComma: false
+BreakBeforeTernaryOperators: true
+BreakConstructorInitializersBeforeComma: false
+BreakConstructorInitializers: BeforeColon
+BreakAfterJavaFieldAnnotations: false
+BreakStringLiterals: true
+ColumnLimit:     80
+CommentPragmas:  '^ IWYU pragma:'
+CompactNamespaces: false
+ConstructorInitializerAllOnOneLineOrOnePerLine: true
+ConstructorInitializerIndentWidth: 4
+ContinuationIndentWidth: 4
+Cpp11BracedListStyle: true
+DerivePointerAlignment: false
+DisableFormat:   false
+ExperimentalAutoDetectBinPacking: false
+FixNamespaceComments: true
+ForEachMacros:
+  - foreach
+  - Q_FOREACH
+  - BOOST_FOREACH
+IncludeCategories:
+  - Regex:           '^<.*\.h>'
+    Priority:        1
+  - Regex:           '^<.*'
+    Priority:        2
+  - Regex:           '.*'
+    Priority:        3
+IncludeIsMainRegex: '([-_](test|unittest))?$'
+IndentCaseLabels: true
+IndentWidth:     4
+IndentWrappedFunctionNames: false
+JavaScriptQuotes: Leave
+JavaScriptWrapImports: true
+KeepEmptyLinesAtTheStartOfBlocks: false
+MacroBlockBegin: ''
+MacroBlockEnd:   ''
+MaxEmptyLinesToKeep: 1
+NamespaceIndentation: None
+ObjCBlockIndentWidth: 2
+ObjCSpaceAfterProperty: false
+ObjCSpaceBeforeProtocolList: false
+PenaltyBreakAssignment: 200
+PenaltyBreakBeforeFirstCallParameter: 1
+PenaltyBreakComment: 300
+PenaltyBreakFirstLessLess: 120
+PenaltyBreakString: 1000
+PenaltyExcessCharacter: 1000000
+PenaltyReturnTypeOnItsOwnLine: 200
+PointerAlignment: Right
+ReflowComments:  true
+SortIncludes:    false
+SortUsingDeclarations: true
+SpaceAfterCStyleCast: false
+SpaceAfterTemplateKeyword: true
+SpaceBeforeAssignmentOperators: true
+SpaceBeforeParens: ControlStatements
+SpaceInEmptyParentheses: false
+SpacesBeforeTrailingComments: 2
+SpacesInAngles:  false
+SpacesInContainerLiterals: true
+SpacesInCStyleCastParentheses: false
+SpacesInParentheses: false
+SpacesInSquareBrackets: false
+Standard:        Auto
+TabWidth:        8
+UseTab:          Never
+...
diff --git a/.github/ISSUE_TEMPLATE b/.github/ISSUE_TEMPLATE
new file mode 100644
index 00000000000..386ed2d8dd5
--- /dev/null
+++ b/.github/ISSUE_TEMPLATE
@@ -0,0 +1,30 @@
+<!-- Please use this template while reporting an issue, providing as much information as possible. Failure to do so may result in a delayed response. Thank you! -->
+
+**Description of problem:**
+
+
+**The exact command to reproduce the issue**:
+
+
+**The full output of the command that failed**:
+<details>
+
+
+
+</details>
+
+**Expected results:**
+
+
+**Additional info:**
+
+
+**- The output of the `gluster volume info` command**:
+<details>
+
+
+
+</details>
+
+**- The operating system / glusterfs version**:
+
diff --git a/.github/PULL_REQUEST_TEMPLATE b/.github/PULL_REQUEST_TEMPLATE
new file mode 100644
index 00000000000..e69de29bb2d
--- /dev/null
+++ b/.github/PULL_REQUEST_TEMPLATE
diff --git a/.github/RELEASE_TRACKER_TEMPLATE b/.github/RELEASE_TRACKER_TEMPLATE
new file mode 100644
index 00000000000..502bbd5556c
--- /dev/null
+++ b/.github/RELEASE_TRACKER_TEMPLATE
@@ -0,0 +1,12 @@
+<!-- Please use this template while creating a tracker issue -->
+
+**Description of problem:**
+A tracker issue to track the issues that will be fixed as a part of this release
+
+
+**Major or minor release**:
+
+
+**Release version**:
+
+
diff --git a/.github/stale.yml b/.github/stale.yml
new file mode 100644
index 00000000000..460e327c6ea
--- /dev/null
+++ b/.github/stale.yml
@@ -0,0 +1,25 @@
+# Number of days of inactivity before an issue becomes stale
+daysUntilStale: 210
+# Number of days of inactivity before a stale issue is closed
+daysUntilClose: 15
+# Issues with these labels will never be considered stale
+exemptLabels:
+  - pinned
+  - security
+# Label to use when marking an issue as stale
+staleLabel: wontfix
+
+# Comment to post when marking an issue as stale. Set to `false` to disable
+markComment: >
+  Thank you for your contributions.
+
+  Noticed that this issue is not having any activity in last ~6 months! We
+  are marking this issue as stale because it has not had recent activity.
+
+  It will be closed in 2 weeks if no one responds with a comment here.
+  
+
+# Comment to post when closing a stale issue. Set to `false` to disable
+closeComment: >
+  Closing this issue as there was no update since my last update on issue.
+  If this is an issue which is still valid, feel free to open it.
diff --git a/.gitignore b/.gitignore
index 15896bfc4ea..fc5ba586f8e 100644
--- a/.gitignore
+++ b/.gitignore
@@ -3,12 +3,21 @@ autom4te.cache
 build
 config.*
 configure
+cscope.*
+tags
 depcomp
+INSTALL
 install-sh
 ltmain.sh
+Makefile
 Makefile.in
 missing
-py-compile
+stamp-h1
+stamp-h2
+test-driver
+*compile
+*.gcda
+*.gcno
 *.sw?
 *~
 *.lo
@@ -16,36 +25,101 @@ py-compile
 *.o
 *.tar.gz
 *.rpm
+*.diff
+*.patch
 .libs
 .deps
-Makefile
-stamp-h1
+.dirstamp
+
+# Softlinks to test and log
+log
+*.vol
+.clang-format
+
+# cmocka unit tests
+*.log
+*.trs
+*_unittest
 
 # Generated files
+site.h
+*.py[co]
 api/examples/__init__.py
-api/examples/__init__.py?
 api/examples/setup.py
-argp-standalone/libargp.a
-contrib/uuid/uuid_types.h
+api/src/gfapi.map
+cli/src/gluster
+contrib/fuse-util/fusermount-glusterfs
+extras/geo-rep/gsync-sync-gfid
+extras/geo-rep/schedule_georep.py
+extras/snap_scheduler/conf.py
 extras/init.d/glusterd-Debian
+extras/init.d/glusterd-FreeBSD
 extras/init.d/glusterd-Redhat
 extras/init.d/glusterd-SuSE
 extras/init.d/glusterd.plist
 extras/ocf/glusterd
 extras/ocf/volume
+extras/run-gluster.tmpfiles
+extras/systemd/glusterd.service
+extras/systemd/gluster-ta-volume.service
+extras/systemd/glusterfssharedstorage.service
 extras/who-wrote-glusterfs/gitdm
+geo-replication/.tox
+geo-replication/gsyncd.conf
+geo-replication/src/gsyncd
+geo-replication/src/peer_gsec_create
+geo-replication/src/peer_mountbroker
+geo-replication/src/peer_mountbroker.py
+geo-replication/src/set_geo_rep_pem_keys.sh
+geo-replication/src/peer_georep-sshkey.py
+geo-replication/syncdaemon.egg-info
+geo-replication/syncdaemon/conf.py
+geo-replication/tests/unit/.coverage
+geo-replication/tests/unit/cover
+geo-replication/tests/unit/coverage.xml
+geo-replication/tests/unit/nosetests.xml
+geo-replication/tests/unit/results.html
 glusterfs-api.pc
 glusterfs.spec
+glusterfsd/src/glusterd
+glusterfsd/src/glusterfs
 glusterfsd/src/glusterfsd
+glusterfsd/src/gf_attach
+heal/src/glfsheal
 libgfchangelog.pc
-libglusterfs/src/spec.lex.c
+libglusterfs/src/graph.lex.c
 libglusterfs/src/y.tab.c
 libglusterfs/src/y.tab.h
+libglusterfs/src/defaults.c
+libglusterfs/src/cli1-xdr.h
+libglusterfs/src/protocol-common.h
 libtool
+# copied XDR for cyclic libglusterfs <-> rpc-header dependency
 run-tests.sh
-ufo/.tox
-ufo/test/unit/.coverage
+!tests/basic/fuse/Makefile
+!tests/basic/gfapi/Makefile
+tests/env.rc
+tests/utils/arequal-checksum
+xlators/features/glupy/src/__init__.py
+xlators/features/glupy/src/setup.py
 xlators/mount/fuse/utils/mount.glusterfs
 xlators/mount/fuse/utils/mount_glusterfs
-xlators/features/glupy/src/setup.py
-api/src/gfapi.map
+extras/peer_add_secret_pub
+tools/gfind_missing_files/gcrawler
+tools/glusterfind/glusterfind
+tools/glusterfind/src/tool.conf
+# Eventing
+events/src/eventsapiconf.py
+extras/systemd/glustereventsd.service
+events/src/eventtypes.py
+libglusterfs/src/eventtypes.h
+extras/init.d/glustereventsd-Debian
+extras/init.d/glustereventsd-FreeBSD
+extras/init.d/glustereventsd-Redhat
+tools/setgfid2path/src/gluster-setgfid2path
+xlators/features/cloudsync/src/cloudsync-autogen-fops.c
+xlators/features/cloudsync/src/cloudsync-autogen-fops.h
+xlators/features/utime/src/utime-autogen-fops.c
+xlators/features/utime/src/utime-autogen-fops.h
+tests/basic/metadisp/ftruncate
+xlators/features/metadisp/src/fops.c
diff --git a/.mailmap b/.mailmap
index 6bcd95dea0f..141a1667ffa 100644
--- a/.mailmap
+++ b/.mailmap
@@ -1,5 +1,7 @@
 # .mailmap, see 'git short-log --help' for details
 #
+# This file needs to match extras/who-wrote-glusterfs/gitdm.aliases.
+#
 # Listing of contributors that filed patches with different email addresses.
 # Format: <name> <main-email> <alias> [<alias> ...]
 #
@@ -8,24 +10,34 @@ Amar Tumballi <amarts@redhat.com> <amar@gluster.com> <amar@del.gluster.com>
 Anand Avati <avati@redhat.com> <avati@gluster.com> <avati@dev.gluster.com> <avati@amp.gluster.com> <avati@blackhole.gluster.com>
 Anush Shetty <ashetty@redhat.com> <anush@gluster.com>
 Csaba Henk <csaba@redhat.com> <csaba@gluster.com> <csaba@lowlife.hu> <csaba@zresearch.com>
+Günther Deschner <gd@redhat.com> <gd@samba.org>
 Harshavardhana <fharshav@redhat.com> <harsha@gluster.com> <harsha@zresearch.com> <harsha@dev.gluster.com> <harsha@harshavardhana.net>
-Kaleb S. KEITHLEY <kkeithle@redhat.com> <kkeithle@f16node1.kkeithle.usersys.redhat.com>
+Ji-Hyeon Gim <potatogim@gluesys.com> <potatogim@potatogim.net>
+Justin Clift <justin@gluster.org> <jclift@redhat.com>
+Kaleb S. KEITHLEY <kkeithle@redhat.com> <kkeithle@f16node1.kkeithle.usersys.redhat.com> <kkeithle@linux.keithley.org>
 Kaushal M <kaushal@redhat.com> <kaushal@gluster.com>
 Kaushik BV <kbudiger@redhat.com> <kaushikbv@gluster.com>
 Krishna Srinivas <ksriniva@redhat.com> <krishna@gluster.com> <krishna@zresearch.com> <krishna@guest-laptop>
 Krishnan Parthasarathi <kparthas@redhat.com> <kp@gluster.com>
 Louis Zuckerman <louiszuckerman@gmail.com> <me@louiszuckerman.com>
 M S Vishwanath Bhat <vbhat@redhat.com> <msvbhat@gmail.com> <vishwanath@gluster.com>
+Michael Adam <madam@redhat.com> <obnox@samba.org>
+Oleksandr Natalenko <oleksandr@natalenko.name> <o.natalenko@lanet.ua>
+Patrick Uiterwijk <puiterwijk@fedoraproject.org> <patrick@puiterwijk.org>
 Pavan Sondur <pavan@gluster.com> <pavan@dev.gluster.com>
 Pete Zaitcev <zaitcev@kotori.zaitcev.us> <zaitcev@yahoo.com>
 Pranith Kumar K <pkarampu@redhat.com> <pranithk@gluster.com>
+Prashanth Pai <ppai@redhat.com> <nullpai@gmail.com>
 Raghavendra Bhat <raghavendra@redhat.com> <raghavendrabhat@gluster.com>
 Raghavendra G <rgowdapp@redhat.com> <raghavendra@gluster.com> <raghavendra@zresearch.com>
 Rahul C S <rahulcs@redhat.com> <rahulcssjce@gmail.com>
 Rajesh Amaravathi <rajesh@redhat.com> <rajesh@gluster.com> <rajesh.amaravathi@gmail.com>
+Ravishankar N <ravishankar@redhat.com> <root@ravi2.(none)>
+Sakshi Bansal <sabansal@redhat.com> <sabansal@localhost.localdomain>
 Shehjar Tikoo <shehjart@gluster.com> <shehjart@zresearch.com>
 Venky Shankar <vshankar@redhat.com> <venky@gluster.com>
 Vijay Bellur <vbellur@redhat.com> <vijay@gluster.com> <vijay@dev.gluster.com>
 Vijaykumar Koppad <vkoppad@redhat.com> <vijaykumar.koppad@gmail.com>
+Vijaikumar Mallikarjuna <vmallika@redhat.com>
 Vikas Gorur <vikas@gluster.com> <vikas@zresearch.com>
 shishir gowda <sgowda@redhat.com> <shishirng@gluster.com>
diff --git a/.testignore b/.testignore
new file mode 100644
index 00000000000..fe8f838bf2b
--- /dev/null
+++ b/.testignore
@@ -0,0 +1,64 @@
+.github/ISSUE_TEMPLATE
+.github/PULL_REQUEST_TEMPLATE
+.github/stale.yml
+.gitignore
+.mailmap
+.testignore
+.clang-format
+rfc.sh
+submit-for-review.sh
+AUTHORS
+CONTRIBUTING.md
+COPYING-GPLV2
+COPYING-LGPLV3
+ChangeLog
+INSTALL
+MAINTAINERS
+NEWS
+README.md
+THANKS
+COMMITMENT
+api/examples/README
+api/examples/getvolfile.py
+api/src/README.Symbol_Versions
+build-aux/checkpatch.pl
+contrib/fuse-lib/COPYING.LIB
+contrib/fuse-util/COPYING
+contrib/macfuse/COPYING.txt
+doc/*
+extras/FreeBSD/README.FreeBSD
+extras/Solaris/README.solaris
+extras/Ubuntu/README.Ubuntu
+extras/benchmarking/README
+extras/cliutils/README.md
+extras/command-completion/README
+extras/create_new_xlator/README.md
+extras/glusterfs.vim
+extras/glusterfs-logrotate
+extras/glusterfs-georep-logrotate
+extras/init.d/glusterd-Debian.in
+extras/init.d/glusterd-FreeBSD.in
+extras/init.d/glusterd-Redhat.in
+extras/init.d/glusterd-SuSE.in
+extras/init.d/glusterd.plist.in
+extras/init.d/glustereventsd-Debian.in
+extras/init.d/glustereventsd-FreeBSD.in
+extras/init.d/glustereventsd-Redhat.in
+extras/init.d/rhel5-load-fuse.modules
+extras/logger.conf.example
+extras/snap_scheduler/README.md
+extras/test/ld-preload-test/README
+extras/who-wrote-glusterfs/*
+extras/distributed-testing/*
+geo-replication/syncdaemon/README.md
+geo-replication/test-requirements.txt
+rpc/xdr/src/.gitignore
+tests/README.md
+xlators/experimental/README.md
+xlators/experimental/dht2/README.md
+xlators/experimental/dht2/TODO.md
+xlators/experimental/posix2/README.md
+xlators/experimental/posix2/TODO.md
+xlators/features/glupy/doc/README.md
+xlators/features/glupy/doc/TESTING
+xlators/features/glupy/doc/test.vol
diff --git a/COMMITMENT b/COMMITMENT
new file mode 100644
index 00000000000..16b75efcf29
--- /dev/null
+++ b/COMMITMENT
@@ -0,0 +1,46 @@
+Common Cure Rights Commitment
+Version 1.0
+
+Before filing or continuing to prosecute any legal proceeding or claim
+(other than a Defensive Action) arising from termination of a Covered
+License, we commit to extend to the person or entity ('you') accused
+of violating the Covered License the following provisions regarding
+cure and reinstatement, taken from GPL version 3. As used here, the
+term 'this License' refers to the specific Covered License being
+enforced.
+
+    However, if you cease all violation of this License, then your
+    license from a particular copyright holder is reinstated (a)
+    provisionally, unless and until the copyright holder explicitly
+    and finally terminates your license, and (b) permanently, if the
+    copyright holder fails to notify you of the violation by some
+    reasonable means prior to 60 days after the cessation.
+
+    Moreover, your license from a particular copyright holder is
+    reinstated permanently if the copyright holder notifies you of the
+    violation by some reasonable means, this is the first time you
+    have received notice of violation of this License (for any work)
+    from that copyright holder, and you cure the violation prior to 30
+    days after your receipt of the notice.
+
+We intend this Commitment to be irrevocable, and binding and
+enforceable against us and assignees of or successors to our
+copyrights.
+
+Definitions
+
+'Covered License' means the GNU General Public License, version 2
+(GPLv2), the GNU Lesser General Public License, version 2.1
+(LGPLv2.1), or the GNU Library General Public License, version 2
+(LGPLv2), all as published by the Free Software Foundation.
+
+'Defensive Action' means a legal proceeding or claim that We bring
+against you in response to a prior proceeding or claim initiated by
+you or your affiliate.
+
+'We' means each contributor to this repository as of the date of
+inclusion of this file, including subsidiaries of a corporate
+contributor.
+
+This work is available under a Creative Commons Attribution-ShareAlike
+4.0 International license (https://creativecommons.org/licenses/by-sa/4.0/).
diff --git a/CONTRIBUTING b/CONTRIBUTING
deleted file mode 100644
index 7bccd88d7e5..00000000000
--- a/CONTRIBUTING
+++ /dev/null
@@ -1,25 +0,0 @@
-        Developer's Certificate of Origin 1.1
-
-        By making a contribution to this project, I certify that:
-
-        (a) The contribution was created in whole or in part by me and I
-            have the right to submit it under the open source license
-            indicated in the file; or
-
-        (b) The contribution is based upon previous work that, to the best
-            of my knowledge, is covered under an appropriate open source
-            license and I have the right under that license to submit that
-            work with modifications, whether created in whole or in part
-            by me, under the same open source license (unless I am
-            permitted to submit under a different license), as indicated
-            in the file; or
-
-        (c) The contribution was provided directly to me by some other
-            person who certified (a), (b) or (c) and I have not modified
-            it.
-
-        (d) I understand and agree that this project and the contribution
-            are public and that a record of the contribution (including all
-            personal information I submit with it, including my sign-off) is
-            maintained indefinitely and may be redistributed consistent with
-            this project or the open source license(s) involved.
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
new file mode 100644
index 00000000000..65fc3497104
--- /dev/null
+++ b/CONTRIBUTING.md
@@ -0,0 +1,114 @@
+# GlusterFS project Contribution guidelines
+
+## Development Workflow
+
+We follow most of the details as per the [document here](https://help.github.com/en/github/collaborating-with-issues-and-pull-requests). If you are not aware of the github workflow, it is recommended to go through them before continuing here.
+
+
+#### Get the Repository setup
+
+0. Fork Repository
+   - Fork [GlusterFS repository](https://github.com/gluster/glusterfs/fork).
+
+1. Clone Repository
+   - Clone the glusterfs repo freshly from github using below steps.
+
+```
+   git clone git@github.com:${username}/glusterfs.git
+   cd glusterfs/
+   git remote add upstream git@github.com:gluster/glusterfs.git
+```
+
+About two tasks are one time for the life time. You can continue to use the same repository for all the work in future.
+
+#### Development & Other flows
+
+0. Issue:
+   - Make sure there is an issue filed for the task you are working on.
+   - If it is not filed, open the issue with all the description.
+   - If it is a bug fix, add label "Type:Bug".
+   - If it is an RFC, provide all the documentation, and request for "DocApproved", and "SpecApproved" label.
+
+1. Code:
+   - Start coding
+   - Build and test locally
+   - Make sure clang-format is installed and is run on the patch.
+
+2. Keep up-to-date
+   - GlusterFS is a large project with many developers, so there would be one or the other patch everyday.
+   - It is critical for developer to be up-to-date with `devel` repo to be Conflict-Free when PR is opened.
+   - Git provides many options to keep up-to-date, below is one of them
+```
+   git fetch upstream
+   git rebase upstream/devel
+```
+   - It is recommended you keep pushing to your repo every day, so you don't loose any work.
+     - It can be done by `./rfc.sh` (or `git push origin HEAD:issueNNN`)
+
+2. Commit Message / PR description:
+   - The name of the branch on your personal fork can start with issueNNNN, followed by anything of your choice.
+   - PRs continue to have the title of format "component: \<title\>", like it is practiced now.
+   - When you open a PR, having a reference Issue for the commit is mandatory in GlusterFS.
+   - Commit message can have, either `Fixes: #NNNN` or `Updates: #NNNN` in a separate line in the commit message.
+     - Here, NNNN is the Issue ID in glusterfs repository.
+   - Each commit needs the author to have the "Signed-off-by: Name \<email\>" line.
+     - Can do this by `-s` option for `git commit`.
+   - If the PR is not ready for review, apply the label `work-in-progress`.
+     - Check the availability of "Draft PR" is present for you, if yes, use that instead.
+
+3. Tests:
+   - All the required smoke tests would be auto-triggered.
+     - Developers get a chance to retrigger the smoke tests using **"/recheck smoke"** as comment.
+   - The "regression" tests would be triggered by a comment **"/run regression"** from developers in the [@gluster-maintainers](https://github.com/orgs/gluster/teams/gluster-maintainers) group.
+     - Ask for help as comment in PR if you have any questions about the process!
+
+4. Review Process:
+   - `+2` : is equivalent to "Approve" from the people in the maintainer's group.
+   - `+1` : can be given by a maintainer/reviewer by explicitly stating that in the comment.
+   - `-1` : provide details on required changes and pick "Request Changes" while submitting your review.
+   - `-2` : done by adding the `DO-NOT-MERGE` label.
+
+   - Any further discussions can happen as comments in the PR.
+
+5. Making changes:
+   - There are 2 approaches to submit changes done after addressing review comments.
+     - Commit changes as a new commit on top of the original commits in the branch, and push the changes to same branch (issueNNNN)
+     - Commit changes into the same commit with `--amend` option, and do a push to the same branch with `--force` option.
+
+6. Merging:
+   - GlusterFS project follows 'Squash and Merge' method
+     - This is mainly to preserve the historic Gerrit method of one patch in `git log` for one URL link.
+     - This also makes every merge a complete patch, which has passed all tests.
+   - The merging of the patch is expected to be done by the maintainers.
+     - It can be done when all the tests (smoke and regression) pass.
+     - When the PR has 'Approved' flag from corresponding maintainer.
+   - If you feel there is delay, feel free to add a comment, discuss the same in Slack channel, or send email.
+
+## By contributing to this project, the contributor would need to agree to below.
+
+### Developer's Certificate of Origin 1.1
+
+By making a contribution to this project, I certify that:
+
+(a) The contribution was created in whole or in part by me and I
+    have the right to submit it under the open source license
+    indicated in the file; or
+
+(b) The contribution is based upon previous work that, to the best
+    of my knowledge, is covered under an appropriate open source
+    license and I have the right under that license to submit that
+    work with modifications, whether created in whole or in part
+    by me, under the same open source license (unless I am
+    permitted to submit under a different license), as indicated
+    in the file; or
+
+(c) The contribution was provided directly to me by some other
+    person who certified (a), (b) or (c) and I have not modified
+    it.
+
+(d) I understand and agree that this project and the contribution
+    are public and that a record of the contribution (including all
+    personal information I submit with it, including my sign-off) is
+    maintained indefinitely and may be redistributed consistent with
+    this project or the open source license(s) involved.
+
diff --git a/INSTALL b/INSTALL
index f70ef15b03f..a56390e54fb 100644
--- a/INSTALL
+++ b/INSTALL
@@ -3,18 +3,29 @@ Installation Instructions
 
 0. If you have cloned from git, run ./autogen.sh.
 
-1. Run ./configure after untaring the package.
+1. Run ./configure.
 
  bash# ./configure
  GlusterFS configure summary
  ===========================
- FUSE client        : yes
- Infiniband verbs   : yes
- epoll IO multiplex : yes
- Berkeley-DB        : yes
- libglusterfsclient : yes
- mod_glusterfs      : yes
- argp-standalone    : no
+ GlusterFS configure summary
+  ===========================
+   FUSE client          : yes
+   Infiniband verbs     : yes
+   epoll IO multiplex   : yes
+   argp-standalone      : no
+   fusermount           : yes
+   readline             : yes
+   georeplication       : yes
+   Linux-AIO            : yes
+   Enable Debug         : no
+   Block Device xlator  : yes
+   glupy                : yes
+   Use syslog           : yes
+   XML output           : yes
+   QEMU Block formats   : yes
+   Encryption xlator    : yes
+
  
 The configure summary will tell you what all components will be built with 
 GlusterFS. Other than 'argp-standalone' if something else says 'no', that 
@@ -26,9 +37,11 @@ be used if the system doesn't have a proper argp package installed.
  bash# make 
  bash# make install
 
-Installation complete :-)
+Installation completed :-)
 
  bash# glusterfs --version
 
 Make sure your version is the latest from the release, and the one you 
 just installed :-)
+
+For more information on GlusterFS installation refer# http://docs.gluster.org/en/latest/Developer-guide/Building-GlusterFS/
diff --git a/MAINTAINERS b/MAINTAINERS
new file mode 100644
index 00000000000..953e8755fd9
--- /dev/null
+++ b/MAINTAINERS
@@ -0,0 +1,510 @@
+GlusterFS Maintainers
+=====================
+
+The intention of this file is not to establish who owns what portions of the
+code base, but to provide a set of names that developers can consult when they
+have a question about a particular subset and also to provide a set of names
+to be CC'd when submitting a patch to obtain appropriate review.
+
+In general, if you have a question about inclusion of a patch, you should
+consult gluster-devel@gluster.org and not any specific individual privately.
+
+Descriptions of section entries:
+
+	M: Main contact that knows and takes care of this area
+	L: Mailing list that is relevant to this area
+	W: Web-page with status/info
+	Q: Patchwork web based patch tracking system site
+	T: SCM tree type and location.  Type is one of: git, hg, quilt, stgit.
+	S: Status, one of the following:
+	   Supported:	Someone is actually paid to look after this.
+	   Maintained:	Someone actually looks after it.
+	   Odd Fixes:	It has a maintainer but they don't have time to do
+			much other than throw the odd patch in. See below.
+	   Orphan:	No current maintainer [but maybe you could take the
+			role as you write your new code].
+	   Obsolete:	Old code. Something tagged obsolete generally means
+			it has been replaced by a better system and you
+			should be using that.
+	F: Files and directories with wildcard patterns.
+	   A trailing slash includes all files and subdirectory files.
+	   F:	drivers/net/	all files in and below drivers/net
+	   F:	drivers/net/*	all files in drivers/net, but not below
+	   F:	*/net/*		all files in "any top level directory"/net
+	   One pattern per line.  Multiple F: lines acceptable.
+	X: Files and directories that are NOT maintained, same rules as F:
+	   Files exclusions are tested before file matches.
+	   Can be useful for excluding a specific subdirectory, for instance:
+	   F:	net/
+	   X:	net/ipv6/
+	   matches all files in and below net excluding net/ipv6/
+	K: Keyword perl extended regex pattern to match content in a
+	   patch or file.  For instance:
+	   K: of_get_profile
+	      matches patches or files that contain "of_get_profile"
+	   K: \b(printk|pr_(info|err))\b
+	      matches patches or files that contain one or more of the words
+	      printk, pr_info or pr_err
+	   One regex pattern per line.  Multiple K: lines acceptable.
+        P: Peer for a component
+
+
+General Project Architects
+--------------------------
+M: Amar Tumballi <amarts@gmail.com>
+M: Xavier Hernandez  <xhernandez@redhat.com>
+P: Pranith Karampuri <pranith.karampuri@phonepe.com>
+P: Atin Mukherjee <amukherj@redhat.com>
+
+xlators:
+--------
+Access Control List (ACL)
+M: Raghavendra Talur <rtalur@redhat.com>
+P: Jiffin Tony Thottan <jthottan@redhat.com>
+S: Maintained
+F: xlators/system/posix-acl/
+
+Arbiter
+M: Ravishankar N <ravishankar@redhat.com>
+P: Pranith Karampuri <pranith.karampuri@phonepe.com>
+S: Maintained
+F: xlators/features/arbiter/
+
+Automatic File Replication (AFR)
+M: Pranith Karampuri <pranith.karampuri@phonepe.com>
+M: Ravishankar N <ravishankar@redhat.com>
+P: Karthik US <ksubrahm@redhat.com>
+S: Maintained
+F: xlators/cluster/afr/
+
+Barrier
+M: Raghavendra Bhat <rabhat@redhat.com>
+P: Atin Mukherjee <amukherj@redhat.com>
+S: Maintained
+F: xlators/features/barrier
+
+BitRot
+M: Kotresh HR <khiremat@redhat.com>
+P: Raghavendra Bhat <rabhat@redhat.com>
+S: Maintained
+F: xlators/features/bit-rot/
+
+Changelog
+M: Aravinda V K <avishwan@redhat.com>
+P: Kotresh HR <khiremat@redhat.com>
+S: Maintained
+F: xlators/features/changelog/
+
+Distributed Hashing Table (DHT)
+P: Susant Palai <spalai@redhat.com>
+S: Maintained
+F: xlators/cluster/dht/
+
+Erasure Coding
+M: Pranith Karampuri <pranith.karampuri@phonepe.com>
+M: Xavier Hernandez  <xhernandez@redhat.com>
+P: Ashish Pandey <aspandey@redhat.com>
+S: Maintained
+F: xlators/cluster/ec/
+
+Error-gen
+M: Raghavendra Talur <rtalur@redhat.com>
+S: Maintained
+F: xlators/debug/error-gen/
+
+FUSE Bridge
+M: Csaba Henk <chenk@redhat.com>
+P: Niels de Vos <ndevos@redhat.com>
+S: Maintained
+F: xlators/mount/
+
+Index
+M: Pranith Karampuri <pranith.karampuri@phonepe.com>
+P: Ravishankar N <ravishankar@redhat.com>
+S: Maintained
+F: xlators/features/index/
+
+IO Cache
+P: Mohammed Rafi KC <rafi.kavungal@iternity.com>
+S: Maintained
+F: xlators/performance/io-cache/
+
+IO Statistics
+M: Krutika Dhananjay <kdhananj@redhat.com>
+M: Shyam Ranganathan <srangana@redhat.com>
+S: Maintained
+F: xlators/debug/io-stats/
+
+IO threads
+M: Pranith Karampuri <pranith.karampuri@phonepe.com>
+P: Ravishankar N <ravishankar@redhat.com>
+S: Maintained
+F: xlators/performance/io-threads/
+
+Leases
+M: Poornima G <pgurusid@redhat.com>
+P: Niels de Vos <ndevos@redhat.com>
+P: Soumya Koduri <skoduri@redhat.com>
+S: Maintained
+F: xlators/features/leases/
+
+Locks
+M: Krutika Dhananjay <kdhananj@redhat.com>
+P: Xavier Hernandez  <xhernandez@redhat.com>
+S: Maintained
+F: xlators/features/locks/
+
+Marker
+M: Kotresh HR <khiremat@redhat.com>
+S: Maintained
+F: xlators/features/marker/
+
+Meta
+M: Mohammed Rafi KC <rafi.kavungal@iternity.com>
+S: Maintained
+F: xlators/features/meta/
+
+Metadata-cache
+M: Poornima G <pgurusid@redhat.com>
+P: Soumya Koduri <skoduri@redhat.com>
+S: Maintained
+F: xlators/performance/md-cache/
+
+Negative-lookup Cache
+M: Poornima G <pgurusid@redhat.com>
+P: Pranith Karampuri <pranith.karampuri@phonepe.com>
+S: Maintained
+F: xlators/performance/nl-cache/
+
+gNFS
+M: Jiffin Tony Thottan <jthottan@redhat.com>
+P: Xie Changlong <xiechanglong@cmss.chinamobile.com>
+P: Amar Tumballi <amarts@gmail.com>
+S: Odd Fixes
+F: xlators/nfs/server/
+
+Open-behind
+S: Maintained
+F: xlators/performance/open-behind/
+
+Posix:
+M: Raghavendra Bhat <raghavendra@redhat.com>
+P: Kotresh HR <khiremat@redhat.com>
+P: Krutika Dhananjay <kdhananj@redhat.com>
+S: Maintained
+F: xlators/storage/posix/
+
+Quick-read
+S: Maintained
+F: xlators/performance/quick-read/
+
+Quota
+M: Shyamsundar Ranganathan <srangana@redhat.com>
+P: Hari Gowtham <hgowtham@redhat.com>
+S: Maintained
+F: xlators/features/quota/
+
+Read-ahead
+P: Csaba Henk <chenk@redhat.com>
+S: Maintained
+F: xlators/performance/read-ahead/
+
+Readdir-ahead
+S: Maintained
+F: xlators/performance/readdir-ahead/
+
+Sharding
+M: Krutika Dhananjay <kdhananj@redhat.com>
+P: Xavier Hernandez  <xhernandez@redhat.com>
+S: Maintained
+F: xlators/features/shard/
+
+Trash
+M: Anoop C S <anoopcs@redhat.com>
+M: Jiffin Tony Thottan <jthottan@redhat.com>
+S: Maintained
+F: xlators/features/trash/
+
+Upcall
+M: Poornima G <pgurusid@redhat.com>
+M: Soumya Koduri <skoduri@redhat.com>
+P: Niels de Vos <ndevos@redhat.com>
+S: Maintained
+F: xlators/features/upcall/
+
+Write-behind
+P: Csaba Henk <chenk@redhat.com>
+S: Maintained
+F: xlators/performance/write-behind/
+
+Write Once Read Many
+P: Karthik US <ksubrahm@redhat.com>
+S: Maintained
+F: xlators/features/read-only/
+
+Cloudsync
+M: Susant Kumar Palai <spalai@redhat.com>
+S: Maintained
+F: xlators/features/cloudsync/
+
+Other bits of code:
+-------------------
+
+Doc
+M: Humble Chirammal <hchiramm@redhat.com>
+M: Raghavendra Talur <rtalur@redhat.com>
+S: Maintained
+F: doc/
+
+Geo Replication
+M: Aravinda V K <avishwan@redhat.com>
+M: Kotresh HR <khiremat@redhat.com>
+M: Sunny Kumar <sunkumar@redhat.com>
+S: Maintained
+F: geo-replication/
+
+Glusterfind
+M: Aravinda VK <avishwan@redhat.com>
+S: Maintained
+F: tools/glusterfind/
+
+libgfapi
+M: Niels de Vos <ndevos@redhat.com>
+P: Poornima G <pgurusid@redhat.com>
+P: Shyamsundar Ranganathan <srangana@redhat.com>
+P: Soumya Koduri <skoduri@redhat.com>
+S: Maintained
+F: api/
+
+libglusterfs
+M: Amar Tumballi <amarts@gmail.com>
+M: Xavier Hernandez  <xhernandez@redhat.com>
+M: Jeff Darcy <jeff@pl.atyp.us>
+P: Kaleb Keithley <kkeithle@redhat.com>
+P: Niels de Vos <ndevos@redhat.com>
+P: Pranith Karampuri <pranith.karampuri@phonepe.com>
+P: Shyamsundar Ranganathan <srangana@redhat.com>
+S: Maintained
+F: libglusterfs/
+
+xxhash
+M: Aravinda VK <avishwan@redhat.com>
+M: Kotresh HR <khiremat@redhat.com>
+P: Yaniv Kaul <ykaul@redhat.com>
+S: Maintained
+F: contrib/xxhash/
+T: https://github.com/Cyan4973/xxHash.git
+
+Management Daemon - glusterd
+M: Atin Mukherjee <amukherj@redhat.com>
+M: Mohit Agrawal <moagrawa@redhat.com>
+M: Sanju Rakonde <srakonde@redhat.com>
+S: Maintained
+F: cli/
+F: xlators/mgmt/glusterd/
+
+Protocol
+M: Niels de Vos <ndevos@redhat.com>
+P: Mohammed Rafi KC <rafi.kavungal@iternity.com>
+S: Maintained
+F: xlators/protocol/
+
+Remote Procedure Call subsystem
+P: Mohit Agrawal <moagrawa@redhat.com>
+S: Maintained
+F: rpc/rpc-lib/
+F: rpc/xdr/
+
+Snapshot
+M: Raghavendra Bhat <raghavendra@redhat.com>
+P: Mohammed Rafi KC <rafi.kavungal@iternity.com>
+P: Sunny Kumar <sunkumar@redhat.com>
+S: Maintained
+F: xlators/mgmt/glusterd/src/glusterd-snap*
+F: extras/snap-scheduler.py
+
+Socket subsystem
+P: Krutika Dhananjay <kdhananj@redhat.com>
+P: Milind Changire <mchangir@redhat.com>
+P: Mohammed Rafi KC <rafi.kavungal@iternity.com>
+P: Mohit Agrawal <moagrawa@redhat.com>
+S: Maintained
+F: rpc/rpc-transport/socket/
+
+Testing - .t framework
+M: Raghavendra Talur <rtalur@redhat.com>
+S: Maintained
+F: tests/
+
+Utilities
+M: Aravinda VK <avishwan@redhat.com>
+P: Niels de Vos <ndevos@redhat.com>
+P: Raghavendra Talur <rtalur@redhat.com>
+P: Sachidanda Urs <surs@redhat.com>
+S: Maintained
+F: extras/
+
+Events APIs
+M: Aravinda VK <avishwan@redhat.com>
+S: Maintained
+F: events/
+F: libglusterfs/src/events*
+F: libglusterfs/src/eventtypes*
+F: extras/systemd/glustereventsd*
+
+Distribution Specific:
+----------------------
+Build:
+M: Niels de Vos <ndevos@redhat.com>
+M: Hari Gowtham <hgowtham@redhat.com>
+P: Anoop C S <anoopcs@redhat.com>
+P: Raghavendra Talur <rtalur@redhat.com>
+P: Rinku Kothiya <rkothiya@redhat.com>
+S: Maintained
+
+Debian packages on download.gluster.org
+M: packaging@gluster.org
+M: Kaleb Keithley <kkeithle@redhat.com>
+P: Sheetal Pamecha <spamecha@redhat.com>
+P: Shwetha Acharya <sacharya@redhat.com>
+S: Maintained
+W: http://download.gluster.org/pub/gluster/glusterfs/LATEST/Debian/Debian.README
+T: https://github.com/gluster/glusterfs-debian.git
+
+OpenSuSE
+M: packaging@gluster.org
+M: Kaleb Keithley <kkeithle@redhat.com>
+P: Sheetal Pamecha <spamecha@redhat.com>
+P: Shwetha Acharya <sacharya@redhat.com>
+S: Maintained
+W: https://build.opensuse.org/repositories/home:glusterfs
+W: https://download.gluster.org/pub/gluster/glusterfs/LATEST/SuSE/SuSE.README
+T: https://github.com/gluster/glusterfs-suse.git
+
+Packages for the CentOS Storage SIG
+M: centos-devel@centos.org
+M: Niels de Vos <ndevos@redhat.com>
+P: Kaleb Keithley <kkeithle@redhat.com>
+S: Maintained
+W: https://wiki.centos.org/SpecialInterestGroup/Storage/Gluster
+T: https://github.com/CentOS-Storage-SIG/glusterfs.git
+
+Ubuntu PPA
+M: packaging@gluster.org
+M: Kaleb Keithley <kkeithle@redhat.com>
+P: Sheetal Pamecha <spamecha@redhat.com>
+P: Shwetha Acharya <sacharya@redhat.com>
+S: Maintained
+W: https://launchpad.net/~gluster
+W: http://download.gluster.org/pub/gluster/glusterfs/LATEST/Ubuntu/Ubuntu.README
+T: https://github.com/gluster/glusterfs-debian.git
+
+Related projects
+----------------
+Gluster Block
+M: Prasanna Kumar Kalever <prasanna.kalever@redhat.com>
+M: Xiubo Li <xiubli@redhat.com>
+S: Maintained
+T: https://github.com/gluster/gluster-block.git
+
+GlusterFS core-utils
+M: Anoop C S <anoopcs@redhat.com>
+S: Maintained
+T: https://github.com/gluster/glusterfs-coreutils.git
+
+NFS-Ganesha FSAL plugin
+M: Jiffin Tony Thottan <jthottan@redhat.com>
+M: Kaleb Keithley <kkeithle@redhat.com>
+M: Soumya Koduri <skoduri@redhat.com>
+S: Maintained
+T: git://github.com/nfs-ganesha/nfs-ganesha.git
+F: src/nfs-ganesha~/src/FSAL/FSAL_GLUSTER/
+
+QEMU integration
+M: Niels de Vos <ndevos@redhat.com>
+M: Prasanna Kumar Kalever <prasanna.kalever@redhat.com>
+S: Maintained
+T: git://git.qemu.org/qemu.git
+F: block/gluster.c
+
+Samba VFS plugin
+M: Anoop C S <anoopcs@redhat.com>
+M: Raghavendra Talur <rtalur@redhat.com>
+M: Michael Adam <madam@redhat.com>
+M: Poornima G <pgurusid@redhat.com>
+S: Maintained
+T: git://git.samba.org/samba.git
+F: source3/modules/vfs_glusterfs.c
+
+Storhaug
+M: Jose A. Rivera <jarrpa@redhat.com>
+P: Kaleb Keithley <kkeithle@redhat.com>
+S: Maintained
+T: https://github.com/linux-ha-storage/storhaug.git
+
+Testing - Glusto-Tests
+M: Jonathan Holloway <jholloway@redhat.com>
+M: Vijay Bhaskar Reddy Avuthu <vavuthu@redhat.com>
+M: Akarsha Rai <akrai@redhat.com>
+S: Maintained
+T: https://github.com/gluster/glusto-tests.git
+
+Wireshark dissectors
+M: Niels de Vos <ndevos@redhat.com>
+S: Maintained
+W: https://forge.gluster.org/wireshark
+T: http://code.wireshark.org/git/wireshark
+F: epan/dissectors/packet-gluster*
+
+Infrastructure
+--------------
+
+Platform
+M: Michael Scherer <misc@redhat.com>
+P: Shyamsundar Ranganathan <srangana@redhat.com>
+P: Amar Tumballi <amarts@gmail.com>
+
+Continuous Integration
+M: Michael Scherer <misc@redhat.com>
+M: Deepshikha Khandelwal <dkhandel@redhat.com>
+P: Niels de Vos <ndevos@redhat.com>
+
+Special Thanks
+--------------
+
+GlusterFS would not be possible without the contributions of:
+
+
+M: Vijay Bellur <vbellur@redhat.com>
+M: Jeff Darcy <jeff@pl.atyp.us>
+M: Shreyas Siravara <sshreyas@fb.com>
+M: Kaushal M <kaushal@redhat.com>
+M: Nigel Babu
+M: Prashanth Pai
+P: Sanoj Unnikrishnan
+P: Milind Changire <mchangir@redhat.com>
+P: Sunil Kumar Acharya <sheggodu@redhat.com>
+M: Samikshan Bairagya <samikshan@gmail.com>
+M: Chris Hertel
+M: M. Mohan Kumar <mohan@in.ibm.com>
+M: Shishir Gowda <gowda.shishir@gmail.com>
+M: Brian Foster <bfoster@redhat.com>
+M: Anand Avati <avati@cs.stanford.edu>
+M: Dennis Schafroth <dennis@schafroth.com>
+M: Harshavardhana <harsha@harshavardhana.net>
+M: Krishnan Parthasarathi
+M: Justin Clift <justin@gluster.org>
+M: Venky Shankar <vshankar@redhat.com>
+M: Shravan Chandrashekar <shravantc99@gmail.com>
+M: Joseph Fernandes
+M: Vijaikumar Mallikarjuna
+M: Anand Subramanian
+M: Bharata B Rao <bharata@linux.vnet.ibm.com>
+M: Rajesh Joseph
+M: Dan Lambright
+M: Jay Vyas
+M: Luis Pabon
+M: Ira Cooper
+M: Shwetha Panduranga
+M: Nithya Balachandran
+M: Raghavendra Gowdappa
diff --git a/Makefile.am b/Makefile.am
index 5ff1b65adeb..98ea5c1038d 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -1,32 +1,55 @@
+SOURCES = site.h
+
 EXTRA_DIST = autogen.sh \
-	COPYING-GPLV2 COPYING-LGPLV3 \
-	INSTALL README AUTHORS THANKS NEWS \
+	COPYING-GPLV2 COPYING-LGPLV3 COMMITMENT \
+	INSTALL README.md AUTHORS THANKS NEWS \
 	glusterfs.spec glusterfs-api.pc.in libgfchangelog.pc.in \
 	run-tests.sh \
-	argp-standalone \
+	build-aux/pkg-version \
+	contrib/umountd \
 	$(shell find $(top_srcdir)/tests -type f -print)
 
-SUBDIRS = $(ARGP_STANDALONE_DIR) libglusterfs rpc api xlators glusterfsd \
-	$(FUSERMOUNT_SUBDIR) doc extras cli heal @SYNCDAEMON_SUBDIR@
+
+SUBDIRS = $(ARGP_STANDALONE_DIR) libglusterfs rpc libglusterd api \
+	glusterfsd xlators $(FUSERMOUNT_SUBDIR) doc extras cli heal \
+	@SYNCDAEMON_SUBDIR@ @UMOUNTD_SUBDIR@ tools events
 
 pkgconfigdir = @pkgconfigdir@
 pkgconfig_DATA = glusterfs-api.pc libgfchangelog.pc
 
-CLEANFILES =
+CLEANFILES = glusterfs-api.pc libgfchangelog.pc contrib/umountd/Makefile
+
+clean-local:
+	find . -name '*.o' -o -name '*.lo' -o -name '.Po' | xargs rm -f
 
 gitclean: distclean
 	find . -name Makefile.in -exec rm -f {} \;
-	find . -name Makefile -exec rm -f {} \;
 	find . -name mount.glusterfs -exec rm -f {} \;
+	find . -name .deps -o -name .libs | xargs rm -rf
 	rm -fr autom4te.cache
 	rm -f missing aclocal.m4 config.h.in config.guess config.sub ltmain.sh install-sh configure depcomp
-	-rm -fr argp-standalone/autom4te.cache
-	-rm -f argp-standalone/aclocal.m4
-	-rm -f argp-standalone/config.h.in
-	-rm -f argp-standalone/configure
-	-rm -f argp-standalone/depcomp
-	-rm -f argp-standalone/install-sh
-	-rm -f argp-standalone/missing
-
-dist-hook:
+
+# dist-hook gets executed with 'make dist', this is the only target getting
+# executed, a dist-hook in other Makefile.am files seem to get ignored.
+dist-hook: gen-VERSION gen-ChangeLog
+	-rm -fr $(distdir)/contrib/umountd/.deps
+	-rm -f $(distdir)/events/src/eventtypes.py
+	-rm -f $(distdir)/tests/env.rc
+	-cp -f $(top_srcdir)/build-aux/config.sub.dist $(distdir)/config.sub
+	-cp -f $(top_srcdir)/build-aux/config.guess.dist $(distdir)/config.guess
+
+.PHONY: gen-VERSION gen-ChangeLog clang-check
+
+clang-check:
+	@$(top_srcdir)/extras/clang-checker.sh
+
+gen-ChangeLog:
 	(cd $(srcdir) && git diff && echo ===== git log ==== && git log) > $(distdir)/ChangeLog
+
+.PHONY : gen-VERSION
+gen-VERSION:
+	if test -d $(top_srcdir)/.git; then             \
+	  cd $(top_srcdir);                             \
+	  ./build-aux/pkg-version --full                \
+	    > $(abs_top_builddir)/$(distdir)/VERSION;   \
+	fi
diff --git a/README b/README
deleted file mode 100644
index ce648743ee9..00000000000
--- a/README
+++ /dev/null
@@ -1 +0,0 @@
-For more info, please visit http://www.gluster.org/.
diff --git a/README.md b/README.md
new file mode 100644
index 00000000000..9d68e033782
--- /dev/null
+++ b/README.md
@@ -0,0 +1,46 @@
+# Gluster
+  Gluster is a software defined distributed storage that can scale to several
+  petabytes. It provides interfaces for object, block and file storage.
+
+## Development
+  The development workflow is documented in [Contributors guide](CONTRIBUTING.md)
+
+## Documentation
+  The Gluster documentation can be found at [Gluster Docs](http://docs.gluster.org).
+
+## Deployment
+  Quick instructions to build and install can be found in [INSTALL](INSTALL) file.
+
+## Testing
+
+  GlusterFS source contains some functional tests under `tests/` directory. All
+  these tests are run against every patch submitted for review. If you want your
+  patch to be tested, please add a `.t` test file as part of your patch submission.
+  You can also submit a patch to only add a `.t` file for the test case you are
+  aware of.
+
+  To run these tests, on your test-machine, just run `./run-tests.sh`. Don't run
+  this on a machine where you have 'production' glusterfs is running, as it would
+  blindly kill all gluster processes in each runs.
+
+  If you are sending a patch, and want to validate one or few specific tests, then
+  run a single test by running the below command.
+
+```
+  bash# /bin/bash ${path_to_gluster}/tests/basic/rpc-coverage.t
+```
+
+  You can also use `prove` tool if available in your machine, as follows.
+
+```
+  bash# prove -vmfe '/bin/bash' ${path_to_gluster}/tests/basic/rpc-coverage.t
+```
+
+
+## Maintainers
+  The list of Gluster maintainers is available in [MAINTAINERS](MAINTAINERS) file.
+
+## License
+  Gluster is dual licensed under [GPLV2](COPYING-GPLV2) and [LGPLV3+](COPYING-LGPLV3).
+
+  Please visit the [Gluster Home Page](http://www.gluster.org/) to find out more about Gluster.
diff --git a/THANKS b/THANKS
index e1ce5105d79..d056d00bb72 100644
--- a/THANKS
+++ b/THANKS
@@ -1 +1 @@
-For all of you, who use the product and help us making it more robust, useful, popular..
+For all of you, who use the product and help us make it more robust, useful, and popular.
diff --git a/api/examples/Makefile.am b/api/examples/Makefile.am
index f1a8d12b29e..7112c81d6a7 100644
--- a/api/examples/Makefile.am
+++ b/api/examples/Makefile.am
@@ -4,7 +4,3 @@ glfsxmp_SOURCES = glfsxmp.c
 glfsxmp_CFLAGS = $(GLFS_CFLAGS) -Wall
 glfsxmp_LDADD = $(GLFS_LIBS) -lrt
 
-# Install __init__.py (a generated file), and gfapi.py into
-# the Python site-packages area
-pygfapidir = $(pythondir)/gluster
-pygfapi_PYTHON = __init__.py gfapi.py
diff --git a/api/examples/__init__.py.in b/api/examples/__init__.py.in
deleted file mode 100644
index 280beb2dcec..00000000000
--- a/api/examples/__init__.py.in
+++ /dev/null
@@ -1 +0,0 @@
-__version__ = "@PACKAGE_VERSION@"
diff --git a/api/examples/getvolfile.py b/api/examples/getvolfile.py
new file mode 100755
index 00000000000..3b2c8ab5a15
--- /dev/null
+++ b/api/examples/getvolfile.py
@@ -0,0 +1,45 @@
+#!/usr/bin/python3
+
+from __future__ import print_function
+import ctypes
+import ctypes.util
+
+api = ctypes.CDLL("libgfapi.so")
+api.glfs_get_volfile.argtypes = [ctypes.c_void_p,
+                                 ctypes.c_void_p,
+                                 ctypes.c_ulong]
+api.glfs_get_volfile.restype = ctypes.c_long
+
+
+def get_volfile(host, volume):
+    # This is set to a large value to exercise the "buffer not big enough"
+    # path.  More realistically, you'd just start with a huge buffer.
+    BUF_LEN = 0
+    fs = api.glfs_new(volume)
+    # api.glfs_set_logging(fs,"/dev/stderr",7)
+    api.glfs_set_volfile_server(fs, "tcp", host, 24007)
+    api.glfs_init(fs)
+    vbuf = ctypes.create_string_buffer(BUF_LEN)
+    vlen = api.glfs_get_volfile(fs, vbuf, BUF_LEN)
+    if vlen < 0:
+        vlen = BUF_LEN - vlen
+        vbuf = ctypes.create_string_buffer(vlen)
+        vlen = api.glfs_get_volfile(fs, vbuf, vlen)
+    api.glfs_fini(fs)
+    if vlen <= 0:
+        return vlen
+    return vbuf.value[:vlen]
+
+if __name__ == "__main__":
+    import sys
+
+    try:
+        res = get_volfile(*sys.argv[1:3])
+    except:
+        print("fetching volfile failed (volume not started?)")
+
+    try:
+        for line in res.split('\n'):
+            print(line)
+    except:
+        print("bad return value %s" % res)
diff --git a/api/examples/gfapi.py b/api/examples/gfapi.py
deleted file mode 100755
index 3ac67f4d5ba..00000000000
--- a/api/examples/gfapi.py
+++ /dev/null
@@ -1,422 +0,0 @@
-#!/usr/bin/python
-
-from ctypes import *
-from ctypes.util import find_library
-import os
-import sys
-import time
-import types
-
-# Looks like ctypes is having trouble with dependencies, so just force them to
-# load with RTLD_GLOBAL until I figure that out.
-glfs = CDLL(find_library("glusterfs"),RTLD_GLOBAL)
-xdr = CDLL(find_library("gfxdr"),RTLD_GLOBAL)
-api = CDLL(find_library("gfapi"),RTLD_GLOBAL)
-
-# Wow, the Linux kernel folks really play nasty games with this structure.  If
-# you look at the man page for stat(2) and then at this definition you'll note
-# two discrepancies.  First, we seem to have st_nlink and st_mode reversed.  In
-# fact that's exactly how they're defined *for 64-bit systems*; for 32-bit
-# they're in the man-page order.  Even uglier, the man page makes no mention of
-# the *nsec fields, but they are very much present and if they're not included
-# then we get memory corruption because libgfapi has a structure definition
-# that's longer than ours and they overwrite some random bit of memory after
-# the space we allocated.  Yes, that's all very disgusting, and I'm still not
-# sure this will really work on 32-bit because all of the field types are so
-# obfuscated behind macros and feature checks.
-class Stat (Structure):
-        _fields_ = [
-                ("st_dev",              c_ulong),
-                ("st_ino",              c_ulong),
-                ("st_nlink",            c_ulong),
-                ("st_mode",             c_uint),
-                ("st_uid",              c_uint),
-                ("st_gid",              c_uint),
-                ("st_rdev",             c_ulong),
-                ("st_size",             c_ulong),
-                ("st_blksize",          c_ulong),
-                ("st_blocks",           c_ulong),
-                ("st_atime",            c_ulong),
-                ("st_atimensec",        c_ulong),
-                ("st_mtime",            c_ulong),
-                ("st_mtimensec",        c_ulong),
-                ("st_ctime",            c_ulong),
-                ("st_ctimensec",        c_ulong),
-        ]
-api.glfs_creat.restype = c_void_p
-api.glfs_open.restype = c_void_p
-api.glfs_lstat.restype = c_int
-api.glfs_lstat.argtypes = [c_void_p, c_char_p, POINTER(Stat)]
-
-class Dirent (Structure):
-        _fields_ = [
-                ("d_ino",       c_ulong),
-                ("d_off",       c_ulong),
-                ("d_reclen",    c_ushort),
-                ("d_type",      c_char),
-                ("d_name",      c_char * 256),
-        ]
-api.glfs_opendir.restype = c_void_p
-api.glfs_readdir_r.restype = c_int
-api.glfs_readdir_r.argtypes = [c_void_p, POINTER(Dirent),
-                               POINTER(POINTER(Dirent))]
-
-# There's a bit of ctypes glitchiness around __del__ functions and module-level
-# variables.  If we unload the module while we still have references to File or
-# Volume objects, the module-level variables might have disappeared by the time
-# __del__ gets called.  Therefore the objects hold references which they
-# release when __del__ is done.  We only actually use the object-local values
-# in __del__; for clarity, we just use the simpler module-level form elsewhere.
-
-class File(object):
-
-        def __init__ (self, fd):
-                # Add a reference so the module-level variable "api" doesn't
-                # get yanked out from under us (see comment above File def'n).
-                self._api = api
-                self.fd = fd
-
-        def __del__ (self):
-                self._api.glfs_close(self.fd)
-                self._api = None
-
-        # File operations, in alphabetical order.
-
-        def fsync (self):
-                return api.glfs_fsync(self.fd)
-
-        def read (self, buflen, flags=0):
-                rbuf = create_string_buffer(buflen)
-                rc = api.glfs_read(self.fd,rbuf,buflen,flags)
-                if rc > 0:
-                        return rbuf.value[:rc]
-                else:
-                        return rc
-
-        def read_buffer (self, buf, flags=0):
-                return api.glfs_read(self.fd,buf,len(buf),flags)
-
-        def write (self, data, flags=0):
-                return api.glfs_write(self.fd,data,len(data),flags)
-
-        def fallocate (self, mode, offset, len):
-            return api.glfs_fallocate(self.fd, mode, offset, len)
-
-        def discard (self, offset, len):
-            return api.glfs_discard(self.fd, offset, len)
-
-
-class Dir(object):
-
-        def __init__ (self, fd):
-                # Add a reference so the module-level variable "api" doesn't
-                # get yanked out from under us (see comment above File def'n).
-                self._api = api
-                self.fd = fd
-                self.cursor = POINTER(Dirent)()
-
-        def __del__ (self):
-                self._api.glfs_closedir(self.fd)
-                self._api = None
-
-        def next (self):
-                entry = Dirent()
-                entry.d_reclen = 256
-                rc = api.glfs_readdir_r(self.fd,byref(entry),byref(self.cursor))
-                if (rc < 0) or (not self.cursor) or (not self.cursor.contents):
-                        return rc
-                return entry
-
-class Volume(object):
-
-        # Housekeeping functions.
-
-        def __init__ (self, host, volid, proto="tcp", port=24007):
-                # Add a reference so the module-level variable "api" doesn't
-                # get yanked out from under us (see comment above File def'n).
-                self._api = api
-                self.fs = api.glfs_new(volid)
-                api.glfs_set_volfile_server(self.fs,proto,host,port)
-
-        def __del__ (self):
-                self._api.glfs_fini(self.fs)
-                self._api = None
-
-        def set_logging (self, path, level):
-                api.glfs_set_logging(self.fs,path,level)
-
-        def mount (self):
-                api.glfs_init(self.fs)
-
-        # File operations, in alphabetical order.
-
-        def creat (self, path, flags, mode):
-                fd = api.glfs_creat(self.fs,path,flags,mode)
-                if not fd:
-                        return fd
-                return File(fd)
-
-        def getxattr (self, path, key, maxlen):
-                buf = create_string_buffer(maxlen)
-                rc = api.glfs_getxattr(self.fs,path,key,buf,maxlen)
-                if rc < 0:
-                        return rc
-                return buf.value[:rc]
-
-        def listxattr (self, path):
-                buf = create_string_buffer(512)
-                rc = api.glfs_listxattr(self.fs,path,buf,512)
-                if rc < 0:
-                        return rc
-                xattrs = []
-                # Parsing character by character is ugly, but it seems like the
-                # easiest way to deal with the "strings separated by NUL in one
-                # buffer" format.
-                i = 0
-                while i < rc:
-                        new_xa = buf.raw[i]
-                        i += 1
-                        while i < rc:
-                                next_char = buf.raw[i]
-                                i += 1
-                                if next_char == '\0':
-                                        xattrs.append(new_xa)
-                                        break
-                                new_xa += next_char
-                xattrs.sort()
-                return xattrs
-
-        def lstat (self, path):
-                x = Stat()
-                rc = api.glfs_lstat(self.fs,path,byref(x))
-                if rc >= 0:
-                        return x
-                else:
-                        return rc
-
-        def mkdir (self, path):
-                return api.glfs_mkdir(self.fs,path)
-
-        def open (self, path, flags):
-                fd = api.glfs_open(self.fs,path,flags)
-                if not fd:
-                        return fd
-                return File(fd)
-
-        def opendir (self, path):
-                fd = api.glfs_opendir(self.fs,path)
-                if not fd:
-                        return fd
-                return Dir(fd)
-
-        def rename (self, opath, npath):
-                return api.glfs_rename(self.fs,opath,npath)
-
-        def rmdir (self, path):
-                return api.glfs_rmdir(self.fs,path)
-
-        def setxattr (self, path, key, value, vlen):
-                return api.glfs_setxattr(self.fs,path,key,value,vlen,0)
-
-        def unlink (self, path):
-                return api.glfs_unlink(self.fs,path)
-
-if __name__ == "__main__":
-        def test_create_write (vol, path, data):
-                mypath = path + ".io"
-                fd = vol.creat(mypath,os.O_WRONLY|os.O_EXCL,0644)
-                if not fd:
-                        return False, "creat error"
-                rc = fd.write(data)
-                if rc != len(data):
-                        return False, "wrote %d/%d bytes" % (rc, len(data))
-                return True, "wrote %d bytes" % rc
-
-        # TBD: this test fails if we do create, open, write, read
-        def test_open_read (vol, path, data):
-                mypath = path + ".io"
-                fd = vol.open(mypath,os.O_RDONLY)
-                if not fd:
-                        return False, "open error"
-                dlen = len(data) * 2
-                buf = fd.read(dlen)
-                if type(buf) == types.IntType:
-                        return False, "read error %d" % buf
-                if len(buf) != len(data):
-                        return False, "read %d/%d bytes" % (len(buf), len(data))
-                return True, "read '%s'" % buf
-
-        def test_lstat (vol, path, data):
-                mypath = path + ".io"
-                sb = vol.lstat(mypath)
-                if type(sb) == types.IntType:
-                        return False, "lstat error %d" % sb
-                if sb.st_size != len(data):
-                        return False, "lstat size is %d, expected %d" % (
-                                sb.st_size, len(data))
-                return True, "lstat got correct size %d" % sb.st_size
-
-        def test_rename (vol, path, data):
-                opath = path + ".io"
-                npath = path + ".tmp"
-                rc = vol.rename(opath,npath)
-                if rc < 0:
-                        return False, "rename error %d" % rc
-                ofd = vol.open(opath,os.O_RDWR)
-                if isinstance(ofd,File):
-                        return False, "old path working after rename"
-                nfd = vol.open(npath,os.O_RDWR)
-                if isinstance(nfd,File):
-                        return False, "new path not working after rename"
-                return True, "rename worked"
-
-        def test_unlink (vol, path, data):
-                mypath = path + ".tmp"
-                rc = vol.unlink(mypath)
-                if rc < 0:
-                        return False, "unlink error %d" % fd
-                fd = vol.open(mypath,os.O_RDWR)
-                if isinstance(fd,File):
-                        return False, "path still usable after unlink"
-                return True, "unlink worked"
-
-        def test_mkdir (vol, path, data):
-                mypath = path + ".dir"
-                rc = vol.mkdir(mypath)
-                if rc < 0:
-                        return False, "mkdir error %d" % rc
-                return True, "mkdir worked"
-
-        def test_create_in_dir (vol, path, data):
-                mypath = path + ".dir/probe"
-                fd = vol.creat(mypath,os.O_RDWR,0644)
-                if not isinstance(fd,File):
-                        return False, "create (in dir) error"
-                return True, "create (in dir) worked"
-
-        def test_dir_listing (vol, path, data):
-                mypath = path + ".dir"
-                fd = vol.opendir(mypath)
-                if not isinstance(fd,Dir):
-                        return False, "opendir error %d" % fd
-                files = []
-                while True:
-                        ent = fd.next()
-                        if not isinstance(ent,Dirent):
-                                break
-                        name = ent.d_name[:ent.d_reclen]
-                        files.append(name)
-                if files != [".", "..", "probe"]:
-                        return False, "wrong directory contents"
-                return True, "directory listing worked"
-
-        def test_unlink_in_dir (vol, path, data):
-                mypath = path + ".dir/probe"
-                rc = vol.unlink(mypath)
-                if rc < 0:
-                        return False, "unlink (in dir) error %d" % rc
-                return True, "unlink (in dir) worked"
-
-        def test_rmdir (vol, path, data):
-                mypath = path + ".dir"
-                rc = vol.rmdir(mypath)
-                if rc < 0:
-                        return False, "rmdir error %d" % rc
-                sb = vol.lstat(mypath)
-                if not isinstance(sb,Stat):
-                        return False, "dir still there after rmdir"
-                return True, "rmdir worked"
-
-        def test_setxattr (vol, path, data):
-                mypath = path + ".xa"
-                fd = vol.creat(mypath,os.O_RDWR|os.O_EXCL,0644)
-                if not fd:
-                        return False, "creat (xattr test) error"
-                key1, key2 = "hello", "goodbye"
-                if vol.setxattr(mypath,"trusted.key1",key1,len(key1)) < 0:
-                        return False, "setxattr (key1) error"
-                if vol.setxattr(mypath,"trusted.key2",key2,len(key2)) < 0:
-                        return False, "setxattr (key2) error"
-                return True, "setxattr worked"
-
-        def test_getxattr (vol, path, data):
-                mypath = path + ".xa"
-                buf = vol.getxattr(mypath,"trusted.key1",32)
-                if type(buf) == types.IntType:
-                        return False, "getxattr error"
-                if buf != "hello":
-                        return False, "wrong getxattr value %s" % buf
-                return True, "getxattr worked"
-
-        def test_listxattr (vol, path, data):
-                mypath = path + ".xa"
-                xattrs = vol.listxattr(mypath)
-                if type(xattrs) == types.IntType:
-                        return False, "listxattr error"
-                if xattrs != ["trusted.key1","trusted.key2"]:
-                        return False, "wrong listxattr value %s" % repr(xattrs)
-                return True, "listxattr worked"
-
-        def test_fallocate (vol, path, data):
-                mypath = path + ".io"
-                fd = vol.creat(mypath,os.O_WRONLY|os.O_EXCL,0644)
-                if not fd:
-                        return False, "creat error"
-		rc = fd.fallocate(0, 0, 1024*1024)
-                if rc != 0:
-                        return False, "fallocate error"
-		rc = fd.discard(4096, 4096)
-		if rc != 0:
-			return False, "discard error"
-                return True, "fallocate/discard worked"
-
-        test_list = (
-                test_create_write,
-                test_open_read,
-                test_lstat,
-                test_rename,
-                test_unlink,
-                test_mkdir,
-                test_create_in_dir,
-                test_dir_listing,
-                test_unlink_in_dir,
-                test_rmdir,
-                test_setxattr,
-                test_getxattr,
-                test_listxattr,
-                test_fallocate,
-        )
-
-        ok_to_fail = (
-                # TBD: this fails opening the new file, even though the file
-                # did get renamed.  Looks like a gfapi bug, not ours.
-                (test_rename, "new path not working after rename"),
-                # TBD: similar, call returns error even though it worked
-                (test_rmdir, "dir still there after rmdir"),
-        )
-
-        volid, path = sys.argv[1:3]
-        data = "fubar"
-        vol = Volume("localhost",volid)
-        vol.set_logging("/dev/null",7)
-        #vol.set_logging("/dev/stderr",7)
-        vol.mount()
-
-        failures = 0
-        expected = 0
-        for t in test_list:
-                rc, msg = t(vol,path,data)
-                if rc:
-                        print "PASS: %s" % msg
-                else:
-                        print "FAIL: %s" % msg
-                        failures += 1
-                        for otf in ok_to_fail:
-                                if (t == otf[0]) and (msg == otf[1]):
-                                        print "  (skipping known failure)"
-                                        expected += 1
-                                        break # from the *inner* for loop
-                        else:
-                                break # from the *outer* for loop
-
-        print "%d failures (%d expected)" % (failures, expected)
diff --git a/api/examples/glfsxmp.c b/api/examples/glfsxmp.c
index 600d72fb55a..a55616ef739 100644
--- a/api/examples/glfsxmp.c
+++ b/api/examples/glfsxmp.c
@@ -1,1598 +1,1811 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <errno.h>
-#include "api/glfs.h"
-#include "api/glfs-handles.h"
+#include <glusterfs/api/glfs.h>
+#include <glusterfs/api/glfs-handles.h>
 #include <string.h>
 #include <time.h>
 
+#define TEST_STR_LEN 2048
 
 int
-test_dirops (glfs_t *fs)
+test_dirops(glfs_t *fs)
 {
-        glfs_fd_t *fd = NULL;
-        char buf[512];
-        struct dirent *entry = NULL;
-
-        fd = glfs_opendir (fs, "/");
-        if (!fd) {
-                fprintf (stderr, "/: %s\n", strerror (errno));
-                return -1;
-        }
-
-        fprintf (stderr, "Entries:\n");
-        while (glfs_readdir_r (fd, (struct dirent *)buf, &entry), entry) {
-                fprintf (stderr, "%s: %lu\n", entry->d_name, glfs_telldir (fd));
-        }
-
-        glfs_closedir (fd);
-        return 0;
+    glfs_fd_t *fd = NULL;
+    char buf[512];
+    struct dirent *entry = NULL;
+
+    fd = glfs_opendir(fs, "/");
+    if (!fd) {
+        fprintf(stderr, "/: %s\n", strerror(errno));
+        return -1;
+    }
+
+    fprintf(stderr, "Entries:\n");
+    while (glfs_readdir_r(fd, (struct dirent *)buf, &entry), entry) {
+        fprintf(stderr, "%s: %lu\n", entry->d_name, glfs_telldir(fd));
+    }
+
+    glfs_closedir(fd);
+    return 0;
 }
 
-
 int
-test_xattr (glfs_t *fs)
+test_xattr(glfs_t *fs)
 {
-        char *filename = "/filename2";
-        char buf[512];
-        char *ptr;
-        int ret;
-
-        ret = glfs_setxattr (fs, filename, "user.testkey", "testval", 8, 0);
-        fprintf (stderr, "setxattr(%s): %d (%s)\n", filename, ret,
-                 strerror (errno));
-
-        ret = glfs_setxattr (fs, filename, "user.testkey2", "testval", 8, 0);
-        fprintf (stderr, "setxattr(%s): %d (%s)\n", filename, ret,
-                 strerror (errno));
-
-        ret = glfs_listxattr (fs, filename, buf, 512);
-        fprintf (stderr, "listxattr(%s): %d (%s)\n", filename, ret,
-                 strerror (errno));
-        if (ret < 0)
-                return -1;
-
-        for (ptr = buf; ptr < buf + ret; ptr++) {
-                printf ("key=%s\n", ptr);
-                ptr += strlen (ptr);
-        }
-
-        return 0;
+    char *filename = "/filename2";
+    char *linkfile = "/linkfile";
+    glfs_fd_t *fd = NULL;
+    char buf[512];
+    char *ptr;
+    int ret;
+
+    ret = glfs_setxattr(fs, filename, "user.testkey", "testval", 8, 0);
+    fprintf(stderr, "setxattr(%s): %d (%s)\n", filename, ret, strerror(errno));
+
+    ret = glfs_setxattr(fs, filename, "user.testkey2", "testval", 8, 0);
+    fprintf(stderr, "setxattr(%s): %d (%s)\n", filename, ret, strerror(errno));
+
+    ret = glfs_getxattr(fs, filename, "user.testkey", buf, 512);
+    fprintf(stderr, "getxattr(%s): %d (%s)\n", filename, ret, strerror(errno));
+    if (ret < 0)
+        return -1;
+
+    ret = glfs_listxattr(fs, filename, buf, 512);
+    fprintf(stderr, "listxattr(%s): %d (%s)\n", filename, ret, strerror(errno));
+    if (ret < 0)
+        return -1;
+
+    ret = glfs_symlink(fs, "filename", linkfile);
+    fprintf(stderr, "symlink(%s %s): %s\n", filename, linkfile,
+            strerror(errno));
+    if (ret < 0)
+        return -1;
+
+    ret = glfs_readlink(fs, linkfile, buf, 512);
+    fprintf(stderr, "readlink(%s) : %d (%s)\n", filename, ret, strerror(errno));
+    if (ret < 0)
+        return -1;
+
+    ret = glfs_lsetxattr(fs, filename, "user.testkey3", "testval", 8, 0);
+    fprintf(stderr, "lsetxattr(%s) : %d (%s)\n", linkfile, ret,
+            strerror(errno));
+    if (ret < 0)
+        return -1;
+
+    ret = glfs_llistxattr(fs, linkfile, buf, 512);
+    fprintf(stderr, "llistxattr(%s): %d (%s)\n", filename, ret,
+            strerror(errno));
+    if (ret < 0)
+        return -1;
+
+    ret = glfs_lgetxattr(fs, filename, "user.testkey3", buf, 512);
+    fprintf(stderr, "lgetxattr(%s): %d (%s)\n", linkfile, ret, strerror(errno));
+    if (ret < 0)
+        return -1;
+
+    for (ptr = buf; ptr < buf + ret; ptr++) {
+        printf("key=%s\n", ptr);
+        ptr += strlen(ptr);
+    }
+
+    ret = glfs_removexattr(fs, filename, "user.testkey2");
+    fprintf(stderr, "removexattr(%s): %d (%s)\n", filename, ret,
+            strerror(errno));
+
+    fd = glfs_open(fs, filename, O_RDWR);
+    fprintf(stderr, "open(%s): (%p) %s\n", filename, fd, strerror(errno));
+
+    ret = glfs_fsetxattr(fd, "user.testkey2", "testval", 8, 0);
+    fprintf(stderr, "fsetxattr(%s): %d (%s)\n", filename, ret, strerror(errno));
+
+    ret = glfs_fgetxattr(fd, "user.testkey2", buf, 512);
+    fprintf(stderr, "fgetxattr(%s): %d (%s)\n", filename, ret, strerror(errno));
+
+    ret = glfs_flistxattr(fd, buf, 512);
+    fprintf(stderr, "flistxattr(%s): %d (%s)\n", filename, ret,
+            strerror(errno));
+    if (ret < 0)
+        return -1;
+
+    for (ptr = buf; ptr < buf + ret; ptr++) {
+        printf("key=%s\n", ptr);
+        ptr += strlen(ptr);
+    }
+
+    ret = glfs_fremovexattr(fd, "user.testkey2");
+    fprintf(stderr, "fremovexattr(%s): %d (%s)\n", filename, ret,
+            strerror(errno));
+
+    glfs_close(fd);
+
+    return 0;
 }
 
-
 int
-test_chdir (glfs_t *fs)
+test_chdir(glfs_t *fs)
 {
-        int ret = -1;
-        char *topdir = "/topdir";
-        char *linkdir = "/linkdir";
-        char *subdir = "./subdir";
-        char *respath = NULL;
-        char pathbuf[4096];
-
-        ret = glfs_mkdir (fs, topdir, 0755);
-        if (ret) {
-                fprintf (stderr, "mkdir(%s): %s\n", topdir, strerror (errno));
-                return -1;
-        }
-
-        respath = glfs_getcwd (fs, pathbuf, 4096);
-        fprintf (stdout, "getcwd() = %s\n", respath);
-
-        ret = glfs_symlink (fs, topdir, linkdir);
-        if (ret) {
-                fprintf (stderr, "symlink(%s, %s): %s\n", topdir, linkdir, strerror (errno));
-                return -1;
-        }
-
-        ret = glfs_chdir (fs, linkdir);
-        if (ret) {
-                fprintf (stderr, "chdir(%s): %s\n", linkdir, strerror (errno));
-                return -1;
-        }
-
-        respath = glfs_getcwd (fs, pathbuf, 4096);
-        fprintf (stdout, "getcwd() = %s\n", respath);
-
-        respath = glfs_realpath (fs, subdir, pathbuf);
-        if (respath) {
-                fprintf (stderr, "realpath(%s) worked unexpectedly: %s\n", subdir, respath);
-                return -1;
-        }
-
-        ret = glfs_mkdir (fs, subdir, 0755);
-        if (ret) {
-                fprintf (stderr, "mkdir(%s): %s\n", subdir, strerror (errno));
-                return -1;
-        }
-
-        respath = glfs_realpath (fs, subdir, pathbuf);
-        if (!respath) {
-                fprintf (stderr, "realpath(%s): %s\n", subdir, strerror (errno));
-        } else {
-                fprintf (stdout, "realpath(%s) = %s\n", subdir, respath);
-        }
-
-        ret = glfs_chdir (fs, subdir);
-        if (ret) {
-                fprintf (stderr, "chdir(%s): %s\n", subdir, strerror (errno));
-                return -1;
-        }
-
-        respath = glfs_getcwd (fs, pathbuf, 4096);
-        fprintf (stdout, "getcwd() = %s\n", respath);
-
-        respath = glfs_realpath (fs, "/linkdir/subdir", pathbuf);
-        if (!respath) {
-                fprintf (stderr, "realpath(/linkdir/subdir): %s\n", strerror (errno));
-        } else {
-                fprintf (stdout, "realpath(/linkdir/subdir) = %s\n", respath);
-        }
-
-        return 0;
+    int ret = -1;
+    char *dir = "/dir";
+    char *topdir = "/topdir";
+    char *linkdir = "/linkdir";
+    char *linkdir2 = "/linkdir2";
+    char *subdir = "./subdir";
+    char *respath = NULL;
+    char pathbuf[4096];
+
+    ret = glfs_mkdir(fs, topdir, 0755);
+    fprintf(stderr, "mkdir(%s): %s\n", topdir, strerror(errno));
+    if (ret)
+        return -1;
+
+    ret = glfs_mkdir(fs, dir, 0755);
+    fprintf(stderr, "mkdir(%s): %s\n", dir, strerror(errno));
+    if (ret)
+        return -1;
+
+    respath = glfs_getcwd(fs, pathbuf, 4096);
+    fprintf(stdout, "getcwd() = %s\n", respath);
+
+    ret = glfs_symlink(fs, "topdir", linkdir);
+    if (ret) {
+        fprintf(stderr, "symlink(%s, %s): %s\n", topdir, linkdir,
+                strerror(errno));
+        return -1;
+    }
+
+    ret = glfs_chdir(fs, linkdir);
+    if (ret) {
+        fprintf(stderr, "chdir(%s): %s\n", linkdir, strerror(errno));
+        return -1;
+    }
+
+    respath = glfs_getcwd(fs, pathbuf, 4096);
+    fprintf(stdout, "getcwd() = %s\n", respath);
+
+    respath = glfs_realpath(fs, subdir, pathbuf);
+    if (respath) {
+        fprintf(stderr, "realpath(%s) worked unexpectedly: %s\n", subdir,
+                respath);
+        return -1;
+    }
+
+    ret = glfs_mkdir(fs, subdir, 0755);
+    if (ret) {
+        fprintf(stderr, "mkdir(%s): %s\n", subdir, strerror(errno));
+        return -1;
+    }
+
+    respath = glfs_realpath(fs, subdir, pathbuf);
+    if (!respath) {
+        fprintf(stderr, "realpath(%s): %s\n", subdir, strerror(errno));
+    } else {
+        fprintf(stdout, "realpath(%s) = %s\n", subdir, respath);
+    }
+
+    ret = glfs_chdir(fs, subdir);
+    if (ret) {
+        fprintf(stderr, "chdir(%s): %s\n", subdir, strerror(errno));
+        return -1;
+    }
+
+    respath = glfs_getcwd(fs, pathbuf, 4096);
+    fprintf(stdout, "getcwd() = %s\n", respath);
+
+    respath = glfs_realpath(fs, "/linkdir/subdir", pathbuf);
+    if (!respath) {
+        fprintf(stderr, "realpath(/linkdir/subdir): %s\n", strerror(errno));
+    } else {
+        fprintf(stdout, "realpath(/linkdir/subdir) = %s\n", respath);
+    }
+
+    return 0;
 }
 
 #ifdef DEBUG
 static void
-peek_stat (struct stat *sb)
+peek_stat(struct stat *sb)
 {
-        printf ("Dumping stat information:\n");
-        printf ("File type:                ");
-
-        switch (sb->st_mode & S_IFMT) {
-                case S_IFBLK:  printf ("block device\n");            break;
-                case S_IFCHR:  printf ("character device\n");        break;
-                case S_IFDIR:  printf ("directory\n");               break;
-                case S_IFIFO:  printf ("FIFO/pipe\n");               break;
-                case S_IFLNK:  printf ("symlink\n");                 break;
-                case S_IFREG:  printf ("regular file\n");            break;
-                case S_IFSOCK: printf ("socket\n");                  break;
-                default:       printf ("unknown?\n");                break;
-        }
-
-        printf ("I-node number:            %ld\n", (long) sb->st_ino);
-
-        printf ("Mode:                     %lo (octal)\n",
-                (unsigned long) sb->st_mode);
-
-        printf ("Link count:               %ld\n", (long) sb->st_nlink);
-        printf ("Ownership:                UID=%ld   GID=%ld\n",
-                (long) sb->st_uid, (long) sb->st_gid);
-
-        printf ("Preferred I/O block size: %ld bytes\n",
-                (long) sb->st_blksize);
-        printf ("File size:                %lld bytes\n",
-                (long long) sb->st_size);
-        printf ("Blocks allocated:         %lld\n",
-                (long long) sb->st_blocks);
-
-        printf ("Last status change:       %s", ctime(&sb->st_ctime));
-        printf ("Last file access:         %s", ctime(&sb->st_atime));
-        printf ("Last file modification:   %s", ctime(&sb->st_mtime));
-
-        return;
+    printf("Dumping stat information:\n");
+    printf("File type:                ");
+
+    switch (sb->st_mode & S_IFMT) {
+        case S_IFBLK:
+            printf("block device\n");
+            break;
+        case S_IFCHR:
+            printf("character device\n");
+            break;
+        case S_IFDIR:
+            printf("directory\n");
+            break;
+        case S_IFIFO:
+            printf("FIFO/pipe\n");
+            break;
+        case S_IFLNK:
+            printf("symlink\n");
+            break;
+        case S_IFREG:
+            printf("regular file\n");
+            break;
+        case S_IFSOCK:
+            printf("socket\n");
+            break;
+        default:
+            printf("unknown?\n");
+            break;
+    }
+
+    printf("I-node number:            %ld\n", (long)sb->st_ino);
+
+    printf("Mode:                     %lo (octal)\n",
+           (unsigned long)sb->st_mode);
+
+    printf("Link count:               %ld\n", (long)sb->st_nlink);
+    printf("Ownership:                UID=%ld   GID=%ld\n", (long)sb->st_uid,
+           (long)sb->st_gid);
+
+    printf("Preferred I/O block size: %ld bytes\n", (long)sb->st_blksize);
+    printf("File size:                %lld bytes\n", (long long)sb->st_size);
+    printf("Blocks allocated:         %lld\n", (long long)sb->st_blocks);
+
+    printf("Last status change:       %s", ctime(&sb->st_ctime));
+    printf("Last file access:         %s", ctime(&sb->st_atime));
+    printf("Last file modification:   %s", ctime(&sb->st_mtime));
+
+    return;
 }
 
 static void
-peek_handle (unsigned char *glid)
+peek_handle(unsigned char *glid)
 {
-        int i;
+    int i;
 
-        for (i = 0; i < GFAPI_HANDLE_LENGTH; i++)
-        {
-                printf (":%02x:", glid[i]);
-        }
-        printf ("\n");
+    for (i = 0; i < GFAPI_HANDLE_LENGTH; i++) {
+        printf(":%02x:", glid[i]);
+    }
+    printf("\n");
 }
-#else /* DEBUG */
+#else  /* DEBUG */
 static void
-peek_stat (struct stat *sb)
+peek_stat(struct stat *sb)
 {
-        return;
+    return;
 }
 
 static void
-peek_handle (unsigned char *id)
+peek_handle(unsigned char *id)
 {
-        return;
+    return;
 }
 #endif /* DEBUG */
 
-glfs_t    *fs = NULL;
-char      *full_parent_name = "/testdir", *parent_name = "testdir";
+glfs_t *fs = NULL;
+char *full_parent_name = "/testdir", *parent_name = "testdir";
 
 void
-test_h_unlink (void)
+test_h_unlink(void)
 {
-        char               *my_dir = "unlinkdir";
-        char               *my_file = "file.txt";
-        char               *my_subdir = "dir1";
-        struct glfs_object *parent = NULL, *leaf = NULL, *dir = NULL,
-                           *subdir = NULL, *subleaf = NULL;
-        struct stat         sb;
-        int                 ret;
-
-        printf ("glfs_h_unlink tests: In Progress\n");
-
-        /* Prepare tests */
-        parent = glfs_h_lookupat (fs, NULL, full_parent_name, &sb);
-        if (parent == NULL) {
-                fprintf (stderr, "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
-                         full_parent_name, NULL, strerror (errno));
-                printf ("glfs_h_lookupat tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        dir = glfs_h_mkdir (fs, parent, my_dir, 0644, &sb);
-        if (dir == NULL) {
-                fprintf (stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
-                         my_dir, parent, strerror (errno));
-                printf ("glfs_h_unlink tests: FAILED\n");
-                goto out;
-        }
-
-        leaf = glfs_h_creat (fs, dir, my_file, O_CREAT, 0644, &sb);
-        if (leaf == NULL) {
-                fprintf (stderr, "glfs_h_creat: error creating %s: from (%p),%s\n",
-                         my_file, dir, strerror (errno));
-                printf ("glfs_h_unlink tests: FAILED\n");
-                goto out;
-        }
-
-        subdir = glfs_h_mkdir (fs, dir, my_subdir, 0644, &sb);
-        if (subdir == NULL) {
-                fprintf (stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
-                         my_subdir, dir, strerror (errno));
-                printf ("glfs_h_unlink tests: FAILED\n");
-                goto out;
-        }
-
-        subleaf = glfs_h_creat (fs, subdir, my_file, O_CREAT, 0644, &sb);
-        if (subleaf == NULL) {
-                fprintf (stderr, "glfs_h_creat: error creating %s: from (%p),%s\n",
-                         my_file, subdir, strerror (errno));
-                printf ("glfs_h_unlink tests: FAILED\n");
-                goto out;
-        }
-
-        /* unlink non empty directory */
-        ret = glfs_h_unlink (fs, dir, my_subdir);
-        if ((ret && errno != ENOTEMPTY) || (ret == 0)) {
-                fprintf (stderr, "glfs_h_unlink: error unlinking %s: it is non empty: %s\n",
-                         my_subdir, strerror (errno));
-                printf ("glfs_h_unlink tests: FAILED\n");
-                goto out;
-        }
-
-        /* unlink regular file */
-        ret = glfs_h_unlink (fs, subdir, my_file);
-        if (ret) {
-                fprintf (stderr, "glfs_h_unlink: error unlinking %s: from (%p),%s\n",
-                         my_file, subdir, strerror (errno));
-                printf ("glfs_h_unlink tests: FAILED\n");
-                goto out;
-        }
-
-        /* unlink directory */
-        ret = glfs_h_unlink (fs, dir, my_subdir);
-        if (ret) {
-                fprintf (stderr, "glfs_h_unlink: error unlinking %s: from (%p),%s\n",
-                         my_subdir, dir, strerror (errno));
-                printf ("glfs_h_unlink tests: FAILED\n");
-                goto out;
-        }
-
-        /* unlink regular file */
-        ret = glfs_h_unlink (fs, dir, my_file);
-        if (ret) {
-                fprintf (stderr, "glfs_h_unlink: error unlinking %s: from (%p),%s\n",
-                         my_file, dir, strerror (errno));
-                printf ("glfs_h_unlink tests: FAILED\n");
-                goto out;
-        }
-
-        /* unlink non-existant regular file */
-        ret = glfs_h_unlink (fs, dir, my_file);
-        if ((ret && errno != ENOENT) || (ret == 0)) {
-                fprintf (stderr, "glfs_h_unlink: error unlinking non-existant %s: invalid errno ,%d, %s\n",
-                         my_file, ret, strerror (errno));
-                printf ("glfs_h_unlink tests: FAILED\n");
-                goto out;
-        }
-
-        /* unlink non-existant directory */
-        ret = glfs_h_unlink (fs, dir, my_subdir);
-        if ((ret && errno != ENOENT) || (ret == 0)) {
-                fprintf (stderr, "glfs_h_unlink: error unlinking non-existant %s:  invalid errno ,%d, %s\n",
-                         my_subdir, ret, strerror (errno));
-                printf ("glfs_h_unlink tests: FAILED\n");
-                goto out;
-        }
-
-        /* unlink directory */
-        ret = glfs_h_unlink (fs, parent, my_dir);
-        if (ret) {
-                fprintf (stderr, "glfs_h_unlink: error unlinking %s: from (%p),%s\n",
-                         my_dir, dir, strerror (errno));
-                printf ("glfs_h_unlink tests: FAILED\n");
-                goto out;
-        }
-
-        printf ("glfs_h_unlink tests: PASSED\n");
+    char *my_dir = "unlinkdir";
+    char *my_file = "file.txt";
+    char *my_subdir = "dir1";
+    struct glfs_object *parent = NULL, *leaf = NULL, *dir = NULL,
+                       *subdir = NULL, *subleaf = NULL;
+    struct stat sb;
+    int ret;
+
+    printf("glfs_h_unlink tests: In Progress\n");
+
+    /* Prepare tests */
+    parent = glfs_h_lookupat(fs, NULL, full_parent_name, &sb, 0);
+    if (parent == NULL) {
+        fprintf(stderr,
+                "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                full_parent_name, NULL, strerror(errno));
+        printf("glfs_h_lookupat tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    dir = glfs_h_mkdir(fs, parent, my_dir, 0755, &sb);
+    if (dir == NULL) {
+        fprintf(stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
+                my_dir, parent, strerror(errno));
+        printf("glfs_h_unlink tests: FAILED\n");
+        goto out;
+    }
+
+    leaf = glfs_h_creat(fs, dir, my_file, O_CREAT, 0644, &sb);
+    if (leaf == NULL) {
+        fprintf(stderr, "glfs_h_creat: error creating %s: from (%p),%s\n",
+                my_file, dir, strerror(errno));
+        printf("glfs_h_unlink tests: FAILED\n");
+        goto out;
+    }
+
+    subdir = glfs_h_mkdir(fs, dir, my_subdir, 0755, &sb);
+    if (subdir == NULL) {
+        fprintf(stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
+                my_subdir, dir, strerror(errno));
+        printf("glfs_h_unlink tests: FAILED\n");
+        goto out;
+    }
+
+    subleaf = glfs_h_creat(fs, subdir, my_file, O_CREAT, 0644, &sb);
+    if (subleaf == NULL) {
+        fprintf(stderr, "glfs_h_creat: error creating %s: from (%p),%s\n",
+                my_file, subdir, strerror(errno));
+        printf("glfs_h_unlink tests: FAILED\n");
+        goto out;
+    }
+
+    /* unlink non empty directory */
+    ret = glfs_h_unlink(fs, dir, my_subdir);
+    if ((ret && errno != ENOTEMPTY) || (ret == 0)) {
+        fprintf(stderr,
+                "glfs_h_unlink: error unlinking %s: it is non empty: %s\n",
+                my_subdir, strerror(errno));
+        printf("glfs_h_unlink tests: FAILED\n");
+        goto out;
+    }
+
+    /* unlink regular file */
+    ret = glfs_h_unlink(fs, subdir, my_file);
+    if (ret) {
+        fprintf(stderr, "glfs_h_unlink: error unlinking %s: from (%p),%s\n",
+                my_file, subdir, strerror(errno));
+        printf("glfs_h_unlink tests: FAILED\n");
+        goto out;
+    }
+
+    /* unlink directory */
+    ret = glfs_h_unlink(fs, dir, my_subdir);
+    if (ret) {
+        fprintf(stderr, "glfs_h_unlink: error unlinking %s: from (%p),%s\n",
+                my_subdir, dir, strerror(errno));
+        printf("glfs_h_unlink tests: FAILED\n");
+        goto out;
+    }
+
+    /* unlink regular file */
+    ret = glfs_h_unlink(fs, dir, my_file);
+    if (ret) {
+        fprintf(stderr, "glfs_h_unlink: error unlinking %s: from (%p),%s\n",
+                my_file, dir, strerror(errno));
+        printf("glfs_h_unlink tests: FAILED\n");
+        goto out;
+    }
+
+    /* unlink non-existent regular file */
+    ret = glfs_h_unlink(fs, dir, my_file);
+    if ((ret && errno != ENOENT) || (ret == 0)) {
+        fprintf(stderr,
+                "glfs_h_unlink: error unlinking non-existent %s: invalid errno "
+                ",%d, %s\n",
+                my_file, ret, strerror(errno));
+        printf("glfs_h_unlink tests: FAILED\n");
+        goto out;
+    }
+
+    /* unlink non-existent directory */
+    ret = glfs_h_unlink(fs, dir, my_subdir);
+    if ((ret && errno != ENOENT) || (ret == 0)) {
+        fprintf(stderr,
+                "glfs_h_unlink: error unlinking non-existent %s:  invalid "
+                "errno ,%d, %s\n",
+                my_subdir, ret, strerror(errno));
+        printf("glfs_h_unlink tests: FAILED\n");
+        goto out;
+    }
+
+    /* unlink directory */
+    ret = glfs_h_unlink(fs, parent, my_dir);
+    if (ret) {
+        fprintf(stderr, "glfs_h_unlink: error unlinking %s: from (%p),%s\n",
+                my_dir, dir, strerror(errno));
+        printf("glfs_h_unlink tests: FAILED\n");
+        goto out;
+    }
+
+    printf("glfs_h_unlink tests: PASSED\n");
 
 out:
-        if (dir)
-                glfs_h_close (dir);
-        if (leaf)
-                glfs_h_close (leaf);
-        if (subdir)
-                glfs_h_close (subdir);
-        if (subleaf)
-                glfs_h_close (subleaf);
-        if (parent)
-                glfs_h_close (parent);
-
-        return;
+    if (dir)
+        glfs_h_close(dir);
+    if (leaf)
+        glfs_h_close(leaf);
+    if (subdir)
+        glfs_h_close(subdir);
+    if (subleaf)
+        glfs_h_close(subleaf);
+    if (parent)
+        glfs_h_close(parent);
+
+    return;
 }
 
 void
-test_h_getsetattrs (void)
+test_h_getsetattrs(void)
 {
-        char               *my_dir = "attrdir";
-        char               *my_file = "attrfile.txt";
-        struct glfs_object *parent = NULL, *leaf = NULL, *dir = NULL;
-        struct stat         sb, retsb;
-        int                 ret, valid;
-        struct timespec     timestamp;
-
-        printf("glfs_h_getattrs and setattrs tests: In Progress\n");
-
-        /* Prepare tests */
-        parent = glfs_h_lookupat (fs, NULL, full_parent_name, &sb);
-        if (parent == NULL) {
-                fprintf (stderr, "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
-                         full_parent_name, NULL, strerror (errno));
-                printf ("glfs_h_lookupat tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        dir = glfs_h_mkdir (fs, parent, my_dir, 0644, &sb);
-        if (dir == NULL) {
-                fprintf (stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
-                         my_dir, parent, strerror (errno));
-                printf ("glfs_h_unlink tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        leaf = glfs_h_creat (fs, dir, my_file, O_CREAT, 0644, &sb);
-        if (leaf == NULL) {
-                fprintf (stderr, "glfs_h_creat: error creating %s: from (%p),%s\n",
-                         my_file, dir, strerror (errno));
-                printf ("glfs_h_unlink tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        ret = glfs_h_getattrs (fs, dir, &retsb);
-        if (ret != 0) {
-                fprintf (stderr, "glfs_h_getattrs: error %s: from (%p),%s\n",
-                         my_dir, dir, strerror (errno));
-                printf ("glfs_h_getattrs and setattrs tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&retsb);
-        /* TODO: Compare stat information */
-
-        retsb.st_mode = 00666;
-        retsb.st_uid = 1000;
-        retsb.st_gid = 1001;
-        ret = clock_gettime (CLOCK_REALTIME, &timestamp);
-        if(ret != 0) {
-                fprintf (stderr, "clock_gettime: error %s\n", strerror (errno));
-                printf ("glfs_h_getattrs and setattrs tests: FAILED\n");
-                goto out;
-        }
-        retsb.st_atim = timestamp;
-        retsb.st_mtim = timestamp;
-        valid = GFAPI_SET_ATTR_MODE | GFAPI_SET_ATTR_UID | GFAPI_SET_ATTR_GID |
-        GFAPI_SET_ATTR_ATIME | GFAPI_SET_ATTR_MTIME;
-        peek_stat (&retsb);
-
-        ret = glfs_h_setattrs (fs, dir, &retsb, valid);
-        if (ret != 0) {
-                fprintf (stderr, "glfs_h_setattrs: error %s: from (%p),%s\n",
-                         my_dir, dir, strerror (errno));
-                printf ("glfs_h_getattrs and setattrs tests: FAILED\n");
-                goto out;
-        }
-
-        memset(&retsb, 0, sizeof (struct stat));
-        ret = glfs_h_stat (fs, dir, &retsb);
-        if (ret != 0) {
-                fprintf (stderr, "glfs_h_stat: error %s: from (%p),%s\n",
-                         my_dir, dir, strerror (errno));
-                printf ("glfs_h_getattrs and setattrs tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&retsb);
-
-        printf ("glfs_h_getattrs and setattrs tests: PASSED\n");
+    char *my_dir = "attrdir";
+    char *my_file = "attrfile.txt";
+    struct glfs_object *parent = NULL, *leaf = NULL, *dir = NULL;
+    struct stat sb, retsb;
+    int ret, valid;
+    struct timespec timestamp;
+
+    printf("glfs_h_getattrs and setattrs tests: In Progress\n");
+
+    /* Prepare tests */
+    parent = glfs_h_lookupat(fs, NULL, full_parent_name, &sb, 0);
+    if (parent == NULL) {
+        fprintf(stderr,
+                "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                full_parent_name, NULL, strerror(errno));
+        printf("glfs_h_lookupat tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    dir = glfs_h_mkdir(fs, parent, my_dir, 0755, &sb);
+    if (dir == NULL) {
+        fprintf(stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
+                my_dir, parent, strerror(errno));
+        printf("glfs_h_unlink tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    leaf = glfs_h_creat(fs, dir, my_file, O_CREAT, 0644, &sb);
+    if (leaf == NULL) {
+        fprintf(stderr, "glfs_h_creat: error creating %s: from (%p),%s\n",
+                my_file, dir, strerror(errno));
+        printf("glfs_h_unlink tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    ret = glfs_h_getattrs(fs, dir, &retsb);
+    if (ret != 0) {
+        fprintf(stderr, "glfs_h_getattrs: error %s: from (%p),%s\n", my_dir,
+                dir, strerror(errno));
+        printf("glfs_h_getattrs and setattrs tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&retsb);
+    /* TODO: Compare stat information */
+
+    retsb.st_mode = 00666;
+    retsb.st_uid = 1000;
+    retsb.st_gid = 1001;
+    ret = clock_gettime(CLOCK_REALTIME, &timestamp);
+    if (ret != 0) {
+        fprintf(stderr, "clock_gettime: error %s\n", strerror(errno));
+        printf("glfs_h_getattrs and setattrs tests: FAILED\n");
+        goto out;
+    }
+    retsb.st_atim = timestamp;
+    retsb.st_mtim = timestamp;
+    valid = GFAPI_SET_ATTR_MODE | GFAPI_SET_ATTR_UID | GFAPI_SET_ATTR_GID |
+            GFAPI_SET_ATTR_ATIME | GFAPI_SET_ATTR_MTIME;
+    peek_stat(&retsb);
+
+    ret = glfs_h_setattrs(fs, dir, &retsb, valid);
+    if (ret != 0) {
+        fprintf(stderr, "glfs_h_setattrs: error %s: from (%p),%s\n", my_dir,
+                dir, strerror(errno));
+        printf("glfs_h_getattrs and setattrs tests: FAILED\n");
+        goto out;
+    }
+
+    memset(&retsb, 0, sizeof(struct stat));
+    ret = glfs_h_stat(fs, dir, &retsb);
+    if (ret != 0) {
+        fprintf(stderr, "glfs_h_stat: error %s: from (%p),%s\n", my_dir, dir,
+                strerror(errno));
+        printf("glfs_h_getattrs and setattrs tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&retsb);
+
+    printf("glfs_h_getattrs and setattrs tests: PASSED\n");
 out:
-        if (parent)
-                glfs_h_close (parent);
-        if (leaf)
-                glfs_h_close (leaf);
-        if (dir)
-                glfs_h_close (dir);
-
-        return;
+    if (parent)
+        glfs_h_close(parent);
+    if (leaf)
+        glfs_h_close(leaf);
+    if (dir)
+        glfs_h_close(dir);
+
+    return;
 }
 
 void
-test_h_truncate (void)
+test_h_truncate(void)
 {
-        char               *my_dir = "truncatedir";
-        char               *my_file = "file.txt";
-        struct glfs_object *root = NULL, *parent = NULL, *leaf = NULL;
-        struct stat         sb;
-        glfs_fd_t          *fd = NULL;
-        char                buf[32];
-        off_t               offset = 0;
-        int                 ret = 0;
-
-        printf("glfs_h_truncate tests: In Progress\n");
-
-        /* Prepare tests */
-        root = glfs_h_lookupat (fs, NULL, full_parent_name, &sb);
-        if (root == NULL) {
-                fprintf (stderr, "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
-                         full_parent_name, NULL, strerror (errno));
-                printf ("glfs_h_truncate tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        parent = glfs_h_mkdir (fs, root, my_dir, 0644, &sb);
-        if (parent == NULL) {
-                fprintf (stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
-                         my_dir, root, strerror (errno));
-                printf ("glfs_h_truncate tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        leaf = glfs_h_creat (fs, parent, my_file, O_CREAT, 0644, &sb);
-        if (leaf == NULL) {
-                fprintf (stderr, "glfs_h_creat: error creating %s: from (%p),%s\n",
-                         my_file, parent, strerror (errno));
-                printf ("glfs_h_truncate tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        fd = glfs_h_open (fs, leaf, O_RDWR);
-        if (fd == NULL) {
-                fprintf (stderr, "glfs_h_open: error on open of %s: %s\n",
-                         my_file, strerror (errno));
-                printf ("glfs_h_truncate tests: FAILED\n");
-                goto out;
-        }
-
-        memcpy (buf, "abcdefghijklmnopqrstuvwxyz012345", 32);
-        ret = glfs_write (fd, buf, 32, 0);
-
-        /* run tests */
-        /* truncate lower */
-        offset = 30;
-        ret = glfs_h_truncate (fs, leaf, offset);
-        if (ret != 0) {
-                fprintf (stderr, "glfs_h_truncate: error creating %s: from (%p),%s\n",
-                         my_file, parent, strerror (errno));
-                printf ("glfs_h_truncate tests: FAILED\n");
-                goto out;
-        }
-        ret = glfs_h_getattrs (fs, leaf, &sb);
-        if (ret != 0) {
-                fprintf (stderr, "glfs_h_getattrs: error for %s (%p),%s\n",
-                         my_file, leaf, strerror (errno));
-                printf ("glfs_h_truncate tests: FAILED\n");
-                goto out;
-        }
-        if (sb.st_size != offset) {
-                fprintf (stderr, "glfs_h_truncate: post size mismatch\n");
-                printf ("glfs_h_truncate tests: FAILED\n");
-                goto out;
-        }
-
-        /* truncate higher */
-        offset = 32;
-        ret = glfs_h_truncate (fs, leaf, offset);
-        if (ret != 0) {
-                fprintf (stderr, "glfs_h_truncate: error creating %s: from (%p),%s\n",
-                         my_file, parent, strerror (errno));
-                printf ("glfs_h_truncate tests: FAILED\n");
-                goto out;
-        }
-        ret = glfs_h_getattrs (fs, leaf, &sb);
-        if (ret != 0) {
-                fprintf (stderr, "glfs_h_getattrs: error for %s (%p),%s\n",
-                         my_file, leaf, strerror (errno));
-                printf ("glfs_h_truncate tests: FAILED\n");
-                goto out;
-        }
-        if (sb.st_size != offset) {
-                fprintf (stderr, "glfs_h_truncate: post size mismatch\n");
-                printf ("glfs_h_truncate tests: FAILED\n");
-                goto out;
-        }
-
-        /* truncate equal */
-        offset = 30;
-        ret = glfs_h_truncate (fs, leaf, offset);
-        if (ret != 0) {
-                fprintf (stderr, "glfs_h_truncate: error creating %s: from (%p),%s\n",
-                         my_file, parent, strerror (errno));
-                printf ("glfs_h_truncate tests: FAILED\n");
-                goto out;
-        }
-        ret = glfs_h_getattrs (fs, leaf, &sb);
-        if (ret != 0) {
-                fprintf (stderr, "glfs_h_getattrs: error for %s (%p),%s\n",
-                         my_file, leaf, strerror (errno));
-                printf ("glfs_h_truncate tests: FAILED\n");
-                goto out;
-        }
-        if (sb.st_size != offset) {
-                fprintf (stderr, "glfs_h_truncate: post size mismatch\n");
-                printf ("glfs_h_truncate tests: FAILED\n");
-                goto out;
-        }
-
-        printf ("glfs_h_truncate tests: PASSED\n");
+    char *my_dir = "truncatedir";
+    char *my_file = "file.txt";
+    struct glfs_object *root = NULL, *parent = NULL, *leaf = NULL;
+    struct stat sb;
+    glfs_fd_t *fd = NULL;
+    char buf[32];
+    off_t offset = 0;
+    int ret = 0;
+
+    printf("glfs_h_truncate tests: In Progress\n");
+
+    /* Prepare tests */
+    root = glfs_h_lookupat(fs, NULL, full_parent_name, &sb, 0);
+    if (root == NULL) {
+        fprintf(stderr,
+                "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                full_parent_name, NULL, strerror(errno));
+        printf("glfs_h_truncate tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    parent = glfs_h_mkdir(fs, root, my_dir, 0755, &sb);
+    if (parent == NULL) {
+        fprintf(stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
+                my_dir, root, strerror(errno));
+        printf("glfs_h_truncate tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    leaf = glfs_h_creat(fs, parent, my_file, O_CREAT, 0644, &sb);
+    if (leaf == NULL) {
+        fprintf(stderr, "glfs_h_creat: error creating %s: from (%p),%s\n",
+                my_file, parent, strerror(errno));
+        printf("glfs_h_truncate tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    fd = glfs_h_open(fs, leaf, O_RDWR);
+    if (fd == NULL) {
+        fprintf(stderr, "glfs_h_open: error on open of %s: %s\n", my_file,
+                strerror(errno));
+        printf("glfs_h_truncate tests: FAILED\n");
+        goto out;
+    }
+
+    memcpy(buf, "abcdefghijklmnopqrstuvwxyz012345", 32);
+    ret = glfs_write(fd, buf, 32, 0);
+
+    /* run tests */
+    /* truncate lower */
+    offset = 30;
+    ret = glfs_h_truncate(fs, leaf, offset);
+    if (ret != 0) {
+        fprintf(stderr, "glfs_h_truncate: error creating %s: from (%p),%s\n",
+                my_file, parent, strerror(errno));
+        printf("glfs_h_truncate tests: FAILED\n");
+        goto out;
+    }
+    ret = glfs_h_getattrs(fs, leaf, &sb);
+    if (ret != 0) {
+        fprintf(stderr, "glfs_h_getattrs: error for %s (%p),%s\n", my_file,
+                leaf, strerror(errno));
+        printf("glfs_h_truncate tests: FAILED\n");
+        goto out;
+    }
+    if (sb.st_size != offset) {
+        fprintf(stderr, "glfs_h_truncate: post size mismatch\n");
+        printf("glfs_h_truncate tests: FAILED\n");
+        goto out;
+    }
+
+    /* truncate higher */
+    offset = 32;
+    ret = glfs_h_truncate(fs, leaf, offset);
+    if (ret != 0) {
+        fprintf(stderr, "glfs_h_truncate: error creating %s: from (%p),%s\n",
+                my_file, parent, strerror(errno));
+        printf("glfs_h_truncate tests: FAILED\n");
+        goto out;
+    }
+    ret = glfs_h_getattrs(fs, leaf, &sb);
+    if (ret != 0) {
+        fprintf(stderr, "glfs_h_getattrs: error for %s (%p),%s\n", my_file,
+                leaf, strerror(errno));
+        printf("glfs_h_truncate tests: FAILED\n");
+        goto out;
+    }
+    if (sb.st_size != offset) {
+        fprintf(stderr, "glfs_h_truncate: post size mismatch\n");
+        printf("glfs_h_truncate tests: FAILED\n");
+        goto out;
+    }
+
+    /* truncate equal */
+    offset = 30;
+    ret = glfs_h_truncate(fs, leaf, offset);
+    if (ret != 0) {
+        fprintf(stderr, "glfs_h_truncate: error creating %s: from (%p),%s\n",
+                my_file, parent, strerror(errno));
+        printf("glfs_h_truncate tests: FAILED\n");
+        goto out;
+    }
+    ret = glfs_h_getattrs(fs, leaf, &sb);
+    if (ret != 0) {
+        fprintf(stderr, "glfs_h_getattrs: error for %s (%p),%s\n", my_file,
+                leaf, strerror(errno));
+        printf("glfs_h_truncate tests: FAILED\n");
+        goto out;
+    }
+    if (sb.st_size != offset) {
+        fprintf(stderr, "glfs_h_truncate: post size mismatch\n");
+        printf("glfs_h_truncate tests: FAILED\n");
+        goto out;
+    }
+
+    printf("glfs_h_truncate tests: PASSED\n");
 out:
-        if (fd)
-                glfs_close (fd);
-        if (root)
-                glfs_h_close (root);
-        if (parent)
-                glfs_h_close (parent);
-        if (leaf)
-                glfs_h_close (leaf);
-
-        return;
+    if (fd)
+        glfs_close(fd);
+    if (root)
+        glfs_h_close(root);
+    if (parent)
+        glfs_h_close(parent);
+    if (leaf)
+        glfs_h_close(leaf);
+
+    return;
 }
 
 void
-test_h_links (void)
+test_h_links(void)
 {
-        char               *my_dir = "linkdir";
-        char               *my_file = "file.txt";
-        char               *my_symlnk = "slnk.txt";
-        char               *my_lnk = "lnk.txt";
-        char               *linksrc_dir = "dir1";
-        char               *linktgt_dir = "dir2";
-        struct glfs_object *root = NULL, *parent = NULL, *leaf = NULL,
-                           *dirsrc = NULL, *dirtgt = NULL, *dleaf = NULL;
-        struct glfs_object *ln1 = NULL;
-        struct stat         sb;
-        int                 ret;
-        char               *buf = NULL;
-
-        printf("glfs_h_link(s) tests: In Progress\n");
-
-        /* Prepare tests */
-        root = glfs_h_lookupat (fs, NULL, full_parent_name, &sb);
-        if (root == NULL) {
-                fprintf (stderr, "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
-                         full_parent_name, NULL, strerror (errno));
-                printf ("glfs_h_link(s) tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        parent = glfs_h_mkdir (fs, root, my_dir, 0644, &sb);
-        if (parent == NULL) {
-                fprintf (stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
-                         my_dir, root, strerror (errno));
-                printf ("glfs_h_link(s) tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        leaf = glfs_h_creat (fs, parent, my_file, O_CREAT, 0644, &sb);
-        if (leaf == NULL) {
-                fprintf (stderr, "glfs_h_creat: error creating %s: from (%p),%s\n",
-                         my_file, parent, strerror (errno));
-                printf ("glfs_h_link(s) tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        dirsrc = glfs_h_mkdir (fs, parent, linksrc_dir, 0644, &sb);
-        if (dirsrc == NULL) {
-                fprintf (stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
-                         linksrc_dir, parent, strerror (errno));
-                printf ("glfs_h_link(s) tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        dirtgt = glfs_h_mkdir (fs, parent, linktgt_dir, 0644, &sb);
-        if (dirtgt == NULL) {
-                fprintf (stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
-                         linktgt_dir, parent, strerror (errno));
-                printf ("glfs_h_link(s) tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        dleaf = glfs_h_creat (fs, dirsrc, my_file, O_CREAT, 0644, &sb);
-        if (dleaf == NULL) {
-                fprintf (stderr, "glfs_h_creat: error creating %s: from (%p),%s\n",
-                         my_file, dirsrc, strerror (errno));
-                printf ("glfs_h_link(s) tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        /* run tests */
-        /* sym link: /testdir/linkdir/file.txt to ./slnk.txt */
-        ln1 = glfs_h_symlink (fs, parent, my_symlnk, "./file.txt", &sb);
-        if (ln1 == NULL) {
-                fprintf (stderr, "glfs_h_symlink: error creating %s: from (%p),%s\n",
-                         my_symlnk, parent, strerror (errno));
-                printf ("glfs_h_link(s) tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        buf = calloc (1024, sizeof(char));
-        if (buf == NULL) {
-                fprintf (stderr, "Error allocating memory\n");
-                printf ("glfs_h_link(s) tests: FAILED\n");
-                goto out;
-        }
-
-        ret = glfs_h_readlink (fs, ln1, buf, 1024);
-        if (ret <= 0) {
-                fprintf (stderr, "glfs_h_readlink: error reading %s: from (%p),%s\n",
-                         my_symlnk, ln1, strerror (errno));
-                printf ("glfs_h_link(s) tests: FAILED\n");
-                goto out;
-        }
-        if (!(strncmp (buf, my_symlnk, strlen (my_symlnk)))) {
-                fprintf (stderr, "glfs_h_readlink: error mismatch in link name: actual %s: retrieved %s\n",
-                         my_symlnk, buf);
-                printf ("glfs_h_link(s) tests: FAILED\n");
-                goto out;
-        }
-
-        /* link: /testdir/linkdir/file.txt to ./lnk.txt */
-        ret = glfs_h_link (fs, leaf, parent, my_lnk);
-        if (ret != 0) {
-                fprintf (stderr, "glfs_h_link: error creating %s: from (%p),%s\n",
-                         my_lnk, parent, strerror (errno));
-                printf ("glfs_h_link(s) tests: FAILED\n");
-                goto out;
-        }
-        /* TODO: Should write content to a file and read from the link */
-
-        /* link: /testdir/linkdir/dir1/file.txt to ../dir2/slnk.txt */
-        ret = glfs_h_link (fs, dleaf, dirtgt, my_lnk);
-        if (ret != 0) {
-                fprintf (stderr, "glfs_h_link: error creating %s: from (%p),%s\n",
-                         my_lnk, dirtgt, strerror (errno));
-                printf ("glfs_h_link(s) tests: FAILED\n");
-                goto out;
-        }
-        /* TODO: Should write content to a file and read from the link */
-
-        printf ("glfs_h_link(s) tests: PASSED\n");
+    char *my_dir = "linkdir";
+    char *my_file = "file.txt";
+    char *my_symlnk = "slnk.txt";
+    char *my_lnk = "lnk.txt";
+    char *linksrc_dir = "dir1";
+    char *linktgt_dir = "dir2";
+    struct glfs_object *root = NULL, *parent = NULL, *leaf = NULL,
+                       *dirsrc = NULL, *dirtgt = NULL, *dleaf = NULL;
+    struct glfs_object *ln1 = NULL;
+    struct stat sb;
+    int ret;
+    char *buf = NULL;
+
+    printf("glfs_h_link(s) tests: In Progress\n");
+
+    /* Prepare tests */
+    root = glfs_h_lookupat(fs, NULL, full_parent_name, &sb, 0);
+    if (root == NULL) {
+        fprintf(stderr,
+                "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                full_parent_name, NULL, strerror(errno));
+        printf("glfs_h_link(s) tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    parent = glfs_h_mkdir(fs, root, my_dir, 0755, &sb);
+    if (parent == NULL) {
+        fprintf(stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
+                my_dir, root, strerror(errno));
+        printf("glfs_h_link(s) tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    leaf = glfs_h_creat(fs, parent, my_file, O_CREAT, 0644, &sb);
+    if (leaf == NULL) {
+        fprintf(stderr, "glfs_h_creat: error creating %s: from (%p),%s\n",
+                my_file, parent, strerror(errno));
+        printf("glfs_h_link(s) tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    dirsrc = glfs_h_mkdir(fs, parent, linksrc_dir, 0755, &sb);
+    if (dirsrc == NULL) {
+        fprintf(stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
+                linksrc_dir, parent, strerror(errno));
+        printf("glfs_h_link(s) tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    dirtgt = glfs_h_mkdir(fs, parent, linktgt_dir, 0755, &sb);
+    if (dirtgt == NULL) {
+        fprintf(stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
+                linktgt_dir, parent, strerror(errno));
+        printf("glfs_h_link(s) tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    dleaf = glfs_h_creat(fs, dirsrc, my_file, O_CREAT, 0644, &sb);
+    if (dleaf == NULL) {
+        fprintf(stderr, "glfs_h_creat: error creating %s: from (%p),%s\n",
+                my_file, dirsrc, strerror(errno));
+        printf("glfs_h_link(s) tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    /* run tests */
+    /* sym link: /testdir/linkdir/file.txt to ./slnk.txt */
+    ln1 = glfs_h_symlink(fs, parent, my_symlnk, "./file.txt", &sb);
+    if (ln1 == NULL) {
+        fprintf(stderr, "glfs_h_symlink: error creating %s: from (%p),%s\n",
+                my_symlnk, parent, strerror(errno));
+        printf("glfs_h_link(s) tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    buf = calloc(1024, sizeof(char));
+    if (buf == NULL) {
+        fprintf(stderr, "Error allocating memory\n");
+        printf("glfs_h_link(s) tests: FAILED\n");
+        goto out;
+    }
+
+    ret = glfs_h_readlink(fs, ln1, buf, 1024);
+    if (ret <= 0) {
+        fprintf(stderr, "glfs_h_readlink: error reading %s: from (%p),%s\n",
+                my_symlnk, ln1, strerror(errno));
+        printf("glfs_h_link(s) tests: FAILED\n");
+        goto out;
+    }
+    if (!(strncmp(buf, my_symlnk, strlen(my_symlnk)))) {
+        fprintf(stderr,
+                "glfs_h_readlink: error mismatch in link name: actual %s: "
+                "retrieved %s\n",
+                my_symlnk, buf);
+        printf("glfs_h_link(s) tests: FAILED\n");
+        goto out;
+    }
+
+    /* link: /testdir/linkdir/file.txt to ./lnk.txt */
+    ret = glfs_h_link(fs, leaf, parent, my_lnk);
+    if (ret != 0) {
+        fprintf(stderr, "glfs_h_link: error creating %s: from (%p),%s\n",
+                my_lnk, parent, strerror(errno));
+        printf("glfs_h_link(s) tests: FAILED\n");
+        goto out;
+    }
+    /* TODO: Should write content to a file and read from the link */
+
+    /* link: /testdir/linkdir/dir1/file.txt to ../dir2/slnk.txt */
+    ret = glfs_h_link(fs, dleaf, dirtgt, my_lnk);
+    if (ret != 0) {
+        fprintf(stderr, "glfs_h_link: error creating %s: from (%p),%s\n",
+                my_lnk, dirtgt, strerror(errno));
+        printf("glfs_h_link(s) tests: FAILED\n");
+        goto out;
+    }
+    /* TODO: Should write content to a file and read from the link */
+
+    printf("glfs_h_link(s) tests: PASSED\n");
 
 out:
-        if (root)
-                glfs_h_close (root);
-        if (parent)
-                glfs_h_close (parent);
-        if (leaf)
-                glfs_h_close (leaf);
-        if (dirsrc)
-                glfs_h_close (dirsrc);
-        if (dirtgt)
-                glfs_h_close (dirtgt);
-        if (dleaf)
-                glfs_h_close (dleaf);
-        if (ln1)
-                glfs_h_close (ln1);
-        if (buf)
-                free (buf);
-
-        return;
+    if (root)
+        glfs_h_close(root);
+    if (parent)
+        glfs_h_close(parent);
+    if (leaf)
+        glfs_h_close(leaf);
+    if (dirsrc)
+        glfs_h_close(dirsrc);
+    if (dirtgt)
+        glfs_h_close(dirtgt);
+    if (dleaf)
+        glfs_h_close(dleaf);
+    if (ln1)
+        glfs_h_close(ln1);
+    if (buf)
+        free(buf);
+
+    return;
 }
 
 void
-test_h_rename (void)
+test_h_rename(void)
 {
-        char               *my_dir = "renamedir";
-        char               *my_file = "file.txt";
-        char               *src_dir = "dir1";
-        char               *tgt_dir = "dir2";
-        struct glfs_object *root = NULL, *parent = NULL, *leaf = NULL,
-                           *dirsrc = NULL, *dirtgt = NULL, *dleaf = NULL;
-        struct stat         sb;
-        int                 ret;
-
-        printf("glfs_h_rename tests: In Progress\n");
-
-        /* Prepare tests */
-        root = glfs_h_lookupat (fs, NULL, full_parent_name, &sb);
-        if (root == NULL) {
-                fprintf (stderr, "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
-                         full_parent_name, NULL, strerror (errno));
-                printf ("glfs_h_rename tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        parent = glfs_h_mkdir (fs, root, my_dir, 0644, &sb);
-        if (parent == NULL) {
-                fprintf (stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
-                         my_dir, root, strerror (errno));
-                printf ("glfs_h_rename tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        leaf = glfs_h_creat (fs, parent, my_file, O_CREAT, 0644, &sb);
-        if (leaf == NULL) {
-                fprintf (stderr, "glfs_h_creat: error creating %s: from (%p),%s\n",
-                         my_file, parent, strerror (errno));
-                printf ("glfs_h_rename tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        dirsrc = glfs_h_mkdir (fs, parent, src_dir, 0644, &sb);
-        if (dirsrc == NULL) {
-                fprintf (stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
-                         src_dir, parent, strerror (errno));
-                printf ("glfs_h_rename tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        dirtgt = glfs_h_mkdir (fs, parent, tgt_dir, 0644, &sb);
-        if (dirtgt == NULL) {
-                fprintf (stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
-                         tgt_dir, parent, strerror (errno));
-                printf ("glfs_h_rename tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        dleaf = glfs_h_creat (fs, dirsrc, my_file, O_CREAT, 0644, &sb);
-        if (dleaf == NULL) {
-                fprintf (stderr, "glfs_h_creat: error creating %s: from (%p),%s\n",
-                         my_file, dirsrc, strerror (errno));
-                printf ("glfs_h_rename tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        /* run tests */
-        /* Rename file.txt -> file1.txt */
-        ret = glfs_h_rename (fs, parent, "file.txt", parent, "file1.txt");
-        if (ret != 0) {
-                fprintf (stderr, "glfs_h_rename: error renaming %s to %s (%s)\n",
-                         "file.txt", "file1.txt", strerror (errno));
-                printf ("glfs_h_rename tests: FAILED\n");
-                goto out;
-        }
-
-        /* rename dir1/file.txt -> file.txt */
-        ret = glfs_h_rename (fs, dirsrc, "file.txt", parent, "file.txt");
-        if (ret != 0) {
-                fprintf (stderr, "glfs_h_rename: error renaming %s/%s to %s (%s)\n",
-                         src_dir, "file.txt", "file.txt", strerror (errno));
-                printf ("glfs_h_rename tests: FAILED\n");
-                goto out;
-        }
-
-        /* rename file1.txt -> file.txt (exists) */
-        ret = glfs_h_rename (fs, parent, "file1.txt", parent, "file.txt");
-        if (ret != 0) {
-                fprintf (stderr, "glfs_h_rename: error renaming %s to %s (%s)\n",
-                         "file.txt", "file.txt", strerror (errno));
-                printf ("glfs_h_rename tests: FAILED\n");
-                goto out;
-        }
-
-        /* rename dir1 -> dir3 */
-        ret = glfs_h_rename (fs, parent, "dir1", parent, "dir3");
-        if (ret != 0) {
-                fprintf (stderr, "glfs_h_rename: error renaming %s to %s (%s)\n",
-                         "dir1", "dir3", strerror (errno));
-                printf ("glfs_h_rename tests: FAILED\n");
-                goto out;
-        }
-
-        /* rename dir2 ->dir3 (exists) */
-        ret = glfs_h_rename (fs, parent, "dir2", parent, "dir3");
-        if (ret != 0) {
-                fprintf (stderr, "glfs_h_rename: error renaming %s to %s (%s)\n",
-                         "dir2", "dir3", strerror (errno));
-                printf ("glfs_h_rename tests: FAILED\n");
-                goto out;
-        }
-
-        /* rename file.txt -> dir3 (fail) */
-        ret = glfs_h_rename (fs, parent, "file.txt", parent, "dir3");
-        if (ret == 0) {
-                fprintf (stderr, "glfs_h_rename: NO error renaming %s to %s (%s)\n",
-                         "file.txt", "dir3", strerror (errno));
-                printf ("glfs_h_rename tests: FAILED\n");
-                goto out;
-        }
-
-        /* rename dir3 -> file.txt (fail) */
-        ret = glfs_h_rename (fs, parent, "dir3", parent, "file.txt");
-        if (ret == 0) {
-                fprintf (stderr, "glfs_h_rename: NO error renaming %s to %s (%s)\n",
-                         "dir3", "file.txt", strerror (errno));
-                printf ("glfs_h_rename tests: FAILED\n");
-                goto out;
-        }
-
-        printf ("glfs_h_rename tests: PASSED\n");
+    char *my_dir = "renamedir";
+    char *my_file = "file.txt";
+    char *src_dir = "dir1";
+    char *tgt_dir = "dir2";
+    struct glfs_object *root = NULL, *parent = NULL, *leaf = NULL,
+                       *dirsrc = NULL, *dirtgt = NULL, *dleaf = NULL;
+    struct stat sb;
+    int ret;
+
+    printf("glfs_h_rename tests: In Progress\n");
+
+    /* Prepare tests */
+    root = glfs_h_lookupat(fs, NULL, full_parent_name, &sb, 0);
+    if (root == NULL) {
+        fprintf(stderr,
+                "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                full_parent_name, NULL, strerror(errno));
+        printf("glfs_h_rename tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    parent = glfs_h_mkdir(fs, root, my_dir, 0755, &sb);
+    if (parent == NULL) {
+        fprintf(stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
+                my_dir, root, strerror(errno));
+        printf("glfs_h_rename tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    leaf = glfs_h_creat(fs, parent, my_file, O_CREAT, 0644, &sb);
+    if (leaf == NULL) {
+        fprintf(stderr, "glfs_h_creat: error creating %s: from (%p),%s\n",
+                my_file, parent, strerror(errno));
+        printf("glfs_h_rename tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    dirsrc = glfs_h_mkdir(fs, parent, src_dir, 0755, &sb);
+    if (dirsrc == NULL) {
+        fprintf(stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
+                src_dir, parent, strerror(errno));
+        printf("glfs_h_rename tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    dirtgt = glfs_h_mkdir(fs, parent, tgt_dir, 0755, &sb);
+    if (dirtgt == NULL) {
+        fprintf(stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
+                tgt_dir, parent, strerror(errno));
+        printf("glfs_h_rename tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    dleaf = glfs_h_creat(fs, dirsrc, my_file, O_CREAT, 0644, &sb);
+    if (dleaf == NULL) {
+        fprintf(stderr, "glfs_h_creat: error creating %s: from (%p),%s\n",
+                my_file, dirsrc, strerror(errno));
+        printf("glfs_h_rename tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    /* run tests */
+    /* Rename file.txt -> file1.txt */
+    ret = glfs_h_rename(fs, parent, "file.txt", parent, "file1.txt");
+    if (ret != 0) {
+        fprintf(stderr, "glfs_h_rename: error renaming %s to %s (%s)\n",
+                "file.txt", "file1.txt", strerror(errno));
+        printf("glfs_h_rename tests: FAILED\n");
+        goto out;
+    }
+
+    /* rename dir1/file.txt -> file.txt */
+    ret = glfs_h_rename(fs, dirsrc, "file.txt", parent, "file.txt");
+    if (ret != 0) {
+        fprintf(stderr, "glfs_h_rename: error renaming %s/%s to %s (%s)\n",
+                src_dir, "file.txt", "file.txt", strerror(errno));
+        printf("glfs_h_rename tests: FAILED\n");
+        goto out;
+    }
+
+    /* rename file1.txt -> file.txt (exists) */
+    ret = glfs_h_rename(fs, parent, "file1.txt", parent, "file.txt");
+    if (ret != 0) {
+        fprintf(stderr, "glfs_h_rename: error renaming %s to %s (%s)\n",
+                "file.txt", "file.txt", strerror(errno));
+        printf("glfs_h_rename tests: FAILED\n");
+        goto out;
+    }
+
+    /* rename dir1 -> dir3 */
+    ret = glfs_h_rename(fs, parent, "dir1", parent, "dir3");
+    if (ret != 0) {
+        fprintf(stderr, "glfs_h_rename: error renaming %s to %s (%s)\n", "dir1",
+                "dir3", strerror(errno));
+        printf("glfs_h_rename tests: FAILED\n");
+        goto out;
+    }
+
+    /* rename dir2 ->dir3 (exists) */
+    ret = glfs_h_rename(fs, parent, "dir2", parent, "dir3");
+    if (ret != 0) {
+        fprintf(stderr, "glfs_h_rename: error renaming %s to %s (%s)\n", "dir2",
+                "dir3", strerror(errno));
+        printf("glfs_h_rename tests: FAILED\n");
+        goto out;
+    }
+
+    /* rename file.txt -> dir3 (fail) */
+    ret = glfs_h_rename(fs, parent, "file.txt", parent, "dir3");
+    if (ret == 0) {
+        fprintf(stderr, "glfs_h_rename: NO error renaming %s to %s (%s)\n",
+                "file.txt", "dir3", strerror(errno));
+        printf("glfs_h_rename tests: FAILED\n");
+        goto out;
+    }
+
+    /* rename dir3 -> file.txt (fail) */
+    ret = glfs_h_rename(fs, parent, "dir3", parent, "file.txt");
+    if (ret == 0) {
+        fprintf(stderr, "glfs_h_rename: NO error renaming %s to %s (%s)\n",
+                "dir3", "file.txt", strerror(errno));
+        printf("glfs_h_rename tests: FAILED\n");
+        goto out;
+    }
+
+    printf("glfs_h_rename tests: PASSED\n");
 
 out:
-        if (root)
-                glfs_h_close (root);
-        if (parent)
-                glfs_h_close (parent);
-        if (leaf)
-                glfs_h_close (leaf);
-        if (dirsrc)
-                glfs_h_close (dirsrc);
-        if (dirtgt)
-                glfs_h_close (dirtgt);
-        if (dleaf)
-                glfs_h_close (dleaf);
-
-        return;
+    if (root)
+        glfs_h_close(root);
+    if (parent)
+        glfs_h_close(parent);
+    if (leaf)
+        glfs_h_close(leaf);
+    if (dirsrc)
+        glfs_h_close(dirsrc);
+    if (dirtgt)
+        glfs_h_close(dirtgt);
+    if (dleaf)
+        glfs_h_close(dleaf);
+
+    return;
 }
 
 void
-assimilatetime (struct timespec *ts, struct timespec ts_st,
-                struct timespec ts_ed)
+assimilatetime(struct timespec *ts, struct timespec ts_st,
+               struct timespec ts_ed)
 {
-        if ((ts_ed.tv_nsec - ts_st.tv_nsec) < 0) {
-                ts->tv_sec += ts_ed.tv_sec - ts_st.tv_sec - 1;
-                ts->tv_nsec += 1000000000 + ts_ed.tv_nsec - ts_st.tv_nsec;
-        } else {
-                ts->tv_sec += ts_ed.tv_sec - ts_st.tv_sec;
-                ts->tv_nsec += ts_ed.tv_nsec - ts_st.tv_nsec;
-        }
-
-        if (ts->tv_nsec > 1000000000) {
-                ts->tv_nsec = ts->tv_nsec - 1000000000;
-                ts->tv_sec += 1;
-        }
-
-        return;
+    if ((ts_ed.tv_nsec - ts_st.tv_nsec) < 0) {
+        ts->tv_sec += ts_ed.tv_sec - ts_st.tv_sec - 1;
+        ts->tv_nsec += 1000000000 + ts_ed.tv_nsec - ts_st.tv_nsec;
+    } else {
+        ts->tv_sec += ts_ed.tv_sec - ts_st.tv_sec;
+        ts->tv_nsec += ts_ed.tv_nsec - ts_st.tv_nsec;
+    }
+
+    if (ts->tv_nsec > 1000000000) {
+        ts->tv_nsec = ts->tv_nsec - 1000000000;
+        ts->tv_sec += 1;
+    }
+
+    return;
 }
 
 #define MAX_FILES_CREATE 10
-#define MAXPATHNAME      512
+#define MAXPATHNAME 512
 void
-test_h_performance (void)
+test_h_performance(void)
 {
-        char               *my_dir = "perftest",
-                           *full_dir_path="/testdir/perftest";
-        char               *my_file = "file_", my_file_name[MAXPATHNAME];
-        struct glfs_object *parent = NULL, *leaf = NULL, *dir = NULL;
-        struct stat         sb;
-        int                 ret, i;
-        struct glfs_fd     *fd;
-        struct timespec     c_ts = {0, 0}, c_ts_st, c_ts_ed;
-        struct timespec     o_ts = {0, 0}, o_ts_st, o_ts_ed;
-
-        printf("glfs_h_performance tests: In Progress\n");
-
-        /* Prepare tests */
-        parent = glfs_h_lookupat (fs, NULL, full_parent_name, &sb);
-        if (parent == NULL) {
-                fprintf (stderr, "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
-                         full_parent_name, NULL, strerror (errno));
-                printf ("glfs_h_performance tests: FAILED\n");
-                goto out;
-        }
-
-        dir = glfs_h_mkdir (fs, parent, my_dir, 0644, &sb);
-        if (dir == NULL) {
-                fprintf (stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
-                         my_dir, parent, strerror (errno));
-                printf ("glfs_h_performance tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        /* create performance */
-        ret = clock_gettime (CLOCK_REALTIME, &o_ts_st);
-        if(ret != 0) {
-                fprintf (stderr, "clock_gettime: error %s\n", strerror (errno));
-                printf ("glfs_h_getattrs and setattrs tests: FAILED\n");
-                goto out;
+    char *my_dir = "perftest", *full_dir_path = "/testdir/perftest";
+    char *my_file = "file_", my_file_name[MAXPATHNAME];
+    struct glfs_object *parent = NULL, *leaf = NULL, *dir = NULL;
+    struct stat sb;
+    int ret, i;
+    struct glfs_fd *fd;
+    struct timespec c_ts = {0, 0}, c_ts_st, c_ts_ed;
+    struct timespec o_ts = {0, 0}, o_ts_st, o_ts_ed;
+
+    printf("glfs_h_performance tests: In Progress\n");
+
+    /* Prepare tests */
+    parent = glfs_h_lookupat(fs, NULL, full_parent_name, &sb, 0);
+    if (parent == NULL) {
+        fprintf(stderr,
+                "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                full_parent_name, NULL, strerror(errno));
+        printf("glfs_h_performance tests: FAILED\n");
+        goto out;
+    }
+
+    dir = glfs_h_mkdir(fs, parent, my_dir, 0755, &sb);
+    if (dir == NULL) {
+        fprintf(stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
+                my_dir, parent, strerror(errno));
+        printf("glfs_h_performance tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    /* create performance */
+    ret = clock_gettime(CLOCK_REALTIME, &o_ts_st);
+    if (ret != 0) {
+        fprintf(stderr, "clock_gettime: error %s\n", strerror(errno));
+        printf("glfs_h_getattrs and setattrs tests: FAILED\n");
+        goto out;
+    }
+
+    for (i = 0; i < MAX_FILES_CREATE; i++) {
+        sprintf(my_file_name, "%s%d", my_file, i);
+
+        ret = clock_gettime(CLOCK_REALTIME, &c_ts_st);
+        if (ret != 0) {
+            fprintf(stderr, "clock_gettime: error %s\n", strerror(errno));
+            printf("glfs_h_getattrs and setattrs tests: FAILED\n");
+            goto out;
         }
 
-        for (i = 0; i < MAX_FILES_CREATE; i++) {
-                sprintf (my_file_name, "%s%d", my_file, i);
-
-                ret = clock_gettime (CLOCK_REALTIME, &c_ts_st);
-                if(ret != 0) {
-                        fprintf (stderr, "clock_gettime: error %s\n",
-                                 strerror (errno));
-                        printf ("glfs_h_getattrs and setattrs tests: FAILED\n");
-                        goto out;
-                }
-
-                leaf = glfs_h_lookupat (fs, dir, my_file_name, &sb);
-                if (leaf != NULL) {
-                        fprintf (stderr, "glfs_h_lookup: exists %s\n",
-                                 my_file_name);
-                        printf ("glfs_h_performance tests: FAILED\n");
-                        goto out;
-                }
-
-                leaf = glfs_h_creat (fs, dir, my_file_name, O_CREAT, 0644, &sb);
-                if (leaf == NULL) {
-                        fprintf (stderr, "glfs_h_creat: error creating %s: from (%p),%s\n",
-                                 my_file, dir, strerror (errno));
-                        printf ("glfs_h_performance tests: FAILED\n");
-                        goto out;
-                }
-
-                ret = clock_gettime (CLOCK_REALTIME, &c_ts_ed);
-                if(ret != 0) {
-                        fprintf (stderr, "clock_gettime: error %s\n",
-                                 strerror (errno));
-                        printf ("glfs_h_getattrs and setattrs tests: FAILED\n");
-                        goto out;
-                }
-
-                assimilatetime (&c_ts, c_ts_st, c_ts_ed);
-                glfs_h_close (leaf); leaf = NULL;
+        leaf = glfs_h_lookupat(fs, dir, my_file_name, &sb, 0);
+        if (leaf != NULL) {
+            fprintf(stderr, "glfs_h_lookup: exists %s\n", my_file_name);
+            printf("glfs_h_performance tests: FAILED\n");
+            goto out;
         }
 
-        ret = clock_gettime (CLOCK_REALTIME, &o_ts_ed);
-        if(ret != 0) {
-                fprintf (stderr, "clock_gettime: error %s\n", strerror (errno));
-                printf ("glfs_h_getattrs and setattrs tests: FAILED\n");
-                goto out;
+        leaf = glfs_h_creat(fs, dir, my_file_name, O_CREAT, 0644, &sb);
+        if (leaf == NULL) {
+            fprintf(stderr, "glfs_h_creat: error creating %s: from (%p),%s\n",
+                    my_file, dir, strerror(errno));
+            printf("glfs_h_performance tests: FAILED\n");
+            goto out;
         }
 
-        assimilatetime (&o_ts, o_ts_st, o_ts_ed);
-
-        printf ("Creation performance (handle based):\n\t# empty files:%d\n",
-                MAX_FILES_CREATE);
-        printf ("\tOverall time:\n\t\tSecs:%ld\n\t\tnSecs:%ld\n",
-                o_ts.tv_sec, o_ts.tv_nsec);
-        printf ("\tcreate call time time:\n\t\tSecs:%ld\n\t\tnSecs:%ld\n",
-                c_ts.tv_sec, c_ts.tv_nsec);
-
-        /* create using path */
-        c_ts.tv_sec = o_ts.tv_sec = 0;
-        c_ts.tv_nsec = o_ts.tv_nsec = 0;
-
-        sprintf (my_file_name, "%s1", full_dir_path);
-        ret = glfs_mkdir (fs, my_file_name, 0644);
+        ret = clock_gettime(CLOCK_REALTIME, &c_ts_ed);
         if (ret != 0) {
-                fprintf (stderr, "glfs_mkdir: error creating %s: from (%p),%s\n",
-                         my_dir, parent, strerror (errno));
-                printf ("glfs_h_performance tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        ret = clock_gettime (CLOCK_REALTIME, &o_ts_st);
-        if(ret != 0) {
-                fprintf (stderr, "clock_gettime: error %s\n", strerror (errno));
-                printf ("glfs_h_getattrs and setattrs tests: FAILED\n");
-                goto out;
+            fprintf(stderr, "clock_gettime: error %s\n", strerror(errno));
+            printf("glfs_h_getattrs and setattrs tests: FAILED\n");
+            goto out;
+        }
+
+        assimilatetime(&c_ts, c_ts_st, c_ts_ed);
+        glfs_h_close(leaf);
+        leaf = NULL;
+    }
+
+    ret = clock_gettime(CLOCK_REALTIME, &o_ts_ed);
+    if (ret != 0) {
+        fprintf(stderr, "clock_gettime: error %s\n", strerror(errno));
+        printf("glfs_h_getattrs and setattrs tests: FAILED\n");
+        goto out;
+    }
+
+    assimilatetime(&o_ts, o_ts_st, o_ts_ed);
+
+    printf("Creation performance (handle based):\n\t# empty files:%d\n",
+           MAX_FILES_CREATE);
+    printf("\tOverall time:\n\t\tSecs:%ld\n\t\tnSecs:%ld\n", o_ts.tv_sec,
+           o_ts.tv_nsec);
+    printf("\tcreate call time time:\n\t\tSecs:%ld\n\t\tnSecs:%ld\n",
+           c_ts.tv_sec, c_ts.tv_nsec);
+
+    /* create using path */
+    c_ts.tv_sec = o_ts.tv_sec = 0;
+    c_ts.tv_nsec = o_ts.tv_nsec = 0;
+
+    sprintf(my_file_name, "%s1", full_dir_path);
+    ret = glfs_mkdir(fs, my_file_name, 0755);
+    if (ret != 0) {
+        fprintf(stderr, "glfs_mkdir: error creating %s: from (%p),%s\n", my_dir,
+                parent, strerror(errno));
+        printf("glfs_h_performance tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    ret = clock_gettime(CLOCK_REALTIME, &o_ts_st);
+    if (ret != 0) {
+        fprintf(stderr, "clock_gettime: error %s\n", strerror(errno));
+        printf("glfs_h_getattrs and setattrs tests: FAILED\n");
+        goto out;
+    }
+
+    for (i = 0; i < MAX_FILES_CREATE; i++) {
+        sprintf(my_file_name, "%s1/%sn%d", full_dir_path, my_file, i);
+
+        ret = clock_gettime(CLOCK_REALTIME, &c_ts_st);
+        if (ret != 0) {
+            fprintf(stderr, "clock_gettime: error %s\n", strerror(errno));
+            printf("glfs_h_getattrs and setattrs tests: FAILED\n");
+            goto out;
         }
 
-        for (i = 0; i < MAX_FILES_CREATE; i++) {
-                sprintf (my_file_name, "%s1/%sn%d", full_dir_path, my_file, i);
-
-                ret = clock_gettime (CLOCK_REALTIME, &c_ts_st);
-                if(ret != 0) {
-                        fprintf (stderr, "clock_gettime: error %s\n",
-                                 strerror (errno));
-                        printf ("glfs_h_getattrs and setattrs tests: FAILED\n");
-                        goto out;
-                }
-
-                ret = glfs_stat (fs, my_file_name, &sb);
-                if (ret == 0) {
-                        fprintf (stderr, "glfs_stat: exists %s\n",
-                                 my_file_name);
-                        printf ("glfs_h_performance tests: FAILED\n");
-                        goto out;
-                }
-
-                fd = glfs_creat (fs, my_file_name, O_CREAT, 0644);
-                if (fd == NULL) {
-                        fprintf (stderr, "glfs_creat: error creating %s: from (%p),%s\n",
-                                 my_file, dir, strerror (errno));
-                        printf ("glfs_h_performance tests: FAILED\n");
-                        goto out;
-                }
-
-                ret = clock_gettime (CLOCK_REALTIME, &c_ts_ed);
-                if(ret != 0) {
-                        fprintf (stderr, "clock_gettime: error %s\n",
-                                 strerror (errno));
-                        printf ("glfs_h_getattrs and setattrs tests: FAILED\n");
-                        goto out;
-                }
-
-                assimilatetime (&c_ts, c_ts_st, c_ts_ed);
-                glfs_close (fd);
+        ret = glfs_stat(fs, my_file_name, &sb);
+        if (ret == 0) {
+            fprintf(stderr, "glfs_stat: exists %s\n", my_file_name);
+            printf("glfs_h_performance tests: FAILED\n");
+            goto out;
         }
 
-        ret = clock_gettime (CLOCK_REALTIME, &o_ts_ed);
-        if(ret != 0) {
-                fprintf (stderr, "clock_gettime: error %s\n", strerror (errno));
-                printf ("glfs_h_getattrs and setattrs tests: FAILED\n");
-                goto out;
+        fd = glfs_creat(fs, my_file_name, O_CREAT, 0644);
+        if (fd == NULL) {
+            fprintf(stderr, "glfs_creat: error creating %s: from (%p),%s\n",
+                    my_file, dir, strerror(errno));
+            printf("glfs_h_performance tests: FAILED\n");
+            goto out;
         }
 
-        assimilatetime (&o_ts, o_ts_st, o_ts_ed);
-
-        printf ("Creation performance (path based):\n\t# empty files:%d\n",
-                MAX_FILES_CREATE);
-        printf ("\tOverall time:\n\t\tSecs:%ld\n\t\tnSecs:%ld\n",
-                o_ts.tv_sec, o_ts.tv_nsec);
-        printf ("\tcreate call time time:\n\t\tSecs:%ld\n\t\tnSecs:%ld\n",
-                c_ts.tv_sec, c_ts.tv_nsec);
+        ret = clock_gettime(CLOCK_REALTIME, &c_ts_ed);
+        if (ret != 0) {
+            fprintf(stderr, "clock_gettime: error %s\n", strerror(errno));
+            printf("glfs_h_getattrs and setattrs tests: FAILED\n");
+            goto out;
+        }
+
+        assimilatetime(&c_ts, c_ts_st, c_ts_ed);
+        glfs_close(fd);
+    }
+
+    ret = clock_gettime(CLOCK_REALTIME, &o_ts_ed);
+    if (ret != 0) {
+        fprintf(stderr, "clock_gettime: error %s\n", strerror(errno));
+        printf("glfs_h_getattrs and setattrs tests: FAILED\n");
+        goto out;
+    }
+
+    assimilatetime(&o_ts, o_ts_st, o_ts_ed);
+
+    printf("Creation performance (path based):\n\t# empty files:%d\n",
+           MAX_FILES_CREATE);
+    printf("\tOverall time:\n\t\tSecs:%ld\n\t\tnSecs:%ld\n", o_ts.tv_sec,
+           o_ts.tv_nsec);
+    printf("\tcreate call time time:\n\t\tSecs:%ld\n\t\tnSecs:%ld\n",
+           c_ts.tv_sec, c_ts.tv_nsec);
 out:
-        return;
+    return;
 }
 
 int
-test_handleops (int argc, char *argv[])
+test_handleops(int argc, char *argv[])
 {
-        int                 ret = 0;
-        glfs_fd_t          *fd = NULL;
-        struct stat         sb = {0, };
-        struct glfs_object *root = NULL, *parent = NULL, *leaf = NULL,
-                           *tmp = NULL;
-        char                readbuf[32], writebuf[32];
-        unsigned char       leaf_handle[GFAPI_HANDLE_LENGTH];
-
-        char *full_leaf_name = "/testdir/testfile.txt",
-             *leaf_name = "testfile.txt",
-             *relative_leaf_name = "testdir/testfile.txt";
-        char *leaf_name1 = "testfile1.txt";
-        char *full_newparent_name = "/testdir/dir1",
-             *newparent_name = "dir1";
-        char *full_newnod_name = "/testdir/nod1",
-             *newnod_name = "nod1";
-
-        /* Initialize test area */
-        ret = glfs_mkdir (fs, full_parent_name, 0644);
-        if (ret != 0 && errno != EEXIST) {
-                fprintf (stderr, "%s: (%p) %s\n", full_parent_name, fd,
-                        strerror (errno));
-                printf ("Test initialization failed on volume %s\n", argv[1]);
-                goto out;
-        }
-        else if (ret != 0) {
-                printf ("Found test directory %s to be existing\n",
-                        full_parent_name);
-                printf ("Cleanup test directory and restart tests\n");
-                goto out;
-        }
-
-        fd = glfs_creat (fs, full_leaf_name, O_CREAT, 0644);
-        if (fd == NULL) {
-                fprintf (stderr, "%s: (%p) %s\n", full_leaf_name, fd,
-                        strerror (errno));
-                printf ("Test initialization failed on volume %s\n", argv[1]);
-                goto out;
-        }
-        glfs_close (fd);
-
-        printf ("Initialized the test area, within volume %s\n", argv[1]);
-
-        /* Handle based APIs test area */
-
-        /* glfs_lookupat test */
-        printf ("glfs_h_lookupat tests: In Progress\n");
-        /* start at root of the volume */
-        root = glfs_h_lookupat (fs, NULL, "/", &sb);
-        if (root == NULL) {
-                fprintf (stderr, "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
-                         "/", NULL, strerror (errno));
-                printf ("glfs_h_lookupat tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        /* lookup a parent within root */
-        parent = glfs_h_lookupat (fs, root, parent_name, &sb);
-        if (parent == NULL) {
-                fprintf (stderr, "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
-                         parent_name, root, strerror (errno));
-                printf ("glfs_h_lookupat tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        /* lookup a leaf/child within the parent */
-        leaf = glfs_h_lookupat (fs, parent, leaf_name, &sb);
-        if (leaf == NULL) {
-                fprintf (stderr, "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
-                         leaf_name, parent, strerror (errno));
-                printf ("glfs_h_lookupat tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        /* reset */
-        glfs_h_close (root); root = NULL;
-        glfs_h_close (leaf); leaf = NULL;
-        glfs_h_close (parent); parent = NULL;
-
-        /* check absolute paths */
-        root = glfs_h_lookupat (fs, NULL, "/", &sb);
-        if (root == NULL) {
-                fprintf (stderr, "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
-                         "/", NULL, strerror (errno));
-                printf ("glfs_h_lookupat tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        parent = glfs_h_lookupat (fs, NULL, full_parent_name, &sb);
-        if (parent == NULL) {
-                fprintf (stderr, "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
-                         full_parent_name, root, strerror (errno));
-                printf ("glfs_h_lookupat tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        leaf = glfs_h_lookupat (fs, NULL, full_leaf_name, &sb);
-        if (leaf == NULL) {
-                fprintf (stderr, "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
-                         full_leaf_name, parent, strerror (errno));
-                printf ("glfs_h_lookupat tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        /* reset */
-        glfs_h_close (leaf); leaf = NULL;
-
-        /* check multiple component paths */
-        leaf = glfs_h_lookupat (fs, root, relative_leaf_name, &sb);
-        if (leaf == NULL) {
-                fprintf (stderr, "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
-                         relative_leaf_name, parent, strerror (errno));
-                goto out;
-        }
-        peek_stat (&sb);
-
-        /* reset */
-        glfs_h_close (root); root = NULL;
-        glfs_h_close (parent); parent = NULL;
-
-        /* check symlinks in path */
-
-        /* TODO: -ve test cases */
-        /* parent invalid
-        * path invalid
-        * path does not exist after some components
-        * no parent, but relative path
-        * parent and full path? -ve?
-        */
-
-        printf ("glfs_h_lookupat tests: PASSED\n");
-
-        /* glfs_openat test */
-        printf ("glfs_h_open tests: In Progress\n");
-        fd = glfs_h_open (fs, leaf, O_RDWR);
-        if (fd == NULL) {
-                fprintf (stderr, "glfs_h_open: error on open of %s: %s\n",
-                         full_leaf_name, strerror (errno));
-                printf ("glfs_h_open tests: FAILED\n");
-                goto out;
-        }
-
-        /* test read/write based on fd */
-        memcpy (writebuf, "abcdefghijklmnopqrstuvwxyz012345", 32);
-        ret = glfs_write (fd, writebuf, 32, 0);
-
-        glfs_lseek (fd, 0, SEEK_SET);
-
-        ret = glfs_read (fd, readbuf, 32, 0);
-        if (memcmp (readbuf, writebuf, 32)) {
-                printf ("Failed to read what I wrote: %s %s\n", readbuf,
-                        writebuf);
-                glfs_close (fd);
-                printf ("glfs_h_open tests: FAILED\n");
-                goto out;
-        }
-
-        glfs_h_close (leaf); leaf = NULL;
-        glfs_close (fd);
-
-        printf ("glfs_h_open tests: PASSED\n");
-
-        /* Create tests */
-        printf ("glfs_h_creat tests: In Progress\n");
-        parent = glfs_h_lookupat (fs, NULL, full_parent_name, &sb);
-        if (parent == NULL) {
-                fprintf (stderr, "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
-                         full_parent_name, root, strerror (errno));
-                printf ("glfs_h_creat tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        leaf = glfs_h_creat (fs, parent, leaf_name1, O_CREAT, 0644, &sb);
-        if (leaf == NULL) {
-                fprintf (stderr, "glfs_h_creat: error on create of %s: from (%p),%s\n",
-                         leaf_name1, parent, strerror (errno));
-                printf ("glfs_h_creat tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        glfs_h_close (leaf); leaf = NULL;
-
-        leaf = glfs_h_creat (fs, parent, leaf_name1, O_CREAT | O_EXCL, 0644,
-                            &sb);
-        if (leaf != NULL || errno != EEXIST) {
-                fprintf (stderr, "glfs_h_creat: existing file, leaf = (%p), errno = %s\n",
-                        leaf, strerror (errno));
-                printf ("glfs_h_creat tests: FAILED\n");
-                if (leaf != NULL) {
-                        glfs_h_close (leaf); leaf = NULL;
-                }
-        }
-
-        tmp = glfs_h_creat (fs, root, parent_name, O_CREAT, 0644, &sb);
-        if (tmp != NULL || !(errno == EISDIR || errno == EINVAL)) {
-                fprintf (stderr, "glfs_h_creat: dir create, tmp = (%p), errno = %s\n",
-                        leaf, strerror (errno));
-                printf ("glfs_h_creat tests: FAILED\n");
-                if (tmp != NULL) {
-                        glfs_h_close (tmp); tmp = NULL;
-                }
-        }
-
-        /* TODO: Other combinations and -ve cases as applicable */
-        printf ("glfs_h_creat tests: PASSED\n");
-
-        /* extract handle and create from handle test */
-        printf ("glfs_h_extract_handle and glfs_h_create_from_handle tests: In Progress\n");
-        /* TODO: Change the lookup to creat below for a GIFD recovery falure,
-         * that needs to be fixed */
-        leaf = glfs_h_lookupat (fs, parent, leaf_name1, &sb);
-        if (leaf == NULL) {
-                fprintf (stderr, "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
-                         leaf_name1, parent, strerror (errno));
-                printf ("glfs_h_extract_handle tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        ret = glfs_h_extract_handle (leaf, leaf_handle,
-                                             GFAPI_HANDLE_LENGTH);
-        if (ret < 0) {
-                fprintf (stderr, "glfs_h_extract_handle: error extracting handle of %s: %s\n",
-                         full_leaf_name, strerror (errno));
-                printf ("glfs_h_extract_handle tests: FAILED\n");
-                goto out;
-        }
-        peek_handle (leaf_handle);
+    int ret = 0;
+    glfs_fd_t *fd = NULL;
+    struct stat sb = {
+        0,
+    };
+    struct glfs_object *root = NULL, *parent = NULL, *leaf = NULL, *tmp = NULL;
+    char readbuf[32], writebuf[32];
+    unsigned char leaf_handle[GFAPI_HANDLE_LENGTH];
+
+    char *full_leaf_name = "/testdir/testfile.txt", *leaf_name = "testfile.txt",
+         *relative_leaf_name = "testdir/testfile.txt";
+    char *leaf_name1 = "testfile1.txt";
+    char *full_newparent_name = "/testdir/dir1", *newparent_name = "dir1";
+    char *full_newnod_name = "/testdir/nod1", *newnod_name = "nod1";
+
+    /* Initialize test area */
+    ret = glfs_mkdir(fs, full_parent_name, 0755);
+    if (ret != 0 && errno != EEXIST) {
+        fprintf(stderr, "%s: (%p) %s\n", full_parent_name, fd, strerror(errno));
+        printf("Test initialization failed on volume %s\n", argv[1]);
+        goto out;
+    } else if (ret != 0) {
+        printf("Found test directory %s to be existing\n", full_parent_name);
+        printf("Cleanup test directory and restart tests\n");
+        goto out;
+    }
+
+    fd = glfs_creat(fs, full_leaf_name, O_CREAT, 0644);
+    if (fd == NULL) {
+        fprintf(stderr, "%s: (%p) %s\n", full_leaf_name, fd, strerror(errno));
+        printf("Test initialization failed on volume %s\n", argv[1]);
+        goto out;
+    }
+    glfs_close(fd);
+
+    printf("Initialized the test area, within volume %s\n", argv[1]);
+
+    /* Handle based APIs test area */
+
+    /* glfs_lookupat test */
+    printf("glfs_h_lookupat tests: In Progress\n");
+    /* start at root of the volume */
+    root = glfs_h_lookupat(fs, NULL, "/", &sb, 0);
+    if (root == NULL) {
+        fprintf(stderr,
+                "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n", "/",
+                NULL, strerror(errno));
+        printf("glfs_h_lookupat tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    /* lookup a parent within root */
+    parent = glfs_h_lookupat(fs, root, parent_name, &sb, 0);
+    if (parent == NULL) {
+        fprintf(stderr,
+                "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                parent_name, root, strerror(errno));
+        printf("glfs_h_lookupat tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    /* lookup a leaf/child within the parent */
+    leaf = glfs_h_lookupat(fs, parent, leaf_name, &sb, 0);
+    if (leaf == NULL) {
+        fprintf(stderr,
+                "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                leaf_name, parent, strerror(errno));
+        printf("glfs_h_lookupat tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    /* reset */
+    glfs_h_close(root);
+    root = NULL;
+    glfs_h_close(leaf);
+    leaf = NULL;
+    glfs_h_close(parent);
+    parent = NULL;
+
+    /* check absolute paths */
+    root = glfs_h_lookupat(fs, NULL, "/", &sb, 0);
+    if (root == NULL) {
+        fprintf(stderr,
+                "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n", "/",
+                NULL, strerror(errno));
+        printf("glfs_h_lookupat tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    parent = glfs_h_lookupat(fs, NULL, full_parent_name, &sb, 0);
+    if (parent == NULL) {
+        fprintf(stderr,
+                "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                full_parent_name, root, strerror(errno));
+        printf("glfs_h_lookupat tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    leaf = glfs_h_lookupat(fs, NULL, full_leaf_name, &sb, 0);
+    if (leaf == NULL) {
+        fprintf(stderr,
+                "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                full_leaf_name, parent, strerror(errno));
+        printf("glfs_h_lookupat tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    /* reset */
+    glfs_h_close(leaf);
+    leaf = NULL;
+
+    /* check multiple component paths */
+    leaf = glfs_h_lookupat(fs, root, relative_leaf_name, &sb, 0);
+    if (leaf == NULL) {
+        fprintf(stderr,
+                "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                relative_leaf_name, parent, strerror(errno));
+        goto out;
+    }
+    peek_stat(&sb);
+
+    /* reset */
+    glfs_h_close(root);
+    root = NULL;
+    glfs_h_close(parent);
+    parent = NULL;
+
+    /* check symlinks in path */
+
+    /* TODO: -ve test cases */
+    /* parent invalid
+     * path invalid
+     * path does not exist after some components
+     * no parent, but relative path
+     * parent and full path? -ve?
+     */
+
+    printf("glfs_h_lookupat tests: PASSED\n");
+
+    /* glfs_openat test */
+    printf("glfs_h_open tests: In Progress\n");
+    fd = glfs_h_open(fs, leaf, O_RDWR);
+    if (fd == NULL) {
+        fprintf(stderr, "glfs_h_open: error on open of %s: %s\n",
+                full_leaf_name, strerror(errno));
+        printf("glfs_h_open tests: FAILED\n");
+        goto out;
+    }
+
+    /* test read/write based on fd */
+    memcpy(writebuf, "abcdefghijklmnopqrstuvwxyz012345", 32);
+    ret = glfs_write(fd, writebuf, 32, 0);
+
+    glfs_lseek(fd, 0, SEEK_SET);
+
+    ret = glfs_read(fd, readbuf, 32, 0);
+    if (memcmp(readbuf, writebuf, 32)) {
+        printf("Failed to read what I wrote: %s %s\n", readbuf, writebuf);
+        glfs_close(fd);
+        printf("glfs_h_open tests: FAILED\n");
+        goto out;
+    }
+
+    glfs_h_close(leaf);
+    leaf = NULL;
+    glfs_close(fd);
+
+    printf("glfs_h_open tests: PASSED\n");
+
+    /* Create tests */
+    printf("glfs_h_creat tests: In Progress\n");
+    parent = glfs_h_lookupat(fs, NULL, full_parent_name, &sb, 0);
+    if (parent == NULL) {
+        fprintf(stderr,
+                "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                full_parent_name, root, strerror(errno));
+        printf("glfs_h_creat tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    leaf = glfs_h_creat(fs, parent, leaf_name1, O_CREAT, 0644, &sb);
+    if (leaf == NULL) {
+        fprintf(stderr, "glfs_h_creat: error on create of %s: from (%p),%s\n",
+                leaf_name1, parent, strerror(errno));
+        printf("glfs_h_creat tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    glfs_h_close(leaf);
+    leaf = NULL;
+
+    leaf = glfs_h_creat(fs, parent, leaf_name1, O_CREAT | O_EXCL, 0644, &sb);
+    if (leaf != NULL || errno != EEXIST) {
+        fprintf(stderr,
+                "glfs_h_creat: existing file, leaf = (%p), errno = %s\n", leaf,
+                strerror(errno));
+        printf("glfs_h_creat tests: FAILED\n");
+        if (leaf != NULL) {
+            glfs_h_close(leaf);
+            leaf = NULL;
+        }
+    }
+
+    tmp = glfs_h_creat(fs, root, parent_name, O_CREAT, 0644, &sb);
+    if (tmp != NULL || !(errno == EISDIR || errno == EINVAL)) {
+        fprintf(stderr, "glfs_h_creat: dir create, tmp = (%p), errno = %s\n",
+                leaf, strerror(errno));
+        printf("glfs_h_creat tests: FAILED\n");
+        if (tmp != NULL) {
+            glfs_h_close(tmp);
+            tmp = NULL;
+        }
+    }
+
+    /* TODO: Other combinations and -ve cases as applicable */
+    printf("glfs_h_creat tests: PASSED\n");
+
+    /* extract handle and create from handle test */
+    printf(
+        "glfs_h_extract_handle and glfs_h_create_from_handle tests: In "
+        "Progress\n");
+    /* TODO: Change the lookup to create below for a GIFD recovery failure,
+     * that needs to be fixed */
+    leaf = glfs_h_lookupat(fs, parent, leaf_name1, &sb, 0);
+    if (leaf == NULL) {
+        fprintf(stderr,
+                "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                leaf_name1, parent, strerror(errno));
+        printf("glfs_h_extract_handle tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    ret = glfs_h_extract_handle(leaf, leaf_handle, GFAPI_HANDLE_LENGTH);
+    if (ret < 0) {
+        fprintf(stderr,
+                "glfs_h_extract_handle: error extracting handle of %s: %s\n",
+                full_leaf_name, strerror(errno));
+        printf("glfs_h_extract_handle tests: FAILED\n");
+        goto out;
+    }
+    peek_handle(leaf_handle);
+
+    glfs_h_close(leaf);
+    leaf = NULL;
+
+    leaf = glfs_h_create_from_handle(fs, leaf_handle, GFAPI_HANDLE_LENGTH, &sb);
+    if (leaf == NULL) {
+        fprintf(
+            stderr,
+            "glfs_h_create_from_handle: error on create of %s: from (%p),%s\n",
+            leaf_name1, leaf_handle, strerror(errno));
+        printf("glfs_h_create_from_handle tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    fd = glfs_h_open(fs, leaf, O_RDWR);
+    if (fd == NULL) {
+        fprintf(stderr, "glfs_h_open: error on open of %s: %s\n",
+                full_leaf_name, strerror(errno));
+        printf("glfs_h_create_from_handle tests: FAILED\n");
+        goto out;
+    }
+
+    /* test read/write based on fd */
+    memcpy(writebuf, "abcdefghijklmnopqrstuvwxyz012345", 32);
+    ret = glfs_write(fd, writebuf, 32, 0);
+
+    glfs_lseek(fd, 0, SEEK_SET);
+
+    ret = glfs_read(fd, readbuf, 32, 0);
+    if (memcmp(readbuf, writebuf, 32)) {
+        printf("Failed to read what I wrote: %s %s\n", writebuf, writebuf);
+        printf("glfs_h_create_from_handle tests: FAILED\n");
+        glfs_close(fd);
+        goto out;
+    }
+
+    glfs_close(fd);
+    glfs_h_close(leaf);
+    leaf = NULL;
+    glfs_h_close(parent);
+    parent = NULL;
+
+    printf(
+        "glfs_h_extract_handle and glfs_h_create_from_handle tests: PASSED\n");
+
+    /* Mkdir tests */
+    printf("glfs_h_mkdir tests: In Progress\n");
+
+    ret = glfs_rmdir(fs, full_newparent_name);
+    if (ret && errno != ENOENT) {
+        fprintf(stderr, "glfs_rmdir: Failed for %s: %s\n", full_newparent_name,
+                strerror(errno));
+        printf("glfs_h_mkdir tests: FAILED\n");
+        goto out;
+    }
+
+    parent = glfs_h_lookupat(fs, NULL, full_parent_name, &sb, 0);
+    if (parent == NULL) {
+        fprintf(stderr,
+                "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                full_parent_name, root, strerror(errno));
+        printf("glfs_h_mkdir tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    leaf = glfs_h_mkdir(fs, parent, newparent_name, 0755, &sb);
+    if (leaf == NULL) {
+        fprintf(stderr, "glfs_h_mkdir: error on mkdir of %s: from (%p),%s\n",
+                newparent_name, parent, strerror(errno));
+        printf("glfs_h_mkdir tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    glfs_h_close(leaf);
+    leaf = NULL;
+
+    leaf = glfs_h_mkdir(fs, parent, newparent_name, 0755, &sb);
+    if (leaf != NULL || errno != EEXIST) {
+        fprintf(stderr,
+                "glfs_h_mkdir: existing directory, leaf = (%p), errno = %s\n",
+                leaf, strerror(errno));
+        printf("glfs_h_mkdir tests: FAILED\n");
+        if (leaf != NULL) {
+            glfs_h_close(leaf);
+            leaf = NULL;
+        }
+    }
+
+    glfs_h_close(parent);
+    parent = NULL;
+
+    printf("glfs_h_mkdir tests: PASSED\n");
+
+    /* Mknod tests */
+    printf("glfs_h_mknod tests: In Progress\n");
+    ret = glfs_unlink(fs, full_newnod_name);
+    if (ret && errno != ENOENT) {
+        fprintf(stderr, "glfs_unlink: Failed for %s: %s\n", full_newnod_name,
+                strerror(errno));
+        printf("glfs_h_mknod tests: FAILED\n");
+        goto out;
+    }
+
+    parent = glfs_h_lookupat(fs, NULL, full_parent_name, &sb, 0);
+    if (parent == NULL) {
+        fprintf(stderr,
+                "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                full_parent_name, root, strerror(errno));
+        printf("glfs_h_mknod tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    leaf = glfs_h_mknod(fs, parent, newnod_name, S_IFIFO, 0, &sb);
+    if (leaf == NULL) {
+        fprintf(stderr, "glfs_h_mkdir: error on mkdir of %s: from (%p),%s\n",
+                newnod_name, parent, strerror(errno));
+        printf("glfs_h_mknod tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    /* TODO: create op on a FIFO node hangs, need to check and fix
+    tmp = glfs_h_creat (fs, parent, newnod_name, O_CREAT, 0644, &sb);
+    if (tmp != NULL || errno != EINVAL) {
+            fprintf (stderr, "glfs_h_creat: node create, tmp = (%p), errno =
+    %s\n", tmp, strerror (errno)); printf ("glfs_h_creat/mknod tests:
+    FAILED\n"); if (tmp != NULL) { glfs_h_close(tmp); tmp = NULL;
+            }
+    } */
+
+    glfs_h_close(leaf);
+    leaf = NULL;
+
+    leaf = glfs_h_mknod(fs, parent, newnod_name, 0644, 0, &sb);
+    if (leaf != NULL || errno != EEXIST) {
+        fprintf(stderr,
+                "glfs_h_mknod: existing node, leaf = (%p), errno = %s\n", leaf,
+                strerror(errno));
+        printf("glfs_h_mknod tests: FAILED\n");
+        if (leaf != NULL) {
+            glfs_h_close(leaf);
+            leaf = NULL;
+        }
+    }
+
+    glfs_h_close(parent);
+    parent = NULL;
+
+    printf("glfs_h_mknod tests: PASSED\n");
+
+    /* unlink tests */
+    test_h_unlink();
+
+    /* TODO: opendir tests */
+
+    /* getattr tests */
+    test_h_getsetattrs();
+
+    /* TODO: setattr tests */
+
+    /* truncate tests */
+    test_h_truncate();
+
+    /* link tests */
+    test_h_links();
+
+    /* rename tests */
+    test_h_rename();
+
+    /* performance tests */
+    test_h_performance();
+
+    /* END: New APIs test area */
 
-        glfs_h_close (leaf); leaf = NULL;
-
-        leaf = glfs_h_create_from_handle (fs, leaf_handle, GFAPI_HANDLE_LENGTH,
-                                          &sb);
-        if (leaf == NULL) {
-                fprintf (stderr, "glfs_h_create_from_handle: error on create of %s: from (%p),%s\n",
-                         leaf_name1, leaf_handle, strerror (errno));
-                printf ("glfs_h_create_from_handle tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        fd = glfs_h_open (fs, leaf, O_RDWR);
-        if (fd == NULL) {
-                fprintf (stderr, "glfs_h_open: error on open of %s: %s\n",
-                         full_leaf_name, strerror (errno));
-                printf ("glfs_h_create_from_handle tests: FAILED\n");
-                goto out;
-        }
-
-        /* test read/write based on fd */
-        memcpy (writebuf, "abcdefghijklmnopqrstuvwxyz012345", 32);
-        ret = glfs_write (fd, writebuf, 32, 0);
-
-        glfs_lseek (fd, 0, SEEK_SET);
-
-        ret = glfs_read (fd, readbuf, 32, 0);
-        if (memcmp (readbuf, writebuf, 32)) {
-                printf ("Failed to read what I wrote: %s %s\n", writebuf,
-                        writebuf);
-                printf ("glfs_h_create_from_handle tests: FAILED\n");
-                glfs_close (fd);
-                goto out;
-        }
-
-        glfs_close (fd);
-        glfs_h_close (leaf); leaf = NULL;
-        glfs_h_close (parent); parent = NULL;
-
-        printf ("glfs_h_extract_handle and glfs_h_create_from_handle tests: PASSED\n");
-
-        /* Mkdir tests */
-        printf ("glfs_h_mkdir tests: In Progress\n");
+out:
+    /* Cleanup glfs handles */
+    if (root)
+        glfs_h_close(root);
+    if (parent)
+        glfs_h_close(parent);
+    if (leaf)
+        glfs_h_close(leaf);
+
+    return ret;
+}
 
-        ret = glfs_rmdir (fs, full_newparent_name);
-        if (ret && errno != ENOENT) {
-                fprintf (stderr, "glfs_rmdir: Failed for %s: %s\n",
-                         full_newparent_name, strerror (errno));
-                printf ("glfs_h_mkdir tests: FAILED\n");
-                goto out;
-        }
+int
+test_write_apis(glfs_t *fs)
+{
+    /* Add more content here */
+    /* Some apis we can get are */
+    /*
+      0. glfs_set_xlator_option()
+
+      Read/Write combinations:
+      . glfs_{p,}readv/{p,}writev
+      . glfs_pread/pwrite
+
+      tests/basic/gfapi/gfapi-async-calls-test.c
+      . glfs_read_async/write_async
+      . glfs_pread_async/pwrite_async
+      . glfs_readv_async/writev_async
+      . glfs_preadv_async/pwritev_async
+
+      . ftruncate/ftruncate_async
+      . fsync/fsync_async
+      . fdatasync/fdatasync_async
+
+    */
+
+    glfs_fd_t *fd = NULL;
+    char *filename = "/filename2";
+    int flags = O_RDWR;
+    char *buf = "some bytes!";
+    char writestr[TEST_STR_LEN];
+    struct iovec iov = {&writestr, TEST_STR_LEN};
+    int ret, i;
+
+    for (i = 0; i < TEST_STR_LEN; i++)
+        writestr[i] = 0x11;
+
+    fd = glfs_open(fs, filename, flags);
+    if (!fd)
+        fprintf(stderr, "open(%s): (%p) %s\n", filename, fd, strerror(errno));
+
+    ret = glfs_writev(fd, &iov, 1, flags);
+    if (ret < 0) {
+        fprintf(stderr, "writev(%s): %d (%s)\n", filename, ret,
+                strerror(errno));
+    }
+
+    ret = glfs_pwrite(fd, buf, 10, 4, flags, NULL, NULL);
+    if (ret < 0) {
+        fprintf(stderr, "pwrite(%s): %d (%s)\n", filename, ret,
+                strerror(errno));
+    }
+
+    ret = glfs_pwritev(fd, &iov, 1, 4, flags);
+    if (ret < 0) {
+        fprintf(stderr, "pwritev(%s): %d (%s)\n", filename, ret,
+                strerror(errno));
+    }
+
+    return 0;
+}
 
-        parent = glfs_h_lookupat (fs, NULL, full_parent_name, &sb);
-        if (parent == NULL) {
-                fprintf (stderr, "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
-                         full_parent_name, root, strerror (errno));
-                printf ("glfs_h_mkdir tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
+int
+test_metadata_ops(glfs_t *fs, glfs_t *fs2)
+{
+    glfs_fd_t *fd = NULL;
+    glfs_fd_t *fd2 = NULL;
+    struct stat sb = {
+        0,
+    };
+    struct glfs_stat gsb = {
+        0,
+    };
+    struct statvfs sfs;
+    char readbuf[32];
+    char writebuf[32];
 
-        leaf = glfs_h_mkdir (fs, parent, newparent_name, 0644, &sb);
-        if (leaf == NULL) {
-                fprintf (stderr, "glfs_h_mkdir: error on mkdir of %s: from (%p),%s\n",
-                         newparent_name, parent, strerror (errno));
-                printf ("glfs_h_mkdir tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        glfs_h_close (leaf); leaf = NULL;
-
-        leaf = glfs_h_mkdir (fs, parent, newparent_name, 0644, &sb);
-        if (leaf != NULL || errno != EEXIST) {
-                fprintf (stderr, "glfs_h_mkdir: existing directory, leaf = (%p), errno = %s\n",
-                         leaf, strerror (errno));
-                printf ("glfs_h_mkdir tests: FAILED\n");
-                if (leaf != NULL) {
-                        glfs_h_close (leaf); leaf = NULL;
-                }
-        }
+    char *filename = "/filename2";
+    int ret;
 
-        glfs_h_close (parent); parent = NULL;
+    ret = glfs_lstat(fs, filename, &sb);
+    fprintf(stderr, "lstat(%s): (%d) %s\n", filename, ret, strerror(errno));
 
-        printf ("glfs_h_mkdir tests: PASSED\n");
+    fd = glfs_creat(fs, filename, O_RDWR, 0644);
+    fprintf(stderr, "creat(%s): (%p) %s\n", filename, fd, strerror(errno));
 
-        /* Mknod tests */
-        printf ("glfs_h_mknod tests: In Progress\n");
-        ret = glfs_unlink (fs, full_newnod_name);
-        if (ret && errno != ENOENT) {
-                fprintf (stderr, "glfs_unlink: Failed for %s: %s\n",
-                         full_newnod_name, strerror (errno));
-                printf ("glfs_h_mknod tests: FAILED\n");
-                goto out;
-        }
+    fd2 = glfs_open(fs2, filename, O_RDWR);
+    fprintf(stderr, "open(%s): (%p) %s\n", filename, fd, strerror(errno));
 
-        parent = glfs_h_lookupat (fs, NULL, full_parent_name, &sb);
-        if (parent == NULL) {
-                fprintf (stderr, "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
-                         full_parent_name, root, strerror (errno));
-                printf ("glfs_h_mknod tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
+    glfs_lseek(fd2, 0, SEEK_SET);
 
-        leaf = glfs_h_mknod (fs, parent, newnod_name, S_IFIFO, 0, &sb);
-        if (leaf == NULL) {
-                fprintf (stderr, "glfs_h_mkdir: error on mkdir of %s: from (%p),%s\n",
-                         newnod_name, parent, strerror (errno));
-                printf ("glfs_h_mknod tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        /* TODO: creat op on a FIFO node hangs, need to check and fix
-        tmp = glfs_h_creat (fs, parent, newnod_name, O_CREAT, 0644, &sb);
-        if (tmp != NULL || errno != EINVAL) {
-                fprintf (stderr, "glfs_h_creat: node create, tmp = (%p), errno = %s\n",
-                        tmp, strerror (errno));
-                printf ("glfs_h_creat/mknod tests: FAILED\n");
-                if (tmp != NULL) {
-                        glfs_h_close(tmp); tmp = NULL;
-                }
-        } */
-
-        glfs_h_close (leaf); leaf = NULL;
-
-        leaf = glfs_h_mknod (fs, parent, newnod_name, 0644, 0, &sb);
-        if (leaf != NULL || errno != EEXIST) {
-                fprintf (stderr, "glfs_h_mknod: existing node, leaf = (%p), errno = %s\n",
-                         leaf, strerror (errno));
-                printf ("glfs_h_mknod tests: FAILED\n");
-                if (leaf != NULL) {
-                        glfs_h_close (leaf); leaf = NULL;
-                }
-        }
+    ret = glfs_read(fd2, readbuf, 32, 0);
 
-        glfs_h_close (parent); parent = NULL;
+    printf("read %d, %s", ret, readbuf);
 
-        printf ("glfs_h_mknod tests: PASSED\n");
+    /* get stat */
+    ret = glfs_fstat(fd2, &sb);
 
-        /* unlink tests */
-        test_h_unlink ();
+    ret = glfs_access(fs, filename, R_OK);
 
-        /* TODO: opendir tests */
+    /* set stat */
+    /* TODO: got some errors, need to fix */
+    /* ret = glfs_fsetattr(fd2, &gsb); */
 
-        /* getattr tests */
-        test_h_getsetattrs ();
+    glfs_close(fd);
+    glfs_close(fd2);
 
-        /* TODO: setattr tests */
+    filename = "/filename3";
+    ret = glfs_mknod(fs, filename, S_IFIFO, 0);
+    fprintf(stderr, "%s: (%d) %s\n", filename, ret, strerror(errno));
 
-        /* truncate tests */
-        test_h_truncate();
+    ret = glfs_lstat(fs, filename, &sb);
+    fprintf(stderr, "%s: (%d) %s\n", filename, ret, strerror(errno));
 
-        /* link tests */
-        test_h_links ();
+    ret = glfs_rename(fs, filename, "/filename4");
+    fprintf(stderr, "rename(%s): (%d) %s\n", filename, ret, strerror(errno));
 
-        /* rename tests */
-        test_h_rename ();
+    ret = glfs_unlink(fs, "/filename4");
+    fprintf(stderr, "unlink(%s): (%d) %s\n", "/filename4", ret,
+            strerror(errno));
 
-        /* performance tests */
-        test_h_performance ();
+    filename = "/dirname2";
+    ret = glfs_mkdir(fs, filename, 0);
+    fprintf(stderr, "%s: (%d) %s\n", filename, ret, strerror(errno));
 
-        /* END: New APIs test area */
+    ret = glfs_lstat(fs, filename, &sb);
+    fprintf(stderr, "lstat(%s): (%d) %s\n", filename, ret, strerror(errno));
 
-out:
-        /* Cleanup glfs handles */
-        if (root)
-                glfs_h_close (root);
-        if (parent)
-                glfs_h_close (parent);
-        if (leaf)
-                glfs_h_close (leaf);
-
-        return ret;
+    ret = glfs_rmdir(fs, filename);
+    fprintf(stderr, "rmdir(%s): (%d) %s\n", filename, ret, strerror(errno));
 }
-
 int
-main (int argc, char *argv[])
+main(int argc, char *argv[])
 {
-        glfs_t    *fs2 = NULL;
-        int        ret = 0;
-        glfs_fd_t *fd = NULL;
-        glfs_fd_t *fd2 = NULL;
-        struct stat sb = {0, };
-        char       readbuf[32];
-        char       writebuf[32];
-
-        char      *filename = "/filename2";
-
-        if (argc != 3) {
-                printf ("Expect following args\n\t%s <volname> <hostname>\n", argv[0]);
-                return -1;
-        }
-
-        fs = glfs_new (argv[1]);
-        if (!fs) {
-                fprintf (stderr, "glfs_new: returned NULL\n");
-                return 1;
-        }
-
-//      ret = glfs_set_volfile (fs, "/tmp/posix.vol");
-
-        ret = glfs_set_volfile_server (fs, "tcp", argv[2], 24007);
-
-//      ret = glfs_set_volfile_server (fs, "unix", "/tmp/gluster.sock", 0);
-
-        ret = glfs_set_logging (fs, "/dev/stderr", 7);
+    glfs_t *fs2 = NULL;
+    int ret = 0;
+    glfs_fd_t *fd = NULL;
+    glfs_fd_t *fd2 = NULL;
+    struct stat sb = {
+        0,
+    };
+    struct glfs_stat gsb = {
+        0,
+    };
+    struct statvfs sfs;
+    char readbuf[32];
+    char writebuf[32];
 
-        ret = glfs_init (fs);
+    char *filename = "/filename2";
 
-        fprintf (stderr, "glfs_init: returned %d\n", ret);
+    if (argc != 3) {
+        printf("Expect following args\n\t%s <volname> <hostname>\n", argv[0]);
+        return -1;
+    }
 
-        sleep (2);
+    fs = glfs_new(argv[1]);
+    if (!fs) {
+        fprintf(stderr, "glfs_new: returned NULL\n");
+        return 1;
+    }
 
-        fs2 = glfs_new (argv[1]);
-        if (!fs2) {
-                fprintf (stderr, "glfs_new: returned NULL\n");
-                return 1;
-        }
-
-
-//      ret = glfs_set_volfile (fs2, "/tmp/posix.vol");
-
-        ret = glfs_set_volfile_server (fs2, "tcp", argv[2], 24007);
+    //      ret = glfs_set_volfile (fs, "/tmp/posix.vol");
 
-        ret = glfs_set_logging (fs2, "/dev/stderr", 7);
+    ret = glfs_set_volfile_server(fs, "tcp", argv[2], 24007);
 
-        ret = glfs_init (fs2);
+    //      ret = glfs_set_volfile_server (fs, "unix", "/tmp/gluster.sock", 0);
 
-        fprintf (stderr, "glfs_init: returned %d\n", ret);
+    ret = glfs_set_logging(fs, "/dev/stderr", 7);
 
-        ret = glfs_lstat (fs, filename, &sb);
-        fprintf (stderr, "%s: (%d) %s\n", filename, ret, strerror (errno));
+    ret = glfs_init(fs);
 
-        fd = glfs_creat (fs, filename, O_RDWR, 0644);
-        fprintf (stderr, "%s: (%p) %s\n", filename, fd, strerror (errno));
+    fprintf(stderr, "glfs_init: returned %d\n", ret);
 
-        fd2 = glfs_open (fs2, filename, O_RDWR);
-        fprintf (stderr, "%s: (%p) %s\n", filename, fd, strerror (errno));
+    if (ret)
+        goto out;
 
-        sprintf (writebuf, "hi there\n");
-        ret = glfs_write (fd, writebuf, 32, 0);
+    sleep(2);
 
-        glfs_lseek (fd2, 0, SEEK_SET);
+    fs2 = glfs_new(argv[1]);
+    if (!fs2) {
+        fprintf(stderr, "glfs_new: returned NULL\n");
+        return 1;
+    }
 
-        ret = glfs_read (fd2, readbuf, 32, 0);
+    //      ret = glfs_set_volfile (fs2, "/tmp/posix.vol");
 
-        printf ("read %d, %s", ret, readbuf);
+    ret = glfs_set_volfile_server(fs2, "tcp", argv[2], 24007);
 
-        glfs_close (fd);
-        glfs_close (fd2);
+    ret = glfs_set_logging(fs2, "/dev/stderr", 7);
 
-        filename = "/filename3";
-        ret = glfs_mknod (fs, filename, S_IFIFO, 0);
-        fprintf (stderr, "%s: (%d) %s\n", filename, ret, strerror (errno));
+    ret = glfs_init(fs2);
 
-        ret = glfs_lstat (fs, filename, &sb);
-        fprintf (stderr, "%s: (%d) %s\n", filename, ret, strerror (errno));
+    fprintf(stderr, "glfs_init: returned %d\n", ret);
 
+    test_metadata_ops(fs, fs2);
 
-        ret = glfs_rename (fs, filename, "/filename4");
-        fprintf (stderr, "rename(%s): (%d) %s\n", filename, ret,
-                 strerror (errno));
+    test_dirops(fs);
 
-        ret = glfs_unlink (fs, "/filename4");
-        fprintf (stderr, "unlink(%s): (%d) %s\n", "/filename4", ret,
-                 strerror (errno));
+    test_xattr(fs);
 
-        filename = "/dirname2";
-        ret = glfs_mkdir (fs, filename, 0);
-        fprintf (stderr, "%s: (%d) %s\n", filename, ret, strerror (errno));
+    test_chdir(fs);
 
-        ret = glfs_lstat (fs, filename, &sb);
-        fprintf (stderr, "lstat(%s): (%d) %s\n", filename, ret, strerror (errno));
+    test_handleops(argc, argv);
+    // done
 
-        ret = glfs_rmdir (fs, filename);
-        fprintf (stderr, "rmdir(%s): (%d) %s\n", filename, ret, strerror (errno));
+    /* Test some extra apis */
+    test_write_apis(fs);
 
-        test_dirops (fs);
+    glfs_statvfs(fs, "/", &sfs);
 
-        test_xattr (fs);
+    glfs_fini(fs);
+    glfs_fini(fs2);
 
-        test_chdir (fs);
-
-        test_handleops (argc, argv);
-        // done
-
-        glfs_fini (fs);
-        glfs_fini (fs2);
-
-        return ret;
+    ret = 0;
+out:
+    return ret;
 }
diff --git a/api/examples/setup.py.in b/api/examples/setup.py.in
deleted file mode 100644
index f22fa1f3099..00000000000
--- a/api/examples/setup.py.in
+++ /dev/null
@@ -1,24 +0,0 @@
-from distutils.core import setup
-
-DESC = """GlusterFS is a clustered file-system capable of scaling to
-several petabytes. It aggregates various storage bricks over Infiniband
-RDMA or TCP/IP interconnect into one large parallel network file system.
-GlusterFS is one of the most sophisticated file systems in terms of
-features and extensibility.  It borrows a powerful concept called
-Translators from GNU Hurd kernel. Much of the code in GlusterFS is in
-user space and easily manageable.
-
-This package contains the Python interface to the libgfapi library."""
-
-setup(
-    name='glusterfs-api',
-    version='@PACKAGE_VERSION@',
-    description='Python client library for the GlusterFS libgfapi',
-    long_description=DESC,
-    author='Gluster Community',
-    author_email='gluster-devel@nongnu.org',
-    license='LGPLv3',
-    url='http://gluster.org/',
-    package_dir={'gluster':''},
-    packages=['gluster']
-)
diff --git a/api/src/Makefile.am b/api/src/Makefile.am
index f7e66e50f5c..7f9a7d17b35 100644
--- a/api/src/Makefile.am
+++ b/api/src/Makefile.am
@@ -1,21 +1,27 @@
 lib_LTLIBRARIES = libgfapi.la
-noinst_HEADERS = glfs-mem-types.h glfs-internal.h
+noinst_HEADERS = glfs-mem-types.h glfs-internal.h gfapi-messages.h
 libgfapi_HEADERS = glfs.h glfs-handles.h
 libgfapidir = $(includedir)/glusterfs/api
 
+EXTRA_DIST = gfapi.map gfapi.aliases
+
 libgfapi_la_SOURCES = glfs.c glfs-mgmt.c glfs-fops.c glfs-resolve.c \
 	glfs-handleops.c
 libgfapi_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la \
 	$(top_builddir)/rpc/rpc-lib/src/libgfrpc.la \
-	$(top_builddir)/rpc/xdr/src/libgfxdr.la \
-	$(GF_LDADD)
+	$(top_builddir)/rpc/xdr/src/libgfxdr.la
 
-libgfapi_la_CPPFLAGS = $(GF_CPPFLAGS) -D__USE_FILE_OFFSET64 \
-	-I$(top_srcdir)/libglusterfs/src \
-	-I$(top_srcdir)/rpc/rpc-lib/src \
-	-I$(top_srcdir)/rpc/xdr/src
+libgfapi_la_LDFLAGS = -version-info $(GFAPI_LT_VERSION) $(GF_LDFLAGS) \
+	$(GFAPI_EXTRA_LDFLAGS) $(ACL_LIBS)
 
-libgfapi_la_LDFLAGS = -Wl,--version-script=gfapi.map
+libgfapi_la_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
+        -I$(top_srcdir)/rpc/rpc-lib/src \
+        -I$(top_srcdir)/rpc/xdr/src \
+        -I$(top_builddir)/rpc/xdr/src \
+        -DDATADIR=\"$(localstatedir)\" \
+        -D__USE_FILE_OFFSET64 -D__USE_LARGEFILE64
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
 
 xlator_LTLIBRARIES = api.la
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/mount
@@ -26,12 +32,12 @@ $(install_xlatorLTLIBRARIES): install-libLTLIBRARIES
 
 api_la_SOURCES = glfs-master.c
 api_la_DEPENDENCIES = libgfapi.la
-api_la_LDFLAGS = -module -avoid-version
+api_la_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
+	-I$(top_srcdir)/rpc/xdr/src \
+	-I$(top_builddir)/rpc/xdr/src
+api_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
+#api_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS) $(GF_LDFLAGS)
 api_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la \
 	$(top_builddir)/rpc/rpc-lib/src/libgfrpc.la \
 	$(top_builddir)/rpc/xdr/src/libgfxdr.la \
 	$(top_builddir)/api/src/libgfapi.la
-
-AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
-
-AM_CFLAGS = -Wall $(GF_CFLAGS)
diff --git a/api/src/README.Symbol_Versions b/api/src/README.Symbol_Versions
new file mode 100644
index 00000000000..b6ec95f9311
--- /dev/null
+++ b/api/src/README.Symbol_Versions
@@ -0,0 +1,3 @@
+
+See ../../doc/developer-guide/gfapi-symbol-versions.md
+
diff --git a/api/src/gfapi-messages.h b/api/src/gfapi-messages.h
new file mode 100644
index 00000000000..b9223940416
--- /dev/null
+++ b/api/src/gfapi-messages.h
@@ -0,0 +1,147 @@
+/*
+ *   Copyright (c) 2015-2018 Red Hat, Inc. <http://www.redhat.com>
+ *   This file is part of GlusterFS.
+ *
+ *   This file is licensed to you under your choice of the GNU Lesser
+ *   General Public License, version 3 or any later version (LGPLv3 or
+ *   later), or the GNU General Public License, version 2 (GPLv2), in all
+ *   cases as published by the Free Software Foundation.
+ *   */
+
+#ifndef _GFAPI_MESSAGES_H__
+#define _GFAPI_MESSAGES_H__
+
+#include <glusterfs/glfs-message-id.h>
+
+/* To add new message IDs, append new identifiers at the end of the list.
+ *
+ * Never remove a message ID. If it's not used anymore, you can rename it or
+ * leave it as it is, but not delete it. This is to prevent reutilization of
+ * IDs by other messages.
+ *
+ * The component name must match one of the entries defined in
+ * glfs-message-id.h.
+ */
+
+GLFS_MSGID(API, API_MSG_MEM_ACCT_INIT_FAILED, API_MSG_MASTER_XLATOR_INIT_FAILED,
+           API_MSG_GFAPI_XLATOR_INIT_FAILED, API_MSG_VOLFILE_OPEN_FAILED,
+           API_MSG_VOL_SPEC_FILE_ERROR, API_MSG_GLFS_FSOBJ_NULL,
+           API_MSG_INVALID_ENTRY, API_MSG_FSMUTEX_LOCK_FAILED,
+           API_MSG_COND_WAIT_FAILED, API_MSG_FSMUTEX_UNLOCK_FAILED,
+           API_MSG_INODE_REFRESH_FAILED, API_MSG_GRAPH_CONSTRUCT_FAILED,
+           API_MSG_API_XLATOR_ERROR, API_MSG_XDR_PAYLOAD_FAILED,
+           API_MSG_GET_VOLINFO_CBK_FAILED, API_MSG_FETCH_VOLUUID_FAILED,
+           API_MSG_INSUFF_SIZE, API_MSG_FRAME_CREAT_FAILED,
+           API_MSG_DICT_SET_FAILED, API_MSG_XDR_DECODE_FAILED,
+           API_MSG_GET_VOLFILE_FAILED, API_MSG_WRONG_OPVERSION,
+           API_MSG_DICT_SERIALIZE_FAILED, API_MSG_REMOTE_HOST_CONN_FAILED,
+           API_MSG_VOLFILE_SERVER_EXHAUST, API_MSG_CREATE_RPC_CLIENT_FAILED,
+           API_MSG_REG_NOTIFY_FUNC_FAILED, API_MSG_REG_CBK_FUNC_FAILED,
+           API_MSG_GET_CWD_FAILED, API_MSG_FGETXATTR_FAILED,
+           API_MSG_LOCKINFO_KEY_MISSING, API_MSG_FSETXATTR_FAILED,
+           API_MSG_FSYNC_FAILED, API_MSG_FDCREATE_FAILED,
+           API_MSG_INODE_PATH_FAILED, API_MSG_SYNCOP_OPEN_FAILED,
+           API_MSG_LOCK_MIGRATE_FAILED, API_MSG_OPENFD_SKIPPED,
+           API_MSG_FIRST_LOOKUP_GRAPH_FAILED, API_MSG_CWD_GRAPH_REF_FAILED,
+           API_MSG_SWITCHED_GRAPH, API_MSG_XDR_RESPONSE_DECODE_FAILED,
+           API_MSG_VOLFILE_INFO, API_MSG_VOLFILE_CONNECTING, API_MSG_NEW_GRAPH,
+           API_MSG_ALLOC_FAILED, API_MSG_CREATE_HANDLE_FAILED,
+           API_MSG_INODE_LINK_FAILED, API_MSG_STATEDUMP_FAILED,
+           API_MSG_XREADDIRP_R_FAILED, API_MSG_LOCK_INSERT_MERGE_FAILED,
+           API_MSG_SETTING_LOCK_TYPE_FAILED, API_MSG_INODE_FIND_FAILED,
+           API_MSG_FDCTX_SET_FAILED, API_MSG_UPCALL_SYNCOP_FAILED,
+           API_MSG_INVALID_ARG, API_MSG_UPCALL_EVENT_NULL_RECEIVED,
+           API_MSG_FLAGS_HANDLE, API_MSG_FDCREATE_FAILED_ON_GRAPH,
+           API_MSG_TRANS_RDMA_DEP, API_MSG_TRANS_NOT_SUPPORTED,
+           API_MSG_FS_NOT_INIT, API_MSG_INVALID_SYSRQ,
+           API_MSG_DECODE_XDR_FAILED, API_MSG_NULL, API_MSG_CALL_NOT_SUCCESSFUL,
+           API_MSG_CALL_NOT_VALID, API_MSG_UNABLE_TO_DEL,
+           API_MSG_REMOTE_HOST_DISCONN, API_MSG_HANDLE_NOT_SET);
+
+#define API_MSG_ALLOC_FAILED_STR "Upcall allocation failed"
+#define API_MSG_LOCK_INSERT_MERGE_FAILED_STR                                   \
+    "Lock insertion and splitting/merging failed"
+#define API_MSG_SETTING_LOCK_TYPE_FAILED_STR "Setting lock type failed"
+
+#define API_MSG_INVALID_ARG_STR "Invalid"
+#define API_MSG_INVALID_ENTRY_STR "Upcall entry validation failed"
+#define API_MSG_INODE_FIND_FAILED_STR "Unable to find inode entry"
+#define API_MSG_CREATE_HANDLE_FAILED_STR "handle creation failed"
+#define API_MSG_UPCALL_EVENT_NULL_RECEIVED_STR                                 \
+    "Upcall_EVENT_NULL received. Skipping it"
+#define API_MSG_UPCALL_SYNCOP_FAILED_STR "Synctask for upcall failed"
+#define API_MSG_FDCREATE_FAILED_STR "Allocating anonymous fd failed"
+#define API_MSG_XREADDIRP_R_FAILED_STR "glfs_x_readdirp_r failed"
+#define API_MSG_FDCTX_SET_FAILED_STR "Setting fd ctx failed"
+#define API_MSG_FLAGS_HANDLE_STR "arg not set. Flags handled are"
+#define API_MSG_INODE_REFRESH_FAILED_STR "inode refresh failed"
+#define API_MSG_INODE_LINK_FAILED_STR "inode linking failed"
+#define API_MSG_GET_CWD_FAILED_STR "Failed to get cwd"
+#define API_MSG_FGETXATTR_FAILED_STR "fgetxattr failed"
+#define API_MSG_LOCKINFO_KEY_MISSING_STR "missing lockinfo key"
+#define API_MSG_FSYNC_FAILED_STR "fsync() failed"
+#define API_MSG_FDCREATE_FAILED_ON_GRAPH_STR "fd_create failed on graph"
+#define API_MSG_INODE_PATH_FAILED_STR "inode_path failed"
+#define API_MSG_SYNCOP_OPEN_FAILED_STR "syncop_open failed"
+#define API_MSG_LOCK_MIGRATE_FAILED_STR "lock migration failed on graph"
+#define API_MSG_OPENFD_SKIPPED_STR "skipping openfd in graph"
+#define API_MSG_FIRST_LOOKUP_GRAPH_FAILED_STR "first lookup on graph failed"
+#define API_MSG_CWD_GRAPH_REF_FAILED_STR "cwd refresh of graph failed"
+#define API_MSG_SWITCHED_GRAPH_STR "switched to graph"
+#define API_MSG_FSETXATTR_FAILED_STR "fsetxattr failed"
+#define API_MSG_MEM_ACCT_INIT_FAILED_STR "Memory accounting init failed"
+#define API_MSG_MASTER_XLATOR_INIT_FAILED_STR                                  \
+    "master xlator for initialization failed"
+#define API_MSG_GFAPI_XLATOR_INIT_FAILED_STR                                   \
+    "failed to initialize gfapi translator"
+#define API_MSG_VOLFILE_OPEN_FAILED_STR "volume file open failed"
+#define API_MSG_VOL_SPEC_FILE_ERROR_STR "Cannot reach volume specification file"
+#define API_MSG_TRANS_RDMA_DEP_STR                                             \
+    "transport RDMA is deprecated, falling back to tcp"
+#define API_MSG_TRANS_NOT_SUPPORTED_STR                                        \
+    "transport is not supported, possible values tcp|unix"
+#define API_MSG_GLFS_FSOBJ_NULL_STR "fs is NULL"
+#define API_MSG_FS_NOT_INIT_STR "fs is not properly initialized"
+#define API_MSG_FSMUTEX_LOCK_FAILED_STR                                        \
+    "pthread lock on glfs mutex, returned error"
+#define API_MSG_FSMUTEX_UNLOCK_FAILED_STR                                      \
+    "pthread unlock on glfs mutex, returned error"
+#define API_MSG_COND_WAIT_FAILED_STR "cond wait failed"
+#define API_MSG_INVALID_SYSRQ_STR "not a valid sysrq"
+#define API_MSG_GRAPH_CONSTRUCT_FAILED_STR "failed to construct the graph"
+#define API_MSG_API_XLATOR_ERROR_STR                                           \
+    "api master xlator cannot be specified in volume file"
+#define API_MSG_STATEDUMP_FAILED_STR "statedump failed"
+#define API_MSG_DECODE_XDR_FAILED_STR                                          \
+    "Failed to decode xdr response for GF_CBK_STATEDUMP"
+#define API_MSG_NULL_STR "NULL"
+#define API_MSG_XDR_PAYLOAD_FAILED_STR "failed to create XDR payload"
+#define API_MSG_CALL_NOT_SUCCESSFUL_STR                                        \
+    "GET_VOLUME_INFO RPC call is not successful"
+#define API_MSG_XDR_RESPONSE_DECODE_FAILED_STR                                 \
+    "Failed to decode xdr response for GET_VOLUME_INFO"
+#define API_MSG_CALL_NOT_VALID_STR                                             \
+    "Response received for GET_VOLUME_INFO RPC is not valid"
+#define API_MSG_GET_VOLINFO_CBK_FAILED_STR                                     \
+    "In GET_VOLUME_INFO cbk, received error"
+#define API_MSG_FETCH_VOLUUID_FAILED_STR "Unable to fetch volume UUID"
+#define API_MSG_INSUFF_SIZE_STR "Insufficient size passed"
+#define API_MSG_FRAME_CREAT_FAILED_STR "failed to create the frame"
+#define API_MSG_DICT_SET_FAILED_STR "failed to set"
+#define API_MSG_XDR_DECODE_FAILED_STR "XDR decoding error"
+#define API_MSG_GET_VOLFILE_FAILED_STR "failed to get the volume file"
+#define API_MSG_VOLFILE_INFO_STR "No change in volfile, continuing"
+#define API_MSG_UNABLE_TO_DEL_STR "unable to delete file"
+#define API_MSG_WRONG_OPVERSION_STR                                            \
+    "Server is operating at an op-version which is not supported"
+#define API_MSG_DICT_SERIALIZE_FAILED_STR "Failed to serialize dictionary"
+#define API_MSG_REMOTE_HOST_CONN_FAILED_STR "Failed to connect to remote-host"
+#define API_MSG_REMOTE_HOST_DISCONN_STR "disconnected from remote-host"
+#define API_MSG_VOLFILE_SERVER_EXHAUST_STR "Exhausted all volfile servers"
+#define API_MSG_VOLFILE_CONNECTING_STR "connecting to next volfile server"
+#define API_MSG_CREATE_RPC_CLIENT_FAILED_STR "failed to create rpc clnt"
+#define API_MSG_REG_NOTIFY_FUNC_FAILED_STR "failed to register notify function"
+#define API_MSG_REG_CBK_FUNC_FAILED_STR "failed to register callback function"
+#define API_MSG_NEW_GRAPH_STR "New graph coming up"
+#define API_MSG_HANDLE_NOT_SET_STR "handle not set. Flags handled for xstat are"
+#endif /* !_GFAPI_MESSAGES_H__ */
diff --git a/api/src/gfapi.aliases b/api/src/gfapi.aliases
new file mode 100644
index 00000000000..bc639e6b99f
--- /dev/null
+++ b/api/src/gfapi.aliases
@@ -0,0 +1,201 @@
+
+_priv_glfs_loc_touchup _glfs_loc_touchup$GFAPI_PRIVATE_3.4.0
+_priv_glfs_active_subvol _glfs_active_subvol$GFAPI_PRIVATE_3.4.0
+_priv_glfs_subvol_done _glfs_subvol_done$GFAPI_PRIVATE_3.4.0
+_priv_glfs_init_done _glfs_init_done$GFAPI_PRIVATE_3.4.0
+_priv_glfs_resolve_at _glfs_resolve_at$GFAPI_PRIVATE_3.4.0
+
+_pub_glfs_new _glfs_new$GFAPI_3.4.0
+_pub_glfs_set_volfile _glfs_set_volfile$GFAPI_3.4.0
+_pub_glfs_set_volfile_server _glfs_set_volfile_server$GFAPI_3.4.0
+_pub_glfs_set_logging _glfs_set_logging$GFAPI_3.4.0
+_pub_glfs_init _glfs_init$GFAPI_3.4.0
+_pub_glfs_fini _glfs_fini$GFAPI_3.4.0
+_pub_glfs_open _glfs_open$GFAPI_3.4.0
+_pub_glfs_creat _glfs_creat$GFAPI_3.4.0
+_pub_glfs_close _glfs_close$GFAPI_3.4.0
+_pub_glfs_from_glfd _glfs_from_glfd$GFAPI_3.4.0
+_pub_glfs_set_xlator_option _glfs_set_xlator_option$GFAPI_3.4.0
+_pub_glfs_read _glfs_read$GFAPI_3.4.0
+_pub_glfs_write _glfs_write$GFAPI_3.4.0
+_pub_glfs_readv _glfs_readv$GFAPI_3.4.0
+_pub_glfs_writev _glfs_writev$GFAPI_3.4.0
+_pub_glfs_pread34 _glfs_pread$GFAPI_3.4.0
+_pub_glfs_pwrite34 _glfs_pwrite$GFAPI_3.4.0
+_pub_glfs_pread_async34 _glfs_pread_async$GFAPI_3.4.0
+_pub_glfs_pwrite_async34 _glfs_pwrite_async$GFAPI_3.4.0
+_pub_glfs_preadv _glfs_preadv$GFAPI_3.4.0
+_pub_glfs_pwritev _glfs_pwritev$GFAPI_3.4.0
+_pub_glfs_lseek _glfs_lseek$GFAPI_3.4.0
+_pub_glfs_ftruncate34 _glfs_ftruncate$GFAPI_3.4.0
+_pub_glfs_ftruncate_async34 _glfs_ftruncate_async$GFAPI_3.4.0
+_pub_glfs_lstat _glfs_lstat$GFAPI_3.4.0
+_pub_glfs_stat _glfs_stat$GFAPI_3.4.0
+_pub_glfs_fstat _glfs_fstat$GFAPI_3.4.0
+_pub_glfs_fsync34 _glfs_fsync$GFAPI_3.4.0
+_pub_glfs_fsync_async34 _glfs_fsync_async$GFAPI_3.4.0
+_pub_glfs_fdatasync34 _glfs_fdatasync$GFAPI_3.4.0
+_pub_glfs_fdatasync_async34 _glfs_fdatasync_async$GFAPI_3.4.0
+_pub_glfs_access _glfs_access$GFAPI_3.4.0
+_pub_glfs_symlink _glfs_symlink$GFAPI_3.4.0
+_pub_glfs_readlink _glfs_readlink$GFAPI_3.4.0
+_pub_glfs_mknod _glfs_mknod$GFAPI_3.4.0
+_pub_glfs_mkdir _glfs_mkdir$GFAPI_3.4.0
+_pub_glfs_unlink _glfs_unlink$GFAPI_3.4.0
+_pub_glfs_rmdir _glfs_rmdir$GFAPI_3.4.0
+_pub_glfs_rename _glfs_rename$GFAPI_3.4.0
+_pub_glfs_link _glfs_link$GFAPI_3.4.0
+_pub_glfs_opendir _glfs_opendir$GFAPI_3.4.0
+_pub_glfs_readdir_r _glfs_readdir_r$GFAPI_3.4.0
+_pub_glfs_readdirplus_r _glfs_readdirplus_r$GFAPI_3.4.0
+_pub_glfs_telldir _glfs_telldir$GFAPI_3.4.0
+_pub_glfs_seekdir _glfs_seekdir$GFAPI_3.4.0
+_pub_glfs_closedir _glfs_closedir$GFAPI_3.4.0
+_pub_glfs_statvfs _glfs_statvfs$GFAPI_3.4.0
+_pub_glfs_chmod _glfs_chmod$GFAPI_3.4.0
+_pub_glfs_fchmod _glfs_fchmod$GFAPI_3.4.0
+_pub_glfs_chown _glfs_chown$GFAPI_3.4.0
+_pub_glfs_lchown _glfs_lchown$GFAPI_3.4.0
+_pub_glfs_fchown _glfs_fchown$GFAPI_3.4.0
+_pub_glfs_utimens _glfs_utimens$GFAPI_3.4.0
+_pub_glfs_lutimens _glfs_lutimens$GFAPI_3.4.0
+_pub_glfs_futimens _glfs_futimens$GFAPI_3.4.0
+_pub_glfs_getxattr _glfs_getxattr$GFAPI_3.4.0
+_pub_glfs_lgetxattr _glfs_lgetxattr$GFAPI_3.4.0
+_pub_glfs_fgetxattr _glfs_fgetxattr$GFAPI_3.4.0
+_pub_glfs_listxattr _glfs_listxattr$GFAPI_3.4.0
+_pub_glfs_llistxattr _glfs_llistxattr$GFAPI_3.4.0
+_pub_glfs_flistxattr _glfs_flistxattr$GFAPI_3.4.0
+_pub_glfs_setxattr _glfs_setxattr$GFAPI_3.4.0
+_pub_glfs_lsetxattr _glfs_lsetxattr$GFAPI_3.4.0
+_pub_glfs_fsetxattr _glfs_fsetxattr$GFAPI_3.4.0
+_pub_glfs_removexattr _glfs_removexattr$GFAPI_3.4.0
+_pub_glfs_lremovexattr _glfs_lremovexattr$GFAPI_3.4.0
+_pub_glfs_fremovexattr _glfs_fremovexattr$GFAPI_3.4.0
+_pub_glfs_getcwd _glfs_getcwd$GFAPI_3.4.0
+_pub_glfs_chdir _glfs_chdir$GFAPI_3.4.0
+_pub_glfs_fchdir _glfs_fchdir$GFAPI_3.4.0
+_pub_glfs_realpath34 _glfs_realpath$GFAPI_3.4.0
+_pub_glfs_posix_lock _glfs_posix_lock$GFAPI_3.4.0
+_pub_glfs_dup _glfs_dup$GFAPI_3.4.0
+
+_pub_glfs_setfsuid _glfs_setfsuid$GFAPI_3.4.2
+_pub_glfs_setfsgid _glfs_setfsgid$GFAPI_3.4.2
+_pub_glfs_setfsgroups _glfs_setfsgroups$GFAPI_3.4.2
+_pub_glfs_h_lookupat34 _glfs_h_lookupat$GFAPI_3.4.2
+_pub_glfs_h_creat _glfs_h_creat$GFAPI_3.4.2
+_pub_glfs_h_mkdir _glfs_h_mkdir$GFAPI_3.4.2
+_pub_glfs_h_mknod _glfs_h_mknod$GFAPI_3.4.2
+_pub_glfs_h_symlink _glfs_h_symlink$GFAPI_3.4.2
+_pub_glfs_h_unlink _glfs_h_unlink$GFAPI_3.4.2
+_pub_glfs_h_close _glfs_h_close$GFAPI_3.4.2
+_pub_glfs_h_truncate _glfs_h_truncate$GFAPI_3.4.2
+_pub_glfs_h_stat _glfs_h_stat$GFAPI_3.4.2
+_pub_glfs_h_getattrs _glfs_h_getattrs$GFAPI_3.4.2
+_pub_glfs_h_setattrs _glfs_h_setattrs$GFAPI_3.4.2
+_pub_glfs_h_readlink _glfs_h_readlink$GFAPI_3.4.2
+_pub_glfs_h_link _glfs_h_link$GFAPI_3.4.2
+_pub_glfs_h_rename _glfs_h_rename$GFAPI_3.4.2
+_pub_glfs_h_extract_handle _glfs_h_extract_handle$GFAPI_3.4.2
+_pub_glfs_h_create_from_handle _glfs_h_create_from_handle$GFAPI_3.4.2
+_pub_glfs_h_opendir _glfs_h_opendir$GFAPI_3.4.2
+_pub_glfs_h_open _glfs_h_open$GFAPI_3.4.2
+
+_pub_glfs_get_volumeid _glfs_get_volumeid$GFAPI_3.5.0
+_pub_glfs_readdir _glfs_readdir$GFAPI_3.5.0
+_pub_glfs_readdirplus _glfs_readdirplus$GFAPI_3.5.0
+_pub_glfs_fallocate _glfs_fallocate$GFAPI_3.5.0
+_pub_glfs_discard _glfs_discard$GFAPI_3.5.0
+_pub_glfs_zerofill _glfs_zerofill$GFAPI_3.5.0
+_pub_glfs_caller_specific_init _glfs_caller_specific_init$GFAPI_3.5.0
+_pub_glfs_h_setxattrs _glfs_h_setxattrs$GFAPI_3.5.0
+
+_pub_glfs_unset_volfile_server _glfs_unset_volfile_server$GFAPI_3.5.1
+_pub_glfs_h_getxattrs _glfs_h_getxattrs$GFAPI_3.5.1
+_pub_glfs_h_removexattrs _glfs_h_removexattrs$GFAPI_3.5.1
+
+_pub_glfs_get_volfile _glfs_get_volfile$GFAPI_3.6.0
+_pub_glfs_h_access _glfs_h_access$GFAPI_3.6.0
+
+_pub_glfs_h_poll_upcall370 _glfs_h_poll_upcall$GFAPI_3.7.0
+_pub_glfs_h_acl_set _glfs_h_acl_set$GFAPI_3.7.0
+_pub_glfs_h_acl_get _glfs_h_acl_get$GFAPI_3.7.0
+_pub_glfs_h_statfs _glfs_h_statfs$GFAPI_3.7.0
+_pub_glfs_h_anonymous_read _glfs_h_anonymous_read$GFAPI_3.7.0
+_pub_glfs_h_anonymous_write _glfs_h_anonymous_write$GFAPI_3.7.0
+
+_priv_glfs_free_from_ctx _glfs_free_from_ctx$GFAPI_PRIVATE_3.7.0
+_priv_glfs_new_from_ctx _glfs_new_from_ctx$GFAPI_PRIVATE_3.7.0
+_priv_glfs_resolve _glfs_resolve$GFAPI_PRIVATE_3.7.0
+_priv_glfs_process_upcall_event _glfs_process_upcall_event$GFAPI_PRIVATE_3.7.0
+
+_pub_glfs_h_lookupat _glfs_h_lookupat$GFAPI_3.7.4
+
+_pub_glfs_truncate _glfs_truncate$GFAPI_3.7.15
+
+_pub_glfs_free _glfs_free$GFAPI_3.7.16
+_pub_glfs_h_poll_upcall _glfs_h_poll_upcall$GFAPI_3.7.16
+_pub_glfs_upcall_get_fs _glfs_upcall_get_fs$GFAPI_3.7.16
+_pub_glfs_upcall_get_reason _glfs_upcall_get_reason$GFAPI_3.7.16
+_pub_glfs_upcall_get_event _glfs_upcall_get_event$GFAPI_3.7.16
+_pub_glfs_upcall_inode_get_object _glfs_upcall_inode_get_object$GFAPI_3.7.16
+_pub_glfs_upcall_inode_get_flags _glfs_upcall_inode_get_flags$GFAPI_3.7.16
+_pub_glfs_upcall_inode_get_stat _glfs_upcall_inode_get_stat$GFAPI_3.7.16
+_pub_glfs_upcall_inode_get_expire _glfs_upcall_inode_get_expire$GFAPI_3.7.16
+_pub_glfs_upcall_inode_get_pobject _glfs_upcall_inode_get_pobject$GFAPI_3.7.16
+_pub_glfs_upcall_inode_get_pstat _glfs_upcall_inode_get_pstat$GFAPI_3.7.16
+_pub_glfs_upcall_inode_get_oldpobject _glfs_upcall_inode_get_oldpobject$GFAPI_3.7.16
+_pub_glfs_upcall_inode_get_oldpstat _glfs_upcall_inode_get_oldpstat$GFAPI_3.7.16
+
+_pub_glfs_realpath _glfs_realpath$GFAPI_3.7.17
+
+_pub_glfs_sysrq _glfs_sysrq$GFAPI_3.10.0
+
+_pub_glfs_fd_set_lkowner _glfs_fd_set_lkowner$GFAPI_3.10.7
+
+_pub_glfs_xreaddirplus_r _glfs_xreaddirplus_r$GFAPI_3.11.0
+_pub_glfs_xreaddirplus_r_get_stat _glfs_xreaddirplus_r_get_stat$GFAPI_3.11.0
+_pub_glfs_xreaddirplus_r_get_object _glfs_xreaddirplus_r_get_object$GFAPI_3.11.0
+_pub_glfs_object_copy _glfs_object_copy$GFAPI_3.11.0
+
+_priv_glfs_ipc _glfs_ipc$GFAPI_3.12.0
+
+_pub_glfs_upcall_register _glfs_upcall_register$GFAPI_3.13.0
+_pub_glfs_upcall_unregister _glfs_upcall_unregister$GFAPI_3.13.0
+
+_pub_glfs_setfsleaseid _glfs_setfsleaseid$GFAPI_4.0.0
+_pub_glfs_file_lock _glfs_file_lock$GFAPI_4.0.0
+_pub_glfs_lease _glfs_lease$GFAPI_4.0.0
+_pub_glfs_h_lease _glfs_h_lease$GFAPI_4.0.0
+_pub_glfs_upcall_lease_get_object _glfs_upcall_lease_get_object$GFAPI_4.1.6
+_pub_glfs_upcall_lease_get_lease_type _glfs_upcall_lease_get_lease_type$GFAPI_4.1.6
+
+_priv_glfs_statx _glfs_statx$GFAPI_6.0
+_priv_glfs_iatt_from_statx _glfs_iatt_from_statx$GFAPI_6.0
+_priv_glfs_setfspid _glfs_setfspid$GFAPI_6.1
+
+_pub_glfs_read_async _glfs_read_async$GFAPI_6.0
+_pub_glfs_write_async _glfs_write_async$GFAPI_6.0
+_pub_glfs_readv_async _glfs_readv_async$GFAPI_6.0
+_pub_glfs_writev_async _glfs_writev_async$GFAPI_6.0
+_pub_glfs_pread _glfs_pread$GFAPI_6.0
+_pub_glfs_pwrite _glfs_pwrite$GFAPI_6.0
+_pub_glfs_pread_async _glfs_pread_async$GFAPI_6.0
+_pub_glfs_pwrite_async _glfs_pwrite_async$GFAPI_6.0
+_pub_glfs_preadv_async _glfs_preadv_async$GFAPI_6.0
+_pub_glfs_pwritev_async _glfs_pwritev_async$GFAPI_6.0
+_pub_glfs_fsync _glfs_fsync$GFAPI_6.0
+_pub_glfs_fsync_async _glfs_fsync_async$GFAPI_6.0
+_pub_glfs_fdatasync _glfs_fdatasync$GFAPI_6.0
+_pub_glfs_fdatasync_async _glfs_fdatasync_async$GFAPI_6.0
+_pub_glfs_ftruncate _glfs_ftruncate$GFAPI_6.0
+_pub_glfs_ftruncate_async _glfs_ftruncate_async$GFAPI_6.0
+_pub_glfs_discard_async _glfs_discard_async$GFAPI_6.0
+_pub_glfs_zerofill_async _glfs_zerofill_async$GFAPI_6.0
+_pub_glfs_copy_file_range _glfs_copy_file_range$GFAPI_6.0
+_pub_glfs_fsetattr _glfs_fsetattr$GFAPI_6.0
+_pub_glfs_setattr _glfs_setattr$GFAPI_6.0
+
+_pub_glfs_set_statedump_path _glfs_set_statedump_path@GFAPI_7.0
+
+_pub_glfs_h_creat_open _glfs_h_creat_open@GFAPI_6.6
diff --git a/api/src/gfapi.map.in b/api/src/gfapi.map
index af89dfcdada..228ac47c084 100644
--- a/api/src/gfapi.map.in
+++ b/api/src/gfapi.map
@@ -39,7 +39,6 @@ GFAPI_3.4.0 {
 		glfs_preadv_async;
 		glfs_pwritev_async;
 		glfs_lseek;
-		glfs_truncate;
 		glfs_ftruncate;
 		glfs_ftruncate_async;
 		glfs_lstat;
@@ -98,7 +97,6 @@ GFAPI_3.4.2 {
 		glfs_setfsuid;
 		glfs_setfsgid;
 		glfs_setfsgroups;
-		glfs_h_lookupat;
 		glfs_h_creat;
 		glfs_h_mkdir;
 		glfs_h_mknod;
@@ -116,6 +114,7 @@ GFAPI_3.4.2 {
 		glfs_h_create_from_handle;
 		glfs_h_opendir;
 		glfs_h_open;
+		glfs_h_lookupat;
 } GFAPI_3.4.0;
 
 GFAPI_3.5.0 {
@@ -139,3 +138,146 @@ GFAPI_3.5.1 {
 		glfs_h_removexattrs;
 } GFAPI_3.5.0;
 
+GFAPI_3.6.0 {
+	global:
+		glfs_get_volfile;
+		glfs_h_access;
+} GFAPI_3.5.1;
+
+GFAPI_3.7.0 {
+	global:
+		glfs_h_poll_upcall;
+		glfs_h_acl_set;
+		glfs_h_acl_get;
+		glfs_h_statfs;
+		glfs_h_anonymous_read;
+		glfs_h_anonymous_write;
+} GFAPI_3.6.0;
+
+GFAPI_PRIVATE_3.7.0 {
+	global:
+		glfs_free_from_ctx;
+		glfs_new_from_ctx;
+		glfs_resolve;
+		glfs_process_upcall_event;
+} GFAPI_3.7.0;
+
+GFAPI_3.7.4 {
+	global:
+		glfs_h_lookupat;
+} GFAPI_PRIVATE_3.7.0;
+
+GFAPI_3.7.15 {
+	global:
+		glfs_truncate;
+} GFAPI_3.7.4;
+
+GFAPI_3.7.16 {
+	global:
+		glfs_free;
+		glfs_upcall_get_fs;
+		glfs_upcall_get_reason;
+		glfs_upcall_get_event;
+		glfs_upcall_inode_get_object;
+		glfs_upcall_inode_get_flags;
+		glfs_upcall_inode_get_stat;
+		glfs_upcall_inode_get_expire;
+		glfs_upcall_inode_get_pobject;
+		glfs_upcall_inode_get_pstat;
+		glfs_upcall_inode_get_oldpobject;
+		glfs_upcall_inode_get_oldpstat;
+		glfs_h_poll_upcall;
+} GFAPI_3.7.15;
+
+GFAPI_3.7.17 {
+	global:
+		glfs_realpath;
+} GFAPI_3.7.16;
+
+GFAPI_3.10.0 {
+	global:
+		glfs_sysrq;
+} GFAPI_3.7.17;
+
+GFAPI_3.10.7 {
+	global:
+		glfs_fd_set_lkowner;
+} GFAPI_3.10.0;
+
+GFAPI_3.11.0 {
+		glfs_xreaddirplus_r;
+		glfs_xreaddirplus_r_get_stat;
+		glfs_xreaddirplus_r_get_object;
+		glfs_object_copy;
+} GFAPI_3.10.7;
+
+GFAPI_PRIVATE_3.12.0 {
+	global:
+		glfs_ipc;
+} GFAPI_3.11.0;
+
+GFAPI_3.13.0 {
+	global:
+		glfs_upcall_register;
+		glfs_upcall_unregister;
+} GFAPI_PRIVATE_3.12.0;
+
+GFAPI_4.0.0 {
+	global:
+		glfs_setfsleaseid;
+		glfs_file_lock;
+		glfs_lease;
+		glfs_h_lease;
+} GFAPI_3.13.0;
+
+GFAPI_4.1.6 {
+	global:
+		glfs_upcall_lease_get_object;
+		glfs_upcall_lease_get_lease_type;
+} GFAPI_4.0.0;
+
+GFAPI_PRIVATE_6.0 {
+	global:
+		glfs_statx;
+		glfs_iatt_from_statx;
+} GFAPI_4.1.6;
+
+GFAPI_6.0 {
+	global:
+		glfs_read_async;
+		glfs_write_async;
+		glfs_readv_async;
+		glfs_writev_async;
+		glfs_pread;
+		glfs_pwrite;
+		glfs_pread_async;
+		glfs_pwrite_async;
+		glfs_preadv_async;
+		glfs_pwritev_async;
+		glfs_fsync;
+		glfs_fsync_async;
+		glfs_fdatasync;
+		glfs_fdatasync_async;
+		glfs_ftruncate;
+		glfs_ftruncate_async;
+		glfs_discard_async;
+		glfs_zerofill_async;
+		glfs_copy_file_range;
+		glfs_setattr;
+		glfs_fsetattr;
+} GFAPI_PRIVATE_6.0;
+
+GFAPI_PRIVATE_6.1 {
+	global:
+		glfs_setfspid;
+} GFAPI_6.0;
+
+GFAPI_6.6 {
+	global:
+		glfs_h_creat_open;
+} GFAPI_PRIVATE_6.1;
+
+GFAPI_7.0 {
+	global:
+		glfs_set_statedump_path;
+} GFAPI_6.6;
diff --git a/api/src/glfs-fops.c b/api/src/glfs-fops.c
index 6726371bd74..6aa3c5602d1 100644
--- a/api/src/glfs-fops.c
+++ b/api/src/glfs-fops.c
@@ -1,5 +1,6 @@
+
 /*
-  Copyright (c) 2012 Red Hat, Inc. <http://www.redhat.com>
+  Copyright (c) 2012-2018 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
   This file is licensed to you under your choice of the GNU Lesser
@@ -8,12 +9,22 @@
   cases as published by the Free Software Foundation.
 */
 
+/* for SEEK_HOLE and SEEK_DATA */
+#ifndef _GNU_SOURCE
+#define _GNU_SOURCE
+#endif
+
+#include <unistd.h>
 
 #include "glfs-internal.h"
 #include "glfs-mem-types.h"
-#include "syncop.h"
+#include <glusterfs/syncop.h>
 #include "glfs.h"
+#include "gfapi-messages.h"
+#include <glusterfs/compat-errno.h>
 #include <limits.h>
+#include "glusterfs3.h"
+#include <glusterfs/iatt.h>
 
 #ifdef NAME_MAX
 #define GF_NAME_MAX NAME_MAX
@@ -21,3422 +32,6414 @@
 #define GF_NAME_MAX 255
 #endif
 
+struct upcall_syncop_args {
+    struct glfs *fs;
+    struct gf_upcall upcall_data;
+};
+
 #define READDIRBUF_SIZE (sizeof(struct dirent) + GF_NAME_MAX + 1)
 
+typedef void (*glfs_io_cbk34)(glfs_fd_t *fd, ssize_t ret, void *data);
+
+/*
+ * This function will mark glfd for deletion and decrement its refcount.
+ */
 int
-glfs_loc_link (loc_t *loc, struct iatt *iatt)
+glfs_mark_glfd_for_deletion(struct glfs_fd *glfd)
 {
-	int ret = -1;
-	inode_t *linked_inode = NULL;
-
-	if (!loc->inode) {
-		errno = EINVAL;
-		return -1;
-	}
+    LOCK(&glfd->lock);
+    {
+        glfd->state = GLFD_CLOSE;
+    }
+    UNLOCK(&glfd->lock);
 
-	linked_inode = inode_link (loc->inode, loc->parent, loc->name, iatt);
-	if (linked_inode) {
-		inode_lookup (linked_inode);
-		inode_unref (linked_inode);
-		ret = 0;
-	} else {
-		ret = -1;
-		errno = ENOMEM;
-	}
+    GF_REF_PUT(glfd);
 
-	return ret;
+    return 0;
 }
 
+/* This function is useful for all async fops. There is chance that glfd is
+ * closed before async fop is completed. When glfd is closed we change the
+ * state to GLFD_CLOSE.
+ *
+ * This function will return _gf_true if the glfd is still valid else return
+ * _gf_false.
+ */
+gf_boolean_t
+glfs_is_glfd_still_valid(struct glfs_fd *glfd)
+{
+    gf_boolean_t ret = _gf_false;
+
+    LOCK(&glfd->lock);
+    {
+        if (glfd->state != GLFD_CLOSE)
+            ret = _gf_true;
+    }
+    UNLOCK(&glfd->lock);
+
+    return ret;
+}
 
 void
-glfs_iatt_to_stat (struct glfs *fs, struct iatt *iatt, struct stat *stat)
+glfd_set_state_bind(struct glfs_fd *glfd)
 {
-	iatt_to_stat (iatt, stat);
-	stat->st_dev = fs->dev_id;
-}
+    LOCK(&glfd->lock);
+    {
+        glfd->state = GLFD_OPEN;
+    }
+    UNLOCK(&glfd->lock);
+
+    fd_bind(glfd->fd);
+    glfs_fd_bind(glfd);
 
+    return;
+}
 
+/*
+ * This routine is called when an upcall event of type
+ * 'GF_UPCALL_CACHE_INVALIDATION' is received.
+ * It makes a copy of the contents of the upcall cache-invalidation
+ * data received into an entry which is stored in the upcall list
+ * maintained by gfapi.
+ */
 int
-glfs_loc_unlink (loc_t *loc)
+glfs_get_upcall_cache_invalidation(struct gf_upcall *to_up_data,
+                                   struct gf_upcall *from_up_data)
 {
-	inode_unlink (loc->inode, loc->parent, loc->name);
+    struct gf_upcall_cache_invalidation *ca_data = NULL;
+    struct gf_upcall_cache_invalidation *f_ca_data = NULL;
+    int ret = -1;
 
-	return 0;
-}
+    GF_VALIDATE_OR_GOTO(THIS->name, to_up_data, out);
+    GF_VALIDATE_OR_GOTO(THIS->name, from_up_data, out);
 
+    f_ca_data = from_up_data->data;
+    GF_VALIDATE_OR_GOTO(THIS->name, f_ca_data, out);
 
-struct glfs_fd *
-pub_glfs_open (struct glfs *fs, const char *path, int flags)
-{
-	int              ret = -1;
-	struct glfs_fd  *glfd = NULL;
-	xlator_t        *subvol = NULL;
-	loc_t            loc = {0, };
-	struct iatt      iatt = {0, };
-	int              reval = 0;
+    ca_data = GF_CALLOC(1, sizeof(*ca_data), glfs_mt_upcall_entry_t);
 
-	__glfs_entry_fs (fs);
+    if (!ca_data) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, errno, API_MSG_ALLOC_FAILED, "entry",
+                NULL);
+        goto out;
+    }
 
-	subvol = priv_glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+    to_up_data->data = ca_data;
 
-	glfd = glfs_fd_new (fs);
-	if (!glfd)
-		goto out;
+    ca_data->flags = f_ca_data->flags;
+    ca_data->expire_time_attr = f_ca_data->expire_time_attr;
+    ca_data->stat = f_ca_data->stat;
+    ca_data->p_stat = f_ca_data->p_stat;
+    ca_data->oldp_stat = f_ca_data->oldp_stat;
 
-retry:
-	ret = glfs_resolve (fs, subvol, path, &loc, &iatt, reval);
+    ret = 0;
+out:
+    return ret;
+}
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+int
+glfs_get_upcall_lease(struct gf_upcall *to_up_data,
+                      struct gf_upcall *from_up_data)
+{
+    struct gf_upcall_recall_lease *ca_data = NULL;
+    struct gf_upcall_recall_lease *f_ca_data = NULL;
+    int ret = -1;
 
-	if (ret)
-		goto out;
+    GF_VALIDATE_OR_GOTO(THIS->name, to_up_data, out);
+    GF_VALIDATE_OR_GOTO(THIS->name, from_up_data, out);
 
-	if (IA_ISDIR (iatt.ia_type)) {
-		ret = -1;
-		errno = EISDIR;
-		goto out;
-	}
+    f_ca_data = from_up_data->data;
+    GF_VALIDATE_OR_GOTO(THIS->name, f_ca_data, out);
 
-	if (!IA_ISREG (iatt.ia_type)) {
-		ret = -1;
-		errno = EINVAL;
-		goto out;
-	}
+    ca_data = GF_CALLOC(1, sizeof(*ca_data), glfs_mt_upcall_entry_t);
 
-	if (glfd->fd) {
-		/* Retry. Safe to touch glfd->fd as we
-		   still have not glfs_fd_bind() yet.
-		*/
-		fd_unref (glfd->fd);
-		glfd->fd = NULL;
-	}
+    if (!ca_data) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, errno, API_MSG_ALLOC_FAILED, "entry",
+                NULL);
+        goto out;
+    }
 
-	glfd->fd = fd_create (loc.inode, getpid());
-	if (!glfd->fd) {
-		ret = -1;
-		errno = ENOMEM;
-		goto out;
-	}
+    to_up_data->data = ca_data;
 
-	ret = syncop_open (subvol, &loc, flags, glfd->fd);
+    ca_data->lease_type = f_ca_data->lease_type;
+    gf_uuid_copy(ca_data->tid, f_ca_data->tid);
+    ca_data->dict = f_ca_data->dict;
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    ret = 0;
 out:
-	loc_wipe (&loc);
-
-	if (ret && glfd) {
-		glfs_fd_destroy (glfd);
-		glfd = NULL;
-	} else if (glfd) {
-                glfd->fd->flags = flags;
-		fd_bind (glfd->fd);
-		glfs_fd_bind (glfd);
-	}
+    return ret;
+}
+int
+glfs_loc_link(loc_t *loc, struct iatt *iatt)
+{
+    int ret = -1;
+    inode_t *old_inode = NULL;
+    uint64_t ctx_value = LOOKUP_NOT_NEEDED;
+
+    if (!loc->inode) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    old_inode = loc->inode;
+
+    /* If the inode already exists in the cache, the inode
+     * returned here points to the existing one. We need
+     * to update loc.inode accordingly.
+     */
+    loc->inode = inode_link(loc->inode, loc->parent, loc->name, iatt);
+    if (loc->inode) {
+        inode_ctx_set(loc->inode, THIS, &ctx_value);
+        inode_lookup(loc->inode);
+        inode_unref(old_inode);
+        ret = 0;
+    } else {
+        ret = -1;
+    }
+
+    return ret;
+}
 
-	priv_glfs_subvol_done (fs, subvol);
+void
+glfs_iatt_to_stat(struct glfs *fs, struct iatt *iatt, struct stat *stat)
+{
+    iatt_to_stat(iatt, stat);
+    stat->st_dev = fs->dev_id;
+}
 
-	return glfd;
+void
+glfs_iatt_to_statx(struct glfs *fs, const struct iatt *iatt,
+                   struct glfs_stat *statx)
+{
+    statx->glfs_st_mask = 0;
+
+    statx->glfs_st_mode = 0;
+    if (IATT_TYPE_VALID(iatt->ia_flags)) {
+        statx->glfs_st_mode |= st_mode_type_from_ia(iatt->ia_type);
+        statx->glfs_st_mask |= GLFS_STAT_TYPE;
+    }
+
+    if (IATT_MODE_VALID(iatt->ia_flags)) {
+        statx->glfs_st_mode |= st_mode_prot_from_ia(iatt->ia_prot);
+        statx->glfs_st_mask |= GLFS_STAT_MODE;
+    }
+
+    if (IATT_NLINK_VALID(iatt->ia_flags)) {
+        statx->glfs_st_nlink = iatt->ia_nlink;
+        statx->glfs_st_mask |= GLFS_STAT_NLINK;
+    }
+
+    if (IATT_UID_VALID(iatt->ia_flags)) {
+        statx->glfs_st_uid = iatt->ia_uid;
+        statx->glfs_st_mask |= GLFS_STAT_UID;
+    }
+
+    if (IATT_GID_VALID(iatt->ia_flags)) {
+        statx->glfs_st_gid = iatt->ia_gid;
+        statx->glfs_st_mask |= GLFS_STAT_GID;
+    }
+
+    if (IATT_ATIME_VALID(iatt->ia_flags)) {
+        statx->glfs_st_atime.tv_sec = iatt->ia_atime;
+        statx->glfs_st_atime.tv_nsec = iatt->ia_atime_nsec;
+        statx->glfs_st_mask |= GLFS_STAT_ATIME;
+    }
+
+    if (IATT_MTIME_VALID(iatt->ia_flags)) {
+        statx->glfs_st_mtime.tv_sec = iatt->ia_mtime;
+        statx->glfs_st_mtime.tv_nsec = iatt->ia_mtime_nsec;
+        statx->glfs_st_mask |= GLFS_STAT_MTIME;
+    }
+
+    if (IATT_CTIME_VALID(iatt->ia_flags)) {
+        statx->glfs_st_ctime.tv_sec = iatt->ia_ctime;
+        statx->glfs_st_ctime.tv_nsec = iatt->ia_ctime_nsec;
+        statx->glfs_st_mask |= GLFS_STAT_CTIME;
+    }
+
+    if (IATT_BTIME_VALID(iatt->ia_flags)) {
+        statx->glfs_st_btime.tv_sec = iatt->ia_btime;
+        statx->glfs_st_btime.tv_nsec = iatt->ia_btime_nsec;
+        statx->glfs_st_mask |= GLFS_STAT_BTIME;
+    }
+
+    if (IATT_INO_VALID(iatt->ia_flags)) {
+        statx->glfs_st_ino = iatt->ia_ino;
+        statx->glfs_st_mask |= GLFS_STAT_INO;
+    }
+
+    if (IATT_SIZE_VALID(iatt->ia_flags)) {
+        statx->glfs_st_size = iatt->ia_size;
+        statx->glfs_st_mask |= GLFS_STAT_SIZE;
+    }
+
+    if (IATT_BLOCKS_VALID(iatt->ia_flags)) {
+        statx->glfs_st_blocks = iatt->ia_blocks;
+        statx->glfs_st_mask |= GLFS_STAT_BLOCKS;
+    }
+
+    /* unconditionally present, encode as is */
+    statx->glfs_st_blksize = iatt->ia_blksize;
+    statx->glfs_st_rdev_major = ia_major(iatt->ia_rdev);
+    statx->glfs_st_rdev_minor = ia_minor(iatt->ia_rdev);
+    statx->glfs_st_dev_major = ia_major(fs->dev_id);
+    statx->glfs_st_dev_minor = ia_minor(fs->dev_id);
+
+    /* At present we do not read any localFS attributes and pass them along,
+     * so setting this to 0. As we start supporting file attributes we can
+     * populate the same here as well */
+    statx->glfs_st_attributes = 0;
+    statx->glfs_st_attributes_mask = 0;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_open, 3.4.0);
+GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_iatt_from_statx, 6.0)
+void
+priv_glfs_iatt_from_statx(struct iatt *iatt, const struct glfs_stat *statx)
+{
+    /* Most code in xlators are not checking validity flags before accessing
+    the items. Hence zero everything before setting valid items */
+    memset(iatt, 0, sizeof(struct iatt));
+
+    if (GLFS_STAT_TYPE_VALID(statx->glfs_st_mask)) {
+        iatt->ia_type = ia_type_from_st_mode(statx->glfs_st_mode);
+        iatt->ia_flags |= IATT_TYPE;
+    }
+
+    if (GLFS_STAT_MODE_VALID(statx->glfs_st_mask)) {
+        iatt->ia_prot = ia_prot_from_st_mode(statx->glfs_st_mode);
+        iatt->ia_flags |= IATT_MODE;
+    }
+
+    if (GLFS_STAT_NLINK_VALID(statx->glfs_st_mask)) {
+        iatt->ia_nlink = statx->glfs_st_nlink;
+        iatt->ia_flags |= IATT_NLINK;
+    }
+
+    if (GLFS_STAT_UID_VALID(statx->glfs_st_mask)) {
+        iatt->ia_uid = statx->glfs_st_uid;
+        iatt->ia_flags |= IATT_UID;
+    }
+
+    if (GLFS_STAT_GID_VALID(statx->glfs_st_mask)) {
+        iatt->ia_gid = statx->glfs_st_gid;
+        iatt->ia_flags |= IATT_GID;
+    }
+
+    if (GLFS_STAT_ATIME_VALID(statx->glfs_st_mask)) {
+        iatt->ia_atime = statx->glfs_st_atime.tv_sec;
+        iatt->ia_atime_nsec = statx->glfs_st_atime.tv_nsec;
+        iatt->ia_flags |= IATT_ATIME;
+    }
+
+    if (GLFS_STAT_MTIME_VALID(statx->glfs_st_mask)) {
+        iatt->ia_mtime = statx->glfs_st_mtime.tv_sec;
+        iatt->ia_mtime_nsec = statx->glfs_st_mtime.tv_nsec;
+        iatt->ia_flags |= IATT_MTIME;
+    }
+
+    if (GLFS_STAT_CTIME_VALID(statx->glfs_st_mask)) {
+        iatt->ia_ctime = statx->glfs_st_ctime.tv_sec;
+        iatt->ia_ctime_nsec = statx->glfs_st_ctime.tv_nsec;
+        iatt->ia_flags |= IATT_CTIME;
+    }
+
+    if (GLFS_STAT_BTIME_VALID(statx->glfs_st_mask)) {
+        iatt->ia_btime = statx->glfs_st_btime.tv_sec;
+        iatt->ia_btime_nsec = statx->glfs_st_btime.tv_nsec;
+        iatt->ia_flags |= IATT_BTIME;
+    }
+
+    if (GLFS_STAT_INO_VALID(statx->glfs_st_mask)) {
+        iatt->ia_ino = statx->glfs_st_ino;
+        iatt->ia_flags |= IATT_INO;
+    }
+
+    if (GLFS_STAT_SIZE_VALID(statx->glfs_st_mask)) {
+        iatt->ia_size = statx->glfs_st_size;
+        iatt->ia_flags |= IATT_SIZE;
+    }
+
+    if (GLFS_STAT_BLOCKS_VALID(statx->glfs_st_mask)) {
+        iatt->ia_blocks = statx->glfs_st_blocks;
+        iatt->ia_flags |= IATT_BLOCKS;
+    }
+
+    /* unconditionally present, encode as is */
+    iatt->ia_blksize = statx->glfs_st_blksize;
+    iatt->ia_rdev = makedev(statx->glfs_st_rdev_major,
+                            statx->glfs_st_rdev_minor);
+    iatt->ia_dev = makedev(statx->glfs_st_dev_major, statx->glfs_st_dev_minor);
+    iatt->ia_attributes = statx->glfs_st_attributes;
+    iatt->ia_attributes_mask = statx->glfs_st_attributes_mask;
+}
 
+void
+glfsflags_from_gfapiflags(struct glfs_stat *stat, int *glvalid)
+{
+    *glvalid = 0;
+    if (stat->glfs_st_mask & GLFS_STAT_MODE) {
+        *glvalid |= GF_SET_ATTR_MODE;
+    }
+
+    if (stat->glfs_st_mask & GLFS_STAT_SIZE) {
+        *glvalid |= GF_SET_ATTR_SIZE;
+    }
+
+    if (stat->glfs_st_mask & GLFS_STAT_UID) {
+        *glvalid |= GF_SET_ATTR_UID;
+    }
+
+    if (stat->glfs_st_mask & GLFS_STAT_GID) {
+        *glvalid |= GF_SET_ATTR_GID;
+    }
+
+    if (stat->glfs_st_mask & GLFS_STAT_ATIME) {
+        *glvalid |= GF_SET_ATTR_ATIME;
+    }
+
+    if (stat->glfs_st_mask & GLFS_STAT_MTIME) {
+        *glvalid |= GF_SET_ATTR_MTIME;
+    }
+}
 
 int
-pub_glfs_close (struct glfs_fd *glfd)
+glfs_loc_unlink(loc_t *loc)
 {
-	xlator_t  *subvol = NULL;
-	int        ret = -1;
-	fd_t      *fd = NULL;
-	struct glfs *fs = NULL;
+    inode_unlink(loc->inode, loc->parent, loc->name);
 
-	__glfs_entry_fd (glfd);
+    /* since glfs_h_* objects hold a reference to inode
+     * it is safe to keep lookup count to '0' */
+    if (!inode_has_dentry(loc->inode))
+        inode_forget(loc->inode, 0);
 
-	subvol = priv_glfs_active_subvol (glfd->fs);
-        if (!subvol) {
-                ret = -1;
-                errno = EIO;
-                goto out;
-        }
+    return 0;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_open, 3.4.0)
+struct glfs_fd *
+pub_glfs_open(struct glfs *fs, const char *path, int flags)
+{
+    int ret = -1;
+    struct glfs_fd *glfd = NULL;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    int reval = 0;
+    dict_t *fop_attr = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    glfd = glfs_fd_new(fs);
+    if (!glfd)
+        goto out;
+
+retry:
+    ret = glfs_resolve(fs, subvol, path, &loc, &iatt, reval);
+
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
+
+    if (ret)
+        goto out;
 
-	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
-	if (!fd) {
-		ret = -1;
-		errno = EBADFD;
-		goto out;
-	}
+    if (IA_ISDIR(iatt.ia_type)) {
+        ret = -1;
+        errno = EISDIR;
+        goto out;
+    }
 
-	ret = syncop_flush (subvol, fd);
+    if (!IA_ISREG(iatt.ia_type)) {
+        ret = -1;
+        errno = EINVAL;
+        goto out;
+    }
+
+    if (glfd->fd) {
+        /* Retry. Safe to touch glfd->fd as we
+           still have not glfs_fd_bind() yet.
+        */
+        fd_unref(glfd->fd);
+        glfd->fd = NULL;
+    }
+
+    glfd->fd = fd_create(loc.inode, getpid());
+    if (!glfd->fd) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+    glfd->fd->flags = flags;
+
+    ret = get_fop_attr_thrd_key(&fop_attr);
+    if (ret)
+        gf_msg_debug("gfapi", 0, "Getting leaseid from thread failed");
+
+    ret = syncop_open(subvol, &loc, flags, glfd->fd, fop_attr, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 out:
-	fs = glfd->fs;
-	glfs_fd_destroy (glfd);
+    loc_wipe(&loc);
 
-	if (fd)
-		fd_unref (fd);
+    if (fop_attr)
+        dict_unref(fop_attr);
 
-	priv_glfs_subvol_done (fs, subvol);
+    if (ret && glfd) {
+        GF_REF_PUT(glfd);
+        glfd = NULL;
+    } else if (glfd) {
+        glfd_set_state_bind(glfd);
+    }
 
-	return ret;
-}
+    glfs_subvol_done(fs, subvol);
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_close, 3.4.0);
+    __GLFS_EXIT_FS;
 
+invalid_fs:
+    return glfd;
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_close, 3.4.0)
 int
-pub_glfs_lstat (struct glfs *fs, const char *path, struct stat *stat)
+pub_glfs_close(struct glfs_fd *glfd)
 {
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	loc_t            loc = {0, };
-	struct iatt      iatt = {0, };
-	int              reval = 0;
+    xlator_t *subvol = NULL;
+    int ret = -1;
+    fd_t *fd = NULL;
+    struct glfs *fs = NULL;
+    dict_t *fop_attr = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    if (glfd->lk_owner.len != 0) {
+        ret = syncopctx_setfslkowner(&glfd->lk_owner);
+        if (ret)
+            goto out;
+    }
+    ret = get_fop_attr_thrd_key(&fop_attr);
+    if (ret)
+        gf_msg_debug("gfapi", 0, "Getting leaseid from thread failed");
+
+    ret = syncop_flush(subvol, fd, fop_attr, NULL);
+    DECODE_SYNCOP_ERR(ret);
+out:
+    fs = glfd->fs;
+
+    if (fd)
+        fd_unref(fd);
+    if (fop_attr)
+        dict_unref(fop_attr);
+
+    glfs_mark_glfd_for_deletion(glfd);
+    glfs_subvol_done(fs, subvol);
+
+    __GLFS_EXIT_FS;
 
-	__glfs_entry_fs (fs);
+invalid_fs:
+    return ret;
+}
 
-	subvol = priv_glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_lstat, 3.4.0)
+int
+pub_glfs_lstat(struct glfs *fs, const char *path, struct stat *stat)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
 retry:
-	ret = glfs_lresolve (fs, subvol, path, &loc, &iatt, reval);
+    ret = glfs_lresolve(fs, subvol, path, &loc, &iatt, reval);
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	if (ret == 0 && stat)
-		glfs_iatt_to_stat (fs, &iatt, stat);
+    if (ret == 0 && stat)
+        glfs_iatt_to_stat(fs, &iatt, stat);
 out:
-	loc_wipe (&loc);
+    loc_wipe(&loc);
 
-	priv_glfs_subvol_done (fs, subvol);
+    glfs_subvol_done(fs, subvol);
 
-	return ret;
-}
-
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_lstat, 3.4.0);
+    __GLFS_EXIT_FS;
 
+invalid_fs:
+    return ret;
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_stat, 3.4.0)
 int
-pub_glfs_stat (struct glfs *fs, const char *path, struct stat *stat)
+pub_glfs_stat(struct glfs *fs, const char *path, struct stat *stat)
 {
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	loc_t            loc = {0, };
-	struct iatt      iatt = {0, };
-	int              reval = 0;
-
-	__glfs_entry_fs (fs);
-
-	subvol = priv_glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
 retry:
-	ret = glfs_resolve (fs, subvol, path, &loc, &iatt, reval);
+    ret = glfs_resolve(fs, subvol, path, &loc, &iatt, reval);
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	if (ret == 0 && stat)
-		glfs_iatt_to_stat (fs, &iatt, stat);
+    if (ret == 0 && stat)
+        glfs_iatt_to_stat(fs, &iatt, stat);
 out:
-	loc_wipe (&loc);
-
-	priv_glfs_subvol_done (fs, subvol);
+    loc_wipe(&loc);
 
-	return ret;
-}
+    glfs_subvol_done(fs, subvol);
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_stat, 3.4.0);
+    __GLFS_EXIT_FS;
 
+invalid_fs:
+    return ret;
+}
 
+GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_statx, 6.0)
 int
-pub_glfs_fstat (struct glfs_fd *glfd, struct stat *stat)
+priv_glfs_statx(struct glfs *fs, const char *path, const unsigned int mask,
+                struct glfs_stat *statxbuf)
 {
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	struct iatt      iatt = {0, };
-	fd_t            *fd = NULL;
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    if (path == NULL) {
+        ret = -1;
+        errno = EINVAL;
+        goto out;
+    }
+
+    if (mask & ~GLFS_STAT_ALL) {
+        ret = -1;
+        errno = EINVAL;
+        goto out;
+    }
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+retry:
+    ret = glfs_resolve(fs, subvol, path, &loc, &iatt, reval);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	__glfs_entry_fd (glfd);
+    if (ret == 0 && statxbuf)
+        glfs_iatt_to_statx(fs, &iatt, statxbuf);
+out:
+    loc_wipe(&loc);
 
-	subvol = priv_glfs_active_subvol (glfd->fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+    glfs_subvol_done(fs, subvol);
 
-	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
-	if (!fd) {
-		ret = -1;
-		errno = EBADFD;
-		goto out;
-	}
+    __GLFS_EXIT_FS;
 
-	ret = syncop_fstat (subvol, fd, &iatt);
+invalid_fs:
+    return ret;
+}
 
-	if (ret == 0 && stat)
-		glfs_iatt_to_stat (glfd->fs, &iatt, stat);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fstat, 3.4.0)
+int
+pub_glfs_fstat(struct glfs_fd *glfd, struct stat *stat)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    struct iatt iatt = {
+        0,
+    };
+    fd_t *fd = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    GF_REF_GET(glfd);
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    ret = syncop_fstat(subvol, fd, &iatt, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    if (ret == 0 && stat)
+        glfs_iatt_to_stat(glfd->fs, &iatt, stat);
 out:
-	if (fd)
-		fd_unref (fd);
-
-	priv_glfs_subvol_done (glfd->fs, subvol);
+    if (fd)
+        fd_unref(fd);
+    if (glfd)
+        GF_REF_PUT(glfd);
 
-	return ret;
-}
+    glfs_subvol_done(glfd->fs, subvol);
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fstat, 3.4.0);
+    __GLFS_EXIT_FS;
 
+invalid_fs:
+    return ret;
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_creat, 3.4.0)
 struct glfs_fd *
-pub_glfs_creat (struct glfs *fs, const char *path, int flags, mode_t mode)
-{
-	int              ret = -1;
-	struct glfs_fd  *glfd = NULL;
-	xlator_t        *subvol = NULL;
-	loc_t            loc = {0, };
-	struct iatt      iatt = {0, };
-	uuid_t           gfid;
-	dict_t          *xattr_req = NULL;
-	int              reval = 0;
-
-	__glfs_entry_fs (fs);
-
-	subvol = priv_glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
-
-	xattr_req = dict_new ();
-	if (!xattr_req) {
-		ret = -1;
-		errno = ENOMEM;
-		goto out;
-	}
-
-	uuid_generate (gfid);
-	ret = dict_set_static_bin (xattr_req, "gfid-req", gfid, 16);
-	if (ret) {
-		ret = -1;
-		errno = ENOMEM;
-		goto out;
-	}
-
-	glfd = glfs_fd_new (fs);
-	if (!glfd)
-		goto out;
-
-	/* This must be glfs_resolve() and NOT glfs_lresolve().
-	   That is because open("name", O_CREAT) where "name"
-	   is a danging symlink must create the dangling
-	   destinataion.
-	*/
+pub_glfs_creat(struct glfs *fs, const char *path, int flags, mode_t mode)
+{
+    int ret = -1;
+    struct glfs_fd *glfd = NULL;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    uuid_t gfid;
+    dict_t *xattr_req = NULL;
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    xattr_req = dict_new();
+    if (!xattr_req) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gf_uuid_generate(gfid);
+    ret = dict_set_gfuuid(xattr_req, "gfid-req", gfid, true);
+    if (ret) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    glfd = glfs_fd_new(fs);
+    if (!glfd)
+        goto out;
+
+    /* This must be glfs_resolve() and NOT glfs_lresolve().
+       That is because open("name", O_CREAT) where "name"
+       is a danging symlink must create the dangling
+       destination.
+    */
 retry:
-	ret = glfs_resolve (fs, subvol, path, &loc, &iatt, reval);
-
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
-
-	if (ret == -1 && errno != ENOENT)
-		/* Any other type of error is fatal */
-		goto out;
-
-	if (ret == -1 && errno == ENOENT && !loc.parent)
-		/* The parent directory or an ancestor even
-		   higher does not exist
-		*/
-		goto out;
-
-	if (loc.inode) {
-		if (flags & O_EXCL) {
-			ret = -1;
-			errno = EEXIST;
-			goto out;
-		}
-
-		if (IA_ISDIR (iatt.ia_type)) {
-			ret = -1;
-			errno = EISDIR;
-			goto out;
-		}
-
-		if (!IA_ISREG (iatt.ia_type)) {
-			ret = -1;
-			errno = EINVAL;
-			goto out;
-		}
-	}
-
-	if (ret == -1 && errno == ENOENT) {
-		loc.inode = inode_new (loc.parent->table);
-		if (!loc.inode) {
-			ret = -1;
-			errno = ENOMEM;
-			goto out;
-		}
-	}
-
-	if (glfd->fd) {
-		/* Retry. Safe to touch glfd->fd as we
-		   still have not glfs_fd_bind() yet.
-		*/
-		fd_unref (glfd->fd);
-		glfd->fd = NULL;
-	}
-
-	glfd->fd = fd_create (loc.inode, getpid());
-	if (!glfd->fd) {
-		ret = -1;
-		errno = ENOMEM;
-		goto out;
-	}
-
-	if (ret == 0) {
-		ret = syncop_open (subvol, &loc, flags, glfd->fd);
-	} else {
-		ret = syncop_create (subvol, &loc, flags, mode, glfd->fd,
-				     xattr_req, &iatt);
-	}
-
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
-
-	if (ret == 0)
-		ret = glfs_loc_link (&loc, &iatt);
-out:
-	loc_wipe (&loc);
-
-	if (xattr_req)
-		dict_unref (xattr_req);
-
-	if (ret && glfd) {
-		glfs_fd_destroy (glfd);
-		glfd = NULL;
-	} else if (glfd) {
-                glfd->fd->flags = flags;
-		fd_bind (glfd->fd);
-		glfs_fd_bind (glfd);
-	}
-
-	priv_glfs_subvol_done (fs, subvol);
-
-	return glfd;
-}
-
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_creat, 3.4.0);
+    ret = glfs_resolve(fs, subvol, path, &loc, &iatt, reval);
 
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-off_t
-pub_glfs_lseek (struct glfs_fd *glfd, off_t offset, int whence)
-{
-	struct stat sb = {0, };
-	int         ret = -1;
-        off_t       off = -1;
-
-	__glfs_entry_fd (glfd);
-
-	switch (whence) {
-	case SEEK_SET:
-		glfd->offset = offset;
-                ret = 0;
-		break;
-	case SEEK_CUR:
-		glfd->offset += offset;
-                ret = 0;
-		break;
-	case SEEK_END:
-		ret = pub_glfs_fstat (glfd, &sb);
-		if (ret) {
-			/* seek cannot fail :O */
-			break;
-		}
-		glfd->offset = sb.st_size + offset;
-		break;
-        default:
-                errno = EINVAL;
-	}
+    if (ret == -1 && errno != ENOENT)
+        /* Any other type of error is fatal */
+        goto out;
 
-        if (ret != -1)
-                off = glfd->offset;
+    if (ret == -1 && errno == ENOENT && !loc.parent)
+        /* The parent directory or an ancestor even
+           higher does not exist
+        */
+        goto out;
 
-        return off;
-}
+    if (loc.inode) {
+        if (flags & O_EXCL) {
+            ret = -1;
+            errno = EEXIST;
+            goto out;
+        }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_lseek, 3.4.0);
+        if (IA_ISDIR(iatt.ia_type)) {
+            ret = -1;
+            errno = EISDIR;
+            goto out;
+        }
 
+        if (!IA_ISREG(iatt.ia_type)) {
+            ret = -1;
+            errno = EINVAL;
+            goto out;
+        }
+    }
+
+    if (ret == -1 && errno == ENOENT) {
+        loc.inode = inode_new(loc.parent->table);
+        if (!loc.inode) {
+            ret = -1;
+            errno = ENOMEM;
+            goto out;
+        }
+    }
 
-ssize_t
-pub_glfs_preadv (struct glfs_fd *glfd, const struct iovec *iovec, int iovcnt,
-                 off_t offset, int flags)
-{
-	xlator_t       *subvol = NULL;
-	ssize_t         ret = -1;
-	ssize_t         size = -1;
-	struct iovec   *iov = NULL;
-	int             cnt = 0;
-	struct iobref  *iobref = NULL;
-	fd_t           *fd = NULL;
+    if (glfd->fd) {
+        /* Retry. Safe to touch glfd->fd as we
+           still have not glfs_fd_bind() yet.
+        */
+        fd_unref(glfd->fd);
+        glfd->fd = NULL;
+    }
+
+    glfd->fd = fd_create(loc.inode, getpid());
+    if (!glfd->fd) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+    glfd->fd->flags = flags;
+
+    if (get_fop_attr_thrd_key(&xattr_req))
+        gf_msg_debug("gfapi", 0, "Getting leaseid from thread failed");
+    if (ret == 0) {
+        ret = syncop_open(subvol, &loc, flags, glfd->fd, xattr_req, NULL);
+        DECODE_SYNCOP_ERR(ret);
+    } else {
+        ret = syncop_create(subvol, &loc, flags, mode, glfd->fd, &iatt,
+                            xattr_req, NULL);
+        DECODE_SYNCOP_ERR(ret);
+    }
+
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
+
+    if (ret == 0)
+        ret = glfs_loc_link(&loc, &iatt);
+out:
+    loc_wipe(&loc);
 
-	__glfs_entry_fd (glfd);
+    if (xattr_req)
+        dict_unref(xattr_req);
 
-	subvol = priv_glfs_active_subvol (glfd->fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+    if (ret && glfd) {
+        GF_REF_PUT(glfd);
+        glfd = NULL;
+    } else if (glfd) {
+        glfd_set_state_bind(glfd);
+    }
 
-	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
-	if (!fd) {
-		ret = -1;
-		errno = EBADFD;
-		goto out;
-	}
+    glfs_subvol_done(fs, subvol);
 
-	size = iov_length (iovec, iovcnt);
+    __GLFS_EXIT_FS;
 
-	ret = syncop_readv (subvol, fd, size, offset, 0, &iov, &cnt, &iobref);
-	if (ret <= 0)
-		goto out;
+invalid_fs:
+    return glfd;
+}
 
-	size = iov_copy (iovec, iovcnt, iov, cnt); /* FIXME!!! */
+#ifdef HAVE_SEEK_HOLE
+static int
+glfs_seek(struct glfs_fd *glfd, off_t offset, int whence)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    fd_t *fd = NULL;
+    gf_seek_what_t what = 0;
+    off_t off = -1;
+
+    switch (whence) {
+        case SEEK_DATA:
+            what = GF_SEEK_DATA;
+            break;
+        case SEEK_HOLE:
+            what = GF_SEEK_HOLE;
+            break;
+        default:
+            /* other SEEK_* do not make sense, all operations get an offset
+             * and the position in the fd is not tracked */
+            errno = EINVAL;
+            goto out;
+    }
 
-	glfd->offset = (offset + size);
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        errno = EIO;
+        goto out;
+    }
 
-	ret = size;
-out:
-        if (iov)
-                GF_FREE (iov);
-        if (iobref)
-                iobref_unref (iobref);
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        errno = EBADFD;
+        goto done;
+    }
 
-	if (fd)
-		fd_unref (fd);
+    ret = syncop_seek(subvol, fd, offset, what, NULL, &off);
+    DECODE_SYNCOP_ERR(ret);
 
-	priv_glfs_subvol_done (glfd->fs, subvol);
+    if (ret != -1)
+        glfd->offset = off;
 
-	return ret;
-}
+done:
+    if (fd)
+        fd_unref(fd);
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_preadv, 3.4.0);
+    glfs_subvol_done(glfd->fs, subvol);
 
+out:
+    return ret;
+}
+#endif
 
-ssize_t
-pub_glfs_read (struct glfs_fd *glfd, void *buf, size_t count, int flags)
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_lseek, 3.4.0)
+off_t
+pub_glfs_lseek(struct glfs_fd *glfd, off_t offset, int whence)
 {
-	struct iovec iov = {0, };
-	ssize_t      ret = 0;
+    struct stat sb = {
+        0,
+    };
+    int ret = -1;
+    off_t off = -1;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    GF_REF_GET(glfd);
+
+    switch (whence) {
+        case SEEK_SET:
+            glfd->offset = offset;
+            ret = 0;
+            break;
+        case SEEK_CUR:
+            glfd->offset += offset;
+            ret = 0;
+            break;
+        case SEEK_END:
+            ret = pub_glfs_fstat(glfd, &sb);
+            if (ret) {
+                /* seek cannot fail :O */
+                break;
+            }
+            glfd->offset = sb.st_size + offset;
+            break;
+#ifdef HAVE_SEEK_HOLE
+        case SEEK_DATA:
+        case SEEK_HOLE:
+            ret = glfs_seek(glfd, offset, whence);
+            break;
+#endif
+        default:
+            errno = EINVAL;
+    }
 
-	iov.iov_base = buf;
-	iov.iov_len = count;
+    if (glfd)
+        GF_REF_PUT(glfd);
 
-	ret = pub_glfs_preadv (glfd, &iov, 1, glfd->offset, flags);
+    __GLFS_EXIT_FS;
 
-	return ret;
-}
+    if (ret != -1)
+        off = glfd->offset;
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_read, 3.4.0);
+    return off;
 
+invalid_fs:
+    return -1;
+}
 
-ssize_t
-pub_glfs_pread (struct glfs_fd *glfd, void *buf, size_t count, off_t offset,
-                int flags)
+static ssize_t
+glfs_preadv_common(struct glfs_fd *glfd, const struct iovec *iovec, int iovcnt,
+                   off_t offset, int flags, struct glfs_stat *poststat)
 {
-	struct iovec iov = {0, };
-	ssize_t      ret = 0;
-
-	iov.iov_base = buf;
-	iov.iov_len = count;
+    xlator_t *subvol = NULL;
+    ssize_t ret = -1;
+    ssize_t size = -1;
+    struct iovec *iov = NULL;
+    int cnt = 0;
+    struct iobref *iobref = NULL;
+    fd_t *fd = NULL;
+    struct iatt iatt = {
+        0,
+    };
+    dict_t *fop_attr = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    GF_REF_GET(glfd);
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    size = iov_length(iovec, iovcnt);
+
+    ret = get_fop_attr_thrd_key(&fop_attr);
+    if (ret)
+        gf_msg_debug("gfapi", 0, "Getting leaseid from thread failed");
+
+    ret = syncop_readv(subvol, fd, size, offset, 0, &iov, &cnt, &iobref, &iatt,
+                       fop_attr, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    if (ret >= 0 && poststat)
+        glfs_iatt_to_statx(glfd->fs, &iatt, poststat);
+
+    if (ret <= 0)
+        goto out;
+
+    size = iov_copy(iovec, iovcnt, iov, cnt); /* FIXME!!! */
+
+    glfd->offset = (offset + size);
+
+    ret = size;
+out:
+    if (iov)
+        GF_FREE(iov);
+    if (iobref)
+        iobref_unref(iobref);
 
-	ret = pub_glfs_preadv (glfd, &iov, 1, offset, flags);
+    if (fd)
+        fd_unref(fd);
+    if (glfd)
+        GF_REF_PUT(glfd);
+    if (fop_attr)
+        dict_unref(fop_attr);
 
-	return ret;
-}
+    glfs_subvol_done(glfd->fs, subvol);
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_pread, 3.4.0);
+    __GLFS_EXIT_FS;
 
+invalid_fs:
+    return ret;
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_preadv, 3.4.0)
 ssize_t
-pub_glfs_readv (struct glfs_fd *glfd, const struct iovec *iov, int count,
-                int flags)
+pub_glfs_preadv(struct glfs_fd *glfd, const struct iovec *iovec, int iovcnt,
+                off_t offset, int flags)
 {
-	ssize_t      ret = 0;
-
-	ret = pub_glfs_preadv (glfd, iov, count, glfd->offset, flags);
-
-	return ret;
+    return glfs_preadv_common(glfd, iovec, iovcnt, offset, flags, NULL);
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_readv, 3.4.0);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_read, 3.4.0)
+ssize_t
+pub_glfs_read(struct glfs_fd *glfd, void *buf, size_t count, int flags)
+{
+    struct iovec iov = {
+        0,
+    };
+    ssize_t ret = 0;
 
+    if (glfd == NULL) {
+        errno = EBADF;
+        return -1;
+    }
 
-struct glfs_io {
-	struct glfs_fd      *glfd;
-	int                  op;
-	off_t                offset;
-	struct iovec        *iov;
-	int                  count;
-	int                  flags;
-	glfs_io_cbk          fn;
-	void                *data;
-};
+    iov.iov_base = buf;
+    iov.iov_len = count;
 
+    ret = pub_glfs_preadv(glfd, &iov, 1, glfd->offset, flags);
 
-static int
-glfs_io_async_cbk (int ret, call_frame_t *frame, void *data)
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC(glfs_pread34, glfs_pread, 3.4.0)
+ssize_t
+pub_glfs_pread34(struct glfs_fd *glfd, void *buf, size_t count, off_t offset,
+                 int flags)
 {
-	struct glfs_io  *gio = data;
+    struct iovec iov = {
+        0,
+    };
+    ssize_t ret = 0;
 
-	gio->fn (gio->glfd, ret, gio->data);
+    iov.iov_base = buf;
+    iov.iov_len = count;
 
-	GF_FREE (gio->iov);
-	GF_FREE (gio);
+    ret = pub_glfs_preadv(glfd, &iov, 1, offset, flags);
 
-	return 0;
+    return ret;
 }
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_pread, 6.0)
 ssize_t
-pub_glfs_pwritev (struct glfs_fd *, const struct iovec *, int, off_t, int);
+pub_glfs_pread(struct glfs_fd *glfd, void *buf, size_t count, off_t offset,
+               int flags, struct glfs_stat *poststat)
+{
+    struct iovec iov = {
+        0,
+    };
+    ssize_t ret = 0;
 
-int
-pub_glfs_ftruncate (struct glfs_fd *, off_t);
+    iov.iov_base = buf;
+    iov.iov_len = count;
 
-int
-pub_glfs_fdatasync (struct glfs_fd *);
+    ret = glfs_preadv_common(glfd, &iov, 1, offset, flags, poststat);
 
-int
-pub_glfs_fsync (struct glfs_fd *glfd);
+    return ret;
+}
 
-int
-pub_glfs_discard (struct glfs_fd *, off_t, size_t);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_readv, 3.4.0)
+ssize_t
+pub_glfs_readv(struct glfs_fd *glfd, const struct iovec *iov, int count,
+               int flags)
+{
+    ssize_t ret = 0;
 
-int
-pub_glfs_zerofill (struct glfs_fd *, off_t, off_t);
+    if (glfd == NULL) {
+        errno = EBADF;
+        return -1;
+    }
 
-static int
-glfs_io_async_task (void *data)
-{
-	struct glfs_io *gio = data;
-	ssize_t         ret = 0;
-
-	switch (gio->op) {
-	case GF_FOP_WRITE:
-		ret = pub_glfs_pwritev (gio->glfd, gio->iov, gio->count,
-				    gio->offset, gio->flags);
-		break;
-	case GF_FOP_FTRUNCATE:
-		ret = pub_glfs_ftruncate (gio->glfd, gio->offset);
-		break;
-	case GF_FOP_FSYNC:
-		if (gio->flags)
-			ret = pub_glfs_fdatasync (gio->glfd);
-		else
-			ret = pub_glfs_fsync (gio->glfd);
-		break;
-	case GF_FOP_DISCARD:
-		ret = pub_glfs_discard (gio->glfd, gio->offset, gio->count);
-		break;
-        case GF_FOP_ZEROFILL:
-                ret = pub_glfs_zerofill(gio->glfd, gio->offset, gio->count);
-                break;
-	}
+    ret = pub_glfs_preadv(glfd, iov, count, glfd->offset, flags);
 
-	return (int) ret;
+    return ret;
 }
 
+struct glfs_io {
+    struct glfs_fd *glfd;
+    int op;
+    off_t offset;
+    struct iovec *iov;
+    int count;
+    int flags;
+    gf_boolean_t oldcb;
+    union {
+        glfs_io_cbk34 fn34;
+        glfs_io_cbk fn;
+    };
+    void *data;
+};
 
-int
-glfs_preadv_async_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		       int op_ret, int op_errno, struct iovec *iovec,
-		       int count, struct iatt *stbuf, struct iobref *iobref,
-		       dict_t *xdata)
+static int
+glfs_io_async_cbk(int op_ret, int op_errno, call_frame_t *frame, void *cookie,
+                  struct iovec *iovec, int count, struct iatt *prebuf,
+                  struct iatt *postbuf)
 {
-	struct glfs_io *gio = NULL;
-	xlator_t       *subvol = NULL;
-	struct glfs    *fs = NULL;
-	struct glfs_fd *glfd = NULL;
-
-
-	gio = frame->local;
-	frame->local = NULL;
-	subvol = cookie;
-	glfd = gio->glfd;
-	fs = glfd->fs;
-
-	if (op_ret <= 0)
-		goto out;
+    struct glfs_io *gio = NULL;
+    xlator_t *subvol = NULL;
+    struct glfs *fs = NULL;
+    struct glfs_fd *glfd = NULL;
+    int ret = -1;
+    struct glfs_stat prestat = {}, *prestatp = NULL;
+    struct glfs_stat poststat = {}, *poststatp = NULL;
+
+    GF_VALIDATE_OR_GOTO("gfapi", frame, inval);
+    GF_VALIDATE_OR_GOTO("gfapi", cookie, inval);
+
+    gio = frame->local;
+    frame->local = NULL;
+    subvol = cookie;
+    glfd = gio->glfd;
+    fs = glfd->fs;
+
+    if (!glfs_is_glfd_still_valid(glfd))
+        goto err;
+
+    if (op_ret <= 0) {
+        goto out;
+    } else if (gio->op == GF_FOP_READ) {
+        if (!iovec) {
+            op_ret = -1;
+            op_errno = EINVAL;
+            goto out;
+        }
 
-	op_ret = iov_copy (gio->iov, gio->count, iovec, count);
+        op_ret = iov_copy(gio->iov, gio->count, iovec, count);
+        glfd->offset = gio->offset + op_ret;
+    } else if (gio->op == GF_FOP_WRITE) {
+        glfd->offset = gio->offset + gio->iov->iov_len;
+    }
 
-	glfd->offset = gio->offset + op_ret;
 out:
-	errno = op_errno;
-	gio->fn (gio->glfd, op_ret, gio->data);
+    errno = op_errno;
+    if (gio->oldcb) {
+        gio->fn34(gio->glfd, op_ret, gio->data);
+    } else {
+        if (prebuf) {
+            prestatp = &prestat;
+            glfs_iatt_to_statx(fs, prebuf, prestatp);
+        }
 
-	GF_FREE (gio->iov);
-	GF_FREE (gio);
-	STACK_DESTROY (frame->root);
-	priv_glfs_subvol_done (fs, subvol);
+        if (postbuf) {
+            poststatp = &poststat;
+            glfs_iatt_to_statx(fs, postbuf, poststatp);
+        }
 
-	return 0;
+        gio->fn(gio->glfd, op_ret, prestatp, poststatp, gio->data);
+    }
+err:
+    fd_unref(glfd->fd);
+    /* Since the async operation is complete
+     * release the ref taken during the start
+     * of async operation
+     */
+    GF_REF_PUT(glfd);
+
+    GF_FREE(gio->iov);
+    GF_FREE(gio);
+    STACK_DESTROY(frame->root);
+    glfs_subvol_done(fs, subvol);
+
+    ret = 0;
+inval:
+    return ret;
 }
 
+static int
+glfs_preadv_async_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int op_ret, int op_errno, struct iovec *iovec, int count,
+                      struct iatt *stbuf, struct iobref *iobref, dict_t *xdata)
+{
+    glfs_io_async_cbk(op_ret, op_errno, frame, cookie, iovec, count, NULL,
+                      stbuf);
 
-int
-pub_glfs_preadv_async (struct glfs_fd *glfd, const struct iovec *iovec,
-                       int count, off_t offset, int flags, glfs_io_cbk fn,
-                       void *data)
+    return 0;
+}
+
+static int
+glfs_preadv_async_common(struct glfs_fd *glfd, const struct iovec *iovec,
+                         int count, off_t offset, int flags, gf_boolean_t oldcb,
+                         glfs_io_cbk fn, void *data)
 {
-	struct glfs_io *gio = NULL;
-	int             ret = 0;
-	call_frame_t   *frame = NULL;
-	xlator_t       *subvol = NULL;
-	glfs_t         *fs = NULL;
-	fd_t           *fd = NULL;
-
-	__glfs_entry_fd (glfd);
-
-	subvol = priv_glfs_active_subvol (glfd->fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
-
-	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
-	if (!fd) {
-		ret = -1;
-		errno = EBADFD;
-		goto out;
-	}
-
-	fs = glfd->fs;
-
-	frame = syncop_create_frame (THIS);
-	if (!frame) {
-		ret = -1;
-		errno = ENOMEM;
-		goto out;
-	}
-
-	gio = GF_CALLOC (1, sizeof (*gio), glfs_mt_glfs_io_t);
-	if (!gio) {
-		ret = -1;
-		errno = ENOMEM;
-		goto out;
-	}
-
-	gio->iov = iov_dup (iovec, count);
-	if (!gio->iov) {
-		ret = -1;
-		errno = ENOMEM;
-		goto out;
-	}
-
-	gio->op     = GF_FOP_READ;
-	gio->glfd   = glfd;
-	gio->count  = count;
-	gio->offset = offset;
-	gio->flags  = flags;
-	gio->fn     = fn;
-	gio->data   = data;
-
-	frame->local = gio;
-
-	STACK_WIND_COOKIE (frame, glfs_preadv_async_cbk, subvol, subvol,
-			   subvol->fops->readv, fd, iov_length (iovec, count),
-			   offset, flags, NULL);
+    struct glfs_io *gio = NULL;
+    int ret = 0;
+    call_frame_t *frame = NULL;
+    xlator_t *subvol = NULL;
+    struct glfs *fs = NULL;
+    fd_t *fd = NULL;
+    dict_t *fop_attr = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    GF_REF_GET(glfd);
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    fs = glfd->fs;
+
+    frame = syncop_create_frame(THIS);
+    if (!frame) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gio = GF_CALLOC(1, sizeof(*gio), glfs_mt_glfs_io_t);
+    if (!gio) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gio->iov = iov_dup(iovec, count);
+    if (!gio->iov) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gio->op = GF_FOP_READ;
+    gio->glfd = glfd;
+    gio->count = count;
+    gio->offset = offset;
+    gio->flags = flags;
+    gio->oldcb = oldcb;
+    gio->fn = fn;
+    gio->data = data;
+
+    frame->local = gio;
+
+    ret = get_fop_attr_thrd_key(&fop_attr);
+    if (ret)
+        gf_msg_debug("gfapi", 0, "Getting leaseid from thread failed");
+
+    STACK_WIND_COOKIE(frame, glfs_preadv_async_cbk, subvol, subvol,
+                      subvol->fops->readv, fd, iov_length(iovec, count), offset,
+                      flags, fop_attr);
 
 out:
-        if (ret) {
-                if (gio) {
-                        GF_FREE (gio->iov);
-                        GF_FREE (gio);
-                }
-                if (frame) {
-                        STACK_DESTROY (frame->root);
-                }
-		priv_glfs_subvol_done (fs, subvol);
-	}
+    if (ret) {
+        if (fd)
+            fd_unref(fd);
+        if (glfd)
+            GF_REF_PUT(glfd);
+        if (gio) {
+            GF_FREE(gio->iov);
+            GF_FREE(gio);
+        }
+        if (frame) {
+            STACK_DESTROY(frame->root);
+        }
+        glfs_subvol_done(fs, subvol);
+    }
+    if (fop_attr)
+        dict_unref(fop_attr);
 
-	if (fd)
-		fd_unref (fd);
+    __GLFS_EXIT_FS;
 
-	return ret;
-}
+    return ret;
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_preadv_async, 3.4.0);
+invalid_fs:
+    return -1;
+}
 
+GFAPI_SYMVER_PUBLIC(glfs_preadv_async34, glfs_preadv_async, 3.4.0)
+int
+pub_glfs_preadv_async34(struct glfs_fd *glfd, const struct iovec *iovec,
+                        int count, off_t offset, int flags, glfs_io_cbk34 fn,
+                        void *data)
+{
+    return glfs_preadv_async_common(glfd, iovec, count, offset, flags, _gf_true,
+                                    (void *)fn, data);
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_preadv_async, 6.0)
 int
-pub_glfs_read_async (struct glfs_fd *glfd, void *buf, size_t count, int flags,
-                     glfs_io_cbk fn, void *data)
+pub_glfs_preadv_async(struct glfs_fd *glfd, const struct iovec *iovec,
+                      int count, off_t offset, int flags, glfs_io_cbk fn,
+                      void *data)
 {
-	struct iovec iov = {0, };
-	ssize_t      ret = 0;
+    return glfs_preadv_async_common(glfd, iovec, count, offset, flags,
+                                    _gf_false, fn, data);
+}
 
-	iov.iov_base = buf;
-	iov.iov_len = count;
+GFAPI_SYMVER_PUBLIC(glfs_read_async34, glfs_read_async, 3.4.0)
+int
+pub_glfs_read_async34(struct glfs_fd *glfd, void *buf, size_t count, int flags,
+                      glfs_io_cbk34 fn, void *data)
+{
+    struct iovec iov = {
+        0,
+    };
+    ssize_t ret = 0;
 
-	ret = pub_glfs_preadv_async (glfd, &iov, 1, glfd->offset, flags, fn, data);
+    if (glfd == NULL) {
+        errno = EBADF;
+        return -1;
+    }
 
-	return ret;
-}
+    iov.iov_base = buf;
+    iov.iov_len = count;
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_read_async, 3.4.0);
+    ret = glfs_preadv_async_common(glfd, &iov, 1, glfd->offset, flags, _gf_true,
+                                   (void *)fn, data);
 
+    return ret;
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_read_async, 6.0)
 int
-pub_glfs_pread_async (struct glfs_fd *glfd, void *buf, size_t count,
-                      off_t offset, int flags, glfs_io_cbk fn, void *data)
+pub_glfs_read_async(struct glfs_fd *glfd, void *buf, size_t count, int flags,
+                    glfs_io_cbk fn, void *data)
 {
-	struct iovec iov = {0, };
-	ssize_t      ret = 0;
+    struct iovec iov = {
+        0,
+    };
+    ssize_t ret = 0;
 
-	iov.iov_base = buf;
-	iov.iov_len = count;
+    if (glfd == NULL) {
+        errno = EBADF;
+        return -1;
+    }
 
-	ret = pub_glfs_preadv_async (glfd, &iov, 1, offset, flags, fn, data);
+    iov.iov_base = buf;
+    iov.iov_len = count;
 
-	return ret;
-}
-
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_pread_async, 3.4.0);
+    ret = glfs_preadv_async_common(glfd, &iov, 1, glfd->offset, flags,
+                                   _gf_false, fn, data);
 
+    return ret;
+}
 
+GFAPI_SYMVER_PUBLIC(glfs_pread_async34, glfs_pread_async, 3.4.0)
 int
-pub_glfs_readv_async (struct glfs_fd *glfd, const struct iovec *iov, int count,
-                      int flags, glfs_io_cbk fn, void *data)
+pub_glfs_pread_async34(struct glfs_fd *glfd, void *buf, size_t count,
+                       off_t offset, int flags, glfs_io_cbk34 fn, void *data)
 {
-	ssize_t      ret = 0;
+    struct iovec iov = {
+        0,
+    };
+    ssize_t ret = 0;
 
-	ret = pub_glfs_preadv_async (glfd, iov, count, glfd->offset, flags,
-				  fn, data);
-	return ret;
-}
+    iov.iov_base = buf;
+    iov.iov_len = count;
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_readv_async, 3.4.0);
+    ret = glfs_preadv_async_common(glfd, &iov, 1, offset, flags, _gf_true,
+                                   (void *)fn, data);
 
+    return ret;
+}
 
-ssize_t
-pub_glfs_pwritev (struct glfs_fd *glfd, const struct iovec *iovec, int iovcnt,
-                  off_t offset, int flags)
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_pread_async, 6.0)
+int
+pub_glfs_pread_async(struct glfs_fd *glfd, void *buf, size_t count,
+                     off_t offset, int flags, glfs_io_cbk fn, void *data)
 {
-	xlator_t       *subvol = NULL;
-	int             ret = -1;
-	size_t          size = -1;
-	struct iobref  *iobref = NULL;
-	struct iobuf   *iobuf = NULL;
-	struct iovec    iov = {0, };
-	fd_t           *fd = NULL;
+    struct iovec iov = {
+        0,
+    };
+    ssize_t ret = 0;
 
-	__glfs_entry_fd (glfd);
+    iov.iov_base = buf;
+    iov.iov_len = count;
 
-	subvol = priv_glfs_active_subvol (glfd->fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+    ret = glfs_preadv_async_common(glfd, &iov, 1, offset, flags, _gf_false, fn,
+                                   data);
 
-	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
-	if (!fd) {
-		ret = -1;
-		errno = EBADFD;
-		goto out;
-	}
+    return ret;
+}
 
-	size = iov_length (iovec, iovcnt);
+GFAPI_SYMVER_PUBLIC(glfs_readv_async34, glfs_readv_async, 3.4.0)
+int
+pub_glfs_readv_async34(struct glfs_fd *glfd, const struct iovec *iov, int count,
+                       int flags, glfs_io_cbk34 fn, void *data)
+{
+    ssize_t ret = 0;
 
-	iobuf = iobuf_get2 (subvol->ctx->iobuf_pool, size);
-	if (!iobuf) {
-		ret = -1;
-		errno = ENOMEM;
-		goto out;
-	}
+    if (glfd == NULL) {
+        errno = EBADF;
+        return -1;
+    }
 
-	iobref = iobref_new ();
-	if (!iobref) {
-		iobuf_unref (iobuf);
-		errno = ENOMEM;
-		ret = -1;
-		goto out;
-	}
+    ret = glfs_preadv_async_common(glfd, iov, count, glfd->offset, flags,
+                                   _gf_true, (void *)fn, data);
+    return ret;
+}
 
-	ret = iobref_add (iobref, iobuf);
-	if (ret) {
-		iobuf_unref (iobuf);
-		iobref_unref (iobref);
-		errno = ENOMEM;
-		ret = -1;
-		goto out;
-	}
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_readv_async, 6.0)
+int
+pub_glfs_readv_async(struct glfs_fd *glfd, const struct iovec *iov, int count,
+                     int flags, glfs_io_cbk fn, void *data)
+{
+    ssize_t ret = 0;
 
-	iov_unload (iobuf_ptr (iobuf), iovec, iovcnt);  /* FIXME!!! */
+    if (glfd == NULL) {
+        errno = EBADF;
+        return -1;
+    }
 
-	iov.iov_base = iobuf_ptr (iobuf);
-	iov.iov_len = size;
+    ret = glfs_preadv_async_common(glfd, iov, count, glfd->offset, flags,
+                                   _gf_false, fn, data);
+    return ret;
+}
 
-	ret = syncop_writev (subvol, fd, &iov, 1, offset, iobref, flags);
+static ssize_t
+glfs_pwritev_common(struct glfs_fd *glfd, const struct iovec *iovec, int iovcnt,
+                    off_t offset, int flags, struct glfs_stat *prestat,
+                    struct glfs_stat *poststat)
+{
+    xlator_t *subvol = NULL;
+    int ret = -1;
+    struct iobref *iobref = NULL;
+    struct iobuf *iobuf = NULL;
+    struct iovec iov = {
+        0,
+    };
+    fd_t *fd = NULL;
+    struct iatt preiatt =
+                    {
+                        0,
+                    },
+                postiatt = {
+                    0,
+                };
+    dict_t *fop_attr = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    GF_REF_GET(glfd);
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    ret = iobuf_copy(subvol->ctx->iobuf_pool, iovec, iovcnt, &iobref, &iobuf,
+                     &iov);
+    if (ret)
+        goto out;
+
+    ret = get_fop_attr_thrd_key(&fop_attr);
+    if (ret)
+        gf_msg_debug("gfapi", 0, "Getting leaseid from thread failed");
+
+    ret = syncop_writev(subvol, fd, &iov, 1, offset, iobref, flags, &preiatt,
+                        &postiatt, fop_attr, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    if (ret >= 0) {
+        if (prestat)
+            glfs_iatt_to_statx(glfd->fs, &preiatt, prestat);
+        if (poststat)
+            glfs_iatt_to_statx(glfd->fs, &postiatt, poststat);
+    }
+
+    if (ret <= 0)
+        goto out;
+
+    glfd->offset = (offset + iov.iov_len);
+out:
+    if (iobuf)
+        iobuf_unref(iobuf);
+    if (iobref)
+        iobref_unref(iobref);
+    if (fd)
+        fd_unref(fd);
+    if (glfd)
+        GF_REF_PUT(glfd);
+    if (fop_attr)
+        dict_unref(fop_attr);
+
+    glfs_subvol_done(glfd->fs, subvol);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
 
-	iobuf_unref (iobuf);
-	iobref_unref (iobref);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_copy_file_range, 6.0)
+ssize_t
+pub_glfs_copy_file_range(struct glfs_fd *glfd_in, off64_t *off_in,
+                         struct glfs_fd *glfd_out, off64_t *off_out, size_t len,
+                         unsigned int flags, struct glfs_stat *statbuf,
+                         struct glfs_stat *prestat, struct glfs_stat *poststat)
+{
+    xlator_t *subvol = NULL;
+    int ret = -1;
+    fd_t *fd_in = NULL;
+    fd_t *fd_out = NULL;
+    struct iatt preiatt =
+                    {
+                        0,
+                    },
+                iattbuf =
+                    {
+                        0,
+                    },
+                postiatt = {
+                    0,
+                };
+    dict_t *fop_attr = NULL;
+    off64_t pos_in;
+    off64_t pos_out;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd_in, invalid_fs);
+    __GLFS_ENTRY_VALIDATE_FD(glfd_out, invalid_fs);
+
+    GF_REF_GET(glfd_in);
+    GF_REF_GET(glfd_out);
+
+    if (glfd_in->fs != glfd_out->fs) {
+        ret = -1;
+        errno = EXDEV;
+        goto out;
+    }
+
+    subvol = glfs_active_subvol(glfd_in->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd_in = glfs_resolve_fd(glfd_in->fs, subvol, glfd_in);
+    if (!fd_in) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    fd_out = glfs_resolve_fd(glfd_out->fs, subvol, glfd_out);
+    if (!fd_out) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    /*
+     * This is based on how the vfs layer in the kernel handles
+     * copy_file_range call. Upon receiving it follows the
+     * below method to consider the offset.
+     * if (off_in != NULL)
+     *    use the value off_in to perform the op
+     * else if off_in == NULL
+     *    use the current file offset position to perform the op
+     *
+     * For gfapi, glfd->offset is used. For a freshly opened
+     * fd, the offset is set to 0.
+     */
+    if (off_in)
+        pos_in = *off_in;
+    else
+        pos_in = glfd_in->offset;
+
+    if (off_out)
+        pos_out = *off_out;
+    else
+        pos_out = glfd_out->offset;
+
+    ret = get_fop_attr_thrd_key(&fop_attr);
+    if (ret)
+        gf_msg_debug("gfapi", 0, "Getting leaseid from thread failed");
+
+    ret = syncop_copy_file_range(subvol, fd_in, pos_in, fd_out, pos_out, len,
+                                 flags, &iattbuf, &preiatt, &postiatt, fop_attr,
+                                 NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    if (ret >= 0) {
+        pos_in += ret;
+        pos_out += ret;
+
+        if (off_in)
+            *off_in = pos_in;
+        if (off_out)
+            *off_out = pos_out;
+
+        if (statbuf)
+            glfs_iatt_to_statx(glfd_in->fs, &iattbuf, statbuf);
+        if (prestat)
+            glfs_iatt_to_statx(glfd_in->fs, &preiatt, prestat);
+        if (poststat)
+            glfs_iatt_to_statx(glfd_in->fs, &postiatt, poststat);
+    }
+
+    if (ret <= 0)
+        goto out;
+
+    /*
+     * If *off_in is NULL, then there is no offset info that can
+     * obtained from the input argument. Hence follow below method.
+     *  If *off_in is NULL, then
+     *     glfd->offset = offset + ret;
+     * else
+     *     do nothing.
+     *
+     * According to the man page of copy_file_range, if off_in is
+     * NULL, then the offset of the source file is advanced by
+     * the return value of the fop. The same applies to off_out as
+     * well. Otherwise, if *off_in is not NULL, then the offset
+     * is not advanced by the filesystem. The entity which sends
+     * the copy_file_range call is supposed to advance the offset
+     * value in its buffer (pointed to by *off_in or *off_out)
+     * by the return value of copy_file_range.
+     */
+    if (!off_in)
+        glfd_in->offset += ret;
+
+    if (!off_out)
+        glfd_out->offset += ret;
 
-	if (ret <= 0)
-		goto out;
+out:
+    if (fd_in)
+        fd_unref(fd_in);
+    if (fd_out)
+        fd_unref(fd_out);
+    if (glfd_in)
+        GF_REF_PUT(glfd_in);
+    if (glfd_out)
+        GF_REF_PUT(glfd_out);
+    if (fop_attr)
+        dict_unref(fop_attr);
+
+    glfs_subvol_done(glfd_in->fs, subvol);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
 
-	glfd->offset = (offset + size);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_pwritev, 3.4.0)
+ssize_t
+pub_glfs_pwritev(struct glfs_fd *glfd, const struct iovec *iovec, int iovcnt,
+                 off_t offset, int flags)
+{
+    return glfs_pwritev_common(glfd, iovec, iovcnt, offset, flags, NULL, NULL);
+}
 
-out:
-	if (fd)
-		fd_unref (fd);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_write, 3.4.0)
+ssize_t
+pub_glfs_write(struct glfs_fd *glfd, const void *buf, size_t count, int flags)
+{
+    struct iovec iov = {
+        0,
+    };
+    ssize_t ret = 0;
 
-	priv_glfs_subvol_done (glfd->fs, subvol);
+    if (glfd == NULL) {
+        errno = EBADF;
+        return -1;
+    }
 
-	return ret;
-}
+    iov.iov_base = (void *)buf;
+    iov.iov_len = count;
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_pwritev, 3.4.0);
+    ret = pub_glfs_pwritev(glfd, &iov, 1, glfd->offset, flags);
 
+    return ret;
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_writev, 3.4.0)
 ssize_t
-pub_glfs_write (struct glfs_fd *glfd, const void *buf, size_t count, int flags)
+pub_glfs_writev(struct glfs_fd *glfd, const struct iovec *iov, int count,
+                int flags)
 {
-	struct iovec iov = {0, };
-	ssize_t      ret = 0;
+    ssize_t ret = 0;
 
-	iov.iov_base = (void *) buf;
-	iov.iov_len = count;
+    if (glfd == NULL) {
+        errno = EBADF;
+        return -1;
+    }
 
-	ret = pub_glfs_pwritev (glfd, &iov, 1, glfd->offset, flags);
+    ret = pub_glfs_pwritev(glfd, iov, count, glfd->offset, flags);
 
-	return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_write, 3.4.0);
-
-
+GFAPI_SYMVER_PUBLIC(glfs_pwrite34, glfs_pwrite, 3.4.0)
 ssize_t
-pub_glfs_writev (struct glfs_fd *glfd, const struct iovec *iov, int count,
-                 int flags)
+pub_glfs_pwrite34(struct glfs_fd *glfd, const void *buf, size_t count,
+                  off_t offset, int flags)
 {
-	ssize_t      ret = 0;
-
-	ret = pub_glfs_pwritev (glfd, iov, count, glfd->offset, flags);
+    struct iovec iov = {
+        0,
+    };
+    ssize_t ret = 0;
 
-	return ret;
-}
+    iov.iov_base = (void *)buf;
+    iov.iov_len = count;
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_writev, 3.4.0);
+    ret = pub_glfs_pwritev(glfd, &iov, 1, offset, flags);
 
+    return ret;
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_pwrite, 6.0)
 ssize_t
-pub_glfs_pwrite (struct glfs_fd *glfd, const void *buf, size_t count,
-                 off_t offset, int flags)
+pub_glfs_pwrite(struct glfs_fd *glfd, const void *buf, size_t count,
+                off_t offset, int flags, struct glfs_stat *prestat,
+                struct glfs_stat *poststat)
 {
-	struct iovec iov = {0, };
-	ssize_t      ret = 0;
+    struct iovec iov = {
+        0,
+    };
+    ssize_t ret = 0;
 
-	iov.iov_base = (void *) buf;
-	iov.iov_len = count;
+    iov.iov_base = (void *)buf;
+    iov.iov_len = count;
 
-	ret = pub_glfs_pwritev (glfd, &iov, 1, offset, flags);
+    ret = glfs_pwritev_common(glfd, &iov, 1, offset, flags, prestat, poststat);
 
-	return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_pwrite, 3.4.0);
+extern glfs_t *
+pub_glfs_from_glfd(glfs_fd_t *);
+
+static int
+glfs_pwritev_async_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int op_ret, int op_errno, struct iatt *prebuf,
+                       struct iatt *postbuf, dict_t *xdata)
+{
+    glfs_io_async_cbk(op_ret, op_errno, frame, cookie, NULL, 0, prebuf,
+                      postbuf);
 
+    return 0;
+}
 
-extern glfs_t *pub_glfs_from_glfd (glfs_fd_t *);
+static int
+glfs_pwritev_async_common(struct glfs_fd *glfd, const struct iovec *iovec,
+                          int count, off_t offset, int flags,
+                          gf_boolean_t oldcb, glfs_io_cbk fn, void *data)
+{
+    struct glfs_io *gio = NULL;
+    int ret = -1;
+    call_frame_t *frame = NULL;
+    xlator_t *subvol = NULL;
+    fd_t *fd = NULL;
+    struct iobref *iobref = NULL;
+    struct iobuf *iobuf = NULL;
+    dict_t *fop_attr = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    /* Need to take explicit ref so that the fd
+     * is not destroyed before the fop is complete
+     */
+    GF_REF_GET(glfd);
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        errno = EBADFD;
+        goto out;
+    }
+
+    gio = GF_CALLOC(1, sizeof(*gio), glfs_mt_glfs_io_t);
+    if (!gio) {
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gio->op = GF_FOP_WRITE;
+    gio->glfd = glfd;
+    gio->offset = offset;
+    gio->flags = flags;
+    gio->oldcb = oldcb;
+    gio->fn = fn;
+    gio->data = data;
+    gio->count = 1;
+    gio->iov = GF_CALLOC(gio->count, sizeof(*(gio->iov)), gf_common_mt_iovec);
+    if (!gio->iov) {
+        errno = ENOMEM;
+        goto out;
+    }
+
+    ret = iobuf_copy(subvol->ctx->iobuf_pool, iovec, count, &iobref, &iobuf,
+                     gio->iov);
+    if (ret)
+        goto out;
+
+    frame = syncop_create_frame(THIS);
+    if (!frame) {
+        errno = ENOMEM;
+        ret = -1;
+        goto out;
+    }
+
+    frame->local = gio;
+
+    ret = get_fop_attr_thrd_key(&fop_attr);
+    if (ret)
+        gf_msg_debug("gfapi", 0, "Getting leaseid from thread failed");
+
+    STACK_WIND_COOKIE(frame, glfs_pwritev_async_cbk, subvol, subvol,
+                      subvol->fops->writev, fd, gio->iov, gio->count, offset,
+                      flags, iobref, fop_attr);
+
+    ret = 0;
+out:
+    if (ret) {
+        if (fd)
+            fd_unref(fd);
+        if (glfd)
+            GF_REF_PUT(glfd);
+        GF_FREE(gio);
+        /*
+         * If there is any error condition check after the frame
+         * creation, we have to destroy the frame root.
+         */
+        glfs_subvol_done(glfd->fs, subvol);
+    }
+    if (fop_attr)
+        dict_unref(fop_attr);
+
+    if (iobuf)
+        iobuf_unref(iobuf);
+    if (iobref)
+        iobref_unref(iobref);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
 
+GFAPI_SYMVER_PUBLIC(glfs_pwritev_async34, glfs_pwritev_async, 3.4.0)
 int
-pub_glfs_pwritev_async (struct glfs_fd *glfd, const struct iovec *iovec,
-                        int count, off_t offset, int flags, glfs_io_cbk fn,
-                        void *data)
+pub_glfs_pwritev_async34(struct glfs_fd *glfd, const struct iovec *iovec,
+                         int count, off_t offset, int flags, glfs_io_cbk34 fn,
+                         void *data)
 {
-	struct glfs_io *gio = NULL;
-	int             ret = 0;
+    return glfs_pwritev_async_common(glfd, iovec, count, offset, flags,
+                                     _gf_true, (void *)fn, data);
+}
 
-	gio = GF_CALLOC (1, sizeof (*gio), glfs_mt_glfs_io_t);
-	if (!gio) {
-		errno = ENOMEM;
-		return -1;
-	}
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_pwritev_async, 6.0)
+int
+pub_glfs_pwritev_async(struct glfs_fd *glfd, const struct iovec *iovec,
+                       int count, off_t offset, int flags, glfs_io_cbk fn,
+                       void *data)
+{
+    return glfs_pwritev_async_common(glfd, iovec, count, offset, flags,
+                                     _gf_false, fn, data);
+}
 
-	gio->iov = iov_dup (iovec, count);
-	if (!gio->iov) {
-		GF_FREE (gio);
-		errno = ENOMEM;
-		return -1;
-	}
+GFAPI_SYMVER_PUBLIC(glfs_write_async34, glfs_write_async, 3.4.0)
+int
+pub_glfs_write_async34(struct glfs_fd *glfd, const void *buf, size_t count,
+                       int flags, glfs_io_cbk34 fn, void *data)
+{
+    struct iovec iov = {
+        0,
+    };
+    ssize_t ret = 0;
 
-	gio->op     = GF_FOP_WRITE;
-	gio->glfd   = glfd;
-	gio->count  = count;
-	gio->offset = offset;
-	gio->flags  = flags;
-	gio->fn     = fn;
-	gio->data   = data;
+    if (glfd == NULL) {
+        errno = EBADF;
+        return -1;
+    }
 
-	ret = synctask_new (pub_glfs_from_glfd (glfd)->ctx->env,
-			    glfs_io_async_task, glfs_io_async_cbk,
-			    NULL, gio);
+    iov.iov_base = (void *)buf;
+    iov.iov_len = count;
 
-	if (ret) {
-		GF_FREE (gio->iov);
-		GF_FREE (gio);
-	}
+    ret = glfs_pwritev_async_common(glfd, &iov, 1, glfd->offset, flags,
+                                    _gf_true, (void *)fn, data);
 
-	return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_pwritev_async, 3.4.0);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_write_async, 6.0)
 int
-pub_glfs_write_async (struct glfs_fd *glfd, const void *buf, size_t count,
-                      int flags, glfs_io_cbk fn, void *data)
+pub_glfs_write_async(struct glfs_fd *glfd, const void *buf, size_t count,
+                     int flags, glfs_io_cbk fn, void *data)
 {
-	struct iovec iov = {0, };
-	ssize_t      ret = 0;
-
-	iov.iov_base = (void *) buf;
-	iov.iov_len = count;
+    struct iovec iov = {
+        0,
+    };
+    ssize_t ret = 0;
 
-	ret = pub_glfs_pwritev_async (glfd, &iov, 1, glfd->offset, flags, fn, data);
+    if (glfd == NULL) {
+        errno = EBADF;
+        return -1;
+    }
 
-	return ret;
-}
+    iov.iov_base = (void *)buf;
+    iov.iov_len = count;
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_write_async, 3.4.0);
+    ret = glfs_pwritev_async_common(glfd, &iov, 1, glfd->offset, flags,
+                                    _gf_false, fn, data);
 
+    return ret;
+}
 
+GFAPI_SYMVER_PUBLIC(glfs_pwrite_async34, glfs_pwrite_async, 3.4.0)
 int
-pub_glfs_pwrite_async (struct glfs_fd *glfd, const void *buf, int count,
-                       off_t offset, int flags, glfs_io_cbk fn, void *data)
+pub_glfs_pwrite_async34(struct glfs_fd *glfd, const void *buf, int count,
+                        off_t offset, int flags, glfs_io_cbk34 fn, void *data)
 {
-	struct iovec iov = {0, };
-	ssize_t      ret = 0;
+    struct iovec iov = {
+        0,
+    };
+    ssize_t ret = 0;
 
-	iov.iov_base = (void *) buf;
-	iov.iov_len = count;
+    iov.iov_base = (void *)buf;
+    iov.iov_len = count;
 
-	ret = pub_glfs_pwritev_async (glfd, &iov, 1, offset, flags, fn, data);
+    ret = glfs_pwritev_async_common(glfd, &iov, 1, offset, flags, _gf_true,
+                                    (void *)fn, data);
 
-	return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_pwrite_async, 3.4.0);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_pwrite_async, 6.0)
 int
-pub_glfs_writev_async (struct glfs_fd *glfd, const struct iovec *iov, int count,
-                       int flags, glfs_io_cbk fn, void *data)
+pub_glfs_pwrite_async(struct glfs_fd *glfd, const void *buf, int count,
+                      off_t offset, int flags, glfs_io_cbk fn, void *data)
 {
-	ssize_t      ret = 0;
+    struct iovec iov = {
+        0,
+    };
+    ssize_t ret = 0;
 
-	ret = pub_glfs_pwritev_async (glfd, iov, count, glfd->offset, flags,
-				   fn, data);
-	return ret;
-}
+    iov.iov_base = (void *)buf;
+    iov.iov_len = count;
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_writev_async, 3.4.0);
+    ret = glfs_pwritev_async_common(glfd, &iov, 1, offset, flags, _gf_false, fn,
+                                    data);
 
+    return ret;
+}
 
+GFAPI_SYMVER_PUBLIC(glfs_writev_async34, glfs_writev_async, 3.4.0)
 int
-pub_glfs_fsync (struct glfs_fd *glfd)
+pub_glfs_writev_async34(struct glfs_fd *glfd, const struct iovec *iov,
+                        int count, int flags, glfs_io_cbk34 fn, void *data)
 {
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	fd_t            *fd = NULL;
-
-	__glfs_entry_fd (glfd);
+    ssize_t ret = 0;
 
-	subvol = priv_glfs_active_subvol (glfd->fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+    if (glfd == NULL) {
+        errno = EBADF;
+        return -1;
+    }
 
-	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
-	if (!fd) {
-		ret = -1;
-		errno = EBADFD;
-		goto out;
-	}
+    ret = glfs_pwritev_async_common(glfd, iov, count, glfd->offset, flags,
+                                    _gf_true, (void *)fn, data);
+    return ret;
+}
 
-	ret = syncop_fsync (subvol, fd, 0);
-out:
-	if (fd)
-		fd_unref (fd);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_writev_async, 6.0)
+int
+pub_glfs_writev_async(struct glfs_fd *glfd, const struct iovec *iov, int count,
+                      int flags, glfs_io_cbk fn, void *data)
+{
+    ssize_t ret = 0;
 
-	priv_glfs_subvol_done (glfd->fs, subvol);
+    if (glfd == NULL) {
+        errno = EBADF;
+        return -1;
+    }
 
-	return ret;
+    ret = glfs_pwritev_async_common(glfd, iov, count, glfd->offset, flags,
+                                    _gf_false, fn, data);
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fsync, 3.4.0);
-
-
 static int
-glfs_fsync_async_common (struct glfs_fd *glfd, glfs_io_cbk fn, void *data,
-			 int dataonly)
+glfs_fsync_common(struct glfs_fd *glfd, struct glfs_stat *prestat,
+                  struct glfs_stat *poststat)
 {
-	struct glfs_io *gio = NULL;
-	int             ret = 0;
-
-	gio = GF_CALLOC (1, sizeof (*gio), glfs_mt_glfs_io_t);
-	if (!gio) {
-		errno = ENOMEM;
-		return -1;
-	}
-
-	gio->op     = GF_FOP_FSYNC;
-	gio->glfd   = glfd;
-	gio->flags  = dataonly;
-	gio->fn     = fn;
-	gio->data   = data;
-
-	ret = synctask_new (pub_glfs_from_glfd (glfd)->ctx->env,
-			    glfs_io_async_task, glfs_io_async_cbk,
-			    NULL, gio);
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    fd_t *fd = NULL;
+    struct iatt preiatt =
+                    {
+                        0,
+                    },
+                postiatt = {
+                    0,
+                };
+    dict_t *fop_attr = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    GF_REF_GET(glfd);
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    ret = get_fop_attr_thrd_key(&fop_attr);
+    if (ret)
+        gf_msg_debug("gfapi", 0, "Getting leaseid from thread failed");
+
+    ret = syncop_fsync(subvol, fd, 0, &preiatt, &postiatt, fop_attr, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    if (ret >= 0) {
+        if (prestat)
+            glfs_iatt_to_statx(glfd->fs, &preiatt, prestat);
+        if (poststat)
+            glfs_iatt_to_statx(glfd->fs, &postiatt, poststat);
+    }
+out:
+    if (fd)
+        fd_unref(fd);
+    if (glfd)
+        GF_REF_PUT(glfd);
+    if (fop_attr)
+        dict_unref(fop_attr);
 
-	if (ret) {
-		GF_FREE (gio->iov);
-		GF_FREE (gio);
-	}
+    glfs_subvol_done(glfd->fs, subvol);
 
-	return ret;
+    __GLFS_EXIT_FS;
 
+invalid_fs:
+    return ret;
 }
 
-
+GFAPI_SYMVER_PUBLIC(glfs_fsync34, glfs_fsync, 3.4.0)
 int
-pub_glfs_fsync_async (struct glfs_fd *glfd, glfs_io_cbk fn, void *data)
+pub_glfs_fsync34(struct glfs_fd *glfd)
 {
-	return glfs_fsync_async_common (glfd, fn, data, 0);
+    return glfs_fsync_common(glfd, NULL, NULL);
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fsync_async, 3.4.0);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fsync, 6.0)
 int
-pub_glfs_fdatasync (struct glfs_fd *glfd)
+pub_glfs_fsync(struct glfs_fd *glfd, struct glfs_stat *prestat,
+               struct glfs_stat *poststat)
 {
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	fd_t            *fd = NULL;
+    return glfs_fsync_common(glfd, prestat, poststat);
+}
 
-	__glfs_entry_fd (glfd);
+static int
+glfs_fsync_async_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                     struct iatt *postbuf, dict_t *xdata)
+{
+    glfs_io_async_cbk(op_ret, op_errno, frame, cookie, NULL, 0, prebuf,
+                      postbuf);
 
-	subvol = priv_glfs_active_subvol (glfd->fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+    return 0;
+}
 
-	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
-	if (!fd) {
-		ret = -1;
-		errno = EBADFD;
-		goto out;
-	}
+static int
+glfs_fsync_async_common(struct glfs_fd *glfd, gf_boolean_t oldcb,
+                        glfs_io_cbk fn, void *data, int dataonly)
+{
+    struct glfs_io *gio = NULL;
+    int ret = 0;
+    call_frame_t *frame = NULL;
+    xlator_t *subvol = NULL;
+    fd_t *fd = NULL;
+
+    /* Need to take explicit ref so that the fd
+     * is not destroyed before the fop is complete
+     */
+    GF_REF_GET(glfd);
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    frame = syncop_create_frame(THIS);
+    if (!frame) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gio = GF_CALLOC(1, sizeof(*gio), glfs_mt_glfs_io_t);
+    if (!gio) {
+        errno = ENOMEM;
+        ret = -1;
+        goto out;
+    }
+
+    gio->op = GF_FOP_FSYNC;
+    gio->glfd = glfd;
+    gio->flags = dataonly;
+    gio->oldcb = oldcb;
+    gio->fn = fn;
+    gio->data = data;
+
+    frame->local = gio;
+
+    STACK_WIND_COOKIE(frame, glfs_fsync_async_cbk, subvol, subvol,
+                      subvol->fops->fsync, fd, dataonly, NULL);
 
-	ret = syncop_fsync (subvol, fd, 1);
 out:
-	if (fd)
-		fd_unref (fd);
-
-	priv_glfs_subvol_done (glfd->fs, subvol);
-
-	return ret;
+    if (ret) {
+        if (fd)
+            fd_unref(fd);
+        GF_REF_PUT(glfd);
+        GF_FREE(gio);
+        if (frame)
+            STACK_DESTROY(frame->root);
+        glfs_subvol_done(glfd->fs, subvol);
+    }
+
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fdatasync, 3.4.0);
-
-
+GFAPI_SYMVER_PUBLIC(glfs_fsync_async34, glfs_fsync_async, 3.4.0)
 int
-pub_glfs_fdatasync_async (struct glfs_fd *glfd, glfs_io_cbk fn, void *data)
+pub_glfs_fsync_async34(struct glfs_fd *glfd, glfs_io_cbk34 fn, void *data)
 {
-	return glfs_fsync_async_common (glfd, fn, data, 1);
-}
+    int ret = -1;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    ret = glfs_fsync_async_common(glfd, _gf_true, (void *)fn, data, 0);
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fdatasync_async, 3.4.0);
+    __GLFS_EXIT_FS;
 
+invalid_fs:
+    return ret;
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fsync_async, 6.0)
 int
-pub_glfs_ftruncate (struct glfs_fd *glfd, off_t offset)
+pub_glfs_fsync_async(struct glfs_fd *glfd, glfs_io_cbk fn, void *data)
 {
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	fd_t            *fd = NULL;
+    int ret = -1;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
 
-	__glfs_entry_fd (glfd);
+    ret = glfs_fsync_async_common(glfd, _gf_false, fn, data, 0);
 
-	subvol = priv_glfs_active_subvol (glfd->fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+    __GLFS_EXIT_FS;
 
-	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
-	if (!fd) {
-		ret = -1;
-		errno = EBADFD;
-		goto out;
-	}
+invalid_fs:
+    return ret;
+}
 
-	ret = syncop_ftruncate (subvol, fd, offset);
+static int
+glfs_fdatasync_common(struct glfs_fd *glfd, struct glfs_stat *prestat,
+                      struct glfs_stat *poststat)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    fd_t *fd = NULL;
+    struct iatt preiatt =
+                    {
+                        0,
+                    },
+                postiatt = {
+                    0,
+                };
+    dict_t *fop_attr = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    GF_REF_GET(glfd);
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    ret = get_fop_attr_thrd_key(&fop_attr);
+    if (ret)
+        gf_msg_debug("gfapi", 0, "Getting leaseid from thread failed");
+
+    ret = syncop_fsync(subvol, fd, 1, &preiatt, &postiatt, fop_attr, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    if (ret >= 0) {
+        if (prestat)
+            glfs_iatt_to_statx(glfd->fs, &preiatt, prestat);
+        if (poststat)
+            glfs_iatt_to_statx(glfd->fs, &postiatt, poststat);
+    }
 out:
-	if (fd)
-		fd_unref (fd);
+    if (fd)
+        fd_unref(fd);
+    if (glfd)
+        GF_REF_PUT(glfd);
+    if (fop_attr)
+        dict_unref(fop_attr);
 
-	priv_glfs_subvol_done (glfd->fs, subvol);
+    glfs_subvol_done(glfd->fs, subvol);
 
-	return ret;
-}
+    __GLFS_EXIT_FS;
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_ftruncate, 3.4.0);
+invalid_fs:
+    return ret;
+}
 
+GFAPI_SYMVER_PUBLIC(glfs_fdatasync34, glfs_fdatasync, 3.4.0)
+int
+pub_glfs_fdatasync34(struct glfs_fd *glfd)
+{
+    return glfs_fdatasync_common(glfd, NULL, NULL);
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fdatasync, 6.0)
 int
-pub_glfs_ftruncate_async (struct glfs_fd *glfd, off_t offset, glfs_io_cbk fn,
-                          void *data)
+pub_glfs_fdatasync(struct glfs_fd *glfd, struct glfs_stat *prestat,
+                   struct glfs_stat *poststat)
 {
-	struct glfs_io *gio = NULL;
-	int             ret = 0;
+    return glfs_fdatasync_common(glfd, prestat, poststat);
+}
 
-	gio = GF_CALLOC (1, sizeof (*gio), glfs_mt_glfs_io_t);
-	if (!gio) {
-		errno = ENOMEM;
-		return -1;
-	}
+GFAPI_SYMVER_PUBLIC(glfs_fdatasync_async34, glfs_fdatasync_async, 3.4.0)
+int
+pub_glfs_fdatasync_async34(struct glfs_fd *glfd, glfs_io_cbk34 fn, void *data)
+{
+    int ret = -1;
 
-	gio->op     = GF_FOP_FTRUNCATE;
-	gio->glfd   = glfd;
-	gio->offset = offset;
-	gio->fn     = fn;
-	gio->data   = data;
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
 
-	ret = synctask_new (pub_glfs_from_glfd (glfd)->ctx->env,
-			    glfs_io_async_task, glfs_io_async_cbk,
-			    NULL, gio);
+    ret = glfs_fsync_async_common(glfd, _gf_true, (void *)fn, data, 1);
 
-	if (ret) {
-		GF_FREE (gio->iov);
-		GF_FREE (gio);
-	}
+    __GLFS_EXIT_FS;
 
-	return ret;
+invalid_fs:
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_ftruncate_async, 3.4.0);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fdatasync_async, 6.0)
 int
-pub_glfs_access (struct glfs *fs, const char *path, int mode)
+pub_glfs_fdatasync_async(struct glfs_fd *glfd, glfs_io_cbk fn, void *data)
 {
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	loc_t            loc = {0, };
-	struct iatt      iatt = {0, };
-	int              reval = 0;
+    int ret = -1;
 
-	__glfs_entry_fs (fs);
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
 
-	subvol = priv_glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
-retry:
-	ret = glfs_resolve (fs, subvol, path, &loc, &iatt, reval);
-
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    ret = glfs_fsync_async_common(glfd, _gf_false, fn, data, 1);
 
-	if (ret)
-		goto out;
+    __GLFS_EXIT_FS;
 
-	ret = syncop_access (subvol, &loc, mode);
+invalid_fs:
+    return ret;
+}
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+static int
+glfs_ftruncate_common(struct glfs_fd *glfd, off_t offset,
+                      struct glfs_stat *prestat, struct glfs_stat *poststat)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    fd_t *fd = NULL;
+    struct iatt preiatt =
+                    {
+                        0,
+                    },
+                postiatt = {
+                    0,
+                };
+    dict_t *fop_attr = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    GF_REF_GET(glfd);
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    ret = get_fop_attr_thrd_key(&fop_attr);
+    if (ret)
+        gf_msg_debug("gfapi", 0, "Getting leaseid from thread failed");
+
+    ret = syncop_ftruncate(subvol, fd, offset, &preiatt, &postiatt, fop_attr,
+                           NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    if (ret >= 0) {
+        if (prestat)
+            glfs_iatt_to_statx(glfd->fs, &preiatt, prestat);
+        if (poststat)
+            glfs_iatt_to_statx(glfd->fs, &postiatt, poststat);
+    }
 out:
-	loc_wipe (&loc);
-
-	priv_glfs_subvol_done (fs, subvol);
+    if (fd)
+        fd_unref(fd);
+    if (glfd)
+        GF_REF_PUT(glfd);
+    if (fop_attr)
+        dict_unref(fop_attr);
 
-	return ret;
-}
+    glfs_subvol_done(glfd->fs, subvol);
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_access, 3.4.0);
+    __GLFS_EXIT_FS;
 
+invalid_fs:
+    return ret;
+}
 
+GFAPI_SYMVER_PUBLIC(glfs_ftruncate34, glfs_ftruncate, 3.4.0)
 int
-pub_glfs_symlink (struct glfs *fs, const char *data, const char *path)
+pub_glfs_ftruncate34(struct glfs_fd *glfd, off_t offset)
 {
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	loc_t            loc = {0, };
-	struct iatt      iatt = {0, };
-	uuid_t           gfid;
-	dict_t          *xattr_req = NULL;
-	int              reval = 0;
-
-	__glfs_entry_fs (fs);
-
-	subvol = priv_glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+    return glfs_ftruncate_common(glfd, offset, NULL, NULL);
+}
 
-	xattr_req = dict_new ();
-	if (!xattr_req) {
-		ret = -1;
-		errno = ENOMEM;
-		goto out;
-	}
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_ftruncate, 6.0)
+int
+pub_glfs_ftruncate(struct glfs_fd *glfd, off_t offset,
+                   struct glfs_stat *prestat, struct glfs_stat *poststat)
+{
+    return glfs_ftruncate_common(glfd, offset, prestat, poststat);
+}
 
-	uuid_generate (gfid);
-	ret = dict_set_static_bin (xattr_req, "gfid-req", gfid, 16);
-	if (ret) {
-		ret = -1;
-		errno = ENOMEM;
-		goto out;
-	}
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_truncate, 3.7.15)
+int
+pub_glfs_truncate(struct glfs *fs, const char *path, off_t length)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
 retry:
-	ret = glfs_lresolve (fs, subvol, path, &loc, &iatt, reval);
-
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
-
-	if (loc.inode) {
-		errno = EEXIST;
-		ret = -1;
-		goto out;
-	}
+    ret = glfs_resolve(fs, subvol, path, &loc, &iatt, reval);
 
-	if (ret == -1 && errno != ENOENT)
-		/* Any other type of error is fatal */
-		goto out;
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	if (ret == -1 && errno == ENOENT && !loc.parent)
-		/* The parent directory or an ancestor even
-		   higher does not exist
-		*/
-		goto out;
+    if (ret)
+        goto out;
 
-	/* ret == -1 && errno == ENOENT */
-	loc.inode = inode_new (loc.parent->table);
-	if (!loc.inode) {
-		ret = -1;
-		errno = ENOMEM;
-		goto out;
-	}
+    ret = syncop_truncate(subvol, &loc, length, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
 
-	ret = syncop_symlink (subvol, &loc, data, xattr_req, &iatt);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
+out:
+    loc_wipe(&loc);
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    glfs_subvol_done(fs, subvol);
 
-	if (ret == 0)
-		ret = glfs_loc_link (&loc, &iatt);
-out:
-	loc_wipe (&loc);
+    __GLFS_EXIT_FS;
 
-	if (xattr_req)
-		dict_unref (xattr_req);
+invalid_fs:
+    return ret;
+}
 
-	priv_glfs_subvol_done (fs, subvol);
+static int
+glfs_ftruncate_async_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                         int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                         struct iatt *postbuf, dict_t *xdata)
+{
+    glfs_io_async_cbk(op_ret, op_errno, frame, cookie, NULL, 0, prebuf,
+                      postbuf);
 
-	return ret;
+    return 0;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_symlink, 3.4.0);
+static int
+glfs_ftruncate_async_common(struct glfs_fd *glfd, off_t offset,
+                            gf_boolean_t oldcb, glfs_io_cbk fn, void *data)
+{
+    struct glfs_io *gio = NULL;
+    int ret = -1;
+    call_frame_t *frame = NULL;
+    xlator_t *subvol = NULL;
+    fd_t *fd = NULL;
+    dict_t *fop_attr = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    /* Need to take explicit ref so that the fd
+     * is not destroyed before the fop is complete
+     */
+    GF_REF_GET(glfd);
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        errno = EBADFD;
+        goto out;
+    }
+
+    frame = syncop_create_frame(THIS);
+    if (!frame) {
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gio = GF_CALLOC(1, sizeof(*gio), glfs_mt_glfs_io_t);
+    if (!gio) {
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gio->op = GF_FOP_FTRUNCATE;
+    gio->glfd = glfd;
+    gio->offset = offset;
+    gio->oldcb = oldcb;
+    gio->fn = fn;
+    gio->data = data;
+
+    frame->local = gio;
+
+    ret = get_fop_attr_thrd_key(&fop_attr);
+    if (ret)
+        gf_msg_debug("gfapi", 0, "Getting leaseid from thread failed");
+
+    STACK_WIND_COOKIE(frame, glfs_ftruncate_async_cbk, subvol, subvol,
+                      subvol->fops->ftruncate, fd, offset, fop_attr);
+
+    ret = 0;
 
+out:
+    if (ret) {
+        if (fd)
+            fd_unref(fd);
+        if (glfd)
+            GF_REF_PUT(glfd);
+        GF_FREE(gio);
+        if (frame)
+            STACK_DESTROY(frame->root);
+        glfs_subvol_done(glfd->fs, subvol);
+    }
+    if (fop_attr)
+        dict_unref(fop_attr);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
 
+GFAPI_SYMVER_PUBLIC(glfs_ftruncate_async34, glfs_ftruncate_async, 3.4.0)
 int
-pub_glfs_readlink (struct glfs *fs, const char *path, char *buf, size_t bufsiz)
+pub_glfs_ftruncate_async34(struct glfs_fd *glfd, off_t offset, glfs_io_cbk34 fn,
+                           void *data)
 {
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	loc_t            loc = {0, };
-	struct iatt      iatt = {0, };
-	int              reval = 0;
-	char            *linkval = NULL;
+    return glfs_ftruncate_async_common(glfd, offset, _gf_true, (void *)fn,
+                                       data);
+}
 
-	__glfs_entry_fs (fs);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_ftruncate_async, 6.0)
+int
+pub_glfs_ftruncate_async(struct glfs_fd *glfd, off_t offset, glfs_io_cbk fn,
+                         void *data)
+{
+    return glfs_ftruncate_async_common(glfd, offset, _gf_false, fn, data);
+}
 
-	subvol = priv_glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_access, 3.4.0)
+int
+pub_glfs_access(struct glfs *fs, const char *path, int mode)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
 retry:
-	ret = glfs_lresolve (fs, subvol, path, &loc, &iatt, reval);
-
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    ret = glfs_resolve(fs, subvol, path, &loc, &iatt, reval);
 
-	if (ret)
-		goto out;
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	if (iatt.ia_type != IA_IFLNK) {
-		ret = -1;
-		errno = EINVAL;
-		goto out;
-	}
+    if (ret)
+        goto out;
 
-	ret = syncop_readlink (subvol, &loc, &linkval, bufsiz);
-	if (ret > 0) {
-		memcpy (buf, linkval, ret);
-		GF_FREE (linkval);
-	}
+    ret = syncop_access(subvol, &loc, mode, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 out:
-	loc_wipe (&loc);
-
-	priv_glfs_subvol_done (fs, subvol);
+    loc_wipe(&loc);
 
-	return ret;
-}
+    glfs_subvol_done(fs, subvol);
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_readlink, 3.4.0);
+    __GLFS_EXIT_FS;
 
+invalid_fs:
+    return ret;
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_symlink, 3.4.0)
 int
-pub_glfs_mknod (struct glfs *fs, const char *path, mode_t mode, dev_t dev)
+pub_glfs_symlink(struct glfs *fs, const char *data, const char *path)
 {
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	loc_t            loc = {0, };
-	struct iatt      iatt = {0, };
-	uuid_t           gfid;
-	dict_t          *xattr_req = NULL;
-	int              reval = 0;
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    uuid_t gfid;
+    dict_t *xattr_req = NULL;
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    xattr_req = dict_new();
+    if (!xattr_req) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gf_uuid_generate(gfid);
+    ret = dict_set_gfuuid(xattr_req, "gfid-req", gfid, true);
+    if (ret) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+retry:
+    ret = glfs_lresolve(fs, subvol, path, &loc, &iatt, reval);
 
-	__glfs_entry_fs (fs);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	subvol = priv_glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+    if (loc.inode) {
+        errno = EEXIST;
+        ret = -1;
+        goto out;
+    }
 
-	xattr_req = dict_new ();
-	if (!xattr_req) {
-		ret = -1;
-		errno = ENOMEM;
-		goto out;
-	}
+    if (ret == -1 && errno != ENOENT)
+        /* Any other type of error is fatal */
+        goto out;
 
-	uuid_generate (gfid);
-	ret = dict_set_static_bin (xattr_req, "gfid-req", gfid, 16);
-	if (ret) {
-		ret = -1;
-		errno = ENOMEM;
-		goto out;
-	}
-retry:
-	ret = glfs_lresolve (fs, subvol, path, &loc, &iatt, reval);
+    if (ret == -1 && errno == ENOENT && !loc.parent)
+        /* The parent directory or an ancestor even
+           higher does not exist
+        */
+        goto out;
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    /* ret == -1 && errno == ENOENT */
+    loc.inode = inode_new(loc.parent->table);
+    if (!loc.inode) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
 
-	if (loc.inode) {
-		errno = EEXIST;
-		ret = -1;
-		goto out;
-	}
+    ret = syncop_symlink(subvol, &loc, data, &iatt, xattr_req, NULL);
+    DECODE_SYNCOP_ERR(ret);
 
-	if (ret == -1 && errno != ENOENT)
-		/* Any other type of error is fatal */
-		goto out;
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	if (ret == -1 && errno == ENOENT && !loc.parent)
-		/* The parent directory or an ancestor even
-		   higher does not exist
-		*/
-		goto out;
+    if (ret == 0)
+        ret = glfs_loc_link(&loc, &iatt);
+out:
+    loc_wipe(&loc);
 
-	/* ret == -1 && errno == ENOENT */
-	loc.inode = inode_new (loc.parent->table);
-	if (!loc.inode) {
-		ret = -1;
-		errno = ENOMEM;
-		goto out;
-	}
+    if (xattr_req)
+        dict_unref(xattr_req);
 
-	ret = syncop_mknod (subvol, &loc, mode, dev, xattr_req, &iatt);
+    glfs_subvol_done(fs, subvol);
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    __GLFS_EXIT_FS;
 
-	if (ret == 0)
-		ret = glfs_loc_link (&loc, &iatt);
-out:
-	loc_wipe (&loc);
+invalid_fs:
+    return ret;
+}
 
-	if (xattr_req)
-		dict_unref (xattr_req);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_readlink, 3.4.0)
+int
+pub_glfs_readlink(struct glfs *fs, const char *path, char *buf, size_t bufsiz)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    int reval = 0;
+    char *linkval = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+retry:
+    ret = glfs_lresolve(fs, subvol, path, &loc, &iatt, reval);
 
-	priv_glfs_subvol_done (fs, subvol);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	return ret;
-}
+    if (ret)
+        goto out;
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_mknod, 3.4.0);
+    if (iatt.ia_type != IA_IFLNK) {
+        ret = -1;
+        errno = EINVAL;
+        goto out;
+    }
 
+    ret = syncop_readlink(subvol, &loc, &linkval, bufsiz, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+    if (ret > 0) {
+        memcpy(buf, linkval, ret);
+        GF_FREE(linkval);
+    }
 
-int
-pub_glfs_mkdir (struct glfs *fs, const char *path, mode_t mode)
-{
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	loc_t            loc = {0, };
-	struct iatt      iatt = {0, };
-	uuid_t           gfid;
-	dict_t          *xattr_req = NULL;
-	int              reval = 0;
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
+out:
+    loc_wipe(&loc);
 
-	__glfs_entry_fs (fs);
+    glfs_subvol_done(fs, subvol);
 
-	subvol = priv_glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+    __GLFS_EXIT_FS;
 
-	xattr_req = dict_new ();
-	if (!xattr_req) {
-		ret = -1;
-		errno = ENOMEM;
-		goto out;
-	}
+invalid_fs:
+    return ret;
+}
 
-	uuid_generate (gfid);
-	ret = dict_set_static_bin (xattr_req, "gfid-req", gfid, 16);
-	if (ret) {
-		ret = -1;
-		errno = ENOMEM;
-		goto out;
-	}
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_mknod, 3.4.0)
+int
+pub_glfs_mknod(struct glfs *fs, const char *path, mode_t mode, dev_t dev)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    uuid_t gfid;
+    dict_t *xattr_req = NULL;
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    xattr_req = dict_new();
+    if (!xattr_req) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gf_uuid_generate(gfid);
+    ret = dict_set_gfuuid(xattr_req, "gfid-req", gfid, true);
+    if (ret) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
 retry:
-	ret = glfs_lresolve (fs, subvol, path, &loc, &iatt, reval);
+    ret = glfs_lresolve(fs, subvol, path, &loc, &iatt, reval);
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	if (loc.inode) {
-		errno = EEXIST;
-		ret = -1;
-		goto out;
-	}
+    if (loc.inode) {
+        errno = EEXIST;
+        ret = -1;
+        goto out;
+    }
 
-	if (ret == -1 && errno != ENOENT)
-		/* Any other type of error is fatal */
-		goto out;
+    if (ret == -1 && errno != ENOENT)
+        /* Any other type of error is fatal */
+        goto out;
 
-	if (ret == -1 && errno == ENOENT && !loc.parent)
-		/* The parent directory or an ancestor even
-		   higher does not exist
-		*/
-		goto out;
+    if (ret == -1 && errno == ENOENT && !loc.parent)
+        /* The parent directory or an ancestor even
+           higher does not exist
+        */
+        goto out;
 
-	/* ret == -1 && errno == ENOENT */
-	loc.inode = inode_new (loc.parent->table);
-	if (!loc.inode) {
-		ret = -1;
-		errno = ENOMEM;
-		goto out;
-	}
+    /* ret == -1 && errno == ENOENT */
+    loc.inode = inode_new(loc.parent->table);
+    if (!loc.inode) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
 
-	ret = syncop_mkdir (subvol, &loc, mode, xattr_req, &iatt);
+    ret = syncop_mknod(subvol, &loc, mode, dev, &iatt, xattr_req, NULL);
+    DECODE_SYNCOP_ERR(ret);
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	if (ret == 0)
-		ret = glfs_loc_link (&loc, &iatt);
+    if (ret == 0)
+        ret = glfs_loc_link(&loc, &iatt);
 out:
-	loc_wipe (&loc);
-
-	if (xattr_req)
-		dict_unref (xattr_req);
+    loc_wipe(&loc);
 
-	priv_glfs_subvol_done (fs, subvol);
+    if (xattr_req)
+        dict_unref(xattr_req);
 
-	return ret;
-}
+    glfs_subvol_done(fs, subvol);
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_mkdir, 3.4.0);
+    __GLFS_EXIT_FS;
 
+invalid_fs:
+    return ret;
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_mkdir, 3.4.0)
 int
-pub_glfs_unlink (struct glfs *fs, const char *path)
+pub_glfs_mkdir(struct glfs *fs, const char *path, mode_t mode)
 {
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	loc_t            loc = {0, };
-	struct iatt      iatt = {0, };
-	int              reval = 0;
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    uuid_t gfid;
+    dict_t *xattr_req = NULL;
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    xattr_req = dict_new();
+    if (!xattr_req) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gf_uuid_generate(gfid);
+    ret = dict_set_gfuuid(xattr_req, "gfid-req", gfid, true);
+    if (ret) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+retry:
+    ret = glfs_lresolve(fs, subvol, path, &loc, &iatt, reval);
 
-	__glfs_entry_fs (fs);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	subvol = priv_glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
-retry:
-	ret = glfs_lresolve (fs, subvol, path, &loc, &iatt, reval);
+    if (loc.inode) {
+        errno = EEXIST;
+        ret = -1;
+        goto out;
+    }
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    if (ret == -1 && errno != ENOENT)
+        /* Any other type of error is fatal */
+        goto out;
 
-	if (ret)
-		goto out;
+    if (ret == -1 && errno == ENOENT && !loc.parent)
+        /* The parent directory or an ancestor even
+           higher does not exist
+        */
+        goto out;
 
-	if (iatt.ia_type == IA_IFDIR) {
-		ret = -1;
-		errno = EISDIR;
-		goto out;
-	}
+    /* ret == -1 && errno == ENOENT */
+    loc.inode = inode_new(loc.parent->table);
+    if (!loc.inode) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
 
-	ret = syncop_unlink (subvol, &loc);
+    ret = syncop_mkdir(subvol, &loc, mode, &iatt, xattr_req, NULL);
+    DECODE_SYNCOP_ERR(ret);
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	if (ret == 0)
-		ret = glfs_loc_unlink (&loc);
+    if (ret == 0)
+        ret = glfs_loc_link(&loc, &iatt);
 out:
-	loc_wipe (&loc);
+    loc_wipe(&loc);
 
-	priv_glfs_subvol_done (fs, subvol);
+    if (xattr_req)
+        dict_unref(xattr_req);
 
-	return ret;
-}
+    glfs_subvol_done(fs, subvol);
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_unlink, 3.4.0);
+    __GLFS_EXIT_FS;
 
+invalid_fs:
+    return ret;
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_unlink, 3.4.0)
 int
-pub_glfs_rmdir (struct glfs *fs, const char *path)
+pub_glfs_unlink(struct glfs *fs, const char *path)
 {
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	loc_t            loc = {0, };
-	struct iatt      iatt = {0, };
-	int              reval = 0;
-
-	__glfs_entry_fs (fs);
-
-	subvol = priv_glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
 retry:
-	ret = glfs_lresolve (fs, subvol, path, &loc, &iatt, reval);
+    ret = glfs_lresolve(fs, subvol, path, &loc, &iatt, reval);
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	if (ret)
-		goto out;
+    if (ret)
+        goto out;
 
-	if (iatt.ia_type != IA_IFDIR) {
-		ret = -1;
-		errno = ENOTDIR;
-		goto out;
-	}
+    if (iatt.ia_type == IA_IFDIR) {
+        ret = -1;
+        errno = EISDIR;
+        goto out;
+    }
 
-	ret = syncop_rmdir (subvol, &loc, 0);
+    /* TODO: Add leaseid */
+    ret = syncop_unlink(subvol, &loc, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	if (ret == 0)
-		ret = glfs_loc_unlink (&loc);
+    if (ret == 0)
+        ret = glfs_loc_unlink(&loc);
 out:
-	loc_wipe (&loc);
-
-	priv_glfs_subvol_done (fs, subvol);
+    loc_wipe(&loc);
 
-	return ret;
-}
+    glfs_subvol_done(fs, subvol);
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_rmdir, 3.4.0);
+    __GLFS_EXIT_FS;
 
+invalid_fs:
+    return ret;
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_rmdir, 3.4.0)
 int
-pub_glfs_rename (struct glfs *fs, const char *oldpath, const char *newpath)
+pub_glfs_rmdir(struct glfs *fs, const char *path)
 {
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	loc_t            oldloc = {0, };
-	loc_t            newloc = {0, };
-	struct iatt      oldiatt = {0, };
-	struct iatt      newiatt = {0, };
-	int              reval = 0;
-
-	__glfs_entry_fs (fs);
-
-	subvol = priv_glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
 retry:
-	ret = glfs_lresolve (fs, subvol, oldpath, &oldloc, &oldiatt, reval);
+    ret = glfs_lresolve(fs, subvol, path, &loc, &iatt, reval);
 
-	ESTALE_RETRY (ret, errno, reval, &oldloc, retry);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	if (ret)
-		goto out;
-retrynew:
-	ret = glfs_lresolve (fs, subvol, newpath, &newloc, &newiatt, reval);
-
-	ESTALE_RETRY (ret, errno, reval, &newloc, retrynew);
-
-	if (ret && errno != ENOENT && newloc.parent)
-		goto out;
-
-	if (newiatt.ia_type != IA_INVAL) {
-                if ((oldiatt.ia_type == IA_IFDIR) !=
-                    (newiatt.ia_type == IA_IFDIR)) {
-                       /* Either both old and new must be dirs,
-                        * or both must be non-dirs. Else, fail.
-                        */
-                       ret = -1;
-                       errno = EISDIR;
-                       goto out;
-                }
-        }
+    if (ret)
+        goto out;
 
-	/* TODO: check if new or old is a prefix of the other, and fail EINVAL */
+    if (iatt.ia_type != IA_IFDIR) {
+        ret = -1;
+        errno = ENOTDIR;
+        goto out;
+    }
 
-	ret = syncop_rename (subvol, &oldloc, &newloc);
+    ret = syncop_rmdir(subvol, &loc, 0, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
 
-	if (ret == -1 && errno == ESTALE) {
-		if (reval < DEFAULT_REVAL_COUNT) {
-			reval++;
-			loc_wipe (&oldloc);
-			loc_wipe (&newloc);
-			goto retry;
-		}
-	}
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	if (ret == 0)
-		inode_rename (oldloc.parent->table, oldloc.parent, oldloc.name,
-			      newloc.parent, newloc.name, oldloc.inode,
-			      &oldiatt);
+    if (ret == 0)
+        ret = glfs_loc_unlink(&loc);
 out:
-	loc_wipe (&oldloc);
-	loc_wipe (&newloc);
-
-	priv_glfs_subvol_done (fs, subvol);
+    loc_wipe(&loc);
 
-	return ret;
-}
+    glfs_subvol_done(fs, subvol);
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_rename, 3.4.0);
+    __GLFS_EXIT_FS;
 
+invalid_fs:
+    return ret;
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_rename, 3.4.0)
 int
-pub_glfs_link (struct glfs *fs, const char *oldpath, const char *newpath)
+pub_glfs_rename(struct glfs *fs, const char *oldpath, const char *newpath)
 {
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	loc_t            oldloc = {0, };
-	loc_t            newloc = {0, };
-	struct iatt      oldiatt = {0, };
-	struct iatt      newiatt = {0, };
-	int              reval = 0;
-
-	__glfs_entry_fs (fs);
-
-	subvol = priv_glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    loc_t oldloc = {
+        0,
+    };
+    loc_t newloc = {
+        0,
+    };
+    struct iatt oldiatt = {
+        0,
+    };
+    struct iatt newiatt = {
+        0,
+    };
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
 retry:
-	ret = glfs_lresolve (fs, subvol, oldpath, &oldloc, &oldiatt, reval);
+    ret = glfs_lresolve(fs, subvol, oldpath, &oldloc, &oldiatt, reval);
 
-	ESTALE_RETRY (ret, errno, reval, &oldloc, retry);
+    ESTALE_RETRY(ret, errno, reval, &oldloc, retry);
 
-	if (ret)
-		goto out;
+    if (ret)
+        goto out;
 retrynew:
-	ret = glfs_lresolve (fs, subvol, newpath, &newloc, &newiatt, reval);
+    ret = glfs_lresolve(fs, subvol, newpath, &newloc, &newiatt, reval);
 
-	ESTALE_RETRY (ret, errno, reval, &newloc, retrynew);
+    ESTALE_RETRY(ret, errno, reval, &newloc, retrynew);
 
-	if (ret == 0) {
-		ret = -1;
-		errno = EEXIST;
-		goto out;
-	}
+    if (ret && errno != ENOENT && newloc.parent)
+        goto out;
 
-	if (oldiatt.ia_type == IA_IFDIR) {
-		ret = -1;
-		errno = EISDIR;
-		goto out;
-	}
+    if (newiatt.ia_type != IA_INVAL) {
+        if ((oldiatt.ia_type == IA_IFDIR) != (newiatt.ia_type == IA_IFDIR)) {
+            /* Either both old and new must be dirs,
+             * or both must be non-dirs. Else, fail.
+             */
+            ret = -1;
+            errno = EISDIR;
+            goto out;
+        }
+    }
 
-        /* Filling the inode of the hard link to be same as that of the
-           original file
-        */
-	if (newloc.inode) {
-		inode_unref (newloc.inode);
-		newloc.inode = NULL;
-	}
-        newloc.inode = inode_ref (oldloc.inode);
+    /* TODO: - check if new or old is a prefix of the other, and fail EINVAL
+     *       - Add leaseid */
 
-	ret = syncop_link (subvol, &oldloc, &newloc);
+    ret = syncop_rename(subvol, &oldloc, &newloc, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
 
-	if (ret == -1 && errno == ESTALE) {
-		loc_wipe (&oldloc);
-		loc_wipe (&newloc);
-		if (reval--)
-			goto retry;
-	}
+    if (ret == -1 && errno == ESTALE) {
+        if (reval < DEFAULT_REVAL_COUNT) {
+            reval++;
+            loc_wipe(&oldloc);
+            loc_wipe(&newloc);
+            goto retry;
+        }
+    }
 
-	if (ret == 0)
-		ret = glfs_loc_link (&newloc, &oldiatt);
+    if (ret == 0) {
+        inode_rename(oldloc.parent->table, oldloc.parent, oldloc.name,
+                     newloc.parent, newloc.name, oldloc.inode, &oldiatt);
+
+        if (newloc.inode && !inode_has_dentry(newloc.inode))
+            inode_forget(newloc.inode, 0);
+    }
 out:
-	loc_wipe (&oldloc);
-	loc_wipe (&newloc);
+    loc_wipe(&oldloc);
+    loc_wipe(&newloc);
+
+    glfs_subvol_done(fs, subvol);
 
-	priv_glfs_subvol_done (fs, subvol);
+    __GLFS_EXIT_FS;
 
-	return ret;
+invalid_fs:
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_link, 3.4.0);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_link, 3.4.0)
+int
+pub_glfs_link(struct glfs *fs, const char *oldpath, const char *newpath)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    loc_t oldloc = {
+        0,
+    };
+    loc_t newloc = {
+        0,
+    };
+    struct iatt oldiatt = {
+        0,
+    };
+    struct iatt newiatt = {
+        0,
+    };
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+retry:
+    ret = glfs_lresolve(fs, subvol, oldpath, &oldloc, &oldiatt, reval);
 
+    ESTALE_RETRY(ret, errno, reval, &oldloc, retry);
 
-struct glfs_fd *
-pub_glfs_opendir (struct glfs *fs, const char *path)
-{
-	int              ret = -1;
-	struct glfs_fd  *glfd = NULL;
-	xlator_t        *subvol = NULL;
-	loc_t            loc = {0, };
-	struct iatt      iatt = {0, };
-	int              reval = 0;
+    if (ret)
+        goto out;
+retrynew:
+    ret = glfs_lresolve(fs, subvol, newpath, &newloc, &newiatt, reval);
+
+    ESTALE_RETRY(ret, errno, reval, &newloc, retrynew);
+
+    if (ret == 0) {
+        ret = -1;
+        errno = EEXIST;
+        goto out;
+    }
+
+    if (oldiatt.ia_type == IA_IFDIR) {
+        ret = -1;
+        errno = EISDIR;
+        goto out;
+    }
+
+    /* Filling the inode of the hard link to be same as that of the
+       original file
+    */
+    if (newloc.inode) {
+        inode_unref(newloc.inode);
+        newloc.inode = NULL;
+    }
+    newloc.inode = inode_ref(oldloc.inode);
+
+    ret = syncop_link(subvol, &oldloc, &newloc, &newiatt, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    if (ret == -1 && errno == ESTALE) {
+        loc_wipe(&oldloc);
+        loc_wipe(&newloc);
+        if (reval--)
+            goto retry;
+    }
+
+    if (ret == 0)
+        ret = glfs_loc_link(&newloc, &newiatt);
+out:
+    loc_wipe(&oldloc);
+    loc_wipe(&newloc);
 
-	__glfs_entry_fs (fs);
+    glfs_subvol_done(fs, subvol);
 
-	subvol = priv_glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+    __GLFS_EXIT_FS;
 
-	glfd = glfs_fd_new (fs);
-	if (!glfd)
-		goto out;
+invalid_fs:
+    return ret;
+}
 
-	INIT_LIST_HEAD (&glfd->entries);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_opendir, 3.4.0)
+struct glfs_fd *
+pub_glfs_opendir(struct glfs *fs, const char *path)
+{
+    int ret = -1;
+    struct glfs_fd *glfd = NULL;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    glfd = glfs_fd_new(fs);
+    if (!glfd)
+        goto out;
+
+    INIT_LIST_HEAD(&glfd->entries);
 retry:
-	ret = glfs_resolve (fs, subvol, path, &loc, &iatt, reval);
+    ret = glfs_resolve(fs, subvol, path, &loc, &iatt, reval);
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	if (ret)
-		goto out;
+    if (ret)
+        goto out;
 
-	if (!IA_ISDIR (iatt.ia_type)) {
-		ret = -1;
-		errno = ENOTDIR;
-		goto out;
-	}
+    if (!IA_ISDIR(iatt.ia_type)) {
+        ret = -1;
+        errno = ENOTDIR;
+        goto out;
+    }
 
-	if (glfd->fd) {
-		/* Retry. Safe to touch glfd->fd as we
-		   still have not glfs_fd_bind() yet.
-		*/
-		fd_unref (glfd->fd);
-		glfd->fd = NULL;
-	}
+    if (glfd->fd) {
+        /* Retry. Safe to touch glfd->fd as we
+           still have not glfs_fd_bind() yet.
+        */
+        fd_unref(glfd->fd);
+        glfd->fd = NULL;
+    }
 
-	glfd->fd = fd_create (loc.inode, getpid());
-	if (!glfd->fd) {
-		ret = -1;
-		errno = ENOMEM;
-		goto out;
-	}
+    glfd->fd = fd_create(loc.inode, getpid());
+    if (!glfd->fd) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
 
-	ret = syncop_opendir (subvol, &loc, glfd->fd);
+    ret = syncop_opendir(subvol, &loc, glfd->fd, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 out:
-	loc_wipe (&loc);
-
-	if (ret && glfd) {
-		glfs_fd_destroy (glfd);
-		glfd = NULL;
-	} else {
-		fd_bind (glfd->fd);
-		glfs_fd_bind (glfd);
-	}
+    loc_wipe(&loc);
 
-	priv_glfs_subvol_done (fs, subvol);
+    if (ret && glfd) {
+        GF_REF_PUT(glfd);
+        glfd = NULL;
+    } else if (glfd) {
+        glfd_set_state_bind(glfd);
+    }
 
-	return glfd;
-}
+    glfs_subvol_done(fs, subvol);
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_opendir, 3.4.0);
+    __GLFS_EXIT_FS;
 
+invalid_fs:
+    return glfd;
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_closedir, 3.4.0)
 int
-pub_glfs_closedir (struct glfs_fd *glfd)
+pub_glfs_closedir(struct glfs_fd *glfd)
 {
-	__glfs_entry_fd (glfd);
+    int ret = -1;
 
-	gf_dirent_free (list_entry (&glfd->entries, gf_dirent_t, list));
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
 
-	glfs_fd_destroy (glfd);
+    gf_dirent_free(list_entry(&glfd->entries, gf_dirent_t, list));
 
-	return 0;
-}
+    glfs_mark_glfd_for_deletion(glfd);
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_closedir, 3.4.0);
+    __GLFS_EXIT_FS;
 
+    ret = 0;
 
-long
-pub_glfs_telldir (struct glfs_fd *fd)
-{
-	return fd->offset;
+invalid_fs:
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_telldir, 3.4.0);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_telldir, 3.4.0)
+long
+pub_glfs_telldir(struct glfs_fd *fd)
+{
+    if (fd == NULL) {
+        errno = EBADF;
+        return -1;
+    }
 
+    return fd->offset;
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_seekdir, 3.4.0)
 void
-pub_glfs_seekdir (struct glfs_fd *fd, long offset)
+pub_glfs_seekdir(struct glfs_fd *fd, long offset)
 {
-	gf_dirent_t *entry = NULL;
-	gf_dirent_t *tmp = NULL;
-
-	if (fd->offset == offset)
-		return;
+    gf_dirent_t *entry = NULL;
+    gf_dirent_t *tmp = NULL;
 
-	fd->offset = offset;
-	fd->next = NULL;
+    if (fd == NULL) {
+        errno = EBADF;
+        return;
+    }
 
-	list_for_each_entry_safe (entry, tmp, &fd->entries, list) {
-		if (entry->d_off != offset)
-			continue;
+    if (fd->offset == offset)
+        return;
 
-		if (&tmp->list != &fd->entries) {
-			/* found! */
-			fd->next = tmp;
-			return;
-		}
-	}
-	/* could not find entry at requested offset in the cache.
-	   next readdir_r() will result in glfd_entry_refresh()
-	*/
-}
+    fd->offset = offset;
+    fd->next = NULL;
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_seekdir, 3.4.0);
+    list_for_each_entry_safe(entry, tmp, &fd->entries, list)
+    {
+        if (entry->d_off != offset)
+            continue;
 
+        if (&tmp->list != &fd->entries) {
+            /* found! */
+            fd->next = tmp;
+            return;
+        }
+    }
+    /* could not find entry at requested offset in the cache.
+       next readdir_r() will result in glfd_entry_refresh()
+    */
+}
 
-int
-pub_glfs_discard_async (struct glfs_fd *glfd, off_t offset, size_t len,
-                        glfs_io_cbk fn, void *data)
+static int
+glfs_discard_async_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno,
+                       struct iatt *preop_stbuf, struct iatt *postop_stbuf,
+                       dict_t *xdata)
 {
-	struct glfs_io *gio = NULL;
-	int             ret = 0;
+    glfs_io_async_cbk(op_ret, op_errno, frame, cookie, NULL, 0, preop_stbuf,
+                      postop_stbuf);
 
-	gio = GF_CALLOC (1, sizeof (*gio), glfs_mt_glfs_io_t);
-	if (!gio) {
-		errno = ENOMEM;
-		return -1;
-	}
-
-	gio->op     = GF_FOP_DISCARD;
-	gio->glfd   = glfd;
-	gio->offset = offset;
-	gio->count  = len;
-	gio->fn     = fn;
-	gio->data   = data;
-
-	ret = synctask_new (pub_glfs_from_glfd (glfd)->ctx->env,
-			    glfs_io_async_task, glfs_io_async_cbk,
-			    NULL, gio);
-
-	if (ret) {
-		GF_FREE (gio->iov);
-		GF_FREE (gio);
-	}
-
-	return ret;
+    return 0;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_discard_async, 3.5.0);
-
+static int
+glfs_discard_async_common(struct glfs_fd *glfd, off_t offset, size_t len,
+                          gf_boolean_t oldcb, glfs_io_cbk fn, void *data)
+{
+    struct glfs_io *gio = NULL;
+    int ret = -1;
+    call_frame_t *frame = NULL;
+    xlator_t *subvol = NULL;
+    fd_t *fd = NULL;
+    dict_t *fop_attr = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    /* Need to take explicit ref so that the fd
+     * is not destroyed before the fop is complete
+     */
+    GF_REF_GET(glfd);
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        errno = EBADFD;
+        goto out;
+    }
+
+    frame = syncop_create_frame(THIS);
+    if (!frame) {
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gio = GF_CALLOC(1, sizeof(*gio), glfs_mt_glfs_io_t);
+    if (!gio) {
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gio->op = GF_FOP_DISCARD;
+    gio->glfd = glfd;
+    gio->offset = offset;
+    gio->count = len;
+    gio->oldcb = oldcb;
+    gio->fn = fn;
+    gio->data = data;
+
+    frame->local = gio;
+    ret = get_fop_attr_thrd_key(&fop_attr);
+    if (ret)
+        gf_msg_debug("gfapi", 0, "Getting leaseid from thread failed");
+
+    STACK_WIND_COOKIE(frame, glfs_discard_async_cbk, subvol, subvol,
+                      subvol->fops->discard, fd, offset, len, fop_attr);
+
+    ret = 0;
+out:
+    if (ret) {
+        if (fd)
+            fd_unref(fd);
+        if (glfd)
+            GF_REF_PUT(glfd);
+        GF_FREE(gio);
+        if (frame)
+            STACK_DESTROY(frame->root);
+        glfs_subvol_done(glfd->fs, subvol);
+    }
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
 
+GFAPI_SYMVER_PUBLIC(glfs_discard_async35, glfs_discard_async, 3.5.0)
 int
-pub_glfs_zerofill_async (struct glfs_fd *glfd, off_t offset, off_t len,
-                         glfs_io_cbk fn, void *data)
+pub_glfs_discard_async35(struct glfs_fd *glfd, off_t offset, size_t len,
+                         glfs_io_cbk34 fn, void *data)
 {
-        struct glfs_io *gio  = NULL;
-        int             ret  = 0;
-
-        gio = GF_CALLOC (1, sizeof (*gio), glfs_mt_glfs_io_t);
-        if (!gio) {
-                errno = ENOMEM;
-                return -1;
-        }
+    return glfs_discard_async_common(glfd, offset, len, _gf_true, (void *)fn,
+                                     data);
+}
 
-        gio->op     = GF_FOP_ZEROFILL;
-        gio->glfd   = glfd;
-        gio->offset = offset;
-        gio->count  = len;
-        gio->fn     = fn;
-        gio->data   = data;
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_discard_async, 6.0)
+int
+pub_glfs_discard_async(struct glfs_fd *glfd, off_t offset, size_t len,
+                       glfs_io_cbk fn, void *data)
+{
+    return glfs_discard_async_common(glfd, offset, len, _gf_false, fn, data);
+}
 
-        ret = synctask_new (pub_glfs_from_glfd (glfd)->ctx->env,
-                            glfs_io_async_task, glfs_io_async_cbk,
-                            NULL, gio);
+static int
+glfs_zerofill_async_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno,
+                        struct iatt *preop_stbuf, struct iatt *postop_stbuf,
+                        dict_t *xdata)
+{
+    glfs_io_async_cbk(op_ret, op_errno, frame, cookie, NULL, 0, preop_stbuf,
+                      postop_stbuf);
 
-        if (ret) {
-                GF_FREE (gio->iov);
-                GF_FREE (gio);
-        }
+    return 0;
+}
 
-        return ret;
+static int
+glfs_zerofill_async_common(struct glfs_fd *glfd, off_t offset, off_t len,
+                           gf_boolean_t oldcb, glfs_io_cbk fn, void *data)
+{
+    struct glfs_io *gio = NULL;
+    int ret = -1;
+    call_frame_t *frame = NULL;
+    xlator_t *subvol = NULL;
+    fd_t *fd = NULL;
+    dict_t *fop_attr = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    /* Need to take explicit ref so that the fd
+     * is not destroyed before the fop is complete
+     */
+    GF_REF_GET(glfd);
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        errno = EBADFD;
+        goto out;
+    }
+
+    frame = syncop_create_frame(THIS);
+    if (!frame) {
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gio = GF_CALLOC(1, sizeof(*gio), glfs_mt_glfs_io_t);
+    if (!gio) {
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gio->op = GF_FOP_ZEROFILL;
+    gio->glfd = glfd;
+    gio->offset = offset;
+    gio->count = len;
+    gio->oldcb = oldcb;
+    gio->fn = fn;
+    gio->data = data;
+
+    frame->local = gio;
+
+    ret = get_fop_attr_thrd_key(&fop_attr);
+    if (ret)
+        gf_msg_debug("gfapi", 0, "Getting leaseid from thread failed");
+
+    STACK_WIND_COOKIE(frame, glfs_zerofill_async_cbk, subvol, subvol,
+                      subvol->fops->zerofill, fd, offset, len, fop_attr);
+    ret = 0;
+out:
+    if (ret) {
+        if (fd)
+            fd_unref(fd);
+        if (glfd)
+            GF_REF_PUT(glfd);
+        GF_FREE(gio);
+        if (frame)
+            STACK_DESTROY(frame->root);
+        glfs_subvol_done(glfd->fs, subvol);
+    }
+    if (fop_attr)
+        dict_unref(fop_attr);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_zerofill_async, 3.5.0);
+GFAPI_SYMVER_PUBLIC(glfs_zerofill_async35, glfs_zerofill_async, 3.5.0)
+int
+pub_glfs_zerofill_async35(struct glfs_fd *glfd, off_t offset, off_t len,
+                          glfs_io_cbk34 fn, void *data)
+{
+    return glfs_zerofill_async_common(glfd, offset, len, _gf_true, (void *)fn,
+                                      data);
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_zerofill_async, 6.0)
+int
+pub_glfs_zerofill_async(struct glfs_fd *glfd, off_t offset, off_t len,
+                        glfs_io_cbk fn, void *data)
+{
+    return glfs_zerofill_async_common(glfd, offset, len, _gf_false, fn, data);
+}
 
 void
-gf_dirent_to_dirent (gf_dirent_t *gf_dirent, struct dirent *dirent)
+gf_dirent_to_dirent(gf_dirent_t *gf_dirent, struct dirent *dirent)
 {
-	dirent->d_ino = gf_dirent->d_ino;
+    dirent->d_ino = gf_dirent->d_ino;
 
 #ifdef _DIRENT_HAVE_D_OFF
-	dirent->d_off = gf_dirent->d_off;
+    dirent->d_off = gf_dirent->d_off;
 #endif
 
 #ifdef _DIRENT_HAVE_D_TYPE
-	dirent->d_type = gf_dirent->d_type;
+    dirent->d_type = gf_dirent->d_type;
 #endif
 
 #ifdef _DIRENT_HAVE_D_NAMLEN
-	dirent->d_namlen = strlen (gf_dirent->d_name);
+    dirent->d_namlen = strlen(gf_dirent->d_name);
 #endif
 
-	strncpy (dirent->d_name, gf_dirent->d_name, GF_NAME_MAX + 1);
+    snprintf(dirent->d_name, NAME_MAX + 1, "%s", gf_dirent->d_name);
 }
 
-
 int
-glfd_entry_refresh (struct glfs_fd *glfd, int plus)
+glfd_entry_refresh(struct glfs_fd *glfd, int plus)
 {
-	xlator_t        *subvol = NULL;
-	gf_dirent_t      entries;
-	gf_dirent_t      old;
-	int              ret = -1;
-	fd_t            *fd = NULL;
-
-	subvol = priv_glfs_active_subvol (glfd->fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
-
-	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
-	if (!fd) {
-		ret = -1;
-		errno = EBADFD;
-		goto out;
-	}
-
-	if (fd->inode->ia_type != IA_IFDIR) {
-		ret = -1;
-		errno = EBADF;
-		goto out;
-	}
-
-	INIT_LIST_HEAD (&entries.list);
-	INIT_LIST_HEAD (&old.list);
+    xlator_t *subvol = NULL;
+    gf_dirent_t entries;
+    gf_dirent_t old;
+    gf_dirent_t *entry = NULL;
+    int ret = -1;
+    fd_t *fd = NULL;
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    if (fd->inode->ia_type != IA_IFDIR) {
+        ret = -1;
+        errno = EBADF;
+        goto out;
+    }
+
+    INIT_LIST_HEAD(&entries.list);
+    INIT_LIST_HEAD(&old.list);
+
+    if (plus)
+        ret = syncop_readdirp(subvol, fd, 131072, glfd->offset, &entries, NULL,
+                              NULL);
+    else
+        ret = syncop_readdir(subvol, fd, 131072, glfd->offset, &entries, NULL,
+                             NULL);
+    DECODE_SYNCOP_ERR(ret);
+    if (ret >= 0) {
+        if (plus) {
+            list_for_each_entry(entry, &entries.list, list)
+            {
+                if ((!entry->inode && (!IA_ISDIR(entry->d_stat.ia_type))) ||
+                    ((entry->d_stat.ia_ctime == 0) &&
+                     strcmp(entry->d_name, ".") &&
+                     strcmp(entry->d_name, ".."))) {
+                    /* entry->inode for directories will be
+                     * always set to null to force a lookup
+                     * on the dentry. Hence to not degrade
+                     * readdir performance, we skip lookups
+                     * for directory entries. Also we will have
+                     * proper stat if directory present on
+                     * hashed subvolume.
+                     *
+                     * In addition, if the stat is invalid, force
+                     * lookup to fetch proper stat.
+                     */
+                    gf_fill_iatt_for_dirent(entry, fd->inode, subvol);
+                }
+            }
 
-	if (plus)
-		ret = syncop_readdirp (subvol, fd, 131072, glfd->offset,
-				       NULL, &entries);
-	else
-		ret = syncop_readdir (subvol, fd, 131072, glfd->offset,
-				      &entries);
-	if (ret >= 0) {
-		if (plus)
-			gf_link_inodes_from_dirent (THIS, fd->inode, &entries);
+            gf_link_inodes_from_dirent(THIS, fd->inode, &entries);
+        }
 
-		list_splice_init (&glfd->entries, &old.list);
-		list_splice_init (&entries.list, &glfd->entries);
+        list_splice_init(&glfd->entries, &old.list);
+        list_splice_init(&entries.list, &glfd->entries);
 
-		/* spurious errno is dangerous for glfd_entry_next() */
-		errno = 0;
-	}
+        /* spurious errno is dangerous for glfd_entry_next() */
+        errno = 0;
+    }
 
-	if (ret > 0)
-		glfd->next = list_entry (glfd->entries.next, gf_dirent_t, list);
+    if (ret > 0)
+        glfd->next = list_entry(glfd->entries.next, gf_dirent_t, list);
 
-	gf_dirent_free (&old);
+    gf_dirent_free(&old);
 out:
-	if (fd)
-		fd_unref (fd);
+    if (fd)
+        fd_unref(fd);
 
-	priv_glfs_subvol_done (glfd->fs, subvol);
+    glfs_subvol_done(glfd->fs, subvol);
 
-	return ret;
+    return ret;
 }
 
-
 gf_dirent_t *
-glfd_entry_next (struct glfs_fd *glfd, int plus)
+glfd_entry_next(struct glfs_fd *glfd, int plus)
 {
-	gf_dirent_t     *entry = NULL;
-	int              ret = -1;
+    gf_dirent_t *entry = NULL;
+    int ret = -1;
 
-	if (!glfd->offset || !glfd->next) {
-		ret = glfd_entry_refresh (glfd, plus);
-		if (ret < 0)
-			return NULL;
-	}
+    if (!glfd->offset || !glfd->next) {
+        ret = glfd_entry_refresh(glfd, plus);
+        if (ret < 0)
+            return NULL;
+    }
 
-	entry = glfd->next;
-	if (!entry)
-		return NULL;
+    entry = glfd->next;
+    if (!entry)
+        return NULL;
 
-	if (&entry->next->list == &glfd->entries)
-		glfd->next = NULL;
-	else
-		glfd->next = entry->next;
+    if (&entry->next->list == &glfd->entries)
+        glfd->next = NULL;
+    else
+        glfd->next = entry->next;
 
-	glfd->offset = entry->d_off;
+    glfd->offset = entry->d_off;
 
-	return entry;
+    return entry;
 }
 
-
-static struct dirent *
-glfs_readdirbuf_get (struct glfs_fd *glfd)
+struct dirent *
+glfs_readdirbuf_get(struct glfs_fd *glfd)
 {
-        struct dirent *buf = NULL;
-
-        LOCK (&glfd->fd->lock);
-        {
-                buf = glfd->readdirbuf;
-                if (buf) {
-                        memset (buf, 0, READDIRBUF_SIZE);
-                        goto unlock;
-                }
-
-                buf = GF_CALLOC (1, READDIRBUF_SIZE, glfs_mt_readdirbuf_t);
-                if (!buf) {
-                        errno = ENOMEM;
-                        goto unlock;
-                }
+    struct dirent *buf = NULL;
+
+    LOCK(&glfd->fd->lock);
+    {
+        buf = glfd->readdirbuf;
+        if (buf) {
+            memset(buf, 0, READDIRBUF_SIZE);
+            goto unlock;
+        }
 
-                glfd->readdirbuf = buf;
+        buf = GF_CALLOC(1, READDIRBUF_SIZE, glfs_mt_readdirbuf_t);
+        if (!buf) {
+            errno = ENOMEM;
+            goto unlock;
         }
+
+        glfd->readdirbuf = buf;
+    }
 unlock:
-        UNLOCK (&glfd->fd->lock);
+    UNLOCK(&glfd->fd->lock);
 
-        return buf;
+    return buf;
 }
 
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_readdirplus_r, 3.4.0)
 int
-pub_glfs_readdirplus_r (struct glfs_fd *glfd, struct stat *stat,
-                        struct dirent *ext, struct dirent **res)
+pub_glfs_readdirplus_r(struct glfs_fd *glfd, struct stat *stat,
+                       struct dirent *ext, struct dirent **res)
 {
-	int              ret = 0;
-	gf_dirent_t     *entry = NULL;
-	struct dirent   *buf = NULL;
+    int ret = 0;
+    gf_dirent_t *entry = NULL;
+    struct dirent *buf = NULL;
 
-	__glfs_entry_fd (glfd);
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
 
-	errno = 0;
+    GF_REF_GET(glfd);
 
-	if (ext)
-		buf = ext;
-	else
-		buf = glfs_readdirbuf_get (glfd);
+    errno = 0;
 
-	if (!buf) {
-		errno = ENOMEM;
-		return -1;
-	}
+    if (ext)
+        buf = ext;
+    else
+        buf = glfs_readdirbuf_get(glfd);
 
-	entry = glfd_entry_next (glfd, !!stat);
-	if (errno)
-		ret = -1;
+    if (!buf) {
+        errno = ENOMEM;
+        ret = -1;
+        goto out;
+    }
 
-	if (res) {
-		if (entry)
-			*res = buf;
-		else
-			*res = NULL;
-	}
+    entry = glfd_entry_next(glfd, !!stat);
+    if (errno)
+        ret = -1;
 
-	if (entry) {
-		gf_dirent_to_dirent (entry, buf);
-		if (stat)
-			glfs_iatt_to_stat (glfd->fs, &entry->d_stat, stat);
-	}
+    if (res) {
+        if (entry)
+            *res = buf;
+        else
+            *res = NULL;
+    }
 
-	return ret;
-}
+    if (entry) {
+        gf_dirent_to_dirent(entry, buf);
+        if (stat)
+            glfs_iatt_to_stat(glfd->fs, &entry->d_stat, stat);
+    }
+
+out:
+    if (glfd)
+        GF_REF_PUT(glfd);
+
+    __GLFS_EXIT_FS;
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_readdirplus_r, 3.4.0);
+    return ret;
 
+invalid_fs:
+    return -1;
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_readdir_r, 3.4.0)
 int
-pub_glfs_readdir_r (struct glfs_fd *glfd, struct dirent *buf,
-                    struct dirent **res)
+pub_glfs_readdir_r(struct glfs_fd *glfd, struct dirent *buf,
+                   struct dirent **res)
 {
-	return pub_glfs_readdirplus_r (glfd, 0, buf, res);
+    return pub_glfs_readdirplus_r(glfd, 0, buf, res);
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_readdir_r, 3.4.0);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_readdirplus, 3.5.0)
 struct dirent *
-pub_glfs_readdirplus (struct glfs_fd *glfd, struct stat *stat)
+pub_glfs_readdirplus(struct glfs_fd *glfd, struct stat *stat)
 {
-        struct dirent *res = NULL;
-        int ret = -1;
+    struct dirent *res = NULL;
+    int ret = -1;
 
-        ret = pub_glfs_readdirplus_r (glfd, stat, NULL, &res);
-        if (ret)
-                return NULL;
+    ret = pub_glfs_readdirplus_r(glfd, stat, NULL, &res);
+    if (ret)
+        return NULL;
 
-        return res;
+    return res;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_readdirplus, 3.5.0);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_readdir, 3.5.0)
 struct dirent *
-pub_glfs_readdir (struct glfs_fd *glfd)
+pub_glfs_readdir(struct glfs_fd *glfd)
 {
-        return pub_glfs_readdirplus (glfd, NULL);
+    return pub_glfs_readdirplus(glfd, NULL);
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_readdir, 3.5.0);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_statvfs, 3.4.0)
 int
-pub_glfs_statvfs (struct glfs *fs, const char *path, struct statvfs *buf)
+pub_glfs_statvfs(struct glfs *fs, const char *path, struct statvfs *buf)
 {
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	loc_t            loc = {0, };
-	struct iatt      iatt = {0, };
-	int              reval = 0;
-
-	__glfs_entry_fs (fs);
-
-	subvol = priv_glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
 retry:
-	ret = glfs_resolve (fs, subvol, path, &loc, &iatt, reval);
+    ret = glfs_resolve(fs, subvol, path, &loc, &iatt, reval);
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	if (ret)
-		goto out;
+    if (ret)
+        goto out;
 
-	ret = syncop_statfs (subvol, &loc, buf);
+    ret = syncop_statfs(subvol, &loc, buf, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 out:
-	loc_wipe (&loc);
+    loc_wipe(&loc);
 
-	priv_glfs_subvol_done (fs, subvol);
+    glfs_subvol_done(fs, subvol);
 
-	return ret;
-}
-
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_statvfs, 3.4.0);
+    __GLFS_EXIT_FS;
 
+invalid_fs:
+    return ret;
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_setattr, 6.0)
 int
-glfs_setattr (struct glfs *fs, const char *path, struct iatt *iatt,
-	      int valid, int follow)
+pub_glfs_setattr(struct glfs *fs, const char *path, struct glfs_stat *stat,
+                 int follow)
 {
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	loc_t            loc = {0, };
-	struct iatt      riatt = {0, };
-	int              reval = 0;
-
-	__glfs_entry_fs (fs);
-
-	subvol = priv_glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+    int ret = -1;
+    int glvalid;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt riatt = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    GF_VALIDATE_OR_GOTO("glfs_setattr", stat, out);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
 retry:
-	if (follow)
-		ret = glfs_resolve (fs, subvol, path, &loc, &riatt, reval);
-	else
-		ret = glfs_lresolve (fs, subvol, path, &loc, &riatt, reval);
+    if (follow)
+        ret = glfs_resolve(fs, subvol, path, &loc, &riatt, reval);
+    else
+        ret = glfs_lresolve(fs, subvol, path, &loc, &riatt, reval);
+
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    if (ret)
+        goto out;
 
-	if (ret)
-		goto out;
+    glfs_iatt_from_statx(&iatt, stat);
+    glfsflags_from_gfapiflags(stat, &glvalid);
 
-	ret = syncop_setattr (subvol, &loc, iatt, valid, 0, 0);
+    /* TODO : Add leaseid */
+    ret = syncop_setattr(subvol, &loc, &iatt, glvalid, 0, 0, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 out:
-	loc_wipe (&loc);
+    loc_wipe(&loc);
 
-	priv_glfs_subvol_done (fs, subvol);
+    glfs_subvol_done(fs, subvol);
 
-	return ret;
-}
+    __GLFS_EXIT_FS;
 
+invalid_fs:
+    return ret;
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fsetattr, 6.0)
 int
-glfs_fsetattr (struct glfs_fd *glfd, struct iatt *iatt, int valid)
+pub_glfs_fsetattr(struct glfs_fd *glfd, struct glfs_stat *stat)
 {
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	fd_t            *fd = NULL;
-
-	__glfs_entry_fd (glfd);
-
-	subvol = priv_glfs_active_subvol (glfd->fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
-
-	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
-	if (!fd) {
-		ret = -1;
-		errno = EBADFD;
-		goto out;
-	}
-
-	ret = syncop_fsetattr (subvol, fd, iatt, valid, 0, 0);
+    int ret = -1;
+    int glvalid;
+    struct iatt iatt = {
+        0,
+    };
+    xlator_t *subvol = NULL;
+    fd_t *fd = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    GF_REF_GET(glfd);
+
+    GF_VALIDATE_OR_GOTO("glfs_fsetattr", stat, out);
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    glfs_iatt_from_statx(&iatt, stat);
+    glfsflags_from_gfapiflags(stat, &glvalid);
+
+    /* TODO : Add leaseid */
+    ret = syncop_fsetattr(subvol, fd, &iatt, glvalid, 0, 0, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
 out:
-	if (fd)
-		fd_unref (fd);
+    if (fd)
+        fd_unref(fd);
+    if (glfd)
+        GF_REF_PUT(glfd);
 
-	priv_glfs_subvol_done (glfd->fs, subvol);
+    glfs_subvol_done(glfd->fs, subvol);
 
-	return ret;
-}
+    __GLFS_EXIT_FS;
 
+invalid_fs:
+    return ret;
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_chmod, 3.4.0)
 int
-pub_glfs_chmod (struct glfs *fs, const char *path, mode_t mode)
+pub_glfs_chmod(struct glfs *fs, const char *path, mode_t mode)
 {
-	int              ret = -1;
-	struct iatt      iatt = {0, };
-	int              valid = 0;
+    int ret = -1;
+    struct glfs_stat stat = {
+        0,
+    };
 
-	iatt.ia_prot = ia_prot_from_st_mode (mode);
-	valid = GF_SET_ATTR_MODE;
+    stat.glfs_st_mode = mode;
+    stat.glfs_st_mask = GLFS_STAT_MODE;
 
-	ret = glfs_setattr (fs, path, &iatt, valid, 1);
+    ret = glfs_setattr(fs, path, &stat, 1);
 
-	return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_chmod, 3.4.0);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fchmod, 3.4.0)
 int
-pub_glfs_fchmod (struct glfs_fd *glfd, mode_t mode)
+pub_glfs_fchmod(struct glfs_fd *glfd, mode_t mode)
 {
-	int              ret = -1;
-	struct iatt      iatt = {0, };
-	int              valid = 0;
+    int ret = -1;
+    struct glfs_stat stat = {
+        0,
+    };
 
-	iatt.ia_prot = ia_prot_from_st_mode (mode);
-	valid = GF_SET_ATTR_MODE;
+    stat.glfs_st_mode = mode;
+    stat.glfs_st_mask = GLFS_STAT_MODE;
 
-	ret = glfs_fsetattr (glfd, &iatt, valid);
+    ret = glfs_fsetattr(glfd, &stat);
 
-	return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fchmod, 3.4.0);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_chown, 3.4.0)
 int
-pub_glfs_chown (struct glfs *fs, const char *path, uid_t uid, gid_t gid)
+pub_glfs_chown(struct glfs *fs, const char *path, uid_t uid, gid_t gid)
 {
-	int              ret = -1;
-	int              valid = 0;
-	struct iatt      iatt = {0, };
+    int ret = 0;
+    struct glfs_stat stat = {
+        0,
+    };
 
-	iatt.ia_uid = uid;
-	iatt.ia_gid = gid;
-	valid = GF_SET_ATTR_UID|GF_SET_ATTR_GID;
+    if (uid != (uid_t)-1) {
+        stat.glfs_st_uid = uid;
+        stat.glfs_st_mask = GLFS_STAT_UID;
+    }
 
-	ret = glfs_setattr (fs, path, &iatt, valid, 1);
+    if (gid != (uid_t)-1) {
+        stat.glfs_st_gid = gid;
+        stat.glfs_st_mask = stat.glfs_st_mask | GLFS_STAT_GID;
+    }
 
-	return ret;
-}
-
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_chown, 3.4.0);
+    if (stat.glfs_st_mask)
+        ret = glfs_setattr(fs, path, &stat, 1);
 
+    return ret;
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_lchown, 3.4.0)
 int
-pub_glfs_lchown (struct glfs *fs, const char *path, uid_t uid, gid_t gid)
+pub_glfs_lchown(struct glfs *fs, const char *path, uid_t uid, gid_t gid)
 {
-	int              ret = -1;
-	int              valid = 0;
-	struct iatt      iatt = {0, };
+    int ret = 0;
+    struct glfs_stat stat = {
+        0,
+    };
 
-	iatt.ia_uid = uid;
-	iatt.ia_gid = gid;
-	valid = GF_SET_ATTR_UID|GF_SET_ATTR_GID;
+    if (uid != (uid_t)-1) {
+        stat.glfs_st_uid = uid;
+        stat.glfs_st_mask = GLFS_STAT_UID;
+    }
 
-	ret = glfs_setattr (fs, path, &iatt, valid, 0);
+    if (gid != (uid_t)-1) {
+        stat.glfs_st_gid = gid;
+        stat.glfs_st_mask = stat.glfs_st_mask | GLFS_STAT_GID;
+    }
 
-	return ret;
-}
-
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_lchown, 3.4.0);
+    if (stat.glfs_st_mask)
+        ret = glfs_setattr(fs, path, &stat, 0);
 
+    return ret;
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fchown, 3.4.0)
 int
-pub_glfs_fchown (struct glfs_fd *glfd, uid_t uid, gid_t gid)
+pub_glfs_fchown(struct glfs_fd *glfd, uid_t uid, gid_t gid)
 {
-	int              ret = -1;
-	int              valid = 0;
-	struct iatt      iatt = {0, };
+    int ret = 0;
+    struct glfs_stat stat = {
+        0,
+    };
 
-	iatt.ia_uid = uid;
-	iatt.ia_gid = gid;
-	valid = GF_SET_ATTR_UID|GF_SET_ATTR_GID;
+    if (uid != (uid_t)-1) {
+        stat.glfs_st_uid = uid;
+        stat.glfs_st_mask = GLFS_STAT_UID;
+    }
 
-	ret = glfs_fsetattr (glfd, &iatt, valid);
+    if (gid != (uid_t)-1) {
+        stat.glfs_st_gid = gid;
+        stat.glfs_st_mask = stat.glfs_st_mask | GLFS_STAT_GID;
+    }
 
-	return ret;
-}
-
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fchown, 3.4.0);
+    if (stat.glfs_st_mask)
+        ret = glfs_fsetattr(glfd, &stat);
 
+    return ret;
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_utimens, 3.4.0)
 int
-pub_glfs_utimens (struct glfs *fs, const char *path, struct timespec times[2])
+pub_glfs_utimens(struct glfs *fs, const char *path,
+                 const struct timespec times[2])
 {
-	int              ret = -1;
-	int              valid = 0;
-	struct iatt      iatt = {0, };
+    int ret = -1;
+    struct glfs_stat stat = {
+        0,
+    };
 
-	iatt.ia_atime = times[0].tv_sec;
-	iatt.ia_atime_nsec = times[0].tv_nsec;
-	iatt.ia_mtime = times[1].tv_sec;
-	iatt.ia_mtime_nsec = times[1].tv_nsec;
+    stat.glfs_st_atime = times[0];
+    stat.glfs_st_mtime = times[1];
 
-	valid = GF_SET_ATTR_ATIME|GF_SET_ATTR_MTIME;
+    stat.glfs_st_mask = GLFS_STAT_ATIME | GLFS_STAT_MTIME;
 
-	ret = glfs_setattr (fs, path, &iatt, valid, 1);
+    ret = glfs_setattr(fs, path, &stat, 1);
 
-	return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_utimens, 3.4.0);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_lutimens, 3.4.0)
 int
-pub_glfs_lutimens (struct glfs *fs, const char *path, struct timespec times[2])
+pub_glfs_lutimens(struct glfs *fs, const char *path,
+                  const struct timespec times[2])
 {
-	int              ret = -1;
-	int              valid = 0;
-	struct iatt      iatt = {0, };
+    int ret = -1;
+    struct glfs_stat stat = {
+        0,
+    };
 
-	iatt.ia_atime = times[0].tv_sec;
-	iatt.ia_atime_nsec = times[0].tv_nsec;
-	iatt.ia_mtime = times[1].tv_sec;
-	iatt.ia_mtime_nsec = times[1].tv_nsec;
+    stat.glfs_st_atime = times[0];
+    stat.glfs_st_mtime = times[1];
 
-	valid = GF_SET_ATTR_ATIME|GF_SET_ATTR_MTIME;
+    stat.glfs_st_mask = GLFS_STAT_ATIME | GLFS_STAT_MTIME;
 
-	ret = glfs_setattr (fs, path, &iatt, valid, 0);
+    ret = glfs_setattr(fs, path, &stat, 0);
 
-	return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_lutimens, 3.4.0);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_futimens, 3.4.0)
 int
-pub_glfs_futimens (struct glfs_fd *glfd, struct timespec times[2])
+pub_glfs_futimens(struct glfs_fd *glfd, const struct timespec times[2])
 {
-	int              ret = -1;
-	int              valid = 0;
-	struct iatt      iatt = {0, };
+    int ret = -1;
+    struct glfs_stat stat = {
+        0,
+    };
 
-	iatt.ia_atime = times[0].tv_sec;
-	iatt.ia_atime_nsec = times[0].tv_nsec;
-	iatt.ia_mtime = times[1].tv_sec;
-	iatt.ia_mtime_nsec = times[1].tv_nsec;
+    stat.glfs_st_atime = times[0];
+    stat.glfs_st_mtime = times[1];
 
-	valid = GF_SET_ATTR_ATIME|GF_SET_ATTR_MTIME;
+    stat.glfs_st_mask = GLFS_STAT_ATIME | GLFS_STAT_MTIME;
 
-	ret = glfs_fsetattr (glfd, &iatt, valid);
+    ret = glfs_fsetattr(glfd, &stat);
 
-	return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_futimens, 3.4.0);
-
-
 int
-glfs_getxattr_process (void *value, size_t size, dict_t *xattr,
-		       const char *name)
+glfs_getxattr_process(void *value, size_t size, dict_t *xattr, const char *name)
 {
-	data_t *data = NULL;
-	int     ret = -1;
-
-	data = dict_get (xattr, (char *)name);
-	if (!data) {
-		errno = ENODATA;
-		ret = -1;
-		goto out;
-	}
-
-	ret = data->len;
-	if (!value || !size)
-		goto out;
-
-	if (size < ret) {
-		ret = -1;
-		errno = ERANGE;
-		goto out;
-	}
-
-	memcpy (value, data->data, ret);
+    data_t *data = NULL;
+    int ret = -1;
+
+    data = dict_get(xattr, (char *)name);
+    if (!data) {
+        errno = ENODATA;
+        ret = -1;
+        goto out;
+    }
+
+    ret = data->len;
+    if (!value || !size)
+        goto out;
+
+    if (size < ret) {
+        ret = -1;
+        errno = ERANGE;
+        goto out;
+    }
+
+    memcpy(value, data->data, ret);
 out:
-	if (xattr)
-		dict_unref (xattr);
-	return ret;
+    return ret;
 }
 
-
 ssize_t
-glfs_getxattr_common (struct glfs *fs, const char *path, const char *name,
-		      void *value, size_t size, int follow)
-{
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	loc_t            loc = {0, };
-	struct iatt      iatt = {0, };
-	dict_t          *xattr = NULL;
-	int              reval = 0;
-
-	__glfs_entry_fs (fs);
-
-	subvol = priv_glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+glfs_getxattr_common(struct glfs *fs, const char *path, const char *name,
+                     void *value, size_t size, int follow)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    dict_t *xattr = NULL;
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    if (!name || *name == '\0') {
+        ret = -1;
+        errno = EINVAL;
+        goto out;
+    }
+
+    if (strlen(name) > GF_XATTR_NAME_MAX) {
+        ret = -1;
+        errno = ENAMETOOLONG;
+        goto out;
+    }
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
 retry:
-	if (follow)
-		ret = glfs_resolve (fs, subvol, path, &loc, &iatt, reval);
-	else
-		ret = glfs_lresolve (fs, subvol, path, &loc, &iatt, reval);
+    if (follow)
+        ret = glfs_resolve(fs, subvol, path, &loc, &iatt, reval);
+    else
+        ret = glfs_lresolve(fs, subvol, path, &loc, &iatt, reval);
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	if (ret)
-		goto out;
+    if (ret)
+        goto out;
 
-	ret = syncop_getxattr (subvol, &loc, &xattr, name);
+    ret = syncop_getxattr(subvol, &loc, &xattr, name, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	if (ret)
-		goto out;
+    if (ret)
+        goto out;
 
-	ret = glfs_getxattr_process (value, size, xattr, name);
+    ret = glfs_getxattr_process(value, size, xattr, name);
 out:
-	loc_wipe (&loc);
+    loc_wipe(&loc);
 
-	priv_glfs_subvol_done (fs, subvol);
+    if (xattr)
+        dict_unref(xattr);
 
-	return ret;
-}
+    glfs_subvol_done(fs, subvol);
+
+    __GLFS_EXIT_FS;
 
+invalid_fs:
+    return ret;
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_getxattr, 3.4.0)
 ssize_t
-pub_glfs_getxattr (struct glfs *fs, const char *path, const char *name,
-                   void *value, size_t size)
+pub_glfs_getxattr(struct glfs *fs, const char *path, const char *name,
+                  void *value, size_t size)
 {
-	return glfs_getxattr_common (fs, path, name, value, size, 1);
+    return glfs_getxattr_common(fs, path, name, value, size, 1);
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_getxattr, 3.4.0);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_lgetxattr, 3.4.0)
 ssize_t
-pub_glfs_lgetxattr (struct glfs *fs, const char *path, const char *name,
-                    void *value, size_t size)
+pub_glfs_lgetxattr(struct glfs *fs, const char *path, const char *name,
+                   void *value, size_t size)
 {
-	return glfs_getxattr_common (fs, path, name, value, size, 0);
+    return glfs_getxattr_common(fs, path, name, value, size, 0);
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_lgetxattr, 3.4.0);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fgetxattr, 3.4.0)
 ssize_t
-pub_glfs_fgetxattr (struct glfs_fd *glfd, const char *name, void *value,
-                    size_t size)
+pub_glfs_fgetxattr(struct glfs_fd *glfd, const char *name, void *value,
+                   size_t size)
 {
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	dict_t          *xattr = NULL;
-	fd_t            *fd = NULL;
-
-	__glfs_entry_fd (glfd);
-
-	subvol = priv_glfs_active_subvol (glfd->fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
-
-	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
-	if (!fd) {
-		ret = -1;
-		errno = EBADFD;
-		goto out;
-	}
-
-	ret = syncop_fgetxattr (subvol, fd, &xattr, name);
-	if (ret)
-		goto out;
-
-	ret = glfs_getxattr_process (value, size, xattr, name);
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    dict_t *xattr = NULL;
+    fd_t *fd = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    GF_REF_GET(glfd);
+
+    if (!name || *name == '\0') {
+        ret = -1;
+        errno = EINVAL;
+        goto out;
+    }
+
+    if (strlen(name) > GF_XATTR_NAME_MAX) {
+        ret = -1;
+        errno = ENAMETOOLONG;
+        goto out;
+    }
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    ret = syncop_fgetxattr(subvol, fd, &xattr, name, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+    if (ret)
+        goto out;
+
+    ret = glfs_getxattr_process(value, size, xattr, name);
 out:
-	if (fd)
-		fd_unref (fd);
+    if (fd)
+        fd_unref(fd);
+    if (glfd)
+        GF_REF_PUT(glfd);
+    if (xattr)
+        dict_unref(xattr);
 
-	priv_glfs_subvol_done (glfd->fs, subvol);
+    glfs_subvol_done(glfd->fs, subvol);
 
-	return ret;
-}
-
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fgetxattr, 3.4.0);
+    __GLFS_EXIT_FS;
 
+invalid_fs:
+    return ret;
+}
 
 int
-glfs_listxattr_process (void *value, size_t size, dict_t *xattr)
+glfs_listxattr_process(void *value, size_t size, dict_t *xattr)
 {
-	int     ret = -1;
+    int ret = -1;
 
-	ret = dict_keys_join (NULL, 0, xattr, NULL);
+    if (!xattr)
+        goto out;
 
-	if (!value || !size)
-		goto out;
+    ret = dict_keys_join(NULL, 0, xattr, NULL);
 
-	if (size < ret) {
-		ret = -1;
-		errno = ERANGE;
-		goto out;
-	}
+    if (!value || !size)
+        goto out;
+
+    if (size < ret) {
+        ret = -1;
+        errno = ERANGE;
+    } else {
+        dict_keys_join(value, size, xattr, NULL);
+    }
 
-	dict_keys_join (value, size, xattr, NULL);
 out:
-	if (xattr)
-		dict_unref (xattr);
-	return ret;
+    return ret;
 }
 
-
 ssize_t
-glfs_listxattr_common (struct glfs *fs, const char *path, void *value,
-		       size_t size, int follow)
+glfs_listxattr_common(struct glfs *fs, const char *path, void *value,
+                      size_t size, int follow)
 {
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	loc_t            loc = {0, };
-	struct iatt      iatt = {0, };
-	dict_t          *xattr = NULL;
-	int              reval = 0;
-
-	__glfs_entry_fs (fs);
-
-	subvol = priv_glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    dict_t *xattr = NULL;
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
 
 retry:
-	if (follow)
-		ret = glfs_resolve (fs, subvol, path, &loc, &iatt, reval);
-	else
-		ret = glfs_lresolve (fs, subvol, path, &loc, &iatt, reval);
+    if (follow)
+        ret = glfs_resolve(fs, subvol, path, &loc, &iatt, reval);
+    else
+        ret = glfs_lresolve(fs, subvol, path, &loc, &iatt, reval);
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	if (ret)
-		goto out;
+    if (ret)
+        goto out;
 
-	ret = syncop_getxattr (subvol, &loc, &xattr, NULL);
+    ret = syncop_getxattr(subvol, &loc, &xattr, NULL, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	if (ret)
-		goto out;
+    if (ret)
+        goto out;
 
-	ret = glfs_listxattr_process (value, size, xattr);
+    ret = glfs_listxattr_process(value, size, xattr);
 out:
-	loc_wipe (&loc);
+    loc_wipe(&loc);
 
-	priv_glfs_subvol_done (fs, subvol);
+    if (xattr)
+        dict_unref(xattr);
 
-	return ret;
-}
+    glfs_subvol_done(fs, subvol);
+
+    __GLFS_EXIT_FS;
 
+invalid_fs:
+    return ret;
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_listxattr, 3.4.0)
 ssize_t
-pub_glfs_listxattr (struct glfs *fs, const char *path, void *value, size_t size)
+pub_glfs_listxattr(struct glfs *fs, const char *path, void *value, size_t size)
 {
-	return glfs_listxattr_common (fs, path, value, size, 1);
+    return glfs_listxattr_common(fs, path, value, size, 1);
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_listxattr, 3.4.0);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_llistxattr, 3.4.0)
 ssize_t
-pub_glfs_llistxattr (struct glfs *fs, const char *path, void *value, size_t size)
+pub_glfs_llistxattr(struct glfs *fs, const char *path, void *value, size_t size)
 {
-	return glfs_listxattr_common (fs, path, value, size, 0);
+    return glfs_listxattr_common(fs, path, value, size, 0);
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_llistxattr, 3.4.0);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_flistxattr, 3.4.0)
+ssize_t
+pub_glfs_flistxattr(struct glfs_fd *glfd, void *value, size_t size)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    dict_t *xattr = NULL;
+    fd_t *fd = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    GF_REF_GET(glfd);
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    ret = syncop_fgetxattr(subvol, fd, &xattr, NULL, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+    if (ret)
+        goto out;
+
+    ret = glfs_listxattr_process(value, size, xattr);
+out:
+    if (fd)
+        fd_unref(fd);
+    if (glfd)
+        GF_REF_PUT(glfd);
+    if (xattr)
+        dict_unref(xattr);
 
+    glfs_subvol_done(glfd->fs, subvol);
 
-ssize_t
-pub_glfs_flistxattr (struct glfs_fd *glfd, void *value, size_t size)
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
+
+int
+glfs_setxattr_common(struct glfs *fs, const char *path, const char *name,
+                     const void *value, size_t size, int flags, int follow)
 {
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	dict_t          *xattr = NULL;
-	fd_t            *fd = NULL;
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    dict_t *xattr = NULL;
+    int reval = 0;
+    void *value_cp = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    if (!name || *name == '\0') {
+        ret = -1;
+        errno = EINVAL;
+        goto out;
+    }
+
+    if (strlen(name) > GF_XATTR_NAME_MAX) {
+        ret = -1;
+        errno = ENAMETOOLONG;
+        goto out;
+    }
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+retry:
+    if (follow)
+        ret = glfs_resolve(fs, subvol, path, &loc, &iatt, reval);
+    else
+        ret = glfs_lresolve(fs, subvol, path, &loc, &iatt, reval);
+
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	__glfs_entry_fd (glfd);
+    if (ret)
+        goto out;
 
-	subvol = priv_glfs_active_subvol (glfd->fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+    value_cp = gf_memdup(value, size);
+    GF_CHECK_ALLOC_AND_LOG(subvol->name, value_cp, ret,
+                           "Failed to"
+                           " duplicate setxattr value",
+                           out);
 
-	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
-	if (!fd) {
-		ret = -1;
-		errno = EBADFD;
-		goto out;
-	}
+    xattr = dict_for_key_value(name, value_cp, size, _gf_false);
+    if (!xattr) {
+        GF_FREE(value_cp);
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
 
-	ret = syncop_fgetxattr (subvol, fd, &xattr, NULL);
-	if (ret)
-		goto out;
+    ret = syncop_setxattr(subvol, &loc, xattr, flags, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
 
-	ret = glfs_listxattr_process (value, size, xattr);
 out:
-	if (fd)
-		fd_unref (fd);
+    loc_wipe(&loc);
+    if (xattr)
+        dict_unref(xattr);
+
+    glfs_subvol_done(fs, subvol);
 
-	priv_glfs_subvol_done (glfd->fs, subvol);
+    __GLFS_EXIT_FS;
 
-	return ret;
+invalid_fs:
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_flistxattr, 3.4.0);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_setxattr, 3.4.0)
+int
+pub_glfs_setxattr(struct glfs *fs, const char *path, const char *name,
+                  const void *value, size_t size, int flags)
+{
+    return glfs_setxattr_common(fs, path, name, value, size, flags, 1);
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_lsetxattr, 3.4.0)
+int
+pub_glfs_lsetxattr(struct glfs *fs, const char *path, const char *name,
+                   const void *value, size_t size, int flags)
+{
+    return glfs_setxattr_common(fs, path, name, value, size, flags, 0);
+}
 
-dict_t *
-dict_for_key_value (const char *name, const char *value, size_t size)
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fsetxattr, 3.4.0)
+int
+pub_glfs_fsetxattr(struct glfs_fd *glfd, const char *name, const void *value,
+                   size_t size, int flags)
 {
-	dict_t *xattr = NULL;
-	int     ret = 0;
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    dict_t *xattr = NULL;
+    fd_t *fd = NULL;
+    void *value_cp = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    GF_REF_GET(glfd);
+
+    if (!name || *name == '\0') {
+        ret = -1;
+        errno = EINVAL;
+        goto out;
+    }
+
+    if (strlen(name) > GF_XATTR_NAME_MAX) {
+        ret = -1;
+        errno = ENAMETOOLONG;
+        goto out;
+    }
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    value_cp = gf_memdup(value, size);
+    GF_CHECK_ALLOC_AND_LOG(subvol->name, value_cp, ret,
+                           "Failed to"
+                           " duplicate setxattr value",
+                           out);
+
+    xattr = dict_for_key_value(name, value_cp, size, _gf_false);
+    if (!xattr) {
+        GF_FREE(value_cp);
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    ret = syncop_fsetxattr(subvol, fd, xattr, flags, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+out:
+    if (xattr)
+        dict_unref(xattr);
 
-	xattr = dict_new ();
-	if (!xattr)
-		return NULL;
+    if (fd)
+        fd_unref(fd);
+    if (glfd)
+        GF_REF_PUT(glfd);
 
-	ret = dict_set_static_bin (xattr, (char *)name, (void *)value, size);
-	if (ret) {
-		dict_destroy (xattr);
-		xattr = NULL;
-	}
+    glfs_subvol_done(glfd->fs, subvol);
 
-	return xattr;
-}
+    __GLFS_EXIT_FS;
 
+invalid_fs:
+    return ret;
+}
 
 int
-glfs_setxattr_common (struct glfs *fs, const char *path, const char *name,
-		      const void *value, size_t size, int flags, int follow)
+glfs_removexattr_common(struct glfs *fs, const char *path, const char *name,
+                        int follow)
 {
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	loc_t            loc = {0, };
-	struct iatt      iatt = {0, };
-	dict_t          *xattr = NULL;
-	int              reval = 0;
-
-	__glfs_entry_fs (fs);
-
-	subvol = priv_glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
 retry:
-	if (follow)
-		ret = glfs_resolve (fs, subvol, path, &loc, &iatt, reval);
-	else
-		ret = glfs_lresolve (fs, subvol, path, &loc, &iatt, reval);
-
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    if (follow)
+        ret = glfs_resolve(fs, subvol, path, &loc, &iatt, reval);
+    else
+        ret = glfs_lresolve(fs, subvol, path, &loc, &iatt, reval);
 
-	if (ret)
-		goto out;
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	xattr = dict_for_key_value (name, value, size);
-	if (!xattr) {
-		ret = -1;
-		errno = ENOMEM;
-		goto out;
-	}
+    if (ret)
+        goto out;
 
-	ret = syncop_setxattr (subvol, &loc, xattr, flags);
+    ret = syncop_removexattr(subvol, &loc, name, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
 out:
-	loc_wipe (&loc);
-	if (xattr)
-		dict_unref (xattr);
+    loc_wipe(&loc);
+
+    glfs_subvol_done(fs, subvol);
 
-	priv_glfs_subvol_done (fs, subvol);
+    __GLFS_EXIT_FS;
 
-	return ret;
+invalid_fs:
+    return ret;
 }
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_removexattr, 3.4.0)
+int
+pub_glfs_removexattr(struct glfs *fs, const char *path, const char *name)
+{
+    return glfs_removexattr_common(fs, path, name, 1);
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_lremovexattr, 3.4.0)
 int
-pub_glfs_setxattr (struct glfs *fs, const char *path, const char *name,
-                   const void *value, size_t size, int flags)
+pub_glfs_lremovexattr(struct glfs *fs, const char *path, const char *name)
 {
-	return glfs_setxattr_common (fs, path, name, value, size, flags, 1);
+    return glfs_removexattr_common(fs, path, name, 0);
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_setxattr, 3.4.0);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fremovexattr, 3.4.0)
+int
+pub_glfs_fremovexattr(struct glfs_fd *glfd, const char *name)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    fd_t *fd = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    GF_REF_GET(glfd);
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    ret = syncop_fremovexattr(subvol, fd, name, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+out:
+    if (fd)
+        fd_unref(fd);
+    if (glfd)
+        GF_REF_PUT(glfd);
+
+    glfs_subvol_done(glfd->fs, subvol);
+
+    __GLFS_EXIT_FS;
 
+invalid_fs:
+    return ret;
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fallocate, 3.5.0)
 int
-pub_glfs_lsetxattr (struct glfs *fs, const char *path, const char *name,
-                    const void *value, size_t size, int flags)
+pub_glfs_fallocate(struct glfs_fd *glfd, int keep_size, off_t offset,
+                   size_t len)
 {
-	return glfs_setxattr_common (fs, path, name, value, size, flags, 0);
-}
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    fd_t *fd = NULL;
+    dict_t *fop_attr = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    GF_REF_GET(glfd);
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    ret = get_fop_attr_thrd_key(&fop_attr);
+    if (ret)
+        gf_msg_debug("gfapi", 0, "Getting leaseid from thread failed");
+
+    ret = syncop_fallocate(subvol, fd, keep_size, offset, len, fop_attr, NULL);
+    DECODE_SYNCOP_ERR(ret);
+out:
+    if (fd)
+        fd_unref(fd);
+    if (glfd)
+        GF_REF_PUT(glfd);
+    if (fop_attr)
+        dict_unref(fop_attr);
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_lsetxattr, 3.4.0);
+    glfs_subvol_done(glfd->fs, subvol);
 
+    __GLFS_EXIT_FS;
 
+invalid_fs:
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_discard, 3.5.0)
 int
-pub_glfs_fsetxattr (struct glfs_fd *glfd, const char *name, const void *value,
-                    size_t size, int flags)
+pub_glfs_discard(struct glfs_fd *glfd, off_t offset, size_t len)
 {
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	dict_t          *xattr = NULL;
-	fd_t            *fd = NULL;
-
-	__glfs_entry_fd (glfd);
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    fd_t *fd = NULL;
+    dict_t *fop_attr = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    GF_REF_GET(glfd);
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    ret = get_fop_attr_thrd_key(&fop_attr);
+    if (ret)
+        gf_msg_debug("gfapi", 0, "Getting leaseid from thread failed");
+
+    ret = syncop_discard(subvol, fd, offset, len, fop_attr, NULL);
+    DECODE_SYNCOP_ERR(ret);
+out:
+    if (fd)
+        fd_unref(fd);
+    if (glfd)
+        GF_REF_PUT(glfd);
+    if (fop_attr)
+        dict_unref(fop_attr);
 
-	subvol = priv_glfs_active_subvol (glfd->fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+    glfs_subvol_done(glfd->fs, subvol);
 
-	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
-	if (!fd) {
-		ret = -1;
-		errno = EBADFD;
-		goto out;
-	}
+    __GLFS_EXIT_FS;
 
-	xattr = dict_for_key_value (name, value, size);
-	if (!xattr) {
-		ret = -1;
-		errno = ENOMEM;
-		goto out;
-	}
+invalid_fs:
+    return ret;
+}
 
-	ret = syncop_fsetxattr (subvol, fd, xattr, flags);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_zerofill, 3.5.0)
+int
+pub_glfs_zerofill(struct glfs_fd *glfd, off_t offset, off_t len)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    fd_t *fd = NULL;
+    dict_t *fop_attr = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    GF_REF_GET(glfd);
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        errno = EBADFD;
+        goto out;
+    }
+
+    ret = get_fop_attr_thrd_key(&fop_attr);
+    if (ret)
+        gf_msg_debug("gfapi", 0, "Getting leaseid from thread failed");
+
+    ret = syncop_zerofill(subvol, fd, offset, len, fop_attr, NULL);
+    DECODE_SYNCOP_ERR(ret);
 out:
-	if (xattr)
-		dict_unref (xattr);
+    if (fd)
+        fd_unref(fd);
+    if (glfd)
+        GF_REF_PUT(glfd);
+    if (fop_attr)
+        dict_unref(fop_attr);
 
-	if (fd)
-		fd_unref (fd);
+    glfs_subvol_done(glfd->fs, subvol);
 
-	priv_glfs_subvol_done (glfd->fs, subvol);
+    __GLFS_EXIT_FS;
 
-	return ret;
+invalid_fs:
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fsetxattr, 3.4.0);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_chdir, 3.4.0)
+int
+pub_glfs_chdir(struct glfs *fs, const char *path)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+retry:
+    ret = glfs_resolve(fs, subvol, path, &loc, &iatt, reval);
+
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
+
+    if (ret)
+        goto out;
+
+    if (!IA_ISDIR(iatt.ia_type)) {
+        ret = -1;
+        errno = ENOTDIR;
+        goto out;
+    }
+
+    glfs_cwd_set(fs, loc.inode);
+
+out:
+    loc_wipe(&loc);
+
+    glfs_subvol_done(fs, subvol);
+
+    __GLFS_EXIT_FS;
 
+invalid_fs:
+    return ret;
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fchdir, 3.4.0)
 int
-glfs_removexattr_common (struct glfs *fs, const char *path, const char *name,
-			 int follow)
+pub_glfs_fchdir(struct glfs_fd *glfd)
 {
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	loc_t            loc = {0, };
-	struct iatt      iatt = {0, };
-	int              reval = 0;
+    int ret = -1;
+    inode_t *inode = NULL;
+    xlator_t *subvol = NULL;
+    fd_t *fd = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    GF_REF_GET(glfd);
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    inode = fd->inode;
+
+    if (!IA_ISDIR(inode->ia_type)) {
+        ret = -1;
+        errno = ENOTDIR;
+        goto out;
+    }
+
+    glfs_cwd_set(glfd->fs, inode);
+    ret = 0;
+out:
+    if (fd)
+        fd_unref(fd);
+    if (glfd)
+        GF_REF_PUT(glfd);
 
-	__glfs_entry_fs (fs);
+    glfs_subvol_done(glfd->fs, subvol);
 
-	subvol = priv_glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
-retry:
-	if (follow)
-		ret = glfs_resolve (fs, subvol, path, &loc, &iatt, reval);
-	else
-		ret = glfs_lresolve (fs, subvol, path, &loc, &iatt, reval);
+    __GLFS_EXIT_FS;
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+invalid_fs:
+    return ret;
+}
+
+static gf_boolean_t warn_realpath = _gf_true; /* log once */
+
+static char *
+glfs_realpath_common(struct glfs *fs, const char *path, char *resolved_path,
+                     gf_boolean_t warn_deprecated)
+{
+    int ret = -1;
+    char *retpath = NULL;
+    char *allocpath = NULL;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    if (resolved_path)
+        retpath = resolved_path;
+    else if (warn_deprecated) {
+        retpath = allocpath = malloc(PATH_MAX + 1);
+        if (warn_realpath) {
+            warn_realpath = _gf_false;
+            gf_log(THIS->name, GF_LOG_WARNING,
+                   "this application "
+                   "is compiled against an old version of "
+                   "libgfapi, it should use glfs_free() to "
+                   "release the path returned by "
+                   "glfs_realpath()");
+        }
+    } else {
+        retpath = allocpath = GLFS_CALLOC(1, PATH_MAX + 1, NULL,
+                                          glfs_mt_realpath_t);
+    }
+
+    if (!retpath) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+retry:
+    ret = glfs_resolve(fs, subvol, path, &loc, &iatt, reval);
 
-	if (ret)
-		goto out;
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	ret = syncop_removexattr (subvol, &loc, name);
+    if (ret)
+        goto out;
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    if (loc.path) {
+        snprintf(retpath, PATH_MAX + 1, "%s", loc.path);
+    }
 
 out:
-	loc_wipe (&loc);
+    loc_wipe(&loc);
 
-	priv_glfs_subvol_done (fs, subvol);
+    if (ret == -1) {
+        if (warn_deprecated && allocpath)
+            free(allocpath);
+        else if (allocpath)
+            GLFS_FREE(allocpath);
+        retpath = NULL;
+    }
 
-	return ret;
+    glfs_subvol_done(fs, subvol);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return retpath;
 }
 
+GFAPI_SYMVER_PUBLIC(glfs_realpath34, glfs_realpath, 3.4.0)
+char *
+pub_glfs_realpath34(struct glfs *fs, const char *path, char *resolved_path)
+{
+    return glfs_realpath_common(fs, path, resolved_path, _gf_true);
+}
 
-int
-pub_glfs_removexattr (struct glfs *fs, const char *path, const char *name)
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_realpath, 3.7.17)
+char *
+pub_glfs_realpath(struct glfs *fs, const char *path, char *resolved_path)
 {
-	return glfs_removexattr_common (fs, path, name, 1);
+    return glfs_realpath_common(fs, path, resolved_path, _gf_false);
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_removexattr, 3.4.0);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_getcwd, 3.4.0)
+char *
+pub_glfs_getcwd(struct glfs *fs, char *buf, size_t n)
+{
+    int ret = -1;
+    inode_t *inode = NULL;
+    char *path = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    if (!buf || n < 2) {
+        ret = -1;
+        errno = EINVAL;
+        goto out;
+    }
+
+    inode = glfs_cwd_get(fs);
+
+    if (!inode) {
+        strncpy(buf, "/", n);
+        ret = 0;
+        goto out;
+    }
+
+    ret = inode_path(inode, 0, &path);
+    if (n <= ret) {
+        ret = -1;
+        errno = ERANGE;
+        goto out;
+    }
+
+    strncpy(buf, path, n);
+    ret = 0;
+out:
+    GF_FREE(path);
 
+    if (inode)
+        inode_unref(inode);
 
-int
-pub_glfs_lremovexattr (struct glfs *fs, const char *path, const char *name)
-{
-	return glfs_removexattr_common (fs, path, name, 0);
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    if (ret < 0)
+        return NULL;
+
+    return buf;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_lremovexattr, 3.4.0);
+static void
+gf_flock_to_flock(struct gf_flock *gf_flock, struct flock *flock)
+{
+    flock->l_type = gf_flock->l_type;
+    flock->l_whence = gf_flock->l_whence;
+    flock->l_start = gf_flock->l_start;
+    flock->l_len = gf_flock->l_len;
+    flock->l_pid = gf_flock->l_pid;
+}
 
+static void
+gf_flock_from_flock(struct gf_flock *gf_flock, struct flock *flock)
+{
+    gf_flock->l_type = flock->l_type;
+    gf_flock->l_whence = flock->l_whence;
+    gf_flock->l_start = flock->l_start;
+    gf_flock->l_len = flock->l_len;
+    gf_flock->l_pid = flock->l_pid;
+}
 
-int
-pub_glfs_fremovexattr (struct glfs_fd *glfd, const char *name)
+static int
+glfs_lock_common(struct glfs_fd *glfd, int cmd, struct flock *flock,
+                 dict_t *xdata)
 {
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	fd_t            *fd = NULL;
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    struct gf_flock gf_flock = {
+        0,
+    };
+    struct gf_flock saved_flock = {
+        0,
+    };
+    fd_t *fd = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    if (!flock) {
+        errno = EINVAL;
+        goto out;
+    }
+
+    GF_REF_GET(glfd);
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    /* Generate glusterfs flock structure from client flock
+     * structure to be processed by server */
+    gf_flock_from_flock(&gf_flock, flock);
+
+    /* Keep another copy of flock for split/merge of locks
+     * at client side */
+    gf_flock_from_flock(&saved_flock, flock);
+
+    if (glfd->lk_owner.len != 0) {
+        ret = syncopctx_setfslkowner(&glfd->lk_owner);
+
+        if (ret)
+            goto out;
+    }
 
-	__glfs_entry_fd (glfd);
+    ret = get_fop_attr_thrd_key(&xdata);
+    if (ret)
+        gf_msg_debug("gfapi", 0, "Getting leaseid from thread failed");
 
-	subvol = priv_glfs_active_subvol (glfd->fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+    ret = syncop_lk(subvol, fd, cmd, &gf_flock, xdata, NULL);
+    DECODE_SYNCOP_ERR(ret);
 
-	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
-	if (!fd) {
-		ret = -1;
-		errno = EBADFD;
-		goto out;
-	}
+    /* Convert back from gf_flock to flock as expected by application */
+    gf_flock_to_flock(&gf_flock, flock);
+
+    if (ret == 0 && (cmd == F_SETLK || cmd == F_SETLKW)) {
+        ret = fd_lk_insert_and_merge(fd, cmd, &saved_flock);
+        if (ret) {
+            gf_smsg(THIS->name, GF_LOG_ERROR, 0,
+                    API_MSG_LOCK_INSERT_MERGE_FAILED, "gfid=%s",
+                    uuid_utoa(fd->inode->gfid), NULL);
+            ret = 0;
+        }
+    }
 
-	ret = syncop_fremovexattr (subvol, fd, name);
 out:
-	if (fd)
-		fd_unref (fd);
+    if (fd)
+        fd_unref(fd);
+    if (glfd)
+        GF_REF_PUT(glfd);
 
-	priv_glfs_subvol_done (glfd->fs, subvol);
+    glfs_subvol_done(glfd->fs, subvol);
 
-	return ret;
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fremovexattr, 3.4.0);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_file_lock, 4.0.0)
+int
+pub_glfs_file_lock(struct glfs_fd *glfd, int cmd, struct flock *flock,
+                   glfs_lock_mode_t lk_mode)
+{
+    int ret = -1;
+    dict_t *xdata_in = NULL;
+
+    if (lk_mode == GLFS_LK_MANDATORY) {
+        /* Create a new dictionary */
+        xdata_in = dict_new();
+        if (xdata_in == NULL) {
+            ret = -1;
+            errno = ENOMEM;
+            goto out;
+        }
+
+        /* Set GF_LK_MANDATORY internally within dictionary to map
+         * GLFS_LK_MANDATORY */
+        ret = dict_set_uint32(xdata_in, GF_LOCK_MODE, GF_LK_MANDATORY);
+        if (ret) {
+            gf_smsg(THIS->name, GF_LOG_ERROR, 0,
+                    API_MSG_SETTING_LOCK_TYPE_FAILED, NULL);
+            ret = -1;
+            errno = ENOMEM;
+            goto out;
+        }
+    }
+
+    ret = glfs_lock_common(glfd, cmd, flock, xdata_in);
+out:
+    if (xdata_in)
+        dict_unref(xdata_in);
+
+    return ret;
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_posix_lock, 3.4.0)
+int
+pub_glfs_posix_lock(struct glfs_fd *glfd, int cmd, struct flock *flock)
+{
+    return glfs_lock_common(glfd, cmd, flock, NULL);
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fd_set_lkowner, 3.10.7)
 int
-pub_glfs_fallocate (struct glfs_fd *glfd, int keep_size, off_t offset, size_t len)
+pub_glfs_fd_set_lkowner(struct glfs_fd *glfd, void *data, int len)
 {
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	fd_t		*fd = NULL;
+    int ret = -1;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
 
-	__glfs_entry_fd (glfd);
+    if (!GF_REF_GET(glfd)) {
+        goto invalid_fs;
+    }
 
-	subvol = priv_glfs_active_subvol (glfd->fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+    GF_VALIDATE_OR_GOTO(THIS->name, data, out);
 
-	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
-	if (!fd) {
-		ret = -1;
-		errno = EBADFD;
-		goto out;
-	}
+    if ((len <= 0) || (len > GFAPI_MAX_LOCK_OWNER_LEN)) {
+        errno = EINVAL;
+        gf_smsg(THIS->name, GF_LOG_ERROR, errno, API_MSG_INVALID_ARG,
+                "lk_owner len=%d", len, NULL);
+        goto out;
+    }
 
-	ret = syncop_fallocate (subvol, fd, keep_size, offset, len);
+    glfd->lk_owner.len = len;
+
+    memcpy(glfd->lk_owner.data, data, len);
+
+    ret = 0;
 out:
-	if (fd)
-		fd_unref(fd);
+    if (glfd)
+        GF_REF_PUT(glfd);
 
-	priv_glfs_subvol_done (glfd->fs, subvol);
+    __GLFS_EXIT_FS;
 
-	return ret;
+invalid_fs:
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fallocate, 3.5.0);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_dup, 3.4.0)
+struct glfs_fd *
+pub_glfs_dup(struct glfs_fd *glfd)
+{
+    xlator_t *subvol = NULL;
+    fd_t *fd = NULL;
+    struct glfs_fd *dupfd = NULL;
+    struct glfs *fs = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    GF_REF_GET(glfd);
+
+    fs = glfd->fs;
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(fs, subvol, glfd);
+    if (!fd) {
+        errno = EBADFD;
+        goto out;
+    }
+
+    dupfd = glfs_fd_new(fs);
+    if (!dupfd) {
+        errno = ENOMEM;
+        goto out;
+    }
+
+    dupfd->fd = fd_ref(fd);
+    dupfd->state = glfd->state;
+out:
+    if (fd)
+        fd_unref(fd);
+    if (dupfd)
+        glfs_fd_bind(dupfd);
+    if (glfd)
+        GF_REF_PUT(glfd);
 
+    glfs_subvol_done(fs, subvol);
 
-int
-pub_glfs_discard (struct glfs_fd *glfd, off_t offset, size_t len)
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return dupfd;
+}
+
+static void
+glfs_enqueue_upcall_data(struct glfs *fs, struct gf_upcall *upcall_data)
 {
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	fd_t		*fd = NULL;
+    int ret = -1;
+    upcall_entry *u_list = NULL;
 
-	__glfs_entry_fd (glfd);
+    if (!fs || !upcall_data)
+        goto out;
 
-	subvol = priv_glfs_active_subvol (glfd->fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+    u_list = GF_CALLOC(1, sizeof(*u_list), glfs_mt_upcall_entry_t);
 
-	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
-	if (!fd) {
-		ret = -1;
-		errno = EBADFD;
-		goto out;
-	}
+    if (!u_list) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, ENOMEM, API_MSG_ALLOC_FAILED, "entry",
+                NULL);
+        goto out;
+    }
 
-	ret = syncop_discard (subvol, fd, offset, len);
-out:
-	if (fd)
-		fd_unref(fd);
+    INIT_LIST_HEAD(&u_list->upcall_list);
 
-	priv_glfs_subvol_done (glfd->fs, subvol);
+    gf_uuid_copy(u_list->upcall_data.gfid, upcall_data->gfid);
+    u_list->upcall_data.event_type = upcall_data->event_type;
 
-	return ret;
-}
+    switch (upcall_data->event_type) {
+        case GF_UPCALL_CACHE_INVALIDATION:
+            ret = glfs_get_upcall_cache_invalidation(&u_list->upcall_data,
+                                                     upcall_data);
+            break;
+        case GF_UPCALL_RECALL_LEASE:
+            ret = glfs_get_upcall_lease(&u_list->upcall_data, upcall_data);
+            break;
+        default:
+            break;
+    }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_discard, 3.5.0);
+    if (ret) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, errno, API_MSG_INVALID_ENTRY, NULL);
+        goto out;
+    }
 
+    pthread_mutex_lock(&fs->upcall_list_mutex);
+    {
+        list_add_tail(&u_list->upcall_list, &fs->upcall_list);
+    }
+    pthread_mutex_unlock(&fs->upcall_list_mutex);
 
-int
-pub_glfs_zerofill (struct glfs_fd *glfd, off_t offset, off_t len)
+    ret = 0;
+
+out:
+    if (ret && u_list) {
+        GF_FREE(u_list->upcall_data.data);
+        GF_FREE(u_list);
+    }
+}
+
+static void
+glfs_free_upcall_lease(void *to_free)
 {
-        int               ret             = -1;
-        xlator_t         *subvol          = NULL;
-        fd_t             *fd              = NULL;
+    struct glfs_upcall_lease *arg = to_free;
 
-        __glfs_entry_fd (glfd);
+    if (!arg)
+        return;
 
-        subvol = priv_glfs_active_subvol (glfd->fs);
-        if (!subvol) {
-                errno = EIO;
-                goto out;
+    if (arg->object)
+        glfs_h_close(arg->object);
+
+    GF_FREE(arg);
+}
+
+int
+glfs_recall_lease_fd(struct glfs *fs, struct gf_upcall *up_data)
+{
+    struct gf_upcall_recall_lease *recall_lease = NULL;
+    xlator_t *subvol = NULL;
+    int ret = 0;
+    inode_t *inode = NULL;
+    struct glfs_fd *glfd = NULL;
+    struct glfs_fd *tmp = NULL;
+    struct list_head glfd_list;
+    fd_t *fd = NULL;
+    uint64_t value = 0;
+    struct glfs_lease lease = {
+        0,
+    };
+
+    GF_VALIDATE_OR_GOTO("gfapi", up_data, out);
+    GF_VALIDATE_OR_GOTO("gfapi", fs, out);
+
+    recall_lease = up_data->data;
+    GF_VALIDATE_OR_GOTO("gfapi", recall_lease, out);
+
+    INIT_LIST_HEAD(&glfd_list);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    gf_msg_debug(THIS->name, 0, "Recall lease received for gfid:%s",
+                 uuid_utoa(up_data->gfid));
+
+    inode = inode_find(subvol->itable, up_data->gfid);
+    if (!inode) {
+        ret = -1;
+        gf_smsg(THIS->name, GF_LOG_ERROR, errno, API_MSG_INODE_FIND_FAILED,
+                "gfid=%s", uuid_utoa(up_data->gfid), "graph_id=%d",
+                subvol->graph->id, NULL);
+        goto out;
+    }
+
+    LOCK(&inode->lock);
+    {
+        list_for_each_entry(fd, &inode->fd_list, inode_list)
+        {
+            ret = fd_ctx_get(fd, subvol, &value);
+            glfd = (struct glfs_fd *)(uintptr_t)value;
+            if (glfd) {
+                gf_msg_trace(THIS->name, 0, "glfd (%p) has held lease", glfd);
+                GF_REF_GET(glfd);
+                list_add_tail(&glfd->list, &glfd_list);
+            }
         }
+    }
+    UNLOCK(&inode->lock);
 
-        fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
-        if (!fd) {
-                errno = EBADFD;
-                goto out;
+    if (!list_empty(&glfd_list)) {
+        list_for_each_entry_safe(glfd, tmp, &glfd_list, list)
+        {
+            LOCK(&glfd->lock);
+            {
+                if (glfd->state != GLFD_CLOSE) {
+                    gf_msg_trace(THIS->name, 0,
+                                 "glfd (%p) has held lease, "
+                                 "calling recall cbk",
+                                 glfd);
+                    glfd->cbk(lease, glfd->cookie);
+                }
+            }
+            UNLOCK(&glfd->lock);
+
+            list_del_init(&glfd->list);
+            GF_REF_PUT(glfd);
         }
+    }
 
-        ret = syncop_zerofill (subvol, fd, offset, len);
 out:
-        if (fd)
-                fd_unref(fd);
+    return ret;
+}
 
-        priv_glfs_subvol_done (glfd->fs, subvol);
+static int
+glfs_recall_lease_upcall(struct glfs *fs, struct glfs_upcall *up_arg,
+                         struct gf_upcall *up_data)
+{
+    struct gf_upcall_recall_lease *recall_lease = NULL;
+    struct glfs_object *object = NULL;
+    xlator_t *subvol = NULL;
+    int ret = -1;
+    struct glfs_upcall_lease *up_lease_arg = NULL;
+
+    GF_VALIDATE_OR_GOTO("gfapi", up_data, out);
+    GF_VALIDATE_OR_GOTO("gfapi", fs, out);
+
+    recall_lease = up_data->data;
+    GF_VALIDATE_OR_GOTO("gfapi", recall_lease, out);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        errno = EIO;
+        goto out;
+    }
+
+    gf_msg_debug(THIS->name, 0, "Recall lease received for gfid:%s",
+                 uuid_utoa(up_data->gfid));
+
+    object = glfs_h_find_handle(fs, up_data->gfid, GFAPI_HANDLE_LENGTH);
+    if (!object) {
+        /* The reason handle creation will fail is because we
+         * couldn't find the inode in the gfapi inode table.
+         *
+         * But since application would have taken inode_ref, the
+         * only case when this can happen is when it has closed
+         * the handle and hence will no more be interested in
+         * the upcall for this particular gfid.
+         */
+        gf_smsg(THIS->name, GF_LOG_DEBUG, errno, API_MSG_CREATE_HANDLE_FAILED,
+                "gfid=%s", uuid_utoa(up_data->gfid), NULL);
+        errno = ESTALE;
+        goto out;
+    }
+
+    up_lease_arg = GF_CALLOC(1, sizeof(struct glfs_upcall_lease),
+                             glfs_mt_upcall_inode_t);
+    up_lease_arg->object = object;
+
+    GF_VALIDATE_OR_GOTO("glfs_recall_lease", up_lease_arg, out);
+
+    up_lease_arg->lease_type = recall_lease->lease_type;
+
+    up_arg->reason = GLFS_UPCALL_RECALL_LEASE;
+    up_arg->event = up_lease_arg;
+    up_arg->free_event = glfs_free_upcall_lease;
+
+    ret = 0;
 
-        return ret;
+out:
+    if (ret) {
+        /* Close p_object and oldp_object as well if being referenced.*/
+        if (object)
+            glfs_h_close(object);
+
+        /* Set reason to prevent applications from using ->event */
+        up_arg->reason = GF_UPCALL_EVENT_NULL;
+    }
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_zerofill, 3.5.0);
-
+static int
+upcall_syncop_args_free(struct upcall_syncop_args *args)
+{
+    dict_t *dict = NULL;
+    struct gf_upcall *upcall_data = NULL;
+
+    if (args) {
+        upcall_data = &args->upcall_data;
+        switch (upcall_data->event_type) {
+            case GF_UPCALL_CACHE_INVALIDATION:
+                dict = ((struct gf_upcall_cache_invalidation *)(upcall_data
+                                                                    ->data))
+                           ->dict;
+                break;
+            case GF_UPCALL_RECALL_LEASE:
+                dict = ((struct gf_upcall_recall_lease *)(upcall_data->data))
+                           ->dict;
+                break;
+        }
+        if (dict)
+            dict_unref(dict);
+
+        GF_FREE(upcall_data->client_uid);
+        GF_FREE(upcall_data->data);
+    }
+    GF_FREE(args);
+    return 0;
+}
 
-int
-pub_glfs_chdir (struct glfs *fs, const char *path)
+static int
+glfs_upcall_syncop_cbk(int ret, call_frame_t *frame, void *opaque)
 {
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	loc_t            loc = {0, };
-	struct iatt      iatt = {0, };
-	int              reval = 0;
+    struct upcall_syncop_args *args = opaque;
 
-	__glfs_entry_fs (fs);
+    (void)upcall_syncop_args_free(args);
 
-	subvol = priv_glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
-retry:
-	ret = glfs_resolve (fs, subvol, path, &loc, &iatt, reval);
+    return 0;
+}
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+static int
+glfs_cbk_upcall_syncop(void *opaque)
+{
+    struct upcall_syncop_args *args = opaque;
+    struct gf_upcall *upcall_data = NULL;
+    struct glfs_upcall *up_arg = NULL;
+    struct glfs *fs;
+    int ret = -1;
+
+    fs = args->fs;
+    upcall_data = &args->upcall_data;
+
+    if (!upcall_data) {
+        goto out;
+    }
+
+    up_arg = GLFS_CALLOC(1, sizeof(struct gf_upcall), glfs_release_upcall,
+                         glfs_mt_upcall_entry_t);
+    if (!up_arg) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, ENOMEM, API_MSG_ALLOC_FAILED, "entry",
+                NULL);
+        goto out;
+    }
+
+    switch (upcall_data->event_type) {
+        case GF_UPCALL_CACHE_INVALIDATION:
+            ret = glfs_h_poll_cache_invalidation(fs, up_arg, upcall_data);
+            break;
+        case GF_UPCALL_RECALL_LEASE:
+            ret = glfs_recall_lease_upcall(fs, up_arg, upcall_data);
+            break;
+        default:
+            errno = EINVAL;
+    }
+
+    /* It could so happen that the file which got
+     * upcall notification may have got deleted by
+     * the same client. In such cases up_arg->reason
+     * is set to GLFS_UPCALL_EVENT_NULL. No need to
+     * send upcall then
+     */
+    if (up_arg->reason == GLFS_UPCALL_EVENT_NULL) {
+        gf_smsg(THIS->name, GF_LOG_DEBUG, errno,
+                API_MSG_UPCALL_EVENT_NULL_RECEIVED, NULL);
+        ret = 0;
+        GLFS_FREE(up_arg);
+        goto out;
+    } else if (ret) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, errno, API_MSG_INVALID_ENTRY, NULL);
+        GLFS_FREE(up_arg);
+        goto out;
+    }
+
+    if (fs->up_cbk && up_arg)
+        (fs->up_cbk)(up_arg, fs->up_data);
+
+    /* application takes care of calling glfs_free on up_arg post
+     * their processing */
 
-	if (ret)
-		goto out;
+out:
+    return ret;
+}
 
-	if (!IA_ISDIR (iatt.ia_type)) {
-		ret = -1;
-		errno = ENOTDIR;
-		goto out;
-	}
+static struct gf_upcall_cache_invalidation *
+gf_copy_cache_invalidation(struct gf_upcall_cache_invalidation *src)
+{
+    struct gf_upcall_cache_invalidation *dst = NULL;
 
-	glfs_cwd_set (fs, loc.inode);
+    if (!src)
+        goto out;
 
-out:
-	loc_wipe (&loc);
+    dst = GF_CALLOC(1, sizeof(struct gf_upcall_cache_invalidation),
+                    glfs_mt_upcall_entry_t);
 
-	priv_glfs_subvol_done (fs, subvol);
+    if (!dst) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, ENOMEM, API_MSG_ALLOC_FAILED, "entry",
+                NULL);
+        goto out;
+    }
 
-	return ret;
-}
+    dst->flags = src->flags;
+    dst->expire_time_attr = src->expire_time_attr;
+    dst->stat = src->stat;
+    dst->p_stat = src->p_stat;
+    dst->oldp_stat = src->oldp_stat;
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_chdir, 3.4.0);
+    if (src->dict)
+        dst->dict = dict_copy_with_ref(src->dict, NULL);
 
+    return dst;
+out:
+    return NULL;
+}
 
-int
-pub_glfs_fchdir (struct glfs_fd *glfd)
+static struct gf_upcall_recall_lease *
+gf_copy_recall_lease(struct gf_upcall_recall_lease *src)
 {
-	int       ret = -1;
-	inode_t  *inode = NULL;
-	xlator_t *subvol = NULL;
-	fd_t     *fd = NULL;
+    struct gf_upcall_recall_lease *dst = NULL;
 
-	__glfs_entry_fd (glfd);
+    if (!src)
+        goto out;
 
-	subvol = priv_glfs_active_subvol (glfd->fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+    dst = GF_CALLOC(1, sizeof(struct gf_upcall_recall_lease),
+                    glfs_mt_upcall_entry_t);
 
-	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
-	if (!fd) {
-		ret = -1;
-		errno = EBADFD;
-		goto out;
-	}
+    if (!dst) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, ENOMEM, API_MSG_ALLOC_FAILED, "entry",
+                NULL);
+        goto out;
+    }
 
-	inode = fd->inode;
+    dst->lease_type = src->lease_type;
+    memcpy(dst->tid, src->tid, 16);
 
-	if (!IA_ISDIR (inode->ia_type)) {
-		ret = -1;
-		errno = ENOTDIR;
-		goto out;
-	}
+    if (src->dict)
+        dst->dict = dict_copy_with_ref(src->dict, NULL);
 
-	glfs_cwd_set (glfd->fs, inode);
-	ret = 0;
+    return dst;
 out:
-	if (fd)
-		fd_unref (fd);
+    return NULL;
+}
 
-	priv_glfs_subvol_done (glfd->fs, subvol);
+static struct upcall_syncop_args *
+upcall_syncop_args_init(struct glfs *fs, struct gf_upcall *upcall_data)
+{
+    struct upcall_syncop_args *args = NULL;
+    int ret = -1;
+    struct gf_upcall *t_data = NULL;
+
+    if (!fs || !upcall_data)
+        goto out;
+
+    args = GF_CALLOC(1, sizeof(struct upcall_syncop_args),
+                     glfs_mt_upcall_entry_t);
+    if (!args) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, ENOMEM, API_MSG_ALLOC_FAILED,
+                "syncop args", NULL);
+        goto out;
+    }
+
+    /* Note: we are not taking any ref on fs here.
+     * Ideally applications have to unregister for upcall events
+     * or stop polling for upcall events before performing
+     * glfs_fini. And as for outstanding synctasks created, we wait
+     * for all syncenv threads to finish tasks before cleaning up the
+     * fs->ctx. Hence it seems safe to process these callback
+     * notification without taking any lock/ref.
+     */
+    args->fs = fs;
+    t_data = &(args->upcall_data);
+    t_data->client_uid = gf_strdup(upcall_data->client_uid);
+
+    gf_uuid_copy(t_data->gfid, upcall_data->gfid);
+    t_data->event_type = upcall_data->event_type;
+
+    switch (t_data->event_type) {
+        case GF_UPCALL_CACHE_INVALIDATION:
+            t_data->data = gf_copy_cache_invalidation(
+                (struct gf_upcall_cache_invalidation *)upcall_data->data);
+            break;
+        case GF_UPCALL_RECALL_LEASE:
+            t_data->data = gf_copy_recall_lease(
+                (struct gf_upcall_recall_lease *)upcall_data->data);
+            break;
+    }
+
+    if (!t_data->data)
+        goto out;
+
+    return args;
+out:
+    if (ret) {
+        if (args) {
+            GF_FREE(args->upcall_data.client_uid);
+            GF_FREE(args);
+        }
+    }
 
-	return ret;
+    return NULL;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fchdir, 3.4.0);
+static void
+glfs_cbk_upcall_data(struct glfs *fs, struct gf_upcall *upcall_data)
+{
+    struct upcall_syncop_args *args = NULL;
+    int ret = -1;
 
+    if (!fs || !upcall_data)
+        goto out;
 
-char *
-pub_glfs_realpath (struct glfs *fs, const char *path, char *resolved_path)
-{
-	int              ret = -1;
-	char            *retpath = NULL;
-	char            *allocpath = NULL;
-	xlator_t        *subvol = NULL;
-	loc_t            loc = {0, };
-	struct iatt      iatt = {0, };
-	int              reval = 0;
-
-	__glfs_entry_fs (fs);
-
-	if (resolved_path)
-		retpath = resolved_path;
-	else
-		retpath = allocpath = malloc (PATH_MAX + 1);
-
-	if (!retpath) {
-		ret = -1;
-		errno = ENOMEM;
-		goto out;
-	}
-
-	subvol = priv_glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
-retry:
-	ret = glfs_resolve (fs, subvol, path, &loc, &iatt, reval);
+    if (!(fs->upcall_events & upcall_data->event_type)) {
+        /* ignore events which application hasn't registered*/
+        goto out;
+    }
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    args = upcall_syncop_args_init(fs, upcall_data);
 
-	if (ret)
-		goto out;
+    if (!args)
+        goto out;
 
-	if (loc.path) {
-		strncpy (retpath, loc.path, PATH_MAX);
-		retpath[PATH_MAX] = 0;
-	}
+    ret = synctask_new(THIS->ctx->env, glfs_cbk_upcall_syncop,
+                       glfs_upcall_syncop_cbk, NULL, args);
+    /* should we retry incase of failure? */
+    if (ret) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, errno, API_MSG_UPCALL_SYNCOP_FAILED,
+                "event_type=%d", upcall_data->event_type, "gfid=%s",
+                (char *)(upcall_data->gfid), NULL);
+        upcall_syncop_args_free(args);
+    }
 
 out:
-	loc_wipe (&loc);
-
-	if (ret == -1) {
-		if (allocpath)
-			free (allocpath);
-		retpath = NULL;
-	}
+    return;
+}
 
-	priv_glfs_subvol_done (fs, subvol);
+/*
+ * This routine is called in case of any notification received
+ * from the server. All the upcall events are queued up in a list
+ * to be read by the applications.
+ *
+ * In case if the application registers a cbk function, that shall
+ * be called by this routine in case of any event received.
+ * The cbk fn is responsible for notifying the
+ * applications the way it desires for each event queued (for eg.,
+ * can raise a signal or broadcast a cond variable etc.)
+ *
+ * Otherwise all the upcall events are queued up in a list
+ * to be read/polled by the applications.
+ */
+GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_process_upcall_event, 3.7.0)
+void
+priv_glfs_process_upcall_event(struct glfs *fs, void *data)
+{
+    glusterfs_ctx_t *ctx = NULL;
+    struct gf_upcall *upcall_data = NULL;
 
-	return retpath;
-}
+    DECLARE_OLD_THIS;
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_realpath, 3.4.0);
+    gf_msg_debug(THIS->name, 0, "Upcall gfapi callback is called");
 
+    __GLFS_ENTRY_VALIDATE_FS(fs, err);
 
-char *
-pub_glfs_getcwd (struct glfs *fs, char *buf, size_t n)
-{
-	int              ret = -1;
-	inode_t         *inode = NULL;
-	char            *path = NULL;
+    if (!data)
+        goto out;
 
-	__glfs_entry_fs (fs);
+    /* Unlike in I/O path, "glfs_fini" would not have freed
+     * 'fs' by the time we take lock as it waits for all epoll
+     * threads to exit including this
+     */
+    pthread_mutex_lock(&fs->mutex);
+    {
+        ctx = fs->ctx;
 
-	if (!buf || n < 2) {
-		ret = -1;
-		errno = EINVAL;
-		goto out;
-	}
+        /* if we're not interested in upcalls (anymore), skip them */
+        if (ctx->cleanup_started || !fs->cache_upcalls) {
+            pthread_mutex_unlock(&fs->mutex);
+            goto out;
+        }
 
-	inode = glfs_cwd_get (fs);
+        fs->pin_refcnt++;
+    }
+    pthread_mutex_unlock(&fs->mutex);
+
+    upcall_data = (struct gf_upcall *)data;
+
+    gf_msg_trace(THIS->name, 0, "Upcall gfapi gfid = %s",
+                 (char *)(upcall_data->gfid));
+
+    /* *
+     * TODO: RECALL LEASE for each glfd
+     *
+     * In case of RECALL_LEASE, we could associate separate
+     * cbk function for each glfd either by
+     * - extending pub_glfs_lease to accept new args (recall_cbk_fn, cookie)
+     * - or by defining new API "glfs_register_recall_cbk_fn (glfd,
+     * recall_cbk_fn, cookie) . In such cases, flag it and instead of calling
+     * below upcall functions, define a new one to go through the glfd list and
+     * invoke each of theirs recall_cbk_fn.
+     * */
+
+    if (fs->up_cbk) { /* upcall cbk registered */
+        (void)glfs_cbk_upcall_data(fs, upcall_data);
+    } else {
+        (void)glfs_enqueue_upcall_data(fs, upcall_data);
+    }
+
+    pthread_mutex_lock(&fs->mutex);
+    {
+        fs->pin_refcnt--;
+    }
+    pthread_mutex_unlock(&fs->mutex);
 
-	if (!inode) {
-		strncpy (buf, "/", n);
-		ret = 0;
-		goto out;
-	}
+out:
+    __GLFS_EXIT_FS;
+err:
+    return;
+}
 
-	ret = inode_path (inode, 0, &path);
-	if (n <= ret) {
-		ret = -1;
-		errno = ERANGE;
-		goto out;
-	}
+ssize_t
+glfs_anonymous_pwritev(struct glfs *fs, struct glfs_object *object,
+                       const struct iovec *iovec, int iovcnt, off_t offset,
+                       int flags)
+{
+    xlator_t *subvol = NULL;
+    struct iobref *iobref = NULL;
+    struct iobuf *iobuf = NULL;
+    struct iovec iov = {
+        0,
+    };
+    inode_t *inode = NULL;
+    fd_t *fd = NULL;
+    int ret = -1;
+    size_t size = -1;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, object);
+    if (!inode) {
+        ret = -1;
+        errno = ESTALE;
+        goto out;
+    }
+
+    fd = fd_anonymous(inode);
+    if (!fd) {
+        ret = -1;
+        gf_smsg("gfapi", GF_LOG_ERROR, ENOMEM, API_MSG_FDCREATE_FAILED, NULL);
+        errno = ENOMEM;
+        goto out;
+    }
+
+    size = iov_length(iovec, iovcnt);
+
+    iobuf = iobuf_get2(subvol->ctx->iobuf_pool, size);
+    if (!iobuf) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    iobref = iobref_new();
+    if (!iobref) {
+        iobuf_unref(iobuf);
+        errno = ENOMEM;
+        ret = -1;
+        goto out;
+    }
+
+    ret = iobref_add(iobref, iobuf);
+    if (ret) {
+        iobuf_unref(iobuf);
+        iobref_unref(iobref);
+        errno = ENOMEM;
+        ret = -1;
+        goto out;
+    }
+
+    iov_unload(iobuf_ptr(iobuf), iovec, iovcnt);
+
+    iov.iov_base = iobuf_ptr(iobuf);
+    iov.iov_len = size;
+
+    /* TODO : set leaseid */
+    ret = syncop_writev(subvol, fd, &iov, 1, offset, iobref, flags, NULL, NULL,
+                        NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    iobuf_unref(iobuf);
+    iobref_unref(iobref);
+
+    if (ret <= 0)
+        goto out;
 
-	strncpy (buf, path, n);
-	ret = 0;
 out:
-	GF_FREE (path);
 
-	if (inode)
-		inode_unref (inode);
+    if (fd)
+        fd_unref(fd);
 
-	if (ret < 0)
-		return NULL;
+    if (inode)
+        inode_unref(inode);
 
-	return buf;
-}
+    glfs_subvol_done(fs, subvol);
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_getcwd, 3.4.0);
+    __GLFS_EXIT_FS;
 
+invalid_fs:
+    return ret;
+}
 
-static void
-gf_flock_to_flock (struct gf_flock *gf_flock, struct flock *flock)
+ssize_t
+glfs_anonymous_preadv(struct glfs *fs, struct glfs_object *object,
+                      const struct iovec *iovec, int iovcnt, off_t offset,
+                      int flags)
 {
-	flock->l_type   = gf_flock->l_type;
-	flock->l_whence = gf_flock->l_whence;
-	flock->l_start  = gf_flock->l_start;
-	flock->l_len    = gf_flock->l_len;
-	flock->l_pid    = gf_flock->l_pid;
-}
+    xlator_t *subvol = NULL;
+    struct iovec *iov = NULL;
+    struct iobref *iobref = NULL;
+    inode_t *inode = NULL;
+    fd_t *fd = NULL;
+    int cnt = 0;
+    ssize_t ret = -1;
+    ssize_t size = -1;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, object);
+    if (!inode) {
+        ret = -1;
+        errno = ESTALE;
+        goto out;
+    }
+
+    fd = fd_anonymous(inode);
+    if (!fd) {
+        ret = -1;
+        gf_smsg("gfapi", GF_LOG_ERROR, ENOMEM, API_MSG_FDCREATE_FAILED, NULL);
+        errno = ENOMEM;
+        goto out;
+    }
+
+    size = iov_length(iovec, iovcnt);
+
+    /* TODO : set leaseid */
+    ret = syncop_readv(subvol, fd, size, offset, flags, &iov, &cnt, &iobref,
+                       NULL, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+    if (ret <= 0)
+        goto out;
+
+    size = iov_copy(iovec, iovcnt, iov, cnt);
+
+    ret = size;
+out:
+    if (iov)
+        GF_FREE(iov);
+    if (iobref)
+        iobref_unref(iobref);
+    if (fd)
+        fd_unref(fd);
 
+    if (inode)
+        inode_unref(inode);
+
+    glfs_subvol_done(fs, subvol);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
 
 static void
-gf_flock_from_flock (struct gf_flock *gf_flock, struct flock *flock)
+glfs_release_xreaddirp_stat(void *ptr)
 {
-	gf_flock->l_type   = flock->l_type;
-	gf_flock->l_whence = flock->l_whence;
-	gf_flock->l_start  = flock->l_start;
-	gf_flock->l_len    = flock->l_len;
-	gf_flock->l_pid    = flock->l_pid;
-}
+    struct glfs_xreaddirp_stat *to_free = ptr;
 
+    if (to_free->object)
+        glfs_h_close(to_free->object);
+}
 
+/*
+ * Given glfd of a directory, this function does readdirp and returns
+ * xstat along with dirents.
+ */
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_xreaddirplus_r, 3.11.0)
 int
-pub_glfs_posix_lock (struct glfs_fd *glfd, int cmd, struct flock *flock)
+pub_glfs_xreaddirplus_r(struct glfs_fd *glfd, uint32_t flags,
+                        struct glfs_xreaddirp_stat **xstat_p,
+                        struct dirent *ext, struct dirent **res)
 {
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	struct gf_flock  gf_flock = {0, };
-	struct gf_flock  saved_flock = {0, };
-	fd_t            *fd = NULL;
+    int ret = -1;
+    gf_dirent_t *entry = NULL;
+    struct dirent *buf = NULL;
+    struct glfs_xreaddirp_stat *xstat = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    GF_REF_GET(glfd);
+
+    GF_VALIDATE_OR_GOTO(THIS->name, xstat_p, out);
+    GF_VALIDATE_OR_GOTO(THIS->name, res, out);
+
+    errno = 0;
+
+    if (ext)
+        buf = ext;
+    else
+        buf = glfs_readdirbuf_get(glfd);
+
+    if (!buf)
+        goto out;
+
+    xstat = GLFS_CALLOC(1, sizeof(struct glfs_xreaddirp_stat),
+                        glfs_release_xreaddirp_stat, glfs_mt_xreaddirp_stat_t);
+
+    if (!xstat)
+        goto out;
+
+    /* this is readdirplus operation */
+    entry = glfd_entry_next(glfd, 1);
+
+    /* XXX: Ideally when we reach EOD, errno should have been
+     * set to ENOENT. But that doesn't seem to be the case.
+     *
+     * The only way to confirm if its EOD at this point is that
+     * errno == 0 and entry == NULL
+     */
+    if (errno)
+        goto out;
+
+    if (!entry) {
+        /* reached EOD, ret = 0  */
+        ret = 0;
+        *res = NULL;
+        *xstat_p = NULL;
+
+        /* free xstat as applications shall not be using it */
+        GLFS_FREE(xstat);
+
+        goto out;
+    }
+
+    *res = buf;
+    gf_dirent_to_dirent(entry, buf);
+
+    if (flags & GFAPI_XREADDIRP_STAT) {
+        glfs_iatt_to_stat(glfd->fs, &entry->d_stat, &xstat->st);
+        xstat->flags_handled |= GFAPI_XREADDIRP_STAT;
+    }
+
+    if ((flags & GFAPI_XREADDIRP_HANDLE) &&
+        /* skip . and .. */
+        strcmp(buf->d_name, ".") && strcmp(buf->d_name, "..")) {
+        /* Now create object.
+         * We can use "glfs_h_find_handle" as well as inodes would have
+         * already got linked as part of 'gf_link_inodes_from_dirent' */
+        xstat->object = glfs_h_create_from_handle(
+            glfd->fs, entry->d_stat.ia_gfid, GFAPI_HANDLE_LENGTH, NULL);
+
+        if (xstat->object) { /* success */
+            /* note: xstat->object->inode->ref is taken
+             * This shall be unref'ed when application does
+             * glfs_free(xstat) */
+            xstat->flags_handled |= GFAPI_XREADDIRP_HANDLE;
+        }
+    }
 
-	__glfs_entry_fd (glfd);
+    ret = xstat->flags_handled;
+    *xstat_p = xstat;
 
-	subvol = priv_glfs_active_subvol (glfd->fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+    gf_msg_debug(THIS->name, 0,
+                 "xreaddirp- requested_flags (%x) , processed_flags (%x)",
+                 flags, xstat->flags_handled);
 
-	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
-	if (!fd) {
-		ret = -1;
-		errno = EBADFD;
-		goto out;
-	}
+out:
+    GF_REF_PUT(glfd);
 
-	gf_flock_from_flock (&gf_flock, flock);
-	gf_flock_from_flock (&saved_flock, flock);
-	ret = syncop_lk (subvol, fd, cmd, &gf_flock);
-	gf_flock_to_flock (&gf_flock, flock);
+    if (ret < 0) {
+        gf_smsg(THIS->name, GF_LOG_WARNING, errno, API_MSG_XREADDIRP_R_FAILED,
+                "reason=%s", strerror(errno), NULL);
 
-	if (ret == 0 && (cmd == F_SETLK || cmd == F_SETLKW))
-		fd_lk_insert_and_merge (fd, cmd, &saved_flock);
-out:
-	if (fd)
-		fd_unref (fd);
+        if (xstat)
+            GLFS_FREE(xstat);
+    }
+
+    __GLFS_EXIT_FS;
 
-	priv_glfs_subvol_done (glfd->fs, subvol);
+    return ret;
 
-	return ret;
+invalid_fs:
+    return -1;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_posix_lock, 3.4.0);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_xreaddirplus_get_stat, 3.11.0)
+struct stat *
+pub_glfs_xreaddirplus_get_stat(struct glfs_xreaddirp_stat *xstat)
+{
+    GF_VALIDATE_OR_GOTO("glfs_xreaddirplus_get_stat", xstat, out);
 
+    if (!xstat->flags_handled & GFAPI_XREADDIRP_STAT)
+        gf_smsg(THIS->name, GF_LOG_ERROR, errno, API_MSG_FLAGS_HANDLE,
+                "GFAPI_XREADDIRP_STAT"
+                "xstat=%p",
+                xstat, "handles=%x", xstat->flags_handled, NULL);
+    return &xstat->st;
 
-struct glfs_fd *
-pub_glfs_dup (struct glfs_fd *glfd)
+out:
+    return NULL;
+}
+
+void
+gf_lease_to_glfs_lease(struct gf_lease *gf_lease, struct glfs_lease *lease)
 {
-	xlator_t  *subvol = NULL;
-	fd_t      *fd = NULL;
-	glfs_fd_t *dupfd = NULL;
-	struct glfs *fs = NULL;
+    u_int lease_type = gf_lease->lease_type;
+    lease->cmd = gf_lease->cmd;
+    lease->lease_type = lease_type;
+    memcpy(lease->lease_id, gf_lease->lease_id, LEASE_ID_SIZE);
+}
 
-	__glfs_entry_fd (glfd);
+void
+glfs_lease_to_gf_lease(struct glfs_lease *lease, struct gf_lease *gf_lease)
+{
+    u_int lease_type = lease->lease_type;
+    gf_lease->cmd = lease->cmd;
+    gf_lease->lease_type = lease_type;
+    memcpy(gf_lease->lease_id, lease->lease_id, LEASE_ID_SIZE);
+}
 
-	fs = glfd->fs;
-	subvol = priv_glfs_active_subvol (fs);
-	if (!subvol) {
-		errno = EIO;
-		goto out;
-	}
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_lease, 4.0.0)
+int
+pub_glfs_lease(struct glfs_fd *glfd, struct glfs_lease *lease,
+               glfs_recall_cbk fn, void *data)
+{
+    int ret = -1;
+    loc_t loc = {
+        0,
+    };
+    xlator_t *subvol = NULL;
+    fd_t *fd = NULL;
+    struct gf_lease gf_lease = {
+        0,
+    };
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    GF_REF_GET(glfd);
+
+    if (!is_valid_lease_id(lease->lease_id)) {
+        ret = -1;
+        errno = EINVAL;
+        goto out;
+    }
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    switch (lease->lease_type) {
+        case GLFS_RD_LEASE:
+            if ((fd->flags != O_RDONLY) && !(fd->flags & O_RDWR)) {
+                ret = -1;
+                errno = EINVAL;
+                goto out;
+            }
+            break;
+        case GLFS_RW_LEASE:
+            if (!((fd->flags & O_WRONLY) || (fd->flags & O_RDWR))) {
+                ret = -1;
+                errno = EINVAL;
+                goto out;
+            }
+            break;
+        default:
+            if (lease->cmd != GLFS_GET_LEASE) {
+                ret = -1;
+                errno = EINVAL;
+                goto out;
+            }
+            break;
+    }
+
+    /* populate loc */
+    GLFS_LOC_FILL_INODE(fd->inode, loc, out);
+
+    glfs_lease_to_gf_lease(lease, &gf_lease);
 
-	fd = glfs_resolve_fd (fs, subvol, glfd);
-	if (!fd) {
-		errno = EBADFD;
-		goto out;
-	}
+    ret = syncop_lease(subvol, &loc, &gf_lease, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
 
-	dupfd = glfs_fd_new (fs);
-	if (!dupfd) {
-		errno = ENOMEM;
-		goto out;
-	}
+    gf_lease_to_glfs_lease(&gf_lease, lease);
+
+    /* TODO: Add leases for client replay
+    if (ret == 0 && (cmd == F_SETLK || cmd == F_SETLKW))
+            fd_lk_insert_and_merge (fd, cmd, &saved_flock);
+    */
+    if (ret == 0) {
+        ret = fd_ctx_set(glfd->fd, subvol, (uint64_t)(long)glfd);
+        if (ret) {
+            gf_smsg(subvol->name, GF_LOG_ERROR, ENOMEM,
+                    API_MSG_FDCTX_SET_FAILED, "fd=%p", glfd->fd, NULL);
+            goto out;
+        }
+        glfd->cbk = fn;
+        glfd->cookie = data;
+    }
 
-	dupfd->fd = fd_ref (fd);
 out:
-	if (fd)
-		fd_unref (fd);
-	if (dupfd)
-		glfs_fd_bind (dupfd);
 
-	priv_glfs_subvol_done (fs, subvol);
+    if (glfd)
+        GF_REF_PUT(glfd);
 
-	return dupfd;
-}
+    if (subvol)
+        glfs_subvol_done(glfd->fs, subvol);
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_dup, 3.4.0);
+    __GLFS_EXIT_FS;
 
+invalid_fs:
+    return ret;
+}
diff --git a/api/src/glfs-handleops.c b/api/src/glfs-handleops.c
index 35cb0c807b4..53c2ee896f9 100644
--- a/api/src/glfs-handleops.c
+++ b/api/src/glfs-handleops.c
@@ -1,5 +1,5 @@
 /*
- *  Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+ *  Copyright (c) 2013-2018 Red Hat, Inc. <http://www.redhat.com>
  *  This file is part of GlusterFS.
  *
  *  This file is licensed to you under your choice of the GNU Lesser
@@ -8,1497 +8,2648 @@
  *  cases as published by the Free Software Foundation.
  */
 
-
 #include "glfs-internal.h"
 #include "glfs-mem-types.h"
-#include "syncop.h"
+#include <glusterfs/syncop.h>
 #include "glfs.h"
 #include "glfs-handles.h"
+#include "gfapi-messages.h"
 
-static void
-glfs_iatt_from_stat (struct stat *stat, int valid, struct iatt *iatt,
-		     int *glvalid)
+int
+glfs_listxattr_process(void *value, size_t size, dict_t *xattr);
+
+void
+glfs_iatt_from_stat(struct stat *stat, int valid, struct iatt *iatt,
+                    int *glvalid)
 {
-	/* validate in args */
-	if ((stat == NULL) || (iatt == NULL) || (glvalid == NULL)) {
-		errno = EINVAL;
-		return;
-	}
-
-	*glvalid = 0;
-
-	if (valid & GFAPI_SET_ATTR_MODE) {
-		iatt->ia_prot = ia_prot_from_st_mode (stat->st_mode);
-		*glvalid |= GF_SET_ATTR_MODE;
-	}
-
-	if (valid & GFAPI_SET_ATTR_UID) {
-		iatt->ia_uid = stat->st_uid;
-		*glvalid |= GF_SET_ATTR_UID;
-	}
-
-	if (valid & GFAPI_SET_ATTR_GID) {
-		iatt->ia_gid = stat->st_gid;
-		*glvalid |= GF_SET_ATTR_GID;
-	}
-
-	if (valid & GFAPI_SET_ATTR_ATIME) {
-		iatt->ia_atime = stat->st_atime;
-		iatt->ia_atime_nsec = ST_ATIM_NSEC (stat);
-		*glvalid |= GF_SET_ATTR_ATIME;
-	}
-
-	if (valid & GFAPI_SET_ATTR_MTIME) {
-		iatt->ia_mtime = stat->st_mtime;
-		iatt->ia_mtime_nsec = ST_MTIM_NSEC (stat);
-		*glvalid |= GF_SET_ATTR_MTIME;
-	}
-
-	return;
+    /* validate in args */
+    if ((stat == NULL) || (iatt == NULL) || (glvalid == NULL)) {
+        errno = EINVAL;
+        return;
+    }
+
+    *glvalid = 0;
+
+    if (valid & GFAPI_SET_ATTR_MODE) {
+        iatt->ia_prot = ia_prot_from_st_mode(stat->st_mode);
+        *glvalid |= GF_SET_ATTR_MODE;
+    }
+
+    if (valid & GFAPI_SET_ATTR_UID) {
+        iatt->ia_uid = stat->st_uid;
+        *glvalid |= GF_SET_ATTR_UID;
+    }
+
+    if (valid & GFAPI_SET_ATTR_GID) {
+        iatt->ia_gid = stat->st_gid;
+        *glvalid |= GF_SET_ATTR_GID;
+    }
+
+    if (valid & GFAPI_SET_ATTR_ATIME) {
+        iatt->ia_atime = stat->st_atime;
+        iatt->ia_atime_nsec = ST_ATIM_NSEC(stat);
+        *glvalid |= GF_SET_ATTR_ATIME;
+    }
+
+    if (valid & GFAPI_SET_ATTR_MTIME) {
+        iatt->ia_mtime = stat->st_mtime;
+        iatt->ia_mtime_nsec = ST_MTIM_NSEC(stat);
+        *glvalid |= GF_SET_ATTR_MTIME;
+    }
+
+    return;
 }
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_lookupat, 3.7.4)
 struct glfs_object *
-pub_glfs_h_lookupat (struct glfs *fs, struct glfs_object *parent,
-		 const char *path, struct stat *stat)
+pub_glfs_h_lookupat(struct glfs *fs, struct glfs_object *parent,
+                    const char *path, struct stat *stat, int follow)
 {
-	int                      ret = 0;
-	xlator_t                *subvol = NULL;
-	inode_t                 *inode = NULL;
-	struct iatt              iatt = {0, };
-	struct glfs_object      *object = NULL;
-	loc_t                    loc = {0, };
-
-	/* validate in args */
-	if ((fs == NULL) || (path == NULL)) {
-		errno = EINVAL;
-		return NULL;
-	}
-
-	__glfs_entry_fs (fs);
-
-	/* get the active volume */
-	subvol = priv_glfs_active_subvol (fs);
-	if (!subvol) {
-		errno = EIO;
-		goto out;
-	}
-
-	/* get/refresh the in arg objects inode in correlation to the xlator */
-	if (parent) {
-		inode = glfs_resolve_inode (fs, subvol, parent);
-		if (!inode) {
-			errno = ESTALE;
-			goto out;
-		}
-	}
-
-	/* fop/op */
-	ret = priv_glfs_resolve_at (fs, subvol, inode, path, &loc, &iatt,
-			       0 /*TODO: links? */, 0);
-
-	/* populate out args */
-	if (!ret) {
-		if (stat)
-			glfs_iatt_to_stat (fs, &iatt, stat);
-
-		ret = glfs_create_object (&loc, &object);
-	}
+    int ret = 0;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    struct iatt iatt = {
+        0,
+    };
+    struct glfs_object *object = NULL;
+    loc_t loc = {
+        0,
+    };
+
+    DECLARE_OLD_THIS;
+
+    /* validate in args */
+    if (path == NULL) {
+        errno = EINVAL;
+        return NULL;
+    }
+
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    if (parent) {
+        inode = glfs_resolve_inode(fs, subvol, parent);
+        if (!inode) {
+            errno = ESTALE;
+            goto out;
+        }
+    }
+
+    /* fop/op */
+    ret = glfs_resolve_at(fs, subvol, inode, path, &loc, &iatt, follow, 0);
+
+    /* populate out args */
+    if (!ret) {
+        if (stat)
+            glfs_iatt_to_stat(fs, &iatt, stat);
+
+        ret = glfs_create_object(&loc, &object);
+    }
 
 out:
-	loc_wipe (&loc);
+    loc_wipe(&loc);
 
-	if (inode)
-		inode_unref (inode);
+    if (inode)
+        inode_unref(inode);
 
-	priv_glfs_subvol_done (fs, subvol);
+    glfs_subvol_done(fs, subvol);
 
-	return object;
-}
+    __GLFS_EXIT_FS;
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_lookupat, 3.4.2);
+invalid_fs:
+    return object;
+}
 
+GFAPI_SYMVER_PUBLIC(glfs_h_lookupat34, glfs_h_lookupat, 3.4.2)
+struct glfs_object *
+pub_glfs_h_lookupat34(struct glfs *fs, struct glfs_object *parent,
+                      const char *path, struct stat *stat)
+{
+    return pub_glfs_h_lookupat(fs, parent, path, stat, 0);
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_statfs, 3.7.0)
 int
-pub_glfs_h_stat (struct glfs *fs, struct glfs_object *object, struct stat *stat)
+pub_glfs_h_statfs(struct glfs *fs, struct glfs_object *object,
+                  struct statvfs *statvfs)
 {
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	inode_t         *inode = NULL;
-	loc_t            loc = {0, };
-	struct iatt      iatt = {0, };
-
-	/* validate in args */
-	if ((fs == NULL) || (object == NULL)) {
-		errno = EINVAL;
-		return -1;
-	}
-
-	__glfs_entry_fs (fs);
-
-	/* get the active volume */
-	subvol = priv_glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
-
-	/* get/refresh the in arg objects inode in correlation to the xlator */
-	inode = glfs_resolve_inode (fs, subvol, object);
-	if (!inode) {
-		errno = ESTALE;
-		goto out;
-	}
-
-	/* populate loc */
-	GLFS_LOC_FILL_INODE (inode, loc, out);
-
-	/* fop/op */
-	ret = syncop_stat (subvol, &loc, &iatt);
-
-	/* populate out args */
-	if (!ret && stat) {
-		glfs_iatt_to_stat (fs, &iatt, stat);
-	}
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    loc_t loc = {
+        0,
+    };
+
+    DECLARE_OLD_THIS;
+
+    /* validate in args */
+    if ((fs == NULL) || (object == NULL || statvfs == NULL)) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, object);
+    if (!inode) {
+        errno = ESTALE;
+        goto out;
+    }
+
+    /* populate loc */
+    GLFS_LOC_FILL_INODE(inode, loc, out);
+
+    /* fop/op */
+    ret = syncop_statfs(subvol, &loc, statvfs, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    loc_wipe(&loc);
+
 out:
-	loc_wipe (&loc);
+    if (inode)
+        inode_unref(inode);
 
-	if (inode)
-		inode_unref (inode);
+    glfs_subvol_done(fs, subvol);
 
-	priv_glfs_subvol_done (fs, subvol);
+    __GLFS_EXIT_FS;
 
-	return ret;
+invalid_fs:
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_stat, 3.4.2);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_stat, 3.4.2)
+int
+pub_glfs_h_stat(struct glfs *fs, struct glfs_object *object, struct stat *stat)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+
+    DECLARE_OLD_THIS;
+
+    /* validate in args */
+    if ((fs == NULL) || (object == NULL)) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, object);
+    if (!inode) {
+        errno = ESTALE;
+        goto out;
+    }
+
+    /* populate loc */
+    GLFS_LOC_FILL_INODE(inode, loc, out);
+
+    /* fop/op */
+    ret = syncop_stat(subvol, &loc, &iatt, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    /* populate out args */
+    if (!ret && stat) {
+        glfs_iatt_to_stat(fs, &iatt, stat);
+    }
+out:
+    loc_wipe(&loc);
 
+    if (inode)
+        inode_unref(inode);
 
+    glfs_subvol_done(fs, subvol);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_getattrs, 3.4.2)
 int
-pub_glfs_h_getattrs (struct glfs *fs, struct glfs_object *object, struct stat *stat)
+pub_glfs_h_getattrs(struct glfs *fs, struct glfs_object *object,
+                    struct stat *stat)
 {
-	int                      ret = 0;
-	xlator_t                *subvol = NULL;
-	inode_t                 *inode = NULL;
-	struct iatt              iatt = {0, };
-
-	/* validate in args */
-	if ((fs == NULL) || (object == NULL)) {
-		errno = EINVAL;
-		return -1;
-	}
-
-	__glfs_entry_fs (fs);
-
-	/* get the active volume */
-	subvol = priv_glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
-
-	/* get/refresh the in arg objects inode in correlation to the xlator */
-	inode = glfs_resolve_inode (fs, subvol, object);
-	if (!inode) {
-		errno = ESTALE;
-		goto out;
-	}
-
-	/* fop/op */
-	ret = glfs_resolve_base (fs, subvol, inode, &iatt);
-
-	/* populate out args */
-	if (!ret && stat) {
-		glfs_iatt_to_stat (fs, &iatt, stat);
-	}
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    struct iatt iatt = {
+        0,
+    };
+
+    /* validate in args */
+    if ((fs == NULL) || (object == NULL)) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, object);
+    if (!inode) {
+        ret = 0;
+        errno = ESTALE;
+        goto out;
+    }
+
+    /* fop/op */
+    ret = glfs_resolve_base(fs, subvol, inode, &iatt);
+
+    /* populate out args */
+    if (!ret && stat) {
+        glfs_iatt_to_stat(fs, &iatt, stat);
+    }
 
 out:
-	if (inode)
-		inode_unref (inode);
+    if (inode)
+        inode_unref(inode);
+
+    glfs_subvol_done(fs, subvol);
 
-	priv_glfs_subvol_done (fs, subvol);
+    __GLFS_EXIT_FS;
 
-	return ret;
+invalid_fs:
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_getattrs, 3.4.2);
+int
+glfs_h_getxattrs_common(struct glfs *fs, struct glfs_object *object,
+                        dict_t **xattr, const char *name,
+                        gf_boolean_t is_listxattr)
+{
+    int ret = 0;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    loc_t loc = {
+        0,
+    };
+
+    /* validate in args */
+    if ((fs == NULL) || (object == NULL)) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    if (!is_listxattr) {
+        if (!name || *name == '\0') {
+            errno = EINVAL;
+            return -1;
+        }
 
+        if (strlen(name) > GF_XATTR_NAME_MAX) {
+            errno = ENAMETOOLONG;
+            return -1;
+        }
+    }
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, object);
+    if (!inode) {
+        errno = ESTALE;
+        goto out;
+    }
+
+    /* populate loc */
+    GLFS_LOC_FILL_INODE(inode, loc, out);
+
+    ret = syncop_getxattr(subvol, &loc, xattr, name, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
 
+out:
+    loc_wipe(&loc);
+
+    if (inode)
+        inode_unref(inode);
+
+    glfs_subvol_done(fs, subvol);
+
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_getxattrs, 3.5.1)
 int
-pub_glfs_h_getxattrs (struct glfs *fs, struct glfs_object *object,
-                      const char *name, void *value, size_t size)
+pub_glfs_h_getxattrs(struct glfs *fs, struct glfs_object *object,
+                     const char *name, void *value, size_t size)
 {
-        int             ret = 0;
-        xlator_t        *subvol = NULL;
-        inode_t         *inode = NULL;
-        loc_t            loc = {0, };
-        dict_t          *xattr = NULL;
-
-        /* validate in args */
-        if ((fs == NULL) || (object == NULL)) {
-                errno = EINVAL;
-                return -1;
-        }
+    int ret = -1;
+    dict_t *xattr = NULL;
 
-        __glfs_entry_fs (fs);
+    /* validate in args */
+    if ((fs == NULL) || (object == NULL)) {
+        errno = EINVAL;
+        return -1;
+    }
 
-        /* get the active volume */
-        subvol = priv_glfs_active_subvol (fs);
-        if (!subvol) {
-                ret = -1;
-                errno = EIO;
-                goto out;
-        }
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
 
-        /* get/refresh the in arg objects inode in correlation to the xlator */
-        inode = glfs_resolve_inode (fs, subvol, object);
-        if (!inode) {
-                errno = ESTALE;
-                goto out;
-        }
+    ret = glfs_h_getxattrs_common(fs, object, &xattr, name, (name == NULL));
+    if (ret)
+        goto out;
 
-        /* populate loc */
-        GLFS_LOC_FILL_INODE (inode, loc, out);
+    /* If @name is NULL, means get all the xattrs (i.e listxattr). */
+    if (name)
+        ret = glfs_getxattr_process(value, size, xattr, name);
+    else
+        ret = glfs_listxattr_process(value, size, xattr);
 
-        ret = syncop_getxattr (subvol, &loc, &xattr, name);
+out:
+    if (xattr)
+        dict_unref(xattr);
 
-        if (ret)
-                goto out;
+    __GLFS_EXIT_FS;
 
-        ret = glfs_getxattr_process (value, size, xattr, name);
+invalid_fs:
+    return ret;
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_setattrs, 3.4.2)
+int
+pub_glfs_h_setattrs(struct glfs *fs, struct glfs_object *object,
+                    struct stat *stat, int valid)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    int glvalid = 0;
+
+    /* validate in args */
+    if ((fs == NULL) || (object == NULL) || (stat == NULL)) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, object);
+    if (!inode) {
+        errno = ESTALE;
+        goto out;
+    }
+
+    /* map valid masks from in args */
+    glfs_iatt_from_stat(stat, valid, &iatt, &glvalid);
+
+    /* populate loc */
+    GLFS_LOC_FILL_INODE(inode, loc, out);
+
+    /* fop/op */
+    ret = syncop_setattr(subvol, &loc, &iatt, glvalid, 0, 0, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
 out:
-        loc_wipe (&loc);
+    loc_wipe(&loc);
 
-        if (inode)
-                inode_unref (inode);
+    if (inode)
+        inode_unref(inode);
 
-        priv_glfs_subvol_done (fs, subvol);
+    glfs_subvol_done(fs, subvol);
 
-        return ret;
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_getxattrs, 3.5.1);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_setxattrs, 3.5.0)
+int
+pub_glfs_h_setxattrs(struct glfs *fs, struct glfs_object *object,
+                     const char *name, const void *value, size_t size,
+                     int flags)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    loc_t loc = {
+        0,
+    };
+    dict_t *xattr = NULL;
+    void *value_cp = NULL;
+
+    /* validate in args */
+    if ((fs == NULL) || (object == NULL) || (name == NULL) || (value == NULL)) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    if (!name || *name == '\0') {
+        errno = EINVAL;
+        return -1;
+    }
+
+    if (strlen(name) > GF_XATTR_NAME_MAX) {
+        errno = ENAMETOOLONG;
+        return -1;
+    }
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, object);
+    if (!inode) {
+        errno = ESTALE;
+        goto out;
+    }
+
+    value_cp = gf_memdup(value, size);
+    GF_CHECK_ALLOC_AND_LOG(subvol->name, value_cp, ret,
+                           "Failed to"
+                           " duplicate setxattr value",
+                           out);
+
+    xattr = dict_for_key_value(name, value_cp, size, _gf_false);
+    if (!xattr) {
+        GF_FREE(value_cp);
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    /* populate loc */
+    GLFS_LOC_FILL_INODE(inode, loc, out);
+
+    /* fop/op */
+    ret = syncop_setxattr(subvol, &loc, xattr, flags, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+out:
+    loc_wipe(&loc);
+
+    if (inode)
+        inode_unref(inode);
+
+    if (xattr)
+        dict_unref(xattr);
 
+    glfs_subvol_done(fs, subvol);
 
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_removexattrs, 3.5.1)
 int
-pub_glfs_h_setattrs (struct glfs *fs, struct glfs_object *object,
-                     struct stat *stat, int valid)
+pub_glfs_h_removexattrs(struct glfs *fs, struct glfs_object *object,
+                        const char *name)
 {
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	inode_t         *inode = NULL;
-	loc_t            loc = {0, };
-	struct iatt      iatt = {0, };
-	int              glvalid = 0;
-
-	/* validate in args */
-	if ((fs == NULL) || (object == NULL) || (stat == NULL)) {
-		errno = EINVAL;
-		return -1;
-	}
-
-	__glfs_entry_fs (fs);
-
-	/* get the active volume */
-	subvol = priv_glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
-
-	/* get/refresh the in arg objects inode in correlation to the xlator */
-	inode = glfs_resolve_inode (fs, subvol, object);
-	if (!inode) {
-		errno = ESTALE;
-		goto out;
-	}
-
-	/* map valid masks from in args */
-	glfs_iatt_from_stat (stat, valid, &iatt, &glvalid);
-
-	/* populate loc */
-	GLFS_LOC_FILL_INODE (inode, loc, out);
-
-	/* fop/op */
-	ret = syncop_setattr (subvol, &loc, &iatt, glvalid, 0, 0);
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    loc_t loc = {
+        0,
+    };
+
+    /* validate in args */
+    if ((fs == NULL) || (object == NULL) || (name == NULL)) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, object);
+    if (!inode) {
+        errno = ESTALE;
+        goto out;
+    }
+
+    /* populate loc */
+    GLFS_LOC_FILL_INODE(inode, loc, out);
+
+    /* fop/op */
+    ret = syncop_removexattr(subvol, &loc, name, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
 out:
-	loc_wipe (&loc);
+    loc_wipe(&loc);
+
+    if (inode)
+        inode_unref(inode);
 
-	if (inode)
-		inode_unref (inode);
+    glfs_subvol_done(fs, subvol);
 
-	priv_glfs_subvol_done (fs, subvol);
+    __GLFS_EXIT_FS;
 
-	return ret;
+invalid_fs:
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_setattrs, 3.4.2);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_open, 3.4.2)
+struct glfs_fd *
+pub_glfs_h_open(struct glfs *fs, struct glfs_object *object, int flags)
+{
+    int ret = -1;
+    struct glfs_fd *glfd = NULL;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    loc_t loc = {
+        0,
+    };
+    dict_t *fop_attr = NULL;
+
+    /* validate in args */
+    if ((fs == NULL) || (object == NULL)) {
+        errno = EINVAL;
+        return NULL;
+    }
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, object);
+    if (!inode) {
+        errno = ESTALE;
+        goto out;
+    }
+
+    /* check types to open */
+    if (IA_ISDIR(inode->ia_type)) {
+        ret = -1;
+        errno = EISDIR;
+        goto out;
+    }
+
+    if (!IA_ISREG(inode->ia_type)) {
+        ret = -1;
+        errno = EINVAL;
+        goto out;
+    }
+
+    glfd = glfs_fd_new(fs);
+    if (!glfd) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    glfd->fd = fd_create(inode, getpid());
+    if (!glfd->fd) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+    glfd->fd->flags = flags;
+
+    /* populate loc */
+    GLFS_LOC_FILL_INODE(inode, loc, out);
+
+    /* fop/op */
+    ret = get_fop_attr_thrd_key(&fop_attr);
+    if (ret)
+        gf_msg_debug("gfapi", 0, "Getting leaseid from thread failed");
+
+    ret = syncop_open(subvol, &loc, flags, glfd->fd, fop_attr, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    glfd->fd->flags = flags;
 
+out:
+    loc_wipe(&loc);
 
-int
-pub_glfs_h_setxattrs (struct glfs *fs, struct glfs_object *object,
-                      const char *name, const void *value, size_t size, int flags)
-{
-        int              ret = -1;
-        xlator_t        *subvol = NULL;
-        inode_t         *inode = NULL;
-        loc_t            loc = {0, };
-        dict_t          *xattr = NULL;
-
-        /* validate in args */
-        if ((fs == NULL) || (object == NULL) ||
-                 (name == NULL) || (value == NULL)) {
-                errno = EINVAL;
-                return -1;
-        }
+    if (inode)
+        inode_unref(inode);
+    if (fop_attr)
+        dict_unref(fop_attr);
 
-        __glfs_entry_fs (fs);
+    if (ret && glfd) {
+        GF_REF_PUT(glfd);
+        glfd = NULL;
+    } else if (glfd) {
+        glfd_set_state_bind(glfd);
+    }
 
-        /* get the active volume */
-        subvol = priv_glfs_active_subvol (fs);
-        if (!subvol) {
-                ret = -1;
-                errno = EIO;
-                goto out;
-        }
+    glfs_subvol_done(fs, subvol);
 
-        /* get/refresh the in arg objects inode in correlation to the xlator */
-        inode = glfs_resolve_inode (fs, subvol, object);
-        if (!inode) {
-                errno = ESTALE;
-                goto out;
-        }
+    __GLFS_EXIT_FS;
 
-        xattr = dict_for_key_value (name, value, size);
-        if (!xattr) {
-                ret = -1;
-                errno = ENOMEM;
-                goto out;
+invalid_fs:
+    return glfd;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_creat, 3.4.2)
+struct glfs_object *
+pub_glfs_h_creat(struct glfs *fs, struct glfs_object *parent, const char *path,
+                 int flags, mode_t mode, struct stat *stat)
+{
+    int ret = -1;
+    fd_t *fd = NULL;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    uuid_t gfid;
+    dict_t *xattr_req = NULL;
+    struct glfs_object *object = NULL;
+
+    /* validate in args */
+    if ((fs == NULL) || (parent == NULL) || (path == NULL)) {
+        errno = EINVAL;
+        return NULL;
+    }
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, parent);
+    if (!inode) {
+        ret = -1;
+        errno = ESTALE;
+        goto out;
+    }
+
+    xattr_req = dict_new();
+    if (!xattr_req) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gf_uuid_generate(gfid);
+    ret = dict_set_gfuuid(xattr_req, "gfid-req", gfid, true);
+    if (ret) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    GLFS_LOC_FILL_PINODE(inode, loc, ret, errno, out, path);
+
+    fd = fd_create(loc.inode, getpid());
+    if (!fd) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+    fd->flags = flags;
+
+    /* fop/op */
+    ret = syncop_create(subvol, &loc, flags, mode, fd, &iatt, xattr_req, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    /* populate out args */
+    if (ret == 0) {
+        ret = glfs_loc_link(&loc, &iatt);
+        if (ret != 0) {
+            goto out;
         }
 
-        /* populate loc */
-        GLFS_LOC_FILL_INODE (inode, loc, out);
+        if (stat)
+            glfs_iatt_to_stat(fs, &iatt, stat);
 
-        /* fop/op */
-        ret = syncop_setxattr (subvol, &loc, xattr, flags);
+        ret = glfs_create_object(&loc, &object);
+    }
 
 out:
-        loc_wipe (&loc);
+    if (ret && object != NULL) {
+        /* Release the held reference */
+        glfs_h_close(object);
+        object = NULL;
+    }
 
-        if (inode)
-                inode_unref (inode);
+    loc_wipe(&loc);
 
-        if (xattr)
-                dict_unref (xattr);
+    if (inode)
+        inode_unref(inode);
 
-        priv_glfs_subvol_done (fs, subvol);
+    if (xattr_req)
+        dict_unref(xattr_req);
 
-        return ret;
-}
+    if (fd)
+        fd_unref(fd);
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_setxattrs, 3.5.0);
+    glfs_subvol_done(fs, subvol);
 
+    __GLFS_EXIT_FS;
 
-int
-pub_glfs_h_removexattrs (struct glfs *fs, struct glfs_object *object, const char *name)
+invalid_fs:
+    return object;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_creat_open, 6.6)
+struct glfs_object *
+pub_glfs_h_creat_open(struct glfs *fs, struct glfs_object *parent,
+                      const char *path, int flags, mode_t mode,
+                      struct stat *stat, struct glfs_fd **out_fd)
 {
-        int              ret = -1;
-        xlator_t        *subvol = NULL;
-        inode_t         *inode = NULL;
-        loc_t            loc = {0, };
-
-        /* validate in args */
-        if ((fs == NULL) || (object == NULL) || (name == NULL)) {
-                errno = EINVAL;
-                return -1;
+    int ret = -1;
+    struct glfs_fd *glfd = NULL;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    uuid_t gfid;
+    dict_t *xattr_req = NULL;
+    struct glfs_object *object = NULL;
+    dict_t *fop_attr = NULL;
+
+    /* validate in args */
+    if ((fs == NULL) || (parent == NULL) || (path == NULL) ||
+        (out_fd == NULL)) {
+        errno = EINVAL;
+        return NULL;
+    }
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, parent);
+    if (!inode) {
+        ret = -1;
+        goto out;
+    }
+
+    xattr_req = dict_new();
+    if (!xattr_req) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gf_uuid_generate(gfid);
+    ret = dict_set_gfuuid(xattr_req, "gfid-req", gfid, true);
+    if (ret) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    GLFS_LOC_FILL_PINODE(inode, loc, ret, errno, out, path);
+
+    glfd = glfs_fd_new(fs);
+    if (!glfd) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    glfd->fd = fd_create(loc.inode, getpid());
+    if (!glfd->fd) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+    glfd->fd->flags = flags;
+
+    ret = get_fop_attr_thrd_key(&fop_attr);
+    if (ret)
+        gf_msg_debug("gfapi", 0, "Getting leaseid from thread failed");
+
+    /* fop/op */
+    ret = syncop_create(subvol, &loc, flags, mode, glfd->fd, &iatt, xattr_req,
+                        NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    /* populate out args */
+    if (ret == 0) {
+        glfd->fd->flags = flags;
+
+        ret = glfs_loc_link(&loc, &iatt);
+        if (ret != 0) {
+            goto out;
         }
 
-        __glfs_entry_fs (fs);
+        if (stat)
+            glfs_iatt_to_stat(fs, &iatt, stat);
 
-        /* get the active volume */
-        subvol = priv_glfs_active_subvol (fs);
-        if (!subvol) {
-                ret = -1;
-                errno = EIO;
-                goto out;
-        }
+        ret = glfs_create_object(&loc, &object);
+    }
 
-        /* get/refresh the in arg objects inode in correlation to the xlator */
-        inode = glfs_resolve_inode (fs, subvol, object);
-        if (!inode) {
-                errno = ESTALE;
-                goto out;
-        }
+out:
+    if (ret && object != NULL) {
+        /* Release the held reference */
+        glfs_h_close(object);
+        object = NULL;
+    }
 
-        /* populate loc */
-        GLFS_LOC_FILL_INODE (inode, loc, out);
+    loc_wipe(&loc);
 
-        /* fop/op */
-        ret = syncop_removexattr (subvol, &loc, name);
+    if (inode)
+        inode_unref(inode);
 
-out:
-        loc_wipe (&loc);
+    if (fop_attr)
+        dict_unref(fop_attr);
 
-        if (inode)
-                inode_unref (inode);
+    if (xattr_req)
+        dict_unref(xattr_req);
 
-        priv_glfs_subvol_done (fs, subvol);
+    if (ret && glfd) {
+        GF_REF_PUT(glfd);
+    } else if (glfd) {
+        glfd_set_state_bind(glfd);
+        *out_fd = glfd;
+    }
 
-        return ret;
-}
+    glfs_subvol_done(fs, subvol);
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_removexattrs, 3.5.1);
+    __GLFS_EXIT_FS;
 
+invalid_fs:
+    return object;
+}
 
-struct glfs_fd *
-pub_glfs_h_open (struct glfs *fs, struct glfs_object *object, int flags)
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_mkdir, 3.4.2)
+struct glfs_object *
+pub_glfs_h_mkdir(struct glfs *fs, struct glfs_object *parent, const char *path,
+                 mode_t mode, struct stat *stat)
 {
-	int              ret = -1;
-	struct glfs_fd  *glfd = NULL;
-	xlator_t        *subvol = NULL;
-	inode_t         *inode = NULL;
-	loc_t            loc = {0, };
-
-	/* validate in args */
-	if ((fs == NULL) || (object == NULL)) {
-		errno = EINVAL;
-		return NULL;
-	}
-
-	__glfs_entry_fs (fs);
-
-	/* get the active volume */
-	subvol = priv_glfs_active_subvol (fs);
-	if (!subvol) {
-		errno = EIO;
-		goto out;
-	}
-
-	/* get/refresh the in arg objects inode in correlation to the xlator */
-	inode = glfs_resolve_inode (fs, subvol, object);
-	if (!inode) {
-		errno = ESTALE;
-		goto out;
-	}
-
-	/* check types to open */
-	if (IA_ISDIR (inode->ia_type)) {
-		ret = -1;
-		errno = EISDIR;
-		goto out;
-	}
-
-	if (!IA_ISREG (inode->ia_type)) {
-		ret = -1;
-		errno = EINVAL;
-		goto out;
-	}
-
-	glfd = glfs_fd_new (fs);
-	if (!glfd) {
-		errno = ENOMEM;
-		goto out;
-	}
-
-	glfd->fd = fd_create (inode, getpid());
-	if (!glfd->fd) {
-		ret = -1;
-		errno = ENOMEM;
-		goto out;
-	}
-
-	/* populate loc */
-	GLFS_LOC_FILL_INODE (inode, loc, out);
-
-	/* fop/op */
-	ret = syncop_open (subvol, &loc, flags, glfd->fd);
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    uuid_t gfid;
+    dict_t *xattr_req = NULL;
+    struct glfs_object *object = NULL;
+
+    /* validate in args */
+    if ((fs == NULL) || (parent == NULL) || (path == NULL)) {
+        errno = EINVAL;
+        return NULL;
+    }
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, parent);
+    if (!inode) {
+        errno = ESTALE;
+        goto out;
+    }
+
+    xattr_req = dict_new();
+    if (!xattr_req) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gf_uuid_generate(gfid);
+    ret = dict_set_gfuuid(xattr_req, "gfid-req", gfid, true);
+    if (ret) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    GLFS_LOC_FILL_PINODE(inode, loc, ret, errno, out, path);
+
+    /* fop/op */
+    ret = syncop_mkdir(subvol, &loc, mode, &iatt, xattr_req, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    /* populate out args */
+    if (ret == 0) {
+        ret = glfs_loc_link(&loc, &iatt);
+        if (ret != 0) {
+            goto out;
+        }
+
+        if (stat)
+            glfs_iatt_to_stat(fs, &iatt, stat);
+
+        ret = glfs_create_object(&loc, &object);
+    }
 
 out:
-	loc_wipe (&loc);
+    if (ret && object != NULL) {
+        glfs_h_close(object);
+        object = NULL;
+    }
 
-	if (inode)
-		inode_unref (inode);
+    loc_wipe(&loc);
 
-	if (ret && glfd) {
-		glfs_fd_destroy (glfd);
-		glfd = NULL;
-	} else {
-		glfd->fd->flags = flags;
-		fd_bind (glfd->fd);
-		glfs_fd_bind (glfd);
-	}
+    if (inode)
+        inode_unref(inode);
 
-	priv_glfs_subvol_done (fs, subvol);
+    if (xattr_req)
+        dict_unref(xattr_req);
 
-	return glfd;
-}
+    glfs_subvol_done(fs, subvol);
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_open, 3.4.2);
+    __GLFS_EXIT_FS;
 
+invalid_fs:
+    return object;
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_mknod, 3.4.2)
 struct glfs_object *
-pub_glfs_h_creat (struct glfs *fs, struct glfs_object *parent, const char *path,
-	          int flags, mode_t mode, struct stat *stat)
+pub_glfs_h_mknod(struct glfs *fs, struct glfs_object *parent, const char *path,
+                 mode_t mode, dev_t dev, struct stat *stat)
 {
-	int                 ret = -1;
-	struct glfs_fd     *glfd = NULL;
-	xlator_t           *subvol = NULL;
-	inode_t            *inode = NULL;
-	loc_t               loc = {0, };
-	struct iatt         iatt = {0, };
-	uuid_t              gfid;
-	dict_t             *xattr_req = NULL;
-	struct glfs_object *object = NULL;
-
-	/* validate in args */
-	if ((fs == NULL) || (parent == NULL) || (path == NULL)) {
-		errno = EINVAL;
-		return NULL;
-	}
-
-	__glfs_entry_fs (fs);
-
-	/* get the active volume */
-	subvol = priv_glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
-
-	/* get/refresh the in arg objects inode in correlation to the xlator */
-	inode = glfs_resolve_inode (fs, subvol, parent);
-	if (!inode) {
-		errno = ESTALE;
-		goto out;
-	}
-
-	xattr_req = dict_new ();
-	if (!xattr_req) {
-		ret = -1;
-		errno = ENOMEM;
-		goto out;
-	}
-
-	uuid_generate (gfid);
-	ret = dict_set_static_bin (xattr_req, "gfid-req", gfid, 16);
-	if (ret) {
-		ret = -1;
-		errno = ENOMEM;
-		goto out;
-	}
-
-	GLFS_LOC_FILL_PINODE (inode, loc, ret, errno, out, path);
-
-	glfd = glfs_fd_new (fs);
-	if (!glfd)
-		goto out;
-
-	glfd->fd = fd_create (loc.inode, getpid());
-	if (!glfd->fd) {
-		ret = -1;
-		errno = ENOMEM;
-		goto out;
-	}
-
-	/* fop/op */
-	ret = syncop_create (subvol, &loc, flags, mode, glfd->fd,
-			     xattr_req, &iatt);
-
-	/* populate out args */
-	if (ret == 0) {
-		/* TODO: If the inode existed in the cache (say file already
-		   exists), then the glfs_loc_link will not update the
-		   loc.inode, as a result we will have a 0000 GFID that we
-		   would copy out to the object, this needs to be fixed.
-		*/
-		ret = glfs_loc_link (&loc, &iatt);
-		if (ret != 0) {
-			goto out;
-		}
-
-		if (stat)
-			glfs_iatt_to_stat (fs, &iatt, stat);
-
-		ret = glfs_create_object (&loc, &object);
-	}
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    uuid_t gfid;
+    dict_t *xattr_req = NULL;
+    struct glfs_object *object = NULL;
+
+    /* validate in args */
+    if ((fs == NULL) || (parent == NULL) || (path == NULL)) {
+        errno = EINVAL;
+        return NULL;
+    }
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, parent);
+    if (!inode) {
+        errno = ESTALE;
+        goto out;
+    }
+
+    xattr_req = dict_new();
+    if (!xattr_req) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gf_uuid_generate(gfid);
+    ret = dict_set_gfuuid(xattr_req, "gfid-req", gfid, true);
+    if (ret) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    GLFS_LOC_FILL_PINODE(inode, loc, ret, errno, out, path);
+
+    /* fop/op */
+    ret = syncop_mknod(subvol, &loc, mode, dev, &iatt, xattr_req, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    /* populate out args */
+    if (ret == 0) {
+        ret = glfs_loc_link(&loc, &iatt);
+        if (ret != 0) {
+            goto out;
+        }
+
+        if (stat)
+            glfs_iatt_to_stat(fs, &iatt, stat);
 
+        ret = glfs_create_object(&loc, &object);
+    }
 out:
-	if (ret && object != NULL) {
-		glfs_h_close (object);
-		object = NULL;
-	}
+    if (ret && object != NULL) {
+        glfs_h_close(object);
+        object = NULL;
+    }
 
-	loc_wipe(&loc);
+    loc_wipe(&loc);
 
-	if (inode)
-		inode_unref (inode);
+    if (inode)
+        inode_unref(inode);
 
-	if (xattr_req)
-		dict_unref (xattr_req);
+    if (xattr_req)
+        dict_unref(xattr_req);
 
-	if (glfd) {
-		glfs_fd_destroy (glfd);
-		glfd = NULL;
-	}
+    glfs_subvol_done(fs, subvol);
 
-	priv_glfs_subvol_done (fs, subvol);
+    __GLFS_EXIT_FS;
 
-	return object;
+invalid_fs:
+    return object;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_creat, 3.4.2);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_unlink, 3.4.2)
+int
+pub_glfs_h_unlink(struct glfs *fs, struct glfs_object *parent, const char *path)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    loc_t loc = {
+        0,
+    };
+
+    /* validate in args */
+    if ((fs == NULL) || (parent == NULL) || (path == NULL)) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, parent);
+    if (!inode) {
+        errno = ESTALE;
+        goto out;
+    }
+
+    ret = glfs_resolve_at(fs, subvol, inode, path, &loc, NULL, 0, 0);
+    if (ret != 0) {
+        goto out;
+    }
+
+    if (!IA_ISDIR(loc.inode->ia_type)) {
+        ret = syncop_unlink(subvol, &loc, NULL, NULL);
+        DECODE_SYNCOP_ERR(ret);
+        if (ret != 0) {
+            goto out;
+        }
+    } else {
+        ret = syncop_rmdir(subvol, &loc, 0, NULL, NULL);
+        DECODE_SYNCOP_ERR(ret);
+        if (ret != 0) {
+            goto out;
+        }
+    }
+
+    if (ret == 0)
+        ret = glfs_loc_unlink(&loc);
 
+out:
+    loc_wipe(&loc);
 
-struct glfs_object *
-pub_glfs_h_mkdir (struct glfs *fs, struct glfs_object *parent, const char *path,
-	      mode_t mode, struct stat *stat)
+    if (inode)
+        inode_unref(inode);
+
+    glfs_subvol_done(fs, subvol);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_opendir, 3.4.2)
+struct glfs_fd *
+pub_glfs_h_opendir(struct glfs *fs, struct glfs_object *object)
 {
-	int                 ret = -1;
-	xlator_t           *subvol = NULL;
-	inode_t            *inode = NULL;
-	loc_t               loc = {0, };
-	struct iatt         iatt = {0, };
-	uuid_t              gfid;
-	dict_t             *xattr_req = NULL;
-	struct glfs_object *object = NULL;
-
-	/* validate in args */
-	if ((fs == NULL) || (parent == NULL) || (path == NULL)) {
-		errno = EINVAL;
-		return NULL;
-	}
-
-	__glfs_entry_fs (fs);
-
-	/* get the active volume */
-	subvol = priv_glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
-
-	/* get/refresh the in arg objects inode in correlation to the xlator */
-	inode = glfs_resolve_inode (fs, subvol, parent);
-	if (!inode) {
-		errno = ESTALE;
-		goto out;
-	}
-
-	xattr_req = dict_new ();
-	if (!xattr_req) {
-		ret = -1;
-		errno = ENOMEM;
-		goto out;
-	}
-
-	uuid_generate (gfid);
-	ret = dict_set_static_bin (xattr_req, "gfid-req", gfid, 16);
-	if (ret) {
-		ret = -1;
-		errno = ENOMEM;
-		goto out;
-	}
-
-	GLFS_LOC_FILL_PINODE (inode, loc, ret, errno, out, path);
-
-	/* fop/op */
-	ret = syncop_mkdir (subvol, &loc, mode, xattr_req, &iatt);
-
-	/* populate out args */
-	if ( ret == 0 )  {
-		ret = glfs_loc_link (&loc, &iatt);
-		if (ret != 0) {
-			goto out;
-		}
-
-		if (stat)
-			glfs_iatt_to_stat (fs, &iatt, stat);
-
-		ret = glfs_create_object (&loc, &object);
-	}
+    int ret = -1;
+    struct glfs_fd *glfd = NULL;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    loc_t loc = {
+        0,
+    };
+
+    /* validate in args */
+    if ((fs == NULL) || (object == NULL)) {
+        errno = EINVAL;
+        return NULL;
+    }
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, object);
+    if (!inode) {
+        errno = ESTALE;
+        goto out;
+    }
+
+    if (!IA_ISDIR(inode->ia_type)) {
+        ret = -1;
+        errno = ENOTDIR;
+        goto out;
+    }
+
+    glfd = glfs_fd_new(fs);
+    if (!glfd)
+        goto out;
+
+    INIT_LIST_HEAD(&glfd->entries);
+
+    glfd->fd = fd_create(inode, getpid());
+    if (!glfd->fd) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    GLFS_LOC_FILL_INODE(inode, loc, out);
+
+    /* fop/op */
+    ret = syncop_opendir(subvol, &loc, glfd->fd, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
 
 out:
-	if (ret && object != NULL) {
-		glfs_h_close (object);
-		object = NULL;
-	}
+    loc_wipe(&loc);
 
-	loc_wipe(&loc);
+    if (inode)
+        inode_unref(inode);
 
-	if (inode)
-		inode_unref (inode);
+    if (ret && glfd) {
+        GF_REF_PUT(glfd);
+        glfd = NULL;
+    } else if (glfd) {
+        glfd_set_state_bind(glfd);
+    }
 
-	if (xattr_req)
-		dict_unref (xattr_req);
+    glfs_subvol_done(fs, subvol);
 
-	priv_glfs_subvol_done (fs, subvol);
+    __GLFS_EXIT_FS;
 
-	return object;
+invalid_fs:
+    return glfd;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_mkdir, 3.4.2);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_access, 3.6.0)
+int
+pub_glfs_h_access(struct glfs *fs, struct glfs_object *object, int mask)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    loc_t loc = {
+        0,
+    };
+
+    DECLARE_OLD_THIS;
+
+    /* validate in args */
+    if ((fs == NULL) || (object == NULL)) {
+        errno = EINVAL;
+        return ret;
+    }
 
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
 
-struct glfs_object *
-pub_glfs_h_mknod (struct glfs *fs, struct glfs_object *parent, const char *path,
-	      mode_t mode, dev_t dev, struct stat *stat)
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, object);
+    if (!inode) {
+        errno = ESTALE;
+        goto out;
+    }
+
+    GLFS_LOC_FILL_INODE(inode, loc, out);
+
+    /* fop/op */
+
+    ret = syncop_access(subvol, &loc, mask, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+out:
+    loc_wipe(&loc);
+
+    if (inode)
+        inode_unref(inode);
+
+    glfs_subvol_done(fs, subvol);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_extract_handle, 3.4.2)
+ssize_t
+pub_glfs_h_extract_handle(struct glfs_object *object, unsigned char *handle,
+                          int len)
 {
-	int                 ret = -1;
-	xlator_t           *subvol = NULL;
-	inode_t            *inode = NULL;
-	loc_t               loc = {0, };
-	struct iatt         iatt = {0, };
-	uuid_t              gfid;
-	dict_t             *xattr_req = NULL;
-	struct glfs_object *object = NULL;
-
-	/* validate in args */
-	if ((fs == NULL) || (parent == NULL) || (path == NULL)) {
-		errno = EINVAL;
-		return NULL;
-	}
-
-	__glfs_entry_fs (fs);
-
-	/* get the active volume */
-	subvol = priv_glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
-
-	/* get/refresh the in arg objects inode in correlation to the xlator */
-	inode = glfs_resolve_inode (fs, subvol, parent);
-	if (!inode) {
-		errno = ESTALE;
-		goto out;
-	}
-
-	xattr_req = dict_new ();
-	if (!xattr_req) {
-		ret = -1;
-		errno = ENOMEM;
-		goto out;
-	}
-
-	uuid_generate (gfid);
-	ret = dict_set_static_bin (xattr_req, "gfid-req", gfid, 16);
-	if (ret) {
-		ret = -1;
-		errno = ENOMEM;
-		goto out;
-	}
-
-	GLFS_LOC_FILL_PINODE (inode, loc, ret, errno, out, path);
-
-	/* fop/op */
-	ret = syncop_mknod (subvol, &loc, mode, dev, xattr_req, &iatt);
-
-	/* populate out args */
-	if (ret == 0) {
-		ret = glfs_loc_link (&loc, &iatt);
-		if (ret != 0) {
-			goto out;
-		}
-
-		if (stat)
-			glfs_iatt_to_stat (fs, &iatt, stat);
-
-		ret = glfs_create_object (&loc, &object);
-	}
+    ssize_t ret = -1;
+
+    /* validate in args */
+    if (object == NULL) {
+        errno = EINVAL;
+        goto out;
+    }
+
+    if (!handle || !len) {
+        ret = GFAPI_HANDLE_LENGTH;
+        goto out;
+    }
+
+    if (len < GFAPI_HANDLE_LENGTH) {
+        errno = ERANGE;
+        goto out;
+    }
+
+    memcpy(handle, object->gfid, GFAPI_HANDLE_LENGTH);
+
+    ret = GFAPI_HANDLE_LENGTH;
+
 out:
-	if (ret && object != NULL) {
-		glfs_h_close (object);
-		object = NULL;
-	}
+    return ret;
+}
 
-	loc_wipe(&loc);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_create_from_handle, 3.4.2)
+struct glfs_object *
+pub_glfs_h_create_from_handle(struct glfs *fs, unsigned char *handle, int len,
+                              struct stat *stat)
+{
+    loc_t loc = {
+        0,
+    };
+    int ret = -1;
+    struct iatt iatt = {
+        0,
+    };
+    inode_t *newinode = NULL;
+    xlator_t *subvol = NULL;
+    struct glfs_object *object = NULL;
+    uint64_t ctx_value = LOOKUP_NOT_NEEDED;
+    gf_boolean_t lookup_needed = _gf_false;
+
+    /* validate in args */
+    if ((fs == NULL) || (handle == NULL) || (len != GFAPI_HANDLE_LENGTH)) {
+        errno = EINVAL;
+        return NULL;
+    }
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        errno = EIO;
+        goto out;
+    }
+
+    memcpy(loc.gfid, handle, GFAPI_HANDLE_LENGTH);
+
+    /* make sure the gfid received is valid */
+    GF_VALIDATE_OR_GOTO("glfs_h_create_from_handle",
+                        !(gf_uuid_is_null(loc.gfid)), out);
+
+    newinode = inode_find(subvol->itable, loc.gfid);
+    if (newinode) {
+        if (!stat) /* No need of lookup */
+            goto found;
+
+        lookup_needed = inode_needs_lookup(newinode, THIS);
+        if (lookup_needed) {
+            loc.inode = newinode;
+        } else {
+            /* populate loc */
+            GLFS_LOC_FILL_INODE(newinode, loc, fill_out);
+
+            /* fop/op */
+            ret = syncop_stat(subvol, &loc, &iatt, NULL, NULL);
+            DECODE_SYNCOP_ERR(ret);
+
+            if (ret) {
+            fill_out:
+                /* Drop the reference hold in inode_find */
+                inode_unref(newinode);
+                goto out;
+            }
 
-	if (inode)
-		inode_unref (inode);
+            glfs_iatt_to_stat(fs, &iatt, stat);
+            goto found;
+        }
+    } else {
+        loc.inode = inode_new(subvol->itable);
+        if (!loc.inode) {
+            errno = ENOMEM;
+            goto out;
+        }
+    }
+
+    ret = syncop_lookup(subvol, &loc, &iatt, 0, 0, 0);
+    DECODE_SYNCOP_ERR(ret);
+    if (ret) {
+        gf_smsg(subvol->name, GF_LOG_WARNING, errno,
+                API_MSG_INODE_REFRESH_FAILED, "gfid=%s", uuid_utoa(loc.gfid),
+                "error=%s", strerror(errno), NULL);
+        goto out;
+    }
+
+    newinode = inode_link(loc.inode, 0, 0, &iatt);
+    if (newinode) {
+        if (newinode == loc.inode) {
+            inode_ctx_set(newinode, THIS, &ctx_value);
+        }
+        inode_lookup(newinode);
+    } else {
+        gf_smsg(subvol->name, GF_LOG_WARNING, errno, API_MSG_INODE_LINK_FAILED,
+                "gfid=%s", uuid_utoa(loc.gfid), NULL);
+        goto out;
+    }
+
+    /* populate stat */
+    if (stat)
+        glfs_iatt_to_stat(fs, &iatt, stat);
+
+found:
+    object = GF_CALLOC(1, sizeof(struct glfs_object), glfs_mt_glfs_object_t);
+    if (object == NULL) {
+        errno = ENOMEM;
+        ret = -1;
+        goto out;
+    }
+
+    /* populate the return object */
+    object->inode = newinode;
+    gf_uuid_copy(object->gfid, object->inode->gfid);
+
+out:
+    /* TODO: Check where the inode ref is being held? */
+    loc_wipe(&loc);
 
-	if (xattr_req)
-		dict_unref (xattr_req);
+    glfs_subvol_done(fs, subvol);
 
-	priv_glfs_subvol_done (fs, subvol);
+    __GLFS_EXIT_FS;
 
-	return object;
+invalid_fs:
+    return object;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_mknod, 3.4.2);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_close, 3.4.2)
+int
+pub_glfs_h_close(struct glfs_object *object)
+{
+    /* since glfs_h_* objects hold a reference to inode
+     * it is safe to keep lookup count to '0' */
+    inode_forget(object->inode, 0);
+    inode_unref(object->inode);
+    GF_FREE(object);
 
+    return 0;
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_truncate, 3.4.2)
 int
-pub_glfs_h_unlink (struct glfs *fs, struct glfs_object *parent, const char *path)
+pub_glfs_h_truncate(struct glfs *fs, struct glfs_object *object, off_t offset)
 {
-	int                 ret = -1;
-	xlator_t           *subvol = NULL;
-	inode_t            *inode = NULL;
-	loc_t               loc = {0, };
-
-	/* validate in args */
-	if ((fs == NULL) || (parent == NULL) || (path == NULL)) {
-		errno = EINVAL;
-		return -1;
-	}
-
-	__glfs_entry_fs (fs);
-
-	/* get the active volume */
-	subvol = priv_glfs_active_subvol (fs);
-	if ( !subvol ) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
-
-	/* get/refresh the in arg objects inode in correlation to the xlator */
-	inode = glfs_resolve_inode (fs, subvol, parent);
-	if (!inode) {
-		errno = ESTALE;
-		goto out;
-	}
-
-	ret = priv_glfs_resolve_at (fs, subvol, inode, path, &loc, NULL, 0 , 0);
-	if (ret != 0) {
-		goto out;
-	}
-
-	if (!IA_ISDIR(loc.inode->ia_type)) {
-		ret = syncop_unlink (subvol, &loc);
-		if (ret != 0) {
-			goto out;
-		}
-	} else {
-		ret = syncop_rmdir (subvol, &loc, 0);
-		if (ret != 0) {
-			goto out;
-		}
-	}
-
-	if (ret == 0)
-		ret = glfs_loc_unlink (&loc);
+    loc_t loc = {
+        0,
+    };
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+
+    DECLARE_OLD_THIS;
+
+    /* validate in args */
+    if (object == NULL) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, object);
+    if (!inode) {
+        errno = ESTALE;
+        goto out;
+    }
+
+    GLFS_LOC_FILL_INODE(inode, loc, out);
+
+    /* fop/op */
+    ret = syncop_truncate(subvol, &loc, (off_t)offset, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    /* populate out args */
+    if (ret == 0)
+        ret = glfs_loc_unlink(&loc);
 
 out:
-	loc_wipe (&loc);
+    loc_wipe(&loc);
+
+    if (inode)
+        inode_unref(inode);
 
-	if (inode)
-		inode_unref (inode);
+    glfs_subvol_done(fs, subvol);
 
-	priv_glfs_subvol_done (fs, subvol);
+    __GLFS_EXIT_FS;
 
-	return ret;
+invalid_fs:
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_unlink, 3.4.2);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_symlink, 3.4.2)
+struct glfs_object *
+pub_glfs_h_symlink(struct glfs *fs, struct glfs_object *parent,
+                   const char *name, const char *data, struct stat *stat)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    uuid_t gfid;
+    dict_t *xattr_req = NULL;
+    struct glfs_object *object = NULL;
+
+    DECLARE_OLD_THIS;
+
+    /* validate in args */
+    if ((parent == NULL) || (name == NULL) || (data == NULL)) {
+        errno = EINVAL;
+        return NULL;
+    }
+
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, parent);
+    if (!inode) {
+        errno = ESTALE;
+        goto out;
+    }
+
+    xattr_req = dict_new();
+    if (!xattr_req) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gf_uuid_generate(gfid);
+    ret = dict_set_gfuuid(xattr_req, "gfid-req", gfid, true);
+    if (ret) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    GLFS_LOC_FILL_PINODE(inode, loc, ret, errno, out, name);
+
+    /* fop/op */
+    ret = syncop_symlink(subvol, &loc, data, &iatt, xattr_req, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    /* populate out args */
+    if (ret == 0) {
+        ret = glfs_loc_link(&loc, &iatt);
+        if (ret != 0) {
+            goto out;
+        }
 
+        if (stat)
+            glfs_iatt_to_stat(fs, &iatt, stat);
 
-struct glfs_fd *
-pub_glfs_h_opendir (struct glfs *fs, struct glfs_object *object)
-{
-	int              ret = -1;
-	struct glfs_fd  *glfd = NULL;
-	xlator_t        *subvol = NULL;
-	inode_t         *inode = NULL;
-	loc_t            loc = {0, };
-
-	/* validate in args */
-	if ((fs == NULL) || (object == NULL)) {
-		errno = EINVAL;
-		return NULL;
-	}
-
-	__glfs_entry_fs (fs);
-
-	/* get the active volume */
-	subvol = priv_glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
-
-	/* get/refresh the in arg objects inode in correlation to the xlator */
-	inode = glfs_resolve_inode (fs, subvol, object);
-	if (!inode) {
-		errno = ESTALE;
-		goto out;
-	}
-
-	if (!IA_ISDIR (inode->ia_type)) {
-		ret = -1;
-		errno = ENOTDIR;
-		goto out;
-	}
-
-	glfd = glfs_fd_new (fs);
-	if (!glfd)
-		goto out;
-
-	INIT_LIST_HEAD (&glfd->entries);
-
-	glfd->fd = fd_create (inode, getpid());
-	if (!glfd->fd) {
-		ret = -1;
-		errno = ENOMEM;
-		goto out;
-	}
-
-	GLFS_LOC_FILL_INODE (inode, loc, out);
-
-	/* fop/op */
-	ret = syncop_opendir (subvol, &loc, glfd->fd);
+        ret = glfs_create_object(&loc, &object);
+    }
 
 out:
-	loc_wipe (&loc);
+    if (ret && object != NULL) {
+        pub_glfs_h_close(object);
+        object = NULL;
+    }
 
-	if (inode)
-		inode_unref (inode);
+    loc_wipe(&loc);
 
-	if (ret && glfd) {
-		glfs_fd_destroy (glfd);
-		glfd = NULL;
-	} else {
-		fd_bind (glfd->fd);
-		glfs_fd_bind (glfd);
-	}
+    if (inode)
+        inode_unref(inode);
 
-	priv_glfs_subvol_done (fs, subvol);
+    if (xattr_req)
+        dict_unref(xattr_req);
 
-	return glfd;
-}
+    glfs_subvol_done(fs, subvol);
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_opendir, 3.4.2);
+    __GLFS_EXIT_FS;
 
+invalid_fs:
+    return object;
+}
 
-ssize_t
-pub_glfs_h_extract_handle (struct glfs_object *object, unsigned char *handle,
-		       int len)
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_readlink, 3.4.2)
+int
+pub_glfs_h_readlink(struct glfs *fs, struct glfs_object *object, char *buf,
+                    size_t bufsiz)
 {
-	ssize_t ret = -1;
+    loc_t loc = {
+        0,
+    };
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    char *linkval = NULL;
+
+    DECLARE_OLD_THIS;
+
+    /* validate in args */
+    if ((object == NULL) || (buf == NULL)) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, object);
+    if (!inode) {
+        errno = ESTALE;
+        goto out;
+    }
+
+    GLFS_LOC_FILL_INODE(inode, loc, out);
+
+    /* fop/op */
+    ret = syncop_readlink(subvol, &loc, &linkval, bufsiz, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    /* populate out args */
+    if (ret > 0)
+        memcpy(buf, linkval, ret);
 
-	/* validate in args */
-	if (object == NULL) {
-		errno = EINVAL;
-		goto out;
-	}
+out:
+    loc_wipe(&loc);
 
-	if (!handle || !len) {
-		ret = GFAPI_HANDLE_LENGTH;
-		goto out;
-	}
+    if (inode)
+        inode_unref(inode);
 
-	if (len < GFAPI_HANDLE_LENGTH)
-	{
-		errno = ERANGE;
-		goto out;
-	}
+    if (linkval)
+        GF_FREE(linkval);
 
-	memcpy (handle, object->gfid, GFAPI_HANDLE_LENGTH);
+    glfs_subvol_done(fs, subvol);
 
-	ret = GFAPI_HANDLE_LENGTH;
+    __GLFS_EXIT_FS;
 
+invalid_fs:
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_link, 3.4.2)
+int
+pub_glfs_h_link(struct glfs *fs, struct glfs_object *linksrc,
+                struct glfs_object *parent, const char *name)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    inode_t *pinode = NULL;
+    loc_t oldloc = {
+        0,
+    };
+    loc_t newloc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+
+    DECLARE_OLD_THIS;
+
+    /* validate in args */
+    if ((linksrc == NULL) || (parent == NULL) || (name == NULL)) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, linksrc);
+    if (!inode) {
+        errno = ESTALE;
+        goto out;
+    }
+
+    if (inode->ia_type == IA_IFDIR) {
+        ret = -1;
+        errno = EISDIR;
+        goto out;
+    }
+
+    GLFS_LOC_FILL_INODE(inode, oldloc, out);
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    pinode = glfs_resolve_inode(fs, subvol, parent);
+    if (!pinode) {
+        errno = ESTALE;
+        goto out;
+    }
+
+    /* setup newloc based on parent */
+    newloc.parent = inode_ref(pinode);
+    newloc.name = name;
+    ret = glfs_loc_touchup(&newloc);
+    if (ret != 0) {
+        errno = EINVAL;
+        goto out;
+    }
+
+    /* Filling the inode of the hard link to be same as that of the
+     * original file
+     */
+    newloc.inode = inode_ref(inode);
+
+    /* fop/op */
+    ret = syncop_link(subvol, &oldloc, &newloc, &iatt, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    if (ret == 0)
+        ret = glfs_loc_link(&newloc, &iatt);
 out:
-	return ret;
+    loc_wipe(&oldloc);
+    loc_wipe(&newloc);
+
+    if (inode)
+        inode_unref(inode);
+
+    if (pinode)
+        inode_unref(pinode);
+
+    glfs_subvol_done(fs, subvol);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_extract_handle, 3.4.2);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_rename, 3.4.2)
+int
+pub_glfs_h_rename(struct glfs *fs, struct glfs_object *olddir,
+                  const char *oldname, struct glfs_object *newdir,
+                  const char *newname)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    inode_t *oldpinode = NULL;
+    inode_t *newpinode = NULL;
+    loc_t oldloc = {
+        0,
+    };
+    loc_t newloc = {
+        0,
+    };
+    struct iatt oldiatt = {
+        0,
+    };
+    struct iatt newiatt = {
+        0,
+    };
+
+    DECLARE_OLD_THIS;
+
+    /* validate in args */
+    if ((olddir == NULL) || (oldname == NULL) || (newdir == NULL) ||
+        (newname == NULL)) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    oldpinode = glfs_resolve_inode(fs, subvol, olddir);
+    if (!oldpinode) {
+        errno = ESTALE;
+        goto out;
+    }
+
+    ret = glfs_resolve_at(fs, subvol, oldpinode, oldname, &oldloc, &oldiatt, 0,
+                          0);
+    if (ret != 0) {
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    newpinode = glfs_resolve_inode(fs, subvol, newdir);
+    if (!newpinode) {
+        errno = ESTALE;
+        goto out;
+    }
+
+    ret = glfs_resolve_at(fs, subvol, newpinode, newname, &newloc, &newiatt, 0,
+                          0);
+
+    if (ret && errno != ENOENT && newloc.parent)
+        goto out;
+
+    if (newiatt.ia_type != IA_INVAL) {
+        if ((oldiatt.ia_type == IA_IFDIR) != (newiatt.ia_type == IA_IFDIR)) {
+            /* Either both old and new must be dirs,
+             * or both must be non-dirs. Else, fail.
+             */
+            ret = -1;
+            errno = EEXIST;
+            goto out;
+        }
+    }
+
+    /* TODO: check if new or old is a prefix of the other, and fail EINVAL */
 
+    ret = syncop_rename(subvol, &oldloc, &newloc, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
 
+    if (ret == 0) {
+        inode_rename(oldloc.parent->table, oldloc.parent, oldloc.name,
+                     newloc.parent, newloc.name, oldloc.inode, &oldiatt);
+
+        if (newloc.inode && !inode_has_dentry(newloc.inode))
+            inode_forget(newloc.inode, 0);
+    }
+
+out:
+    loc_wipe(&oldloc);
+    loc_wipe(&newloc);
+
+    if (oldpinode)
+        inode_unref(oldpinode);
+
+    if (newpinode)
+        inode_unref(newpinode);
+
+    glfs_subvol_done(fs, subvol);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
+
+/*
+ * Given a handle/gfid, find if the corresponding inode is present in
+ * the inode table. If yes create and return the corresponding glfs_object.
+ */
 struct glfs_object *
-pub_glfs_h_create_from_handle (struct glfs *fs, unsigned char *handle, int len,
-			   struct stat *stat)
+glfs_h_find_handle(struct glfs *fs, unsigned char *handle, int len)
 {
-	loc_t               loc = {0, };
-	int                 ret = -1;
-	struct iatt         iatt = {0, };
-	inode_t            *newinode = NULL;
-	xlator_t           *subvol = NULL;
-	struct glfs_object *object = NULL;
-
-	/* validate in args */
-	if ((fs == NULL) || (handle == NULL) || (len != GFAPI_HANDLE_LENGTH)) {
-		errno = EINVAL;
-		return NULL;
-	}
-
-	__glfs_entry_fs (fs);
-
-	/* get the active volume */
-	subvol = priv_glfs_active_subvol (fs);
-	if (!subvol) {
-		errno = EIO;
-		goto out;
-	}
-
-	memcpy (loc.gfid, handle, GFAPI_HANDLE_LENGTH);
-
-	newinode = inode_find (subvol->itable, loc.gfid);
-	if (newinode)
-		loc.inode = newinode;
-	else {
-		loc.inode = inode_new (subvol->itable);
-		if (!loc.inode) {
-			errno = ENOMEM;
-			goto out;
-		}
-	}
-
-	ret = syncop_lookup (subvol, &loc, 0, &iatt, 0, 0);
-	if (ret) {
-		gf_log (subvol->name, GF_LOG_WARNING,
-			"inode refresh of %s failed: %s",
-			uuid_utoa (loc.gfid), strerror (errno));
-		goto out;
-	}
-
-	newinode = inode_link (loc.inode, 0, 0, &iatt);
-	if (newinode)
-		inode_lookup (newinode);
-	else {
-		gf_log (subvol->name, GF_LOG_WARNING,
-			"inode linking of %s failed: %s",
-			uuid_utoa (loc.gfid), strerror (errno));
-		errno = EINVAL;
-		goto out;
-	}
-
-	/* populate stat */
-	if (stat)
-		glfs_iatt_to_stat (fs, &iatt, stat);
-
-	object = GF_CALLOC (1, sizeof(struct glfs_object),
-			    glfs_mt_glfs_object_t);
-	if (object == NULL) {
-		errno = ENOMEM;
-		ret = -1;
-		goto out;
-	}
-
-	/* populate the return object */
-	object->inode = newinode;
-	uuid_copy (object->gfid, object->inode->gfid);
+    inode_t *newinode = NULL;
+    xlator_t *subvol = NULL;
+    struct glfs_object *object = NULL;
+    uuid_t gfid;
+
+    /* validate in args */
+    if ((fs == NULL) || (handle == NULL) || (len != GFAPI_HANDLE_LENGTH)) {
+        errno = EINVAL;
+        return NULL;
+    }
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        errno = EIO;
+        goto out;
+    }
+
+    memcpy(gfid, handle, GFAPI_HANDLE_LENGTH);
+
+    /* make sure the gfid received is valid */
+    GF_VALIDATE_OR_GOTO("glfs_h_find_handle", !(gf_uuid_is_null(gfid)), out);
+
+    newinode = inode_find(subvol->itable, gfid);
+    if (!newinode) {
+        goto out;
+    }
+
+    object = GF_CALLOC(1, sizeof(struct glfs_object), glfs_mt_glfs_object_t);
+    if (object == NULL) {
+        errno = ENOMEM;
+        goto out;
+    }
+
+    /* populate the return object. The ref taken here
+     * is un'refed when the application does glfs_h_close() */
+    object->inode = inode_ref(newinode);
+    gf_uuid_copy(object->gfid, object->inode->gfid);
 
 out:
-	/* TODO: Check where the inode ref is being held? */
-	loc_wipe (&loc);
+    /* inode_find takes a reference. Unref it. */
+    if (newinode)
+        inode_unref(newinode);
 
-	priv_glfs_subvol_done (fs, subvol);
+    glfs_subvol_done(fs, subvol);
 
-	return object;
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return object;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_create_from_handle, 3.4.2);
+static void
+glfs_free_upcall_inode(void *to_free)
+{
+    struct glfs_upcall_inode *arg = to_free;
+
+    if (!arg)
+        return;
 
+    if (arg->object)
+        glfs_h_close(arg->object);
+    if (arg->p_object)
+        glfs_h_close(arg->p_object);
+    if (arg->oldp_object)
+        glfs_h_close(arg->oldp_object);
+
+    GF_FREE(arg);
+}
 
 int
-pub_glfs_h_close (struct glfs_object *object)
+glfs_h_poll_cache_invalidation(struct glfs *fs, struct glfs_upcall *up_arg,
+                               struct gf_upcall *upcall_data)
 {
-	/* Release the held reference */
-	inode_unref (object->inode);
-	GF_FREE (object);
+    int ret = -1;
+    struct glfs_object *p_object = NULL;
+    struct glfs_object *oldp_object = NULL;
+    struct glfs_object *object = NULL;
+    struct gf_upcall_cache_invalidation *ca_data = NULL;
+    struct glfs_upcall_inode *up_inode_arg = NULL;
+
+    ca_data = upcall_data->data;
+    GF_VALIDATE_OR_GOTO("glfs_h_poll_cache_invalidation", ca_data, out);
+
+    object = glfs_h_find_handle(fs, upcall_data->gfid, GFAPI_HANDLE_LENGTH);
+    if (!object) {
+        /* The reason handle creation will fail is because we
+         * couldn't find the inode in the gfapi inode table.
+         *
+         * But since application would have taken inode_ref, the
+         * only case when this can happen is when it has closed
+         * the handle and hence will no more be interested in
+         * the upcall for this particular gfid.
+         */
+        gf_smsg(THIS->name, GF_LOG_DEBUG, errno, API_MSG_CREATE_HANDLE_FAILED,
+                "gfid=%s", uuid_utoa(upcall_data->gfid), NULL);
+        errno = ESTALE;
+        goto out;
+    }
+
+    up_inode_arg = GF_CALLOC(1, sizeof(struct glfs_upcall_inode),
+                             glfs_mt_upcall_inode_t);
+    GF_VALIDATE_OR_GOTO("glfs_h_poll_cache_invalidation", up_inode_arg, out);
+
+    up_inode_arg->object = object;
+    up_inode_arg->flags = ca_data->flags;
+    up_inode_arg->expire_time_attr = ca_data->expire_time_attr;
+
+    /* XXX: Update stat as well in case of UP_*_TIMES.
+     * This will be addressed as part of INODE_UPDATE */
+    if (ca_data->flags & GFAPI_INODE_UPDATE_FLAGS) {
+        glfs_iatt_to_stat(fs, &ca_data->stat, &up_inode_arg->buf);
+    }
+
+    if (ca_data->flags & GFAPI_UP_PARENT_TIMES) {
+        p_object = glfs_h_find_handle(fs, ca_data->p_stat.ia_gfid,
+                                      GFAPI_HANDLE_LENGTH);
+        if (!p_object) {
+            gf_smsg(THIS->name, GF_LOG_DEBUG, errno,
+                    API_MSG_CREATE_HANDLE_FAILED, "gfid=%s",
+                    uuid_utoa(ca_data->p_stat.ia_gfid), NULL);
+            errno = ESTALE;
+            goto out;
+        }
+
+        glfs_iatt_to_stat(fs, &ca_data->p_stat, &up_inode_arg->p_buf);
+    }
+    up_inode_arg->p_object = p_object;
+
+    /* In case of RENAME, update old parent as well */
+    if (ca_data->flags & GFAPI_UP_RENAME) {
+        oldp_object = glfs_h_find_handle(fs, ca_data->oldp_stat.ia_gfid,
+                                         GFAPI_HANDLE_LENGTH);
+        if (!oldp_object) {
+            gf_smsg(THIS->name, GF_LOG_DEBUG, errno,
+                    API_MSG_CREATE_HANDLE_FAILED, "gfid=%s",
+                    uuid_utoa(ca_data->oldp_stat.ia_gfid), NULL);
+            errno = ESTALE;
+            /* By the time we receive upcall old parent_dir may
+             * have got removed. We still need to send upcall
+             * for the file/dir and current parent handles. */
+            up_inode_arg->oldp_object = NULL;
+            ret = 0;
+        }
+
+        glfs_iatt_to_stat(fs, &ca_data->oldp_stat, &up_inode_arg->oldp_buf);
+    }
+    up_inode_arg->oldp_object = oldp_object;
+
+    up_arg->reason = GLFS_UPCALL_INODE_INVALIDATE;
+    up_arg->event = up_inode_arg;
+    up_arg->free_event = glfs_free_upcall_inode;
 
-	return 0;
+    ret = 0;
+
+out:
+    if (ret) {
+        /* Close p_object and oldp_object as well if being referenced.*/
+        if (object)
+            glfs_h_close(object);
+
+        /* Set reason to prevent applications from using ->event */
+        up_arg->reason = GLFS_UPCALL_EVENT_NULL;
+        GF_FREE(up_inode_arg);
+    }
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_close, 3.4.2);
+void
+glfs_release_upcall(void *ptr)
+{
+    struct glfs_upcall *to_free = ptr;
 
+    if (to_free->event)
+        to_free->free_event(to_free->event);
+}
 
+/*
+ * This API is used to poll for upcall events stored in the upcall list.
+ * Current users of this API is NFS-Ganesha. In case of any event received, it
+ * will be mapped appropriately into 'glfs_upcall' along with the handle object
+ * to be passed to NFS-Ganesha.
+ *
+ * On success, applications need to check if up_arg is not-NULL or errno is not
+ * ENOENT. glfs_upcall_get_reason() can be used to decide what kind of event
+ * has been received.
+ *
+ * Current supported upcall_events:
+ *      GLFS_UPCALL_INODE_INVALIDATE
+ *
+ * After processing the event, applications need to free 'up_arg' by calling
+ * glfs_free().
+ *
+ * Also similar to I/Os, the application should ideally stop polling before
+ * calling glfs_fini(..). Hence making an assumption that 'fs' & ctx structures
+ * cannot be freed while in this routine.
+ */
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_poll_upcall, 3.7.16)
 int
-pub_glfs_h_truncate (struct glfs *fs, struct glfs_object *object, off_t offset)
+pub_glfs_h_poll_upcall(struct glfs *fs, struct glfs_upcall **up_arg)
 {
-	loc_t               loc = {0, };
-	int                 ret = -1;
-	xlator_t           *subvol = NULL;
-	inode_t            *inode = NULL;
-
-	/* validate in args */
-	if ((fs == NULL) || (object == NULL)) {
-		errno = EINVAL;
-		return -1;
-	}
-
-	__glfs_entry_fs (fs);
-
-	/* get the active volume */
-	subvol = priv_glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
-
-	/* get/refresh the in arg objects inode in correlation to the xlator */
-	inode = glfs_resolve_inode (fs, subvol, object);
-	if (!inode) {
-		errno = ESTALE;
-		goto out;
-	}
-
-	GLFS_LOC_FILL_INODE (inode, loc, out);
-
-	/* fop/op */
-	ret = syncop_truncate (subvol, &loc, (off_t)offset);
-
-	/* populate out args */
-	if (ret == 0)
-		ret = glfs_loc_unlink (&loc);
+    upcall_entry *u_list = NULL;
+    upcall_entry *tmp = NULL;
+    xlator_t *subvol = NULL;
+    glusterfs_ctx_t *ctx = NULL;
+    int ret = -1;
+    struct gf_upcall *upcall_data = NULL;
+
+    DECLARE_OLD_THIS;
+
+    if (!up_arg) {
+        errno = EINVAL;
+        goto err;
+    }
+
+    __GLFS_ENTRY_VALIDATE_FS(fs, err);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        errno = EIO;
+        goto restore;
+    }
+
+    /* Ideally applications should stop polling before calling
+     * 'glfs_fini'. Yet cross check if cleanup has started. */
+    pthread_mutex_lock(&fs->mutex);
+    {
+        ctx = fs->ctx;
+
+        if (ctx->cleanup_started) {
+            pthread_mutex_unlock(&fs->mutex);
+            goto out;
+        }
 
-out:
-	loc_wipe (&loc);
+        fs->pin_refcnt++;
+
+        /* once we call this function, the applications seems to be
+         * interested in events, enable caching them */
+        fs->cache_upcalls = _gf_true;
+    }
+    pthread_mutex_unlock(&fs->mutex);
+
+    pthread_mutex_lock(&fs->upcall_list_mutex);
+    {
+        list_for_each_entry_safe(u_list, tmp, &fs->upcall_list, upcall_list)
+        {
+            list_del_init(&u_list->upcall_list);
+            upcall_data = &u_list->upcall_data;
+            break;
+        }
+    }
+    /* No other thread can delete this entry. So unlock it */
+    pthread_mutex_unlock(&fs->upcall_list_mutex);
+
+    if (upcall_data) {
+        switch (upcall_data->event_type) {
+            case GF_UPCALL_CACHE_INVALIDATION:
+                *up_arg = GLFS_CALLOC(1, sizeof(struct gf_upcall),
+                                      glfs_release_upcall,
+                                      glfs_mt_upcall_entry_t);
+                if (!*up_arg) {
+                    errno = ENOMEM;
+                    break; /* goto free u_list */
+                }
+
+                /* XXX: Need to revisit this to support
+                 * GLFS_UPCALL_INODE_UPDATE if required. */
+                ret = glfs_h_poll_cache_invalidation(fs, *up_arg, upcall_data);
+                if (ret || (*up_arg)->reason == GLFS_UPCALL_EVENT_NULL) {
+                    /* It could so happen that the file which got
+                     * upcall notification may have got deleted by
+                     * the same client. Irrespective of the error,
+                     * return with an error or success+ENOENT. */
+                    if ((*up_arg)->reason == GLFS_UPCALL_EVENT_NULL)
+                        errno = ENOENT;
+
+                    GLFS_FREE(*up_arg);
+                    *up_arg = NULL;
+                }
+                break;
+            case GF_UPCALL_RECALL_LEASE:
+                gf_log("glfs_h_poll_upcall", GF_LOG_DEBUG,
+                       "UPCALL_RECALL_LEASE is not implemented yet");
+            /* fallthrough till we support leases */
+            case GF_UPCALL_EVENT_NULL:
+                /* no 'default:' label, to force handling all upcall events */
+                errno = ENOENT;
+                break;
+        }
 
-	if (inode)
-		inode_unref (inode);
+        GF_FREE(u_list->upcall_data.data);
+        GF_FREE(u_list);
+    } else {
+        /* fs->upcall_list was empty, no upcall events cached */
+        errno = ENOENT;
+    }
 
-	priv_glfs_subvol_done (fs, subvol);
+    ret = 0;
 
-	return ret;
+out:
+    pthread_mutex_lock(&fs->mutex);
+    {
+        fs->pin_refcnt--;
+    }
+    pthread_mutex_unlock(&fs->mutex);
+
+    glfs_subvol_done(fs, subvol);
+
+restore:
+    __GLFS_EXIT_FS;
+err:
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_truncate, 3.4.2);
+static gf_boolean_t log_upcall370 = _gf_true; /* log once */
 
-
-struct glfs_object *
-pub_glfs_h_symlink (struct glfs *fs, struct glfs_object *parent, const char *name,
-		const char *data, struct stat *stat)
+/* The old glfs_h_poll_upcall interface requires intimate knowledge of the
+ * structures that are returned to the calling application. This is not
+ * recommended, as the returned structures need to returned correctly (handles
+ * closed, memory free'd with the unavailable GF_FREE(), and possibly more.)
+ *
+ * To the best of our knowledge, only NFS-Ganesha uses the upcall events
+ * through gfapi. We keep this backwards compatibility function around so that
+ * applications using the existing implementation do not break.
+ *
+ * WARNING: this function will be removed in the future.
+ */
+GFAPI_SYMVER_PUBLIC(glfs_h_poll_upcall370, glfs_h_poll_upcall, 3.7.0)
+int
+pub_glfs_h_poll_upcall370(struct glfs *fs, struct glfs_callback_arg *up_arg)
 {
-	int                 ret = -1;
-	xlator_t           *subvol = NULL;
-	inode_t            *inode = NULL;
-	loc_t               loc = {0, };
-	struct iatt         iatt = {0, };
-	uuid_t              gfid;
-	dict_t             *xattr_req = NULL;
-	struct glfs_object *object = NULL;
-
-	/* validate in args */
-	if ((fs == NULL) || (parent == NULL) || (name == NULL) ||
-		(data == NULL)) {
-		errno = EINVAL;
-		return NULL;
-	}
-
-	__glfs_entry_fs (fs);
-
-	/* get the active volume */
-	subvol = priv_glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
-
-	/* get/refresh the in arg objects inode in correlation to the xlator */
-	inode = glfs_resolve_inode (fs, subvol, parent);
-	if (!inode) {
-		errno = ESTALE;
-		goto out;
-	}
-
-	xattr_req = dict_new ();
-	if (!xattr_req) {
-		ret = -1;
-		errno = ENOMEM;
-		goto out;
-	}
-
-	uuid_generate (gfid);
-	ret = dict_set_static_bin (xattr_req, "gfid-req", gfid, 16);
-	if (ret) {
-		ret = -1;
-		errno = ENOMEM;
-		goto out;
-	}
-
-	GLFS_LOC_FILL_PINODE (inode, loc, ret, errno, out, name);
-
-	/* fop/op */
-	ret = syncop_symlink (subvol, &loc, data, xattr_req, &iatt);
-
-	/* populate out args */
-	if (ret == 0) {
-		/* TODO: If the inode existed in the cache (say file already
-		 * exists), then the glfs_loc_link will not update the
-		 * loc.inode, as a result we will have a 0000 GFID that we
-		 * would copy out to the object, this needs to be fixed.
-		 */
-		ret = glfs_loc_link (&loc, &iatt);
-		if (ret != 0) {
-			goto out;
-		}
-
-		if (stat)
-			glfs_iatt_to_stat (fs, &iatt, stat);
-
-		ret = glfs_create_object (&loc, &object);
-	}
+    struct glfs_upcall *upcall = NULL;
+    int ret = -1;
+
+    if (log_upcall370) {
+        log_upcall370 = _gf_false;
+        gf_log(THIS->name, GF_LOG_WARNING,
+               "this application is "
+               "compiled against an old version of libgfapi, it "
+               "should use glfs_free() to release the structure "
+               "returned by glfs_h_poll_upcall() - for more details, "
+               "see http://review.gluster.org/14701");
+    }
+
+    ret = pub_glfs_h_poll_upcall(fs, &upcall);
+    if (ret == 0) {
+        up_arg->fs = fs;
+        if ((errno == ENOENT) || !upcall || !upcall->event) {
+            up_arg->reason = GLFS_UPCALL_EVENT_NULL;
+            goto out;
+        }
+
+        up_arg->reason = upcall->reason;
+
+        if (upcall->reason == GLFS_UPCALL_INODE_INVALIDATE) {
+            struct glfs_callback_inode_arg *cb_inode = NULL;
+            struct glfs_upcall_inode *up_inode = NULL;
+
+            cb_inode = GF_CALLOC(1, sizeof(struct glfs_callback_inode_arg),
+                                 glfs_mt_upcall_inode_t);
+            if (!cb_inode) {
+                errno = ENOMEM;
+                ret = -1;
+                goto out;
+            }
+
+            up_inode = upcall->event;
+
+            /* copy attributes one by one, the memory layout might
+             * be different between the old glfs_callback_inode_arg
+             * and new glfs_upcall_inode */
+            cb_inode->object = up_inode->object;
+            cb_inode->flags = up_inode->flags;
+            memcpy(&cb_inode->buf, &up_inode->buf, sizeof(struct stat));
+            cb_inode->expire_time_attr = up_inode->expire_time_attr;
+            cb_inode->p_object = up_inode->p_object;
+            memcpy(&cb_inode->p_buf, &up_inode->p_buf, sizeof(struct stat));
+            cb_inode->oldp_object = up_inode->oldp_object;
+            memcpy(&cb_inode->oldp_buf, &up_inode->oldp_buf,
+                   sizeof(struct stat));
+
+            up_arg->event_arg = cb_inode;
+        }
+    }
 
 out:
-	if (ret && object != NULL) {
-		glfs_h_close (object);
-		object = NULL;
-	}
+    if (upcall) {
+        /* we can not use glfs_free() here, objects need to stay */
+        GF_FREE(upcall->event);
+        GF_FREE(upcall);
+    }
 
-	loc_wipe(&loc);
+    return ret;
+}
 
-	if (inode)
-		inode_unref (inode);
+#ifdef HAVE_ACL_LIBACL_H
+#include <glusterfs/glusterfs-acl.h>
+#include <acl/libacl.h>
 
-	if (xattr_req)
-		dict_unref (xattr_req);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_acl_set, 3.7.0)
+int
+pub_glfs_h_acl_set(struct glfs *fs, struct glfs_object *object,
+                   const acl_type_t type, const acl_t acl)
+{
+    int ret = -1;
+    char *acl_s = NULL;
+    const char *acl_key = NULL;
+    struct glfs_object *new_object = NULL;
+
+    DECLARE_OLD_THIS;
+
+    if (!object || !acl) {
+        errno = EINVAL;
+        return ret;
+    }
+
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    acl_key = gf_posix_acl_get_key(type);
+    if (!acl_key)
+        goto out;
+
+    acl_s = acl_to_any_text(acl, NULL, ',', TEXT_ABBREVIATE | TEXT_NUMERIC_IDS);
+    if (!acl_s)
+        goto out;
+
+    if (IA_ISLNK(object->inode->ia_type)) {
+        new_object = glfs_h_resolve_symlink(fs, object);
+        if (new_object == NULL)
+            goto out;
+    } else
+        new_object = object;
 
-	priv_glfs_subvol_done (fs, subvol);
+    ret = pub_glfs_h_setxattrs(fs, new_object, acl_key, acl_s,
+                               strlen(acl_s) + 1, 0);
 
-	return object;
+    acl_free(acl_s);
+
+out:
+    if (IA_ISLNK(object->inode->ia_type) && new_object)
+        glfs_h_close(new_object);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_symlink, 3.4.2);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_acl_get, 3.7.0)
+acl_t
+pub_glfs_h_acl_get(struct glfs *fs, struct glfs_object *object,
+                   const acl_type_t type)
+{
+    int ret = 0;
+    acl_t acl = NULL;
+    char *acl_s = NULL;
+    dict_t *xattr = NULL;
+    const char *acl_key = NULL;
+    struct glfs_object *new_object = NULL;
+
+    DECLARE_OLD_THIS;
+
+    if (!object) {
+        errno = EINVAL;
+        return NULL;
+    }
+
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
 
+    acl_key = gf_posix_acl_get_key(type);
+    if (!acl_key)
+        goto out;
 
+    if (IA_ISLNK(object->inode->ia_type)) {
+        new_object = glfs_h_resolve_symlink(fs, object);
+        if (new_object == NULL)
+            goto out;
+    } else
+        new_object = object;
+
+    ret = glfs_h_getxattrs_common(fs, new_object, &xattr, acl_key, _gf_false);
+    if (ret)
+        goto out;
+
+    ret = dict_get_str(xattr, (char *)acl_key, &acl_s);
+    if (ret)
+        goto out;
+
+    acl = acl_from_text(acl_s);
+
+out:
+    if (xattr)
+        dict_unref(xattr);
+
+    if (IA_ISLNK(object->inode->ia_type) && new_object)
+        glfs_h_close(new_object);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return acl;
+}
+#else /* !HAVE_ACL_LIBACL_H */
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_acl_get, 3.7.0)
+acl_t
+pub_glfs_h_acl_get(struct glfs *fs, struct glfs_object *object,
+                   const acl_type_t type)
+{
+    errno = ENOTSUP;
+    return NULL;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_acl_set, 3.7.0)
 int
-pub_glfs_h_readlink (struct glfs *fs, struct glfs_object *object, char *buf,
-		 size_t bufsiz)
+pub_glfs_h_acl_set(struct glfs *fs, struct glfs_object *object,
+                   const acl_type_t type, const acl_t acl)
 {
-	loc_t               loc = {0, };
-	int                 ret = -1;
-	xlator_t           *subvol = NULL;
-	inode_t            *inode = NULL;
-	char               *linkval = NULL;
-
-	/* validate in args */
-	if ((fs == NULL) || (object == NULL) || (buf == NULL)) {
-		errno = EINVAL;
-		return -1;
-	}
-
-	__glfs_entry_fs (fs);
-
-	/* get the active volume */
-	subvol = priv_glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
-
-	/* get/refresh the in arg objects inode in correlation to the xlator */
-	inode = glfs_resolve_inode (fs, subvol, object);
-	if (!inode) {
-		errno = ESTALE;
-		goto out;
-	}
-
-	GLFS_LOC_FILL_INODE (inode, loc, out);
-
-	/* fop/op */
-	ret = syncop_readlink (subvol, &loc, &linkval, bufsiz);
-
-	/* populate out args */
-	if (ret > 0)
-		memcpy (buf, linkval, ret);
+    errno = ENOTSUP;
+    return -1;
+}
+#endif
 
-out:
-	loc_wipe (&loc);
+/* The API to perform read using anonymous fd */
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_anonymous_read, 3.7.0)
+ssize_t
+pub_glfs_h_anonymous_read(struct glfs *fs, struct glfs_object *object,
+                          const void *buf, size_t count, off_t offset)
+{
+    struct iovec iov = {
+        0,
+    };
+    ssize_t ret = 0;
 
-	if (inode)
-		inode_unref (inode);
+    /* validate in args */
+    if ((fs == NULL) || (object == NULL)) {
+        errno = EINVAL;
+        return -1;
+    }
 
-	if (linkval)
-		GF_FREE (linkval);
+    iov.iov_base = (void *)buf;
+    iov.iov_len = count;
 
-	priv_glfs_subvol_done (fs, subvol);
+    ret = glfs_anonymous_preadv(fs, object, &iov, 1, offset, 0);
 
-	return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_readlink, 3.4.2);
+/* The API to perform write using anonymous fd */
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_anonymous_write, 3.7.0)
+ssize_t
+pub_glfs_h_anonymous_write(struct glfs *fs, struct glfs_object *object,
+                           const void *buf, size_t count, off_t offset)
+{
+    struct iovec iov = {
+        0,
+    };
+    ssize_t ret = 0;
+
+    /* validate in args */
+    if ((fs == NULL) || (object == NULL)) {
+        errno = EINVAL;
+        return -1;
+    }
 
+    iov.iov_base = (void *)buf;
+    iov.iov_len = count;
 
-int
-pub_glfs_h_link (struct glfs *fs, struct glfs_object *linksrc,
-	     struct glfs_object *parent, const char *name)
+    ret = glfs_anonymous_pwritev(fs, object, &iov, 1, offset, 0);
+
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_object_copy, 3.11.0)
+struct glfs_object *
+pub_glfs_object_copy(struct glfs_object *src)
 {
-	int                 ret = -1;
-	xlator_t           *subvol = NULL;
-	inode_t            *inode = NULL;
-	inode_t            *pinode = NULL;
-	loc_t               oldloc = {0, };
-	loc_t               newloc = {0, };
-
-	/* validate in args */
-	if ((fs == NULL) || (linksrc == NULL) || (parent == NULL) ||
-		(name == NULL)) {
-		errno = EINVAL;
-		return -1;
-	}
-
-	__glfs_entry_fs (fs);
-
-	/* get the active volume */
-	subvol = priv_glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
-
-	/* get/refresh the in arg objects inode in correlation to the xlator */
-	inode = glfs_resolve_inode (fs, subvol, linksrc);
-	if (!inode) {
-		errno = ESTALE;
-		goto out;
-	}
-
-	if (inode->ia_type == IA_IFDIR) {
-		ret = -1;
-		errno = EISDIR;
-		goto out;
-	}
-
-	GLFS_LOC_FILL_INODE (inode, oldloc, out);
-
-	/* get/refresh the in arg objects inode in correlation to the xlator */
-	pinode = glfs_resolve_inode (fs, subvol, parent);
-	if (!pinode) {
-		errno = ESTALE;
-		goto out;
-	}
-
-	/* setup newloc based on parent */
-	newloc.parent = inode_ref (pinode);
-	newloc.name = name;
-	ret = priv_glfs_loc_touchup (&newloc);
-	if (ret != 0) {
-		errno = EINVAL;
-		goto out;
-	}
-
-	/* Filling the inode of the hard link to be same as that of the
-	 * original file
-	 */
-	newloc.inode = inode_ref (inode);
-
-	/* fop/op */
-	ret = syncop_link (subvol, &oldloc, &newloc);
-
-	if (ret == 0)
-		/* TODO: No iatt to pass as there has been no lookup */
-		ret = glfs_loc_link (&newloc, NULL);
-out:
-	loc_wipe (&oldloc);
-	loc_wipe (&newloc);
+    struct glfs_object *object = NULL;
 
-	if (inode)
-		inode_unref (inode);
+    GF_VALIDATE_OR_GOTO("glfs_dup_object", src, out);
 
-	if (pinode)
-		inode_unref (pinode);
+    object = GF_CALLOC(1, sizeof(struct glfs_object), glfs_mt_glfs_object_t);
+    if (object == NULL) {
+        errno = ENOMEM;
+        gf_smsg(THIS->name, GF_LOG_WARNING, errno, API_MSG_CREATE_HANDLE_FAILED,
+                "glfs_dup_object gfid=%s", uuid_utoa(src->inode->gfid), NULL);
+        return NULL;
+    }
 
-	priv_glfs_subvol_done (fs, subvol);
+    object->inode = inode_ref(src->inode);
+    gf_uuid_copy(object->gfid, src->inode->gfid);
 
-	return ret;
+out:
+    return object;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_link, 3.4.2);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_xreaddirplus_get_object, 3.11.0)
+struct glfs_object *
+pub_glfs_xreaddirplus_get_object(struct glfs_xreaddirp_stat *xstat)
+{
+    GF_VALIDATE_OR_GOTO("glfs_xreaddirplus_get_object", xstat, out);
+
+    if (!(xstat->flags_handled & GFAPI_XREADDIRP_HANDLE))
+        gf_smsg(THIS->name, GF_LOG_ERROR, errno, API_MSG_HANDLE_NOT_SET,
+                "GFAPI_XREADDIRP_HANDLE xstat=%p", xstat, "handle=%x",
+                xstat->flags_handled, NULL);
+
+    return xstat->object;
 
+out:
+    return NULL;
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_lease, 4.0.0)
 int
-pub_glfs_h_rename (struct glfs *fs, struct glfs_object *olddir, const char *oldname,
-	       struct glfs_object *newdir, const char *newname)
+pub_glfs_h_lease(struct glfs *fs, struct glfs_object *object,
+                 struct glfs_lease *lease)
 {
-	int                 ret = -1;
-	xlator_t           *subvol = NULL;
-	inode_t            *oldpinode = NULL;
-	inode_t            *newpinode = NULL;
-	loc_t               oldloc = {0, };
-	loc_t               newloc = {0, };
-	struct iatt         oldiatt = {0, };
-	struct iatt         newiatt = {0, };
-
-	/* validate in args */
-	if ((fs == NULL) || (olddir == NULL) || (oldname == NULL) ||
-		(newdir == NULL) || (newname == NULL)) {
-		errno = EINVAL;
-		return -1;
-	}
-
-	__glfs_entry_fs (fs);
-
-	/* get the active volume */
-	subvol = priv_glfs_active_subvol (fs);
-	if ( !subvol ) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
-
-	/* get/refresh the in arg objects inode in correlation to the xlator */
-	oldpinode = glfs_resolve_inode (fs, subvol, olddir);
-	if (!oldpinode) {
-		errno = ESTALE;
-		goto out;
-	}
-
-	ret = priv_glfs_resolve_at (fs, subvol, oldpinode, oldname, &oldloc,
-			       &oldiatt, 0 , 0);
-	if (ret != 0) {
-		goto out;
-	}
-
-	/* get/refresh the in arg objects inode in correlation to the xlator */
-	newpinode = glfs_resolve_inode (fs, subvol, newdir);
-	if (!newpinode) {
-		errno = ESTALE;
-		goto out;
-	}
-
-	ret = priv_glfs_resolve_at (fs, subvol, newpinode, newname, &newloc,
-			       &newiatt, 0, 0);
-
-	if (ret && errno != ENOENT && newloc.parent)
-		goto out;
-
-	if (newiatt.ia_type != IA_INVAL) {
-		if ((oldiatt.ia_type == IA_IFDIR) !=
-			(newiatt.ia_type == IA_IFDIR)) {
-			/* Either both old and new must be dirs,
-			 * or both must be non-dirs. Else, fail.
-			 */
-			ret = -1;
-			errno = EISDIR;
-			goto out;
-		}
-	}
-
-	/* TODO: check if new or old is a prefix of the other, and fail EINVAL */
-
-	ret = syncop_rename (subvol, &oldloc, &newloc);
-
-	if (ret == 0)
-		inode_rename (oldloc.parent->table, oldloc.parent, oldloc.name,
-			      newloc.parent, newloc.name, oldloc.inode,
-			      &oldiatt);
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct gf_lease gf_lease = {
+        0,
+    };
+
+    /* validate in args */
+    if ((fs == NULL) || (object == NULL)) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, object);
+    if (!inode) {
+        errno = ESTALE;
+        goto out;
+    }
+
+    /* populate loc */
+    GLFS_LOC_FILL_INODE(inode, loc, out);
+
+    glfs_lease_to_gf_lease(lease, &gf_lease);
+
+    ret = syncop_lease(subvol, &loc, &gf_lease, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    gf_lease_to_glfs_lease(&gf_lease, lease);
 
 out:
-	loc_wipe (&oldloc);
-	loc_wipe (&newloc);
+    loc_wipe(&loc);
 
-	if (oldpinode)
-		inode_unref (oldpinode);
+    if (inode)
+        inode_unref(inode);
 
-	if (newpinode)
-		inode_unref (newpinode);
+    glfs_subvol_done(fs, subvol);
 
-	priv_glfs_subvol_done (fs, subvol);
+    __GLFS_EXIT_FS;
 
-	return ret;
+invalid_fs:
+    return ret;
 }
-
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_rename, 3.4.2);
-
diff --git a/api/src/glfs-handles.h b/api/src/glfs-handles.h
index d095d9e622f..4d039b9c76b 100644
--- a/api/src/glfs-handles.h
+++ b/api/src/glfs-handles.h
@@ -1,5 +1,5 @@
 /*
-  Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+  Copyright (c) 2013-2018 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
   This file is licensed to you under your choice of the GNU Lesser
@@ -46,20 +46,41 @@
  *
  */
 
-/* Values for valid falgs to be used when using XXXsetattr, to set multiple
- attribute values passed via the related stat structure.
- */
-#define GFAPI_SET_ATTR_MODE  0x1
-#define GFAPI_SET_ATTR_UID   0x2
-#define GFAPI_SET_ATTR_GID   0x4
-#define GFAPI_SET_ATTR_SIZE  0x8
-#define GFAPI_SET_ATTR_ATIME 0x10
-#define GFAPI_SET_ATTR_MTIME 0x20
-
 /* Handle length for object handles returned from glfs_h_extract_handle or
  * glfs_h_create_from_handle */
 #define GFAPI_HANDLE_LENGTH 16
 
+/* These flags should be in sync to the ones defined in upcall.h */
+#define GFAPI_UP_NLINK 0x00000001 /* update nlink */
+#define GFAPI_UP_MODE 0x00000002  /* update mode and ctime */
+#define GFAPI_UP_OWN 0x00000004   /* update mode,uid,gid and ctime */
+#define GFAPI_UP_SIZE 0x00000008  /* update fsize */
+#define GFAPI_UP_TIMES 0x00000010 /* update all times */
+#define GFAPI_UP_ATIME 0x00000020 /* update atime only */
+#define GFAPI_UP_PERM                                                          \
+    0x00000040 /* update fields needed for                                     \
+                  permission checking */
+#define GFAPI_UP_RENAME                                                        \
+    0x00000080 /* this is a rename op -                                        \
+                  delete the cache entry */
+#define GFAPI_UP_FORGET                                                        \
+    0x00000100                           /* inode_forget on server side -      \
+                                            invalidate the cache entry */
+#define GFAPI_UP_PARENT_TIMES 0x00000200 /* update parent dir times */
+
+#define GFAPI_INODE_UPDATE_FLAGS                                               \
+    (GFAPI_UP_NLINK | GFAPI_UP_MODE | GFAPI_UP_OWN | GFAPI_UP_SIZE |           \
+     GFAPI_UP_TIMES | GFAPI_UP_ATIME)
+
+/* Portability non glibc c++ build systems */
+#ifndef __THROW
+#if defined __cplusplus
+#define __THROW throw()
+#else
+#define __THROW
+#endif
+#endif
+
 __BEGIN_DECLS
 
 /*
@@ -73,81 +94,261 @@ __BEGIN_DECLS
 struct glfs_object;
 typedef struct glfs_object glfs_object_t;
 
-/* Handle based operations */
-/* Operations that generate handles */
-struct glfs_object *glfs_h_lookupat (struct glfs *fs,
-				     struct glfs_object *parent,
-				     const char *path, struct stat *stat);
+/* Functions for getting details about the glfs_upcall_inode
+ *
+ * None of the pointers returned by the below functions should be free()'d,
+ * glfs_free()'d or glfs_h_close()'d by the application.
+ *
+ * Releasing of the structures is done by passing the glfs_upcall pointer
+ * to glfs_free().
+ */
+struct glfs_upcall_inode;
+typedef struct glfs_upcall_inode glfs_upcall_inode_t;
+
+glfs_object_t *
+glfs_upcall_inode_get_object(glfs_upcall_inode_t *arg) __THROW
+    GFAPI_PUBLIC(glfs_upcall_inode_get_object, 3.7.16);
+
+uint64_t
+glfs_upcall_inode_get_flags(glfs_upcall_inode_t *arg) __THROW
+    GFAPI_PUBLIC(glfs_upcall_inode_get_flags, 3.7.16);
+
+struct stat *
+glfs_upcall_inode_get_stat(glfs_upcall_inode_t *arg) __THROW
+    GFAPI_PUBLIC(glfs_upcall_inode_get_stat, 3.7.16);
+
+uint64_t
+glfs_upcall_inode_get_expire(glfs_upcall_inode_t *arg) __THROW
+    GFAPI_PUBLIC(glfs_upcall_inode_get_expire, 3.7.16);
 
-struct glfs_object *glfs_h_creat (struct glfs *fs, struct glfs_object *parent,
-				  const char *path, int flags, mode_t mode,
-				  struct stat *sb);
+glfs_object_t *
+glfs_upcall_inode_get_pobject(glfs_upcall_inode_t *arg) __THROW
+    GFAPI_PUBLIC(glfs_upcall_inode_get_pobject, 3.7.16);
 
-struct glfs_object *glfs_h_mkdir (struct glfs *fs, struct glfs_object *parent,
-				  const char *path, mode_t flags,
-				  struct stat *sb);
+struct stat *
+glfs_upcall_inode_get_pstat(glfs_upcall_inode_t *arg) __THROW
+    GFAPI_PUBLIC(glfs_upcall_inode_get_pstat, 3.7.16);
 
-struct glfs_object *glfs_h_mknod (struct glfs *fs, struct glfs_object *parent,
-				  const char *path, mode_t mode, dev_t dev,
-				  struct stat *sb);
+glfs_object_t *
+glfs_upcall_inode_get_oldpobject(glfs_upcall_inode_t *arg) __THROW
+    GFAPI_PUBLIC(glfs_upcall_inode_get_oldpobject, 3.7.16);
 
-struct glfs_object *glfs_h_symlink (struct glfs *fs, struct glfs_object *parent,
-				    const char *name, const char *data,
-				    struct stat *stat);
+struct stat *
+glfs_upcall_inode_get_oldpstat(glfs_upcall_inode_t *arg) __THROW
+    GFAPI_PUBLIC(glfs_upcall_inode_get_oldpstat, 3.7.16);
+
+/* Handle based operations */
+/* Operations that generate handles */
+glfs_object_t *
+glfs_h_lookupat(glfs_t *fs, glfs_object_t *parent, const char *path,
+                struct stat *stat, int follow) __THROW
+    GFAPI_PUBLIC(glfs_h_lookupat, 3.7.4);
+
+glfs_object_t *
+glfs_h_creat(glfs_t *fs, glfs_object_t *parent, const char *path, int flags,
+             mode_t mode, struct stat *sb) __THROW
+    GFAPI_PUBLIC(glfs_h_creat, 3.4.2);
+
+glfs_object_t *
+glfs_h_mkdir(glfs_t *fs, glfs_object_t *parent, const char *path, mode_t flags,
+             struct stat *sb) __THROW GFAPI_PUBLIC(glfs_h_mkdir, 3.4.2);
+
+glfs_object_t *
+glfs_h_mknod(glfs_t *fs, glfs_object_t *parent, const char *path, mode_t mode,
+             dev_t dev, struct stat *sb) __THROW
+    GFAPI_PUBLIC(glfs_h_mknod, 3.4.2);
+
+glfs_object_t *
+glfs_h_symlink(glfs_t *fs, glfs_object_t *parent, const char *name,
+               const char *data, struct stat *stat) __THROW
+    GFAPI_PUBLIC(glfs_h_symlink, 3.4.2);
 
 /* Operations on the actual objects */
-int glfs_h_unlink (struct glfs *fs, struct glfs_object *parent,
-		   const char *path);
+int
+glfs_h_unlink(glfs_t *fs, glfs_object_t *parent, const char *path) __THROW
+    GFAPI_PUBLIC(glfs_h_unlink, 3.4.2);
 
-int glfs_h_close (struct glfs_object *object);
+int
+glfs_h_close(glfs_object_t *object) __THROW GFAPI_PUBLIC(glfs_h_close, 3.4.2);
 
-int glfs_caller_specific_init (void *uid_caller_key, void *gid_caller_key,
-			       void *future);
+int
+glfs_caller_specific_init(void *uid_caller_key, void *gid_caller_key,
+                          void *future) __THROW
+    GFAPI_PUBLIC(glfs_caller_specific_init, 3.5.0);
 
-int glfs_h_truncate (struct glfs *fs, struct glfs_object *object, off_t offset);
+int
+glfs_h_truncate(glfs_t *fs, glfs_object_t *object, off_t offset) __THROW
+    GFAPI_PUBLIC(glfs_h_truncate, 3.4.2);
 
-int glfs_h_stat(struct glfs *fs, struct glfs_object *object, struct stat *stat);
+int
+glfs_h_stat(glfs_t *fs, glfs_object_t *object, struct stat *stat) __THROW
+    GFAPI_PUBLIC(glfs_h_stat, 3.4.2);
 
-int glfs_h_getattrs (struct glfs *fs, struct glfs_object *object,
-		     struct stat *stat);
+int
+glfs_h_statfs(glfs_t *fs, glfs_object_t *object, struct statvfs *stat) __THROW
+    GFAPI_PUBLIC(glfs_h_statfs, 3.7.0);
 
-int glfs_h_getxattrs (struct glfs *fs, struct glfs_object *object,
-                      const char *name, void *value,
-                      size_t size) __THROW;
+int
+glfs_h_getattrs(glfs_t *fs, glfs_object_t *object, struct stat *stat) __THROW
+    GFAPI_PUBLIC(glfs_h_getattrs, 3.4.2);
 
-int glfs_h_setattrs (struct glfs *fs, struct glfs_object *object,
-		     struct stat *sb, int valid);
+int
+glfs_h_getxattrs(glfs_t *fs, glfs_object_t *object, const char *name,
+                 void *value, size_t size) __THROW
+    GFAPI_PUBLIC(glfs_h_getxattrs, 3.5.1);
 
-int glfs_h_setxattrs (struct glfs *fs, struct glfs_object *object,
-                      const char *name, const void *value,
-                      size_t size, int flags) __THROW;
+int
+glfs_h_setattrs(glfs_t *fs, glfs_object_t *object, struct stat *sb,
+                int valid) __THROW GFAPI_PUBLIC(glfs_h_setattrs, 3.4.2);
 
-int glfs_h_readlink (struct glfs *fs, struct glfs_object *object, char *buf,
-		     size_t bufsiz);
+int
+glfs_h_setxattrs(glfs_t *fs, glfs_object_t *object, const char *name,
+                 const void *value, size_t size, int flags) __THROW
+    GFAPI_PUBLIC(glfs_h_setxattrs, 3.5.0);
 
-int glfs_h_link (struct glfs *fs, struct glfs_object *linktgt,
-		 struct glfs_object *parent, const char *name);
+int
+glfs_h_readlink(glfs_t *fs, glfs_object_t *object, char *buf,
+                size_t bufsiz) __THROW GFAPI_PUBLIC(glfs_h_readlink, 3.4.2);
 
-int glfs_h_rename (struct glfs *fs, struct glfs_object *olddir,
-		   const char *oldname, struct glfs_object *newdir,
-		   const char *newname);
+int
+glfs_h_link(glfs_t *fs, glfs_object_t *linktgt, glfs_object_t *parent,
+            const char *name) __THROW GFAPI_PUBLIC(glfs_h_link, 3.4.2);
 
-int glfs_h_removexattrs (struct glfs *fs, struct glfs_object *object,
-                         const char *name) __THROW;
+int
+glfs_h_rename(glfs_t *fs, glfs_object_t *olddir, const char *oldname,
+              glfs_object_t *newdir, const char *newname) __THROW
+    GFAPI_PUBLIC(glfs_h_rename, 3.4.2);
+
+int
+glfs_h_removexattrs(glfs_t *fs, glfs_object_t *object, const char *name) __THROW
+    GFAPI_PUBLIC(glfs_h_removexattrs, 3.5.1);
 
 /* Operations enabling opaque invariant handle to object transitions */
-ssize_t glfs_h_extract_handle (struct glfs_object *object,
-			       unsigned char *handle, int len);
+ssize_t
+glfs_h_extract_handle(glfs_object_t *object, unsigned char *handle,
+                      int len) __THROW
+    GFAPI_PUBLIC(glfs_h_extract_handle, 3.4.2);
 
-struct glfs_object *glfs_h_create_from_handle (struct glfs *fs,
-					       unsigned char *handle, int len,
-					       struct stat *stat);
+/* Given a handle, looks up the inode and creates glfs_object.
+ * In addition, if provided 'stat', copies the inode attributes
+ */
+glfs_object_t *
+glfs_h_create_from_handle(glfs_t *fs, unsigned char *handle, int len,
+                          struct stat *stat) __THROW
+    GFAPI_PUBLIC(glfs_h_create_from_handle, 3.4.2);
 
 /* Operations enabling object handles to fd transitions */
-struct glfs_fd *glfs_h_opendir (struct glfs *fs, struct glfs_object *object);
+glfs_fd_t *
+glfs_h_opendir(glfs_t *fs, glfs_object_t *object) __THROW
+    GFAPI_PUBLIC(glfs_h_opendir, 3.4.2);
+
+glfs_fd_t *
+glfs_h_open(glfs_t *fs, glfs_object_t *object, int flags) __THROW
+    GFAPI_PUBLIC(glfs_h_open, 3.4.2);
+
+int
+glfs_h_access(glfs_t *fs, glfs_object_t *object, int mask) __THROW
+    GFAPI_PUBLIC(glfs_h_access, 3.6.0);
+
+struct glfs_object *
+glfs_h_creat_open(struct glfs *fs, struct glfs_object *parent, const char *path,
+                  int flags, mode_t mode, struct stat *stat,
+                  struct glfs_fd **out_fd) __THROW
+    GFAPI_PUBLIC(glfs_h_creat_open, 6.6);
+/*
+  SYNOPSIS
+
+  glfs_h_poll_upcall: Poll for upcall events given a 'glfs' object.
+
+  DESCRIPTION
+
+  This API is used to poll for upcall events stored in the
+  upcall list. Current users of this API is NFS-Ganesha.
+  In case of any event received, it will be mapped appropriately
+  into 'glfs_upcall' along with the handle('glfs_object') to be
+  passed to NFS-Ganesha.
+
+  In case of success, applications need to check the value of
+  cbk->handle to be NON NULL before processing the upcall
+  events.
+
+  PARAMETERS
+
+  @fs: glfs object to poll the upcall events for
+  @cbk: Pointer that will contain an upcall event for use by the application.
+        Application is responsible for free'ing the structure with glfs_free().
+
+  RETURN VALUES
+
+  0   : Success.
+  -1  : Error condition, mostly due to out of memory.
+
+*/
+
+int
+glfs_h_poll_upcall(glfs_t *fs, glfs_upcall_t **cbk) __THROW
+    GFAPI_PUBLIC(glfs_h_poll_upcall, 3.7.16);
+
+int
+glfs_h_acl_set(glfs_t *fs, glfs_object_t *object, const acl_type_t type,
+               const acl_t acl) __THROW GFAPI_PUBLIC(glfs_h_acl_set, 3.7.0);
+
+acl_t
+glfs_h_acl_get(glfs_t *fs, glfs_object_t *object, const acl_type_t type) __THROW
+    GFAPI_PUBLIC(glfs_h_acl_get, 3.7.0);
+
+size_t
+glfs_h_anonymous_write(glfs_t *fs, glfs_object_t *object, const void *buf,
+                       size_t count, off_t offset) __THROW
+    GFAPI_PUBLIC(glfs_h_anonymous_write, 3.7.0);
+
+ssize_t
+glfs_h_anonymous_read(glfs_t *fs, glfs_object_t *object, const void *buf,
+                      size_t count, off_t offset) __THROW
+    GFAPI_PUBLIC(glfs_h_anonymous_read, 3.7.0);
+
+/*
+ * Caution: The object returned by this object gets freed as part
+ * of 'glfs_free(xstat)'. Make sure to have a copy using 'glfs_object_copy()'
+ * to use post that.
+ */
+glfs_object_t *
+glfs_xreaddirplus_get_object(struct glfs_xreaddirp_stat *xstat) __THROW
+    GFAPI_PUBLIC(glfs_xreaddirplus_get_object, 3.11.0);
+
+/* Applications should close the object returned by this routine
+ * explicitly using 'glfs_h_close()'
+ */
+glfs_object_t *
+glfs_object_copy(glfs_object_t *src) __THROW
+    GFAPI_PUBLIC(glfs_object_copy, 3.11.0);
+
+int
+glfs_h_lease(glfs_t *fs, glfs_object_t *object, glfs_lease_t *lease) __THROW
+    GFAPI_PUBLIC(glfs_h_lease, 4.0.0);
+
+glfs_object_t *
+glfs_h_find_handle(glfs_t *fs, unsigned char *handle, int len) __THROW
+    GFAPI_PUBLIC(glfs_h_lease, 4.0.0);
+
+/* Functions for getting details about the glfs_upcall_lease
+ *
+ * None of the pointers returned by the below functions should be free()'d,
+ * glfs_free()'d or glfs_h_close()'d by the application.
+ *
+ * Releasing of the structures is done by passing the glfs_upcall pointer
+ * to glfs_free().
+ */
+struct glfs_upcall_lease;
+typedef struct glfs_upcall_lease glfs_upcall_lease_t;
+
+glfs_object_t *
+glfs_upcall_lease_get_object(glfs_upcall_lease_t *arg) __THROW
+    GFAPI_PUBLIC(glfs_upcall_lease_get_object, 4.1.6);
 
-struct glfs_fd *glfs_h_open (struct glfs *fs, struct glfs_object *object,
-			     int flags);
+uint32_t
+glfs_upcall_lease_get_lease_type(glfs_upcall_lease_t *arg) __THROW
+    GFAPI_PUBLIC(glfs_upcall_lease_get_lease_type, 4.1.6);
 
 __END_DECLS
 
diff --git a/api/src/glfs-internal.h b/api/src/glfs-internal.h
index 6b4b0cbee53..7cc3b18a104 100644
--- a/api/src/glfs-internal.h
+++ b/api/src/glfs-internal.h
@@ -1,5 +1,5 @@
 /*
-  Copyright (c) 2012 Red Hat, Inc. <http://www.redhat.com>
+  Copyright (c) 2012-2018 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
   This file is licensed to you under your choice of the GNU Lesser
@@ -8,136 +8,484 @@
   cases as published by the Free Software Foundation.
 */
 
-
 #ifndef _GLFS_INTERNAL_H
 #define _GLFS_INTERNAL_H
 
-#include "xlator.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/upcall-utils.h>
+#include "glfs-handles.h"
+#include <glusterfs/refcount.h>
+#include <glusterfs/syncop.h>
 
 #define GLFS_SYMLINK_MAX_FOLLOW 2048
 
 #define DEFAULT_REVAL_COUNT 1
 
-#define ESTALE_RETRY(ret,errno,reval,loc,label) do {	\
-	if (ret == -1 && errno == ESTALE) {	        \
-		if (reval < DEFAULT_REVAL_COUNT) {	\
-			reval++;			\
-			loc_wipe (loc);			\
-			goto label;			\
-		}					\
-	}						\
-	} while (0)
-
-#define GLFS_LOC_FILL_INODE(oinode, loc, label) do {   \
-	loc.inode = inode_ref (oinode);                \
-	uuid_copy (loc.gfid, oinode->gfid);            \
-	ret = priv_glfs_loc_touchup (&loc);            \
-	if (ret != 0) {                                \
-		errno = EINVAL;                        \
-		goto label;                            \
-	}                                              \
-	} while (0)
-
-#define GLFS_LOC_FILL_PINODE(pinode, loc, ret, errno, label, path) do {   \
-	loc.inode = inode_new (pinode->table);                            \
-	if (!loc.inode) {                                                 \
-		ret = -1;                                                 \
-		errno = ENOMEM;                                           \
-		goto label;                                               \
-	}                                                                 \
-	loc.parent = inode_ref (pinode);                                  \
-	loc.name = path;                                                  \
-	ret = priv_glfs_loc_touchup (&loc);                               \
-	if (ret != 0) {                                                   \
-		errno = EINVAL;                                           \
-		goto label;                                               \
-	}                                                                 \
-	} while (0)
+/*
+ * According to  pthread mutex and conditional variable ( cond,
+ * child_down_count, upcall mutex and mutex) initialization of struct glfs
+ * members, below GLFS_INIT_* flags are set in 'pthread_flags' member of struct
+ * glfs. The flags are set from glfs_init() and  glfs_new_from_ctx() functions
+ * as part of fs inititialization.
+ *
+ * These flag bits are validated in glfs_fini() to destroy all or partially
+ * initialized mutex and conditional variables of glfs object.
+ * If you introduce new pthread mutex or conditional variable in glfs object,
+ * please make sure you have a flag bit intorduced here for proper cleanup
+ * in glfs_fini().
+ *
+ */
+
+#define PTHREAD_MUTEX_INIT(mutex, attr, flags, mask, label)                    \
+    do {                                                                       \
+        int __ret = -1;                                                        \
+        __ret = pthread_mutex_init(mutex, attr);                               \
+        if (__ret == 0)                                                        \
+            flags |= mask;                                                     \
+        else                                                                   \
+            goto label;                                                        \
+    } while (0)
+
+#define PTHREAD_MUTEX_DESTROY(mutex, flags, mask)                              \
+    do {                                                                       \
+        if (flags & mask)                                                      \
+            (void)pthread_mutex_destroy(mutex);                                \
+    } while (0)
+
+#define PTHREAD_COND_INIT(cond, attr, flags, mask, label)                      \
+    do {                                                                       \
+        int __ret = -1;                                                        \
+        __ret = pthread_cond_init(cond, attr);                                 \
+        if (__ret == 0)                                                        \
+            flags |= mask;                                                     \
+        else                                                                   \
+            goto label;                                                        \
+    } while (0)
+
+#define PTHREAD_COND_DESTROY(cond, flags, mask)                                \
+    do {                                                                       \
+        if (flags & mask)                                                      \
+            (void)pthread_cond_destroy(cond);                                  \
+    } while (0)
+
+#define GLFS_INIT_MUTEX 0x00000001        /* pthread_mutex_flag */
+#define GLFS_INIT_COND 0x00000002         /* pthread_cond_flag */
+#define GLFS_INIT_COND_CHILD 0x00000004   /* pthread_cond_child_down_flag */
+#define GLFS_INIT_MUTEX_UPCALL 0x00000008 /* pthread_mutex_upcall_flag */
+
+#ifndef GF_DARWIN_HOST_OS
+#ifndef GFAPI_PUBLIC
+#define GFAPI_PUBLIC(sym, ver) /**/
+#endif
+#ifndef GFAPI_PRIVATE
+#define GFAPI_PRIVATE(sym, ver) /**/
+#endif
+#if __GNUC__ >= 10
+#define GFAPI_SYMVER_PUBLIC_DEFAULT(fn, ver)                                   \
+    __attribute__((__symver__(STR(fn) "@@GFAPI_" STR(ver))))
+
+#define GFAPI_SYMVER_PRIVATE_DEFAULT(fn, ver)                                  \
+    __attribute__((__symver__(STR(fn) "@@GFAPI_PRIVATE_" STR(ver))))
+
+#define GFAPI_SYMVER_PUBLIC(fn1, fn2, ver)                                     \
+    __attribute__((__symver__(STR(fn2) "@GFAPI_" STR(ver))))
+
+#define GFAPI_SYMVER_PRIVATE(fn1, fn2, ver)                                    \
+    __attribute__((__symver__(STR(fn2) "@GFAPI_PRIVATE_" STR(ver))))
+
+#else
+#define GFAPI_SYMVER_PUBLIC_DEFAULT(fn, ver)                                   \
+    asm(".symver pub_" STR(fn) ", " STR(fn) "@@GFAPI_" STR(ver));
+
+#define GFAPI_SYMVER_PRIVATE_DEFAULT(fn, ver)                                  \
+    asm(".symver priv_" STR(fn) ", " STR(fn) "@@GFAPI_PRIVATE_" STR(ver));
+
+#define GFAPI_SYMVER_PUBLIC(fn1, fn2, ver)                                     \
+    asm(".symver pub_" STR(fn1) ", " STR(fn2) "@GFAPI_" STR(ver));
+
+#define GFAPI_SYMVER_PRIVATE(fn1, fn2, ver)                                    \
+    asm(".symver priv_" STR(fn1) ", " STR(fn2) "@GFAPI_PRIVATE_" STR(ver));
+#endif
+#define STR(str) #str
+#else
+#ifndef GFAPI_PUBLIC
+#define GFAPI_PUBLIC(sym, ver) __asm("_" __STRING(sym) "$GFAPI_" __STRING(ver));
+#endif
+#ifndef GFAPI_PRIVATE
+#define GFAPI_PRIVATE(sym, ver)                                                \
+    __asm("_" __STRING(sym) "$GFAPI_PRIVATE_" __STRING(ver));
+#endif
+#define GFAPI_SYMVER_PUBLIC_DEFAULT(fn, dotver)  /**/
+#define GFAPI_SYMVER_PRIVATE_DEFAULT(fn, dotver) /**/
+#define GFAPI_SYMVER_PUBLIC(fn1, fn2, dotver)    /**/
+#define GFAPI_SYMVER_PRIVATE(fn1, fn2, dotver)   /**/
+#endif
+
+#define ESTALE_RETRY(ret, errno, reval, loc, label)                            \
+    do {                                                                       \
+        if (ret == -1 && errno == ESTALE) {                                    \
+            if (reval < DEFAULT_REVAL_COUNT) {                                 \
+                reval++;                                                       \
+                loc_wipe(loc);                                                 \
+                goto label;                                                    \
+            }                                                                  \
+        }                                                                      \
+    } while (0)
+
+#define GLFS_LOC_FILL_INODE(oinode, loc, label)                                \
+    do {                                                                       \
+        loc.inode = inode_ref(oinode);                                         \
+        gf_uuid_copy(loc.gfid, oinode->gfid);                                  \
+        ret = glfs_loc_touchup(&loc);                                          \
+        if (ret != 0) {                                                        \
+            errno = EINVAL;                                                    \
+            goto label;                                                        \
+        }                                                                      \
+    } while (0)
+
+#define GLFS_LOC_FILL_PINODE(pinode, loc, ret, errno, label, path)             \
+    do {                                                                       \
+        loc.inode = inode_new(pinode->table);                                  \
+        if (!loc.inode) {                                                      \
+            ret = -1;                                                          \
+            errno = ENOMEM;                                                    \
+            goto label;                                                        \
+        }                                                                      \
+        loc.parent = inode_ref(pinode);                                        \
+        loc.name = path;                                                       \
+        ret = glfs_loc_touchup(&loc);                                          \
+        if (ret != 0) {                                                        \
+            errno = EINVAL;                                                    \
+            goto label;                                                        \
+        }                                                                      \
+    } while (0)
 
 struct glfs;
 
-typedef int (*glfs_init_cbk) (struct glfs *fs, int ret);
-
-struct glfs {
-	char               *volname;
-        uuid_t              vol_uuid;
+struct _upcall_entry {
+    struct list_head upcall_list;
+    struct gf_upcall upcall_data;
+};
+typedef struct _upcall_entry upcall_entry;
 
-	glusterfs_ctx_t    *ctx;
+typedef int (*glfs_init_cbk)(struct glfs *fs, int ret);
 
-	pthread_t           poller;
+struct glfs {
+    char *volname;
+    uuid_t vol_uuid;
+
+    glusterfs_ctx_t *ctx;
+
+    pthread_t poller;
+
+    glfs_init_cbk init_cbk;
+    pthread_mutex_t mutex;
+    pthread_cond_t cond;
+    pthread_cond_t child_down_cond; /* for broadcasting CHILD_DOWN */
+    int init;
+    int ret;
+    int err;
+
+    xlator_t *active_subvol; /* active graph */
+    xlator_t *mip_subvol;    /* graph for which migration is in
+                              * progress */
+    xlator_t *next_subvol;   /* Any new graph is put to
+                              * next_subvol, the graph in
+                              * next_subvol can either be moved
+                              * to mip_subvol (if any IO picks it
+                              * up for migration), or be
+                              * destroyed (if there is a new
+                              * graph, and this was never picked
+                              * for migration) */
+    xlator_t *old_subvol;
+
+    char *oldvolfile;
+    ssize_t oldvollen;
+
+    inode_t *cwd;
+
+    uint32_t dev_id; /* Used to fill st_dev in struct stat */
+
+    struct list_head openfds;
+
+    gf_boolean_t migration_in_progress;
+
+    gf_boolean_t cache_upcalls; /* add upcalls to the upcall_list? */
+    struct list_head upcall_list;
+    pthread_mutex_t upcall_list_mutex; /* mutex for upcall entry list */
+
+    uint32_t pin_refcnt;
+    uint32_t pthread_flags; /* GLFS_INIT_* # defines set this flag */
+
+    uint32_t upcall_events; /* Mask of upcall events application
+                             * is interested in */
+    glfs_upcall_cbk up_cbk; /* upcall cbk function to be registered */
+    void *up_data;          /* Opaque data provided by application
+                             * during upcall registration */
+    struct list_head waitq; /* waiting synctasks */
+};
 
-	glfs_init_cbk       init_cbk;
-	pthread_mutex_t     mutex;
-	pthread_cond_t      cond;
-	int                 init;
-	int                 ret;
-	int                 err;
+/* This enum is used to maintain the state of glfd. In case of async fops
+ * fd might be closed before the actual fop is complete. Therefore we need
+ * to track whether the fd is closed or not, instead actually closing it.*/
+enum glfs_fd_state { GLFD_INIT, GLFD_OPEN, GLFD_CLOSE };
 
-	xlator_t           *active_subvol;
-	xlator_t           *next_subvol;
-	xlator_t           *old_subvol;
+struct glfs_fd {
+    struct list_head openfds;
+    struct list_head list;
+    GF_REF_DECL;
+    struct glfs *fs;
+    enum glfs_fd_state state;
+    off_t offset;
+    fd_t *fd; /* Currently guared by @fs->mutex. TODO: per-glfd lock */
+    struct list_head entries;
+    gf_dirent_t *next;
+    struct dirent *readdirbuf;
+    gf_lkowner_t lk_owner;
+    glfs_leaseid_t lease_id; /* Stores lease_id of client in glfd */
+    gf_lock_t lock;          /* lock taken before updating fd state */
+    glfs_recall_cbk cbk;
+    void *cookie;
+};
 
-	char               *oldvolfile;
-	ssize_t             oldvollen;
+/* glfs object handle introduced for the alternate gfapi implementation based
+   on glfs handles/gfid/inode
+*/
+struct glfs_object {
+    inode_t *inode;
+    uuid_t gfid;
+};
 
-	inode_t            *cwd;
+struct glfs_upcall {
+    struct glfs *fs;                /* glfs object */
+    enum glfs_upcall_reason reason; /* Upcall event type */
+    void *event;                    /* changes based in the event type */
+    void (*free_event)(void *);     /* free event after the usage */
+};
 
-	uint32_t            dev_id; /* Used to fill st_dev in struct stat */
+struct glfs_upcall_inode {
+    struct glfs_object *object;      /* Object which need to be acted upon */
+    int flags;                       /* Cache UPDATE/INVALIDATE flags */
+    struct stat buf;                 /* Latest stat of this entry */
+    unsigned int expire_time_attr;   /* the amount of time for which
+                                      * the application need to cache
+                                      * this entry */
+    struct glfs_object *p_object;    /* parent Object to be updated */
+    struct stat p_buf;               /* Latest stat of parent dir handle */
+    struct glfs_object *oldp_object; /* Old parent Object to be updated */
+    struct stat oldp_buf;            /* Latest stat of old parent dir handle */
+};
 
-	struct list_head    openfds;
+struct glfs_upcall_lease {
+    struct glfs_object *object; /* Object which need to be acted upon */
+    uint32_t lease_type;        /* Lease type to which client can downgrade to*/
+};
 
-	gf_boolean_t        migration_in_progress;
+struct glfs_upcall_lease_fd {
+    uint32_t lease_type; /* Lease type to which client can downgrade to*/
+    void *fd_cookie;     /* Object which need to be acted upon */
 };
 
-struct glfs_fd {
-	struct list_head   openfds;
-	struct glfs       *fs;
-	off_t              offset;
-	fd_t              *fd; /* Currently guared by @fs->mutex. TODO: per-glfd lock */
-	struct list_head   entries;
-	gf_dirent_t       *next;
-	struct dirent     *readdirbuf;
+struct glfs_xreaddirp_stat {
+    struct stat
+        st; /* Stat for that dirent - corresponds to GFAPI_XREADDIRP_STAT */
+    struct glfs_object *object; /* handled for GFAPI_XREADDIRP_HANDLE */
+    uint32_t flags_handled;     /* final set of flags successfulyy handled */
 };
 
-/* glfs object handle introduced for the alternate gfapi implementation based
-   on glfs handles/gfid/inode
-*/
-struct glfs_object {
-        inode_t         *inode;
-        uuid_t          gfid;
+#define DEFAULT_EVENT_POOL_SIZE 16384
+#define GF_MEMPOOL_COUNT_OF_DICT_T 4096
+#define GF_MEMPOOL_COUNT_OF_DATA_T (GF_MEMPOOL_COUNT_OF_DICT_T * 4)
+#define GF_MEMPOOL_COUNT_OF_DATA_PAIR_T (GF_MEMPOOL_COUNT_OF_DICT_T * 4)
+
+#define GF_MEMPOOL_COUNT_OF_LRU_BUF_T 256
+
+typedef void(glfs_mem_release_t)(void *ptr);
+
+struct glfs_mem_header {
+    uint32_t magic;
+    size_t nmemb;
+    size_t size;
+    glfs_mem_release_t *release;
 };
 
-#define DEFAULT_EVENT_POOL_SIZE           16384
-#define GF_MEMPOOL_COUNT_OF_DICT_T        4096
-#define GF_MEMPOOL_COUNT_OF_DATA_T        (GF_MEMPOOL_COUNT_OF_DICT_T * 4)
-#define GF_MEMPOOL_COUNT_OF_DATA_PAIR_T   (GF_MEMPOOL_COUNT_OF_DICT_T * 4)
+#define GLFS_MEM_HEADER_SIZE (sizeof(struct glfs_mem_header))
+#define GLFS_MEM_HEADER_MAGIC 0x20170830
 
-int glfs_mgmt_init (struct glfs *fs);
-void priv_glfs_init_done (struct glfs *fs, int ret);
-int glfs_process_volfp (struct glfs *fs, FILE *fp);
-int glfs_resolve (struct glfs *fs, xlator_t *subvol, const char *path, loc_t *loc,
-		  struct iatt *iatt, int reval);
-int glfs_lresolve (struct glfs *fs, xlator_t *subvol, const char *path, loc_t *loc,
-		   struct iatt *iatt, int reval);
-fd_t *glfs_resolve_fd (struct glfs *fs, xlator_t *subvol, struct glfs_fd *glfd);
+static inline void *
+__glfs_calloc(size_t nmemb, size_t size, glfs_mem_release_t release,
+              uint32_t type, const char *typestr)
+{
+    struct glfs_mem_header *header = NULL;
 
-fd_t *__glfs_migrate_fd (struct glfs *fs, xlator_t *subvol, struct glfs_fd *glfd);
+    header = __gf_calloc(nmemb, (size + GLFS_MEM_HEADER_SIZE), type, typestr);
+    if (!header)
+        return NULL;
 
-int glfs_first_lookup (xlator_t *subvol);
+    header->magic = GLFS_MEM_HEADER_MAGIC;
+    header->nmemb = nmemb;
+    header->size = size;
+    header->release = release;
 
-static inline void
-__glfs_entry_fs (struct glfs *fs)
+    return header + 1;
+}
+
+static inline void *
+__glfs_malloc(size_t size, glfs_mem_release_t release, uint32_t type,
+              const char *typestr)
 {
-	THIS = fs->ctx->master;
+    struct glfs_mem_header *header = NULL;
+
+    header = __gf_malloc((size + GLFS_MEM_HEADER_SIZE), type, typestr);
+    if (!header)
+        return NULL;
+
+    header->magic = GLFS_MEM_HEADER_MAGIC;
+    header->nmemb = 1;
+    header->size = size;
+    header->release = release;
+
+    return header + 1;
 }
 
+static inline void *
+__glfs_realloc(void *ptr, size_t size)
+{
+    struct glfs_mem_header *old_header = NULL;
+    struct glfs_mem_header *new_header = NULL;
+    struct glfs_mem_header tmp_header;
+    void *new_ptr = NULL;
+
+    GF_ASSERT(NULL != ptr);
+
+    old_header = (struct glfs_mem_header *)(ptr - GLFS_MEM_HEADER_SIZE);
+    GF_ASSERT(old_header->magic == GLFS_MEM_HEADER_MAGIC);
+    tmp_header = *old_header;
+
+    new_ptr = __gf_realloc(old_header, (size + GLFS_MEM_HEADER_SIZE));
+    if (!new_ptr)
+        return NULL;
+
+    new_header = (struct glfs_mem_header *)new_ptr;
+    *new_header = tmp_header;
+    new_header->size = size;
+
+    return new_header + 1;
+}
 
 static inline void
-__glfs_entry_fd (struct glfs_fd *fd)
+__glfs_free(void *free_ptr)
 {
-	THIS = fd->fd->inode->table->xl->ctx->master;
+    struct glfs_mem_header *header = NULL;
+    void *release_ptr = NULL;
+    int i = 0;
+
+    if (!free_ptr)
+        return;
+
+    header = (struct glfs_mem_header *)(free_ptr - GLFS_MEM_HEADER_SIZE);
+    GF_ASSERT(header->magic == GLFS_MEM_HEADER_MAGIC);
+
+    if (header->release) {
+        release_ptr = free_ptr;
+        for (i = 0; i < header->nmemb; i++) {
+            header->release(release_ptr);
+            release_ptr += header->size;
+        }
+    }
+
+    __gf_free(header);
 }
 
+#define GLFS_CALLOC(nmemb, size, release, type)                                \
+    __glfs_calloc(nmemb, size, release, type, #type)
+
+#define GLFS_MALLOC(size, release, type)                                       \
+    __glfs_malloc(size, release, type, #type)
+
+#define GLFS_REALLOC(ptr, size) __glfs_realloc(ptr, size)
+
+#define GLFS_FREE(free_ptr) __glfs_free(free_ptr)
+
+int
+glfs_mgmt_init(struct glfs *fs);
+void
+glfs_init_done(struct glfs *fs, int ret) GFAPI_PRIVATE(glfs_init_done, 3.4.0);
+int
+glfs_process_volfp(struct glfs *fs, FILE *fp);
+int
+glfs_resolve(struct glfs *fs, xlator_t *subvol, const char *path, loc_t *loc,
+             struct iatt *iatt, int reval) GFAPI_PRIVATE(glfs_resolve, 3.7.0);
+int
+glfs_lresolve(struct glfs *fs, xlator_t *subvol, const char *path, loc_t *loc,
+              struct iatt *iatt, int reval);
+fd_t *
+glfs_resolve_fd(struct glfs *fs, xlator_t *subvol, struct glfs_fd *glfd);
+
+fd_t *
+__glfs_migrate_fd(struct glfs *fs, xlator_t *subvol, struct glfs_fd *glfd);
+
+int
+glfs_first_lookup(xlator_t *subvol);
+
+void
+glfs_process_upcall_event(struct glfs *fs, void *data)
+    GFAPI_PRIVATE(glfs_process_upcall_event, 3.7.0);
+
+#define __GLFS_ENTRY_VALIDATE_FS(fs, label)                                    \
+    do {                                                                       \
+        if (!fs) {                                                             \
+            errno = EINVAL;                                                    \
+            goto label;                                                        \
+        }                                                                      \
+        old_THIS = THIS;                                                       \
+        THIS = fs->ctx->master;                                                \
+    } while (0)
+
+#define __GLFS_EXIT_FS                                                         \
+    do {                                                                       \
+        THIS = old_THIS;                                                       \
+    } while (0)
+
+#define __GLFS_ENTRY_VALIDATE_FD(glfd, label)                                  \
+    do {                                                                       \
+        if (!glfd || !glfd->fd || !glfd->fd->inode ||                          \
+            glfd->state != GLFD_OPEN) {                                        \
+            errno = EBADF;                                                     \
+            goto label;                                                        \
+        }                                                                      \
+        old_THIS = THIS;                                                       \
+        THIS = glfd->fd->inode->table->xl->ctx->master;                        \
+    } while (0)
+
+#define __GLFS_LOCK_WAIT(fs)                                                   \
+    do {                                                                       \
+        struct synctask *task = NULL;                                          \
+                                                                               \
+        task = synctask_get();                                                 \
+                                                                               \
+        if (task) {                                                            \
+            list_add_tail(&task->waitq, &fs->waitq);                           \
+            pthread_mutex_unlock(&fs->mutex);                                  \
+            synctask_yield(task, NULL);                                        \
+            pthread_mutex_lock(&fs->mutex);                                    \
+        } else {                                                               \
+            /* non-synctask */                                                 \
+            pthread_cond_wait(&fs->cond, &fs->mutex);                          \
+        }                                                                      \
+    } while (0)
+
+#define __GLFS_SYNCTASK_WAKE(fs)                                               \
+    do {                                                                       \
+        struct synctask *waittask = NULL;                                      \
+                                                                               \
+        while (!list_empty(&fs->waitq)) {                                      \
+            waittask = list_entry(fs->waitq.next, struct synctask, waitq);     \
+            list_del_init(&waittask->waitq);                                   \
+            synctask_wake(waittask);                                           \
+        }                                                                      \
+    } while (0)
 
 /*
   By default all lock attempts from user context must
@@ -146,69 +494,263 @@ __glfs_entry_fd (struct glfs_fd *fd)
   we can give up the mutex during syncop calls so
   that bottom up calls (particularly CHILD_UP notify)
   can do a mutex_lock() on @glfs without deadlocking
-  the filesystem
+  the filesystem.
+
+  All the fops should wait for graph migration to finish
+  before starting the fops. Therefore these functions should
+  call glfs_lock with wait_for_migration as true. But waiting
+  for migration to finish in call-back path can result thread
+  dead-locks. The reason for this is we only have finite
+  number of epoll threads. so if we wait on epoll threads
+  there will not be any thread left to handle outstanding
+  rpc replies.
 */
 static inline int
-glfs_lock (struct glfs *fs)
+glfs_lock(struct glfs *fs, gf_boolean_t wait_for_migration)
 {
-	pthread_mutex_lock (&fs->mutex);
+    pthread_mutex_lock(&fs->mutex);
 
-	while (!fs->init)
-		pthread_cond_wait (&fs->cond, &fs->mutex);
+    while (!fs->init)
+        __GLFS_LOCK_WAIT(fs);
 
-	while (fs->migration_in_progress)
-		pthread_cond_wait (&fs->cond, &fs->mutex);
+    while (wait_for_migration && fs->migration_in_progress)
+        __GLFS_LOCK_WAIT(fs);
 
-	return 0;
+    return 0;
 }
 
-
 static inline void
-glfs_unlock (struct glfs *fs)
+glfs_unlock(struct glfs *fs)
 {
-	pthread_mutex_unlock (&fs->mutex);
+    pthread_mutex_unlock(&fs->mutex);
 }
 
+struct glfs_fd *
+glfs_fd_new(struct glfs *fs);
+void
+glfs_fd_bind(struct glfs_fd *glfd);
+void
+glfd_set_state_bind(struct glfs_fd *glfd);
+
+xlator_t *
+glfs_active_subvol(struct glfs *fs) GFAPI_PRIVATE(glfs_active_subvol, 3.4.0);
+xlator_t *
+__glfs_active_subvol(struct glfs *fs);
+void
+glfs_subvol_done(struct glfs *fs, xlator_t *subvol)
+    GFAPI_PRIVATE(glfs_subvol_done, 3.4.0);
+
+inode_t *
+glfs_refresh_inode(xlator_t *subvol, inode_t *inode);
+
+inode_t *
+glfs_cwd_get(struct glfs *fs);
+int
+glfs_cwd_set(struct glfs *fs, inode_t *inode);
+inode_t *
+glfs_resolve_inode(struct glfs *fs, xlator_t *subvol,
+                   struct glfs_object *object);
+int
+glfs_create_object(loc_t *loc, struct glfs_object **retobject);
+int
+__glfs_cwd_set(struct glfs *fs, inode_t *inode);
+
+int
+glfs_resolve_base(struct glfs *fs, xlator_t *subvol, inode_t *inode,
+                  struct iatt *iatt);
+
+int
+glfs_resolve_at(struct glfs *fs, xlator_t *subvol, inode_t *at,
+                const char *origpath, loc_t *loc, struct iatt *iatt, int follow,
+                int reval) GFAPI_PRIVATE(glfs_resolve_at, 3.4.0);
+int
+glfs_loc_touchup(loc_t *loc) GFAPI_PRIVATE(glfs_loc_touchup, 3.4.0);
+void
+glfs_iatt_to_stat(struct glfs *fs, struct iatt *iatt, struct stat *stat);
+void
+glfs_iatt_from_stat(struct stat *stat, int valid, struct iatt *iatt,
+                    int *gvalid);
+int
+glfs_loc_link(loc_t *loc, struct iatt *iatt);
+int
+glfs_loc_unlink(loc_t *loc);
+int
+glfs_getxattr_process(void *value, size_t size, dict_t *xattr,
+                      const char *name);
 
-void glfs_fd_destroy (struct glfs_fd *glfd);
+/* Sends RPC call to glusterd to fetch required volume info */
+int
+glfs_get_volume_info(struct glfs *fs);
 
-struct glfs_fd *glfs_fd_new (struct glfs *fs);
-void glfs_fd_bind (struct glfs_fd *glfd);
+/*
+  SYNOPSIS
 
-xlator_t *priv_glfs_active_subvol (struct glfs *fs);
-xlator_t *__glfs_active_subvol (struct glfs *fs);
-void priv_glfs_subvol_done (struct glfs *fs, xlator_t *subvol);
+       glfs_new_from_ctx: Creates a virtual mount object by taking a
+       glusterfs_ctx_t object.
 
-inode_t *glfs_refresh_inode (xlator_t *subvol, inode_t *inode);
+  DESCRIPTION
 
-inode_t *glfs_cwd_get (struct glfs *fs);
-int glfs_cwd_set (struct glfs *fs, inode_t *inode);
-inode_t *glfs_resolve_inode (struct glfs *fs, xlator_t *subvol,
-			     struct glfs_object *object);
-int glfs_create_object (loc_t *loc, struct glfs_object **retobject);
-int __glfs_cwd_set (struct glfs *fs, inode_t *inode);
+       glfs_new_from_ctx() is not same as glfs_new(). It takes the
+       glusterfs_ctx_t object instead of creating one by glusterfs_ctx_new().
+       Again the usage is restricted to NFS MOUNT over UDP i.e. in
+       glfs_resolve_at() which would take fs object as input but never use
+       (purpose is not to change the ABI of glfs_resolve_at()).
 
-int glfs_resolve_base (struct glfs *fs, xlator_t *subvol, inode_t *inode,
-		       struct iatt *iatt);
-int priv_glfs_resolve_at (struct glfs *fs, xlator_t *subvol, inode_t *at,
-                     const char *origpath, loc_t *loc, struct iatt *iatt,
-                     int follow, int reval);
-int priv_glfs_loc_touchup (loc_t *loc);
-void glfs_iatt_to_stat (struct glfs *fs, struct iatt *iatt, struct stat *stat);
-int glfs_loc_link (loc_t *loc, struct iatt *iatt);
-int glfs_loc_unlink (loc_t *loc);
-dict_t * dict_for_key_value (const char *name, const char *value, size_t size);
-int glfs_getxattr_process (void *value, size_t size, dict_t *xattr,
-                           const char *name);
+  PARAMETERS
 
-/* Sends RPC call to glusterd to fetch required volume info */
-int glfs_get_volume_info (struct glfs *fs);
+       @ctx: glusterfs_ctx_t object
 
-#define GFAPI_SYMVER_PUBLIC_DEFAULT(fn, dotver) \
-        asm(".symver pub_"STR(fn)", "STR(fn)"@@GFAPI_"STR(dotver))
+  RETURN VALUES
 
-#define GFAPI_SYMVER_PRIVATE_DEFAULT(fn, dotver) \
-        asm(".symver priv_"STR(fn)", "STR(fn)"@@GFAPI_PRIVATE_"STR(dotver))
-#define STR(str) #str
+       fs     : Pointer to the newly created glfs_t object.
+       NULL   : Otherwise.
+*/
+
+struct glfs *
+glfs_new_from_ctx(glusterfs_ctx_t *ctx) GFAPI_PRIVATE(glfs_new_from_ctx, 3.7.0);
+
+/*
+  SYNOPSIS
+
+       glfs_free_from_ctx: Free up the memory occupied by glfs_t object
+       created by glfs_new_from_ctx().
+
+  DESCRIPTION
+
+       The glfs_t object allocated by glfs_new_from_ctx() must be released
+       by the caller using this routine. The usage can be found
+       at glfs_fini() or NFS, MOUNT over UDP i.e.
+                        __mnt3udp_get_export_subdir_inode ()
+                                => glfs_resolve_at().
+
+  PARAMETERS
+
+       @fs: The glfs_t object to be deallocated.
+
+  RETURN VALUES
+
+       void
+*/
+
+void
+glfs_free_from_ctx(struct glfs *fs) GFAPI_PRIVATE(glfs_free_from_ctx, 3.7.0);
+
+int
+glfs_recall_lease_fd(struct glfs *fs, struct gf_upcall *up_data);
+
+int
+glfs_get_upcall_cache_invalidation(struct gf_upcall *to_up_data,
+                                   struct gf_upcall *from_up_data);
+int
+glfs_h_poll_cache_invalidation(struct glfs *fs, struct glfs_upcall *up_arg,
+                               struct gf_upcall *upcall_data);
+
+ssize_t
+glfs_anonymous_preadv(struct glfs *fs, struct glfs_object *object,
+                      const struct iovec *iovec, int iovcnt, off_t offset,
+                      int flags);
+ssize_t
+glfs_anonymous_pwritev(struct glfs *fs, struct glfs_object *object,
+                       const struct iovec *iovec, int iovcnt, off_t offset,
+                       int flags);
+
+struct glfs_object *
+glfs_h_resolve_symlink(struct glfs *fs, struct glfs_object *object);
+
+/* Deprecated structures that were passed to client applications, replaced by
+ * accessor functions. Do not use these in new applications, and update older
+ * usage.
+ *
+ * See http://review.gluster.org/14701 for more details.
+ *
+ * WARNING: These structures will be removed in the future.
+ */
+struct glfs_callback_arg {
+    struct glfs *fs;
+    enum glfs_upcall_reason reason;
+    void *event_arg;
+};
 
+struct glfs_callback_inode_arg {
+    struct glfs_object *object;      /* Object which need to be acted upon */
+    int flags;                       /* Cache UPDATE/INVALIDATE flags */
+    struct stat buf;                 /* Latest stat of this entry */
+    unsigned int expire_time_attr;   /* the amount of time for which
+                                      * the application need to cache
+                                      * this entry
+                                      */
+    struct glfs_object *p_object;    /* parent Object to be updated */
+    struct stat p_buf;               /* Latest stat of parent dir handle */
+    struct glfs_object *oldp_object; /* Old parent Object
+                                      * to be updated */
+    struct stat oldp_buf;            /* Latest stat of old parent
+                                      * dir handle */
+};
+struct dirent *
+glfs_readdirbuf_get(struct glfs_fd *glfd);
+
+gf_dirent_t *
+glfd_entry_next(struct glfs_fd *glfd, int plus);
+
+void
+gf_dirent_to_dirent(gf_dirent_t *gf_dirent, struct dirent *dirent);
+
+void
+gf_lease_to_glfs_lease(struct gf_lease *gf_lease, struct glfs_lease *lease);
+
+void
+glfs_lease_to_gf_lease(struct glfs_lease *lease, struct gf_lease *gf_lease);
+
+void
+glfs_release_upcall(void *ptr);
+
+int
+get_fop_attr_glfd(dict_t **fop_attr, struct glfs_fd *glfd);
+
+int
+set_fop_attr_glfd(struct glfs_fd *glfd);
+
+int
+get_fop_attr_thrd_key(dict_t **fop_attr);
+
+void
+unset_fop_attr(dict_t **fop_attr);
+
+/*
+  SYNOPSIS
+  glfs_statx: Fetch extended file attributes for the given path.
+
+  DESCRIPTION
+  This function fetches extended file attributes for the given path.
+
+  PARAMETERS
+  @fs: The 'virtual mount' object referencing a volume, under which file exists.
+  @path: Path of the file within the virtual mount.
+  @mask: Requested extended file attributes mask, (See mask defines above)
+
+  RETURN VALUES
+  -1 : Failure. @errno will be set with the type of failure.
+  0  : Filled in statxbuf with appropriate masks for valid items in the
+       structure.
+
+  ERRNO VALUES
+    EINVAL: fs is invalid
+    EINVAL: mask has unsupported bits set
+    Other errors as returned by stat(2)
+ */
+
+int
+glfs_statx(struct glfs *fs, const char *path, unsigned int mask,
+           struct glfs_stat *statxbuf) GFAPI_PRIVATE(glfs_statx, 6.0);
+
+void
+glfs_iatt_from_statx(struct iatt *, const struct glfs_stat *)
+    GFAPI_PRIVATE(glfs_iatt_from_statx, 6.0);
+
+/*
+ * This API is a per thread setting, similar to glfs_setfs{u/g}id, because of
+ * the call to syncopctx_setfspid.
+ */
+int
+glfs_setfspid(struct glfs *, pid_t) GFAPI_PRIVATE(glfs_setfspid, 6.1);
 #endif /* !_GLFS_INTERNAL_H */
diff --git a/api/src/glfs-master.c b/api/src/glfs-master.c
index 89017bab84d..100dcc16cc0 100644
--- a/api/src/glfs-master.c
+++ b/api/src/glfs-master.c
@@ -1,5 +1,5 @@
 /*
-  Copyright (c) 2012 Red Hat, Inc. <http://www.redhat.com>
+  Copyright (c) 2012-2016 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
   This file is licensed to you under your choice of the GNU Lesser
@@ -8,153 +8,176 @@
   cases as published by the Free Software Foundation.
 */
 
-#include <unistd.h>
-#include <string.h>
-#include <stdlib.h>
 #include <stdio.h>
-#include <inttypes.h>
-#include <limits.h>
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "xlator.h"
-#include "glusterfs.h"
+#include <glusterfs/glusterfs.h>
 
 #include "glfs-internal.h"
 #include "glfs-mem-types.h"
-
-
-extern void
-glfs_subvol_done (struct glfs *, xlator_t *);
+#include "gfapi-messages.h"
 
 int
-graph_setup (struct glfs *fs, glusterfs_graph_t *graph)
+graph_setup(struct glfs *fs, glusterfs_graph_t *graph)
 {
-	xlator_t      *new_subvol = NULL;
-	xlator_t      *old_subvol = NULL;
-	inode_table_t *itable = NULL;
-	int            ret = -1;
-
-	new_subvol = graph->top;
-
-	/* This is called in a bottom-up context, it should specifically
-	   NOT be glfs_lock()
-	*/
-	pthread_mutex_lock (&fs->mutex);
-	{
-		if (new_subvol->switched ||
-		    new_subvol == fs->active_subvol ||
-		    new_subvol == fs->next_subvol) {
-			/* Spurious CHILD_UP event on old graph */
-			ret = 0;
-			goto unlock;
-		}
-
-		if (!new_subvol->itable) {
-			itable = inode_table_new (131072, new_subvol);
-			if (!itable) {
-				errno = ENOMEM;
-				ret = -1;
-				goto unlock;
-			}
-
-			new_subvol->itable = itable;
-		}
-
-		old_subvol = fs->next_subvol;
-		fs->next_subvol = new_subvol;
-		fs->next_subvol->winds++; /* first ref */
-		ret = 0;
-	}
+    xlator_t *new_subvol = NULL;
+    xlator_t *old_subvol = NULL;
+    inode_table_t *itable = NULL;
+    int ret = -1;
+
+    new_subvol = graph->top;
+
+    /* This is called in a bottom-up context, it should specifically
+       NOT be glfs_lock()
+    */
+    pthread_mutex_lock(&fs->mutex);
+    {
+        if (new_subvol->switched || new_subvol == fs->active_subvol ||
+            new_subvol == fs->next_subvol || new_subvol == fs->mip_subvol) {
+            /* Spurious CHILD_UP event on old graph */
+            ret = 0;
+            goto unlock;
+        }
+
+        if (!new_subvol->itable) {
+            itable = inode_table_new(131072, new_subvol);
+            if (!itable) {
+                errno = ENOMEM;
+                ret = -1;
+                goto unlock;
+            }
+
+            new_subvol->itable = itable;
+        }
+
+        old_subvol = fs->next_subvol;
+        fs->next_subvol = new_subvol;
+        fs->next_subvol->winds++; /* first ref */
+        ret = 0;
+    }
 unlock:
-	pthread_mutex_unlock (&fs->mutex);
+    pthread_mutex_unlock(&fs->mutex);
 
-	if (old_subvol)
-		/* wasn't picked up so far, skip */
-		glfs_subvol_done (fs, old_subvol);
+    if (old_subvol)
+        /* wasn't picked up so far, skip */
+        glfs_subvol_done(fs, old_subvol);
 
-	return ret;
+    return ret;
 }
 
-
-extern void
-glfs_init_done (struct glfs *fs, int ret);
-
 int
-notify (xlator_t *this, int event, void *data, ...)
+notify(xlator_t *this, int event, void *data, ...)
 {
-	glusterfs_graph_t   *graph = NULL;
-	struct glfs	    *fs = NULL;
-
-	graph = data;
-	fs = this->private;
-
-	switch (event) {
-	case GF_EVENT_GRAPH_NEW:
-		gf_log (this->name, GF_LOG_INFO, "New graph %s (%d) coming up",
-			uuid_utoa ((unsigned char *)graph->graph_uuid),
-			graph->id);
-		break;
-	case GF_EVENT_CHILD_UP:
-		graph_setup (fs, graph);
-		glfs_init_done (fs, 0);
-		break;
-	case GF_EVENT_CHILD_DOWN:
-		graph_setup (fs, graph);
-		glfs_init_done (fs, 1);
-		break;
-	case GF_EVENT_CHILD_CONNECTING:
-		break;
-	default:
-		gf_log (this->name, GF_LOG_DEBUG,
-			"got notify event %d", event);
-		break;
-	}
-
-	return 0;
+    glusterfs_graph_t *graph = NULL;
+    struct glfs *fs = NULL;
+
+    graph = data;
+    fs = this->private;
+
+    switch (event) {
+        case GF_EVENT_GRAPH_NEW:
+            gf_smsg(this->name, GF_LOG_INFO, 0, API_MSG_NEW_GRAPH,
+                    "graph-uuid=%s",
+                    uuid_utoa((unsigned char *)graph->graph_uuid), "id=%d",
+                    graph->id, NULL);
+            break;
+        case GF_EVENT_CHILD_UP:
+            pthread_mutex_lock(&fs->mutex);
+            {
+                graph->used = 1;
+            }
+            pthread_mutex_unlock(&fs->mutex);
+            graph_setup(fs, graph);
+            glfs_init_done(fs, 0);
+            break;
+        case GF_EVENT_CHILD_DOWN:
+            pthread_mutex_lock(&fs->mutex);
+            {
+                graph->used = 0;
+                pthread_cond_broadcast(&fs->child_down_cond);
+            }
+            pthread_mutex_unlock(&fs->mutex);
+            glfs_init_done(fs, 1);
+            break;
+        case GF_EVENT_CHILD_CONNECTING:
+            break;
+        case GF_EVENT_UPCALL:
+            glfs_process_upcall_event(fs, data);
+            break;
+        default:
+            gf_msg_debug(this->name, 0, "got notify event %d", event);
+            break;
+    }
+
+    return 0;
 }
 
-
 int
-mem_acct_init (xlator_t *this)
+mem_acct_init(xlator_t *this)
 {
-	int ret = -1;
+    int ret = -1;
 
-	if (!this)
-		return ret;
+    if (!this)
+        return ret;
 
-	ret = xlator_mem_acct_init (this, glfs_mt_end + 1);
-	if (ret) {
-		gf_log (this->name, GF_LOG_ERROR, "Failed to initialise "
-                        "memory accounting");
-		return ret;
-	}
+    ret = xlator_mem_acct_init(this, glfs_mt_end + 1);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, ENOMEM, API_MSG_MEM_ACCT_INIT_FAILED,
+                NULL);
+        return ret;
+    }
 
-	return 0;
+    return 0;
 }
 
-
 int
-init (xlator_t *this)
+init(xlator_t *this)
 {
-	return 0;
+    return 0;
 }
 
-
 void
-fini (xlator_t *this)
+fini(xlator_t *this)
+{
+}
+
+/* place-holder fops */
+int
+glfs_forget(xlator_t *this, inode_t *inode)
 {
+    return 0;
+}
 
+int
+glfs_release(xlator_t *this, fd_t *fd)
+{
+    return 0;
 }
 
+int
+glfs_releasedir(xlator_t *this, fd_t *fd)
+{
+    return 0;
+}
 
 struct xlator_dumpops dumpops;
 
-
 struct xlator_fops fops;
 
-
-struct xlator_cbks cbks;
+struct xlator_cbks cbks = {
+    .forget = glfs_forget,
+    .release = glfs_release,
+    .releasedir = glfs_releasedir,
+};
+
+xlator_api_t xlator_api = {
+    .init = init,
+    .fini = fini,
+    .notify = notify,
+    .mem_acct_init = mem_acct_init,
+    .op_version = {1},
+    .dumpops = &dumpops,
+    .fops = &fops,
+    .cbks = &cbks,
+    .identifier = "glfs-api",
+    .category = GF_MAINTAINED,
+};
diff --git a/api/src/glfs-mem-types.h b/api/src/glfs-mem-types.h
index 76f4fc774cc..bfa325a3ad9 100644
--- a/api/src/glfs-mem-types.h
+++ b/api/src/glfs-mem-types.h
@@ -1,5 +1,5 @@
 /*
-  Copyright (c) 2012 Red Hat, Inc. <http://www.redhat.com>
+  Copyright (c) 2012-2017 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
   This file is licensed to you under your choice of the GNU Lesser
@@ -11,22 +11,25 @@
 #ifndef _GLFS_MEM_TYPES_H
 #define _GLFS_MEM_TYPES_H
 
-#include "mem-types.h"
+#include <glusterfs/mem-types.h>
 
 #define GF_MEM_TYPE_START (gf_common_mt_end + 1)
 
 enum glfs_mem_types_ {
-        glfs_mt_glfs_t = GF_MEM_TYPE_START,
-        glfs_mt_call_pool_t,
-        glfs_mt_xlator_t,
-	glfs_mt_glfs_fd_t,
-	glfs_mt_glfs_io_t,
-	glfs_mt_volfile_t,
-	glfs_mt_xlator_cmdline_option_t,
-        glfs_mt_server_cmdline_t,
-	glfs_mt_glfs_object_t,
-	glfs_mt_readdirbuf_t,
-	glfs_mt_end
-
+    glfs_mt_call_pool_t = GF_MEM_TYPE_START,
+    glfs_mt_xlator_t,
+    glfs_mt_glfs_fd_t,
+    glfs_mt_glfs_io_t,
+    glfs_mt_volfile_t,
+    glfs_mt_xlator_cmdline_option_t,
+    glfs_mt_server_cmdline_t,
+    glfs_mt_glfs_object_t,
+    glfs_mt_readdirbuf_t,
+    glfs_mt_upcall_entry_t,
+    glfs_mt_acl_t,
+    glfs_mt_upcall_inode_t,
+    glfs_mt_realpath_t,
+    glfs_mt_xreaddirp_stat_t,
+    glfs_mt_end
 };
 #endif
diff --git a/api/src/glfs-mgmt.c b/api/src/glfs-mgmt.c
index c3ace84fb98..7c82b8cd162 100644
--- a/api/src/glfs-mgmt.c
+++ b/api/src/glfs-mgmt.c
@@ -1,5 +1,5 @@
 /*
-  Copyright (c) 2012 Red Hat, Inc. <http://www.redhat.com>
+  Copyright (c) 2012-2018 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
   This file is licensed to you under your choice of the GNU Lesser
@@ -8,7 +8,6 @@
   cases as published by the Free Software Foundation.
 */
 
-
 #include <stdio.h>
 #include <sys/types.h>
 #include <sys/wait.h>
@@ -16,824 +15,1035 @@
 #include <signal.h>
 #include <pthread.h>
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif /* _CONFIG_H */
-
-#include "glusterfs.h"
+#include <glusterfs/glusterfs.h>
 #include "glfs.h"
-#include "stack.h"
-#include "dict.h"
-#include "event.h"
-#include "defaults.h"
+#include <glusterfs/dict.h>
 
 #include "rpc-clnt.h"
 #include "protocol-common.h"
-#include "glusterfs3.h"
-#include "portmap-xdr.h"
 #include "xdr-generic.h"
+#include "rpc-common-xdr.h"
 
-#include "syncop.h"
-#include "xlator.h"
+#include <glusterfs/syncop.h>
 
 #include "glfs-internal.h"
-#include "glfs-mem-types.h"
+#include "gfapi-messages.h"
+#include <glusterfs/syscall.h>
 
-int glfs_volfile_fetch (struct glfs *fs);
-int32_t glfs_get_volume_info_rpc (call_frame_t *frame, xlator_t *this,
-                                  struct glfs *fs);
+int
+glfs_volfile_fetch(struct glfs *fs);
+int32_t
+glfs_get_volume_info_rpc(call_frame_t *frame, xlator_t *this, struct glfs *fs);
 
 int
-glfs_process_volfp (struct glfs *fs, FILE *fp)
+glfs_process_volfp(struct glfs *fs, FILE *fp)
 {
-	glusterfs_graph_t  *graph = NULL;
-	int		    ret = -1;
-	xlator_t	   *trav = NULL;
-	glusterfs_ctx_t	   *ctx = NULL;
-
-	ctx = fs->ctx;
-	graph = glusterfs_graph_construct (fp);
-	if (!graph) {
-		gf_log ("glfs", GF_LOG_ERROR, "failed to construct the graph");
-		goto out;
-	}
-
-	for (trav = graph->first; trav; trav = trav->next) {
-		if (strcmp (trav->type, "mount/fuse") == 0) {
-			gf_log ("glfs", GF_LOG_ERROR,
-				"fuse xlator cannot be specified "
-				"in volume file");
-			goto out;
-		}
-	}
-
-	ret = glusterfs_graph_prepare (graph, ctx);
-	if (ret) {
-		glusterfs_graph_destroy (graph);
-		goto out;
-	}
-
-	ret = glusterfs_graph_activate (graph, ctx);
-
-	if (ret) {
-		glusterfs_graph_destroy (graph);
-		goto out;
-	}
-
-	ret = 0;
+    glusterfs_graph_t *graph = NULL;
+    int ret = -1;
+    xlator_t *trav = NULL;
+    glusterfs_ctx_t *ctx = NULL;
+
+    ctx = fs->ctx;
+    graph = glusterfs_graph_construct(fp);
+    if (!graph) {
+        gf_smsg("glfs", GF_LOG_ERROR, errno, API_MSG_GRAPH_CONSTRUCT_FAILED,
+                NULL);
+        goto out;
+    }
+
+    for (trav = graph->first; trav; trav = trav->next) {
+        if (strcmp(trav->type, "mount/api") == 0) {
+            gf_smsg("glfs", GF_LOG_ERROR, EINVAL, API_MSG_API_XLATOR_ERROR,
+                    NULL);
+            goto out;
+        }
+    }
+
+    ret = glusterfs_graph_prepare(graph, ctx, fs->volname);
+    if (ret) {
+        glusterfs_graph_destroy(graph);
+        goto out;
+    }
+
+    ret = glusterfs_graph_activate(graph, ctx);
+
+    if (ret) {
+        glusterfs_graph_destroy(graph);
+        goto out;
+    }
+
+    gf_log_dump_graph(fp, graph);
+
+    ret = 0;
 out:
-	if (fp)
-		fclose (fp);
+    if (fp)
+        fclose(fp);
 
-	if (!ctx->active) {
-		ret = -1;
-	}
+    if (!ctx->active) {
+        ret = -1;
+    }
 
-	return ret;
+    return ret;
 }
 
-
 int
-mgmt_cbk_spec (struct rpc_clnt *rpc, void *mydata, void *data)
+mgmt_cbk_spec(struct rpc_clnt *rpc, void *mydata, void *data)
 {
-	struct glfs *fs = NULL;
-	xlator_t    *this = NULL;
+    struct glfs *fs = NULL;
+    xlator_t *this = NULL;
 
-	this = mydata;
-	fs = this->private;
+    this = mydata;
+    fs = this->private;
 
-	glfs_volfile_fetch (fs);
+    glfs_volfile_fetch(fs);
 
-	return 0;
+    return 0;
 }
 
-
 int
-mgmt_cbk_event (struct rpc_clnt *rpc, void *mydata, void *data)
+mgmt_cbk_event(struct rpc_clnt *rpc, void *mydata, void *data)
 {
-	return 0;
+    return 0;
 }
 
+static int
+mgmt_cbk_statedump(struct rpc_clnt *rpc, void *mydata, void *data)
+{
+    struct glfs *fs = NULL;
+    xlator_t *this = NULL;
+    gf_statedump target_pid = {
+        0,
+    };
+    struct iovec *iov = NULL;
+    int ret = -1;
+
+    this = mydata;
+    if (!this) {
+        gf_smsg("glfs", GF_LOG_ERROR, EINVAL, API_MSG_NULL, "mydata", NULL);
+        errno = EINVAL;
+        goto out;
+    }
+
+    fs = this->private;
+    if (!fs) {
+        gf_smsg("glfs", GF_LOG_ERROR, EINVAL, API_MSG_NULL, "glfs", NULL);
+        errno = EINVAL;
+        goto out;
+    }
+
+    iov = (struct iovec *)data;
+    if (!iov) {
+        gf_smsg("glfs", GF_LOG_ERROR, EINVAL, API_MSG_NULL, "iovec data", NULL);
+        errno = EINVAL;
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &target_pid, (xdrproc_t)xdr_gf_statedump);
+    if (ret < 0) {
+        gf_smsg("glfs", GF_LOG_ERROR, EINVAL, API_MSG_DECODE_XDR_FAILED, NULL);
+        goto out;
+    }
+
+    gf_msg_trace("glfs", 0, "statedump requested for pid: %d", target_pid.pid);
+
+    if ((uint64_t)getpid() == target_pid.pid) {
+        gf_msg_debug("glfs", 0, "Taking statedump for pid: %d", target_pid.pid);
+
+        ret = glfs_sysrq(fs, GLFS_SYSRQ_STATEDUMP);
+        if (ret < 0) {
+            gf_smsg("glfs", GF_LOG_INFO, 0, API_MSG_STATEDUMP_FAILED, NULL);
+        }
+    }
+out:
+    return ret;
+}
 
-rpcclnt_cb_actor_t mgmt_cbk_actors[GF_CBK_MAXVALUE] = {
-	[GF_CBK_FETCHSPEC] = {"FETCHSPEC", GF_CBK_FETCHSPEC, mgmt_cbk_spec },
-	[GF_CBK_EVENT_NOTIFY] = {"EVENTNOTIFY", GF_CBK_EVENT_NOTIFY,
-				 mgmt_cbk_event},
+static rpcclnt_cb_actor_t mgmt_cbk_actors[GF_CBK_MAXVALUE] = {
+    [GF_CBK_FETCHSPEC] = {"FETCHSPEC", mgmt_cbk_spec, GF_CBK_FETCHSPEC},
+    [GF_CBK_EVENT_NOTIFY] = {"EVENTNOTIFY", mgmt_cbk_event,
+                             GF_CBK_EVENT_NOTIFY},
+    [GF_CBK_STATEDUMP] = {"STATEDUMP", mgmt_cbk_statedump, GF_CBK_STATEDUMP},
 };
 
-
-struct rpcclnt_cb_program mgmt_cbk_prog = {
-	.progname  = "GlusterFS Callback",
-	.prognum   = GLUSTER_CBK_PROGRAM,
-	.progver   = GLUSTER_CBK_VERSION,
-	.actors	   = mgmt_cbk_actors,
-	.numactors = GF_CBK_MAXVALUE,
+static struct rpcclnt_cb_program mgmt_cbk_prog = {
+    .progname = "GlusterFS Callback",
+    .prognum = GLUSTER_CBK_PROGRAM,
+    .progver = GLUSTER_CBK_VERSION,
+    .actors = mgmt_cbk_actors,
+    .numactors = GF_CBK_MAXVALUE,
 };
 
-char *clnt_handshake_procs[GF_HNDSK_MAXVALUE] = {
-	[GF_HNDSK_NULL]		= "NULL",
-	[GF_HNDSK_SETVOLUME]	= "SETVOLUME",
-	[GF_HNDSK_GETSPEC]	= "GETSPEC",
-	[GF_HNDSK_PING]		= "PING",
-	[GF_HNDSK_EVENT_NOTIFY] = "EVENTNOTIFY",
-        [GF_HNDSK_GET_VOLUME_INFO] = "GETVOLUMEINFO",
+static char *clnt_handshake_procs[GF_HNDSK_MAXVALUE] = {
+    [GF_HNDSK_NULL] = "NULL",
+    [GF_HNDSK_SETVOLUME] = "SETVOLUME",
+    [GF_HNDSK_GETSPEC] = "GETSPEC",
+    [GF_HNDSK_PING] = "PING",
+    [GF_HNDSK_EVENT_NOTIFY] = "EVENTNOTIFY",
+    [GF_HNDSK_GET_VOLUME_INFO] = "GETVOLUMEINFO",
 };
 
-rpc_clnt_prog_t clnt_handshake_prog = {
-	.progname  = "GlusterFS Handshake",
-	.prognum   = GLUSTER_HNDSK_PROGRAM,
-	.progver   = GLUSTER_HNDSK_VERSION,
-	.procnames = clnt_handshake_procs,
+static rpc_clnt_prog_t clnt_handshake_prog = {
+    .progname = "GlusterFS Handshake",
+    .prognum = GLUSTER_HNDSK_PROGRAM,
+    .progver = GLUSTER_HNDSK_VERSION,
+    .procnames = clnt_handshake_procs,
 };
 
-
 int
-mgmt_submit_request (void *req, call_frame_t *frame,
-		     glusterfs_ctx_t *ctx,
-		     rpc_clnt_prog_t *prog, int procnum,
-		     fop_cbk_fn_t cbkfn, xdrproc_t xdrproc)
+mgmt_submit_request(void *req, call_frame_t *frame, glusterfs_ctx_t *ctx,
+                    rpc_clnt_prog_t *prog, int procnum, fop_cbk_fn_t cbkfn,
+                    xdrproc_t xdrproc)
 {
-	int			ret	    = -1;
-	int			count	   = 0;
-	struct iovec		iov	    = {0, };
-	struct iobuf		*iobuf = NULL;
-	struct iobref		*iobref = NULL;
-	ssize_t			xdr_size = 0;
-
-	iobref = iobref_new ();
-	if (!iobref) {
-		goto out;
-	}
-
-	if (req) {
-		xdr_size = xdr_sizeof (xdrproc, req);
-
-		iobuf = iobuf_get2 (ctx->iobuf_pool, xdr_size);
-		if (!iobuf) {
-			goto out;
-		};
-
-		iobref_add (iobref, iobuf);
-
-		iov.iov_base = iobuf->ptr;
-		iov.iov_len  = iobuf_pagesize (iobuf);
-
-		/* Create the xdr payload */
-		ret = xdr_serialize_generic (iov, req, xdrproc);
-		if (ret == -1) {
-			gf_log (THIS->name, GF_LOG_WARNING,
-				"failed to create XDR payload");
-			goto out;
-		}
-		iov.iov_len = ret;
-		count = 1;
-	}
-
-	/* Send the msg */
-	ret = rpc_clnt_submit (ctx->mgmt, prog, procnum, cbkfn,
-			       &iov, count,
-			       NULL, 0, iobref, frame, NULL, 0, NULL, 0, NULL);
+    int ret = -1;
+    int count = 0;
+    struct iovec iov = {
+        0,
+    };
+    struct iobuf *iobuf = NULL;
+    struct iobref *iobref = NULL;
+    ssize_t xdr_size = 0;
+
+    iobref = iobref_new();
+    if (!iobref) {
+        goto out;
+    }
+
+    if (req) {
+        xdr_size = xdr_sizeof(xdrproc, req);
+
+        iobuf = iobuf_get2(ctx->iobuf_pool, xdr_size);
+        if (!iobuf) {
+            goto out;
+        };
+
+        iobref_add(iobref, iobuf);
+
+        iov.iov_base = iobuf->ptr;
+        iov.iov_len = iobuf_pagesize(iobuf);
+
+        /* Create the xdr payload */
+        ret = xdr_serialize_generic(iov, req, xdrproc);
+        if (ret == -1) {
+            gf_smsg(THIS->name, GF_LOG_WARNING, 0, API_MSG_XDR_PAYLOAD_FAILED,
+                    NULL);
+            goto out;
+        }
+        iov.iov_len = ret;
+        count = 1;
+    }
+
+    /* Send the msg */
+    ret = rpc_clnt_submit(ctx->mgmt, prog, procnum, cbkfn, &iov, count, NULL, 0,
+                          iobref, frame, NULL, 0, NULL, 0, NULL);
 
 out:
-	if (iobref)
-		iobref_unref (iobref);
+    if (iobref)
+        iobref_unref(iobref);
 
-	if (iobuf)
-		iobuf_unref (iobuf);
-	return ret;
+    if (iobuf)
+        iobuf_unref(iobuf);
+    return ret;
 }
 
 /*
  * Callback routine for 'GF_HNDSK_GET_VOLUME_INFO' rpc request
  */
 int
-mgmt_get_volinfo_cbk (struct rpc_req *req, struct iovec *iov,
-                      int count, void *myframe)
+mgmt_get_volinfo_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                     void *myframe)
 {
-        int                        ret                  = 0;
-        char                       *volume_id_str       = NULL;
-        dict_t                     *dict                = NULL;
-        char                       key[1024]            = {0};
-        gf_get_volume_info_rsp     rsp                  = {0,};
-        call_frame_t               *frame               = NULL;
-        glusterfs_ctx_t            *ctx                 = NULL;
-        struct glfs                *fs                  = NULL;
-        struct syncargs            *args;
-
-        frame = myframe;
-        ctx = frame->this->ctx;
-        args = frame->local;
-
-        if (!ctx) {
-                gf_log (frame->this->name, GF_LOG_ERROR, "NULL context");
-                errno = EINVAL;
-                ret = -1;
-                goto out;
-        }
-
-        fs = ((xlator_t *)ctx->master)->private;
-
-        if (-1 == req->rpc_status) {
-                gf_log (frame->this->name, GF_LOG_ERROR,
-                        "GET_VOLUME_INFO RPC call is not successfull");
-                errno = EINVAL;
-                ret = -1;
-                goto out;
-        }
-
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_get_volume_info_rsp);
-
-        if (ret < 0) {
-                gf_log (frame->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response for GET_VOLUME_INFO");
-                goto out;
-        }
-
-        gf_log (frame->this->name, GF_LOG_DEBUG,
-                "Received resp to GET_VOLUME_INFO RPC: %d", rsp.op_ret);
-
-        if (rsp.op_ret == -1) {
-                errno = rsp.op_errno;
-                ret = -1;
-                goto out;
-        }
-
-        if (!rsp.dict.dict_len) {
-                gf_log (frame->this->name, GF_LOG_ERROR,
-                        "Response received for GET_VOLUME_INFO RPC call is not valid");
-                ret = -1;
-                errno = EINVAL;
-                goto out;
-        }
-
-        dict = dict_new ();
-
-        if (!dict) {
-                ret = -1;
-                errno = ENOMEM;
-                goto out;
-        }
-
-        ret = dict_unserialize (rsp.dict.dict_val,
-                                rsp.dict.dict_len,
-                                &dict);
-
-        if (ret) {
-                errno = ENOMEM;
-                goto out;
-        }
-
-        snprintf (key, sizeof (key), "volume_id");
-        ret = dict_get_str (dict, key, &volume_id_str);
-        if (ret) {
-                errno = EINVAL;
-                goto out;
-        }
-
-        ret = 0;
+    int ret = 0;
+    char *volume_id_str = NULL;
+    dict_t *dict = NULL;
+    gf_get_volume_info_rsp rsp = {
+        0,
+    };
+    call_frame_t *frame = NULL;
+    glusterfs_ctx_t *ctx = NULL;
+    struct glfs *fs = NULL;
+    struct syncargs *args;
+
+    frame = myframe;
+    ctx = frame->this->ctx;
+    args = frame->local;
+
+    if (!ctx) {
+        gf_smsg(frame->this->name, GF_LOG_ERROR, EINVAL, API_MSG_NULL,
+                "context", NULL);
+        errno = EINVAL;
+        ret = -1;
+        goto out;
+    }
+
+    fs = ((xlator_t *)ctx->master)->private;
+
+    if (-1 == req->rpc_status) {
+        gf_smsg(frame->this->name, GF_LOG_ERROR, EINVAL,
+                API_MSG_CALL_NOT_SUCCESSFUL, NULL);
+        errno = EINVAL;
+        ret = -1;
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_get_volume_info_rsp);
+
+    if (ret < 0) {
+        gf_smsg(frame->this->name, GF_LOG_ERROR, 0,
+                API_MSG_XDR_RESPONSE_DECODE_FAILED, NULL);
+        goto out;
+    }
+
+    gf_msg_debug(frame->this->name, 0,
+                 "Received resp to GET_VOLUME_INFO "
+                 "RPC: %d",
+                 rsp.op_ret);
+
+    if (rsp.op_ret == -1) {
+        errno = rsp.op_errno;
+        ret = -1;
+        goto out;
+    }
+
+    if (!rsp.dict.dict_len) {
+        gf_smsg(frame->this->name, GF_LOG_ERROR, EINVAL, API_MSG_CALL_NOT_VALID,
+                NULL);
+        ret = -1;
+        errno = EINVAL;
+        goto out;
+    }
+
+    dict = dict_new();
+
+    if (!dict) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len, &dict);
+
+    if (ret) {
+        errno = ENOMEM;
+        goto out;
+    }
+
+    ret = dict_get_str_sizen(dict, "volume_id", &volume_id_str);
+    if (ret) {
+        errno = EINVAL;
+        goto out;
+    }
+
+    ret = 0;
 out:
-        if (volume_id_str) {
-                gf_log (frame->this->name, GF_LOG_DEBUG,
-                        "Volume Id: %s", volume_id_str);
-                pthread_mutex_lock (&fs->mutex);
-                uuid_parse (volume_id_str, fs->vol_uuid);
-                pthread_mutex_unlock (&fs->mutex);
-        }
+    if (volume_id_str) {
+        gf_msg_debug(frame->this->name, 0, "Volume Id: %s", volume_id_str);
+        pthread_mutex_lock(&fs->mutex);
+        gf_uuid_parse(volume_id_str, fs->vol_uuid);
+        pthread_mutex_unlock(&fs->mutex);
+    }
 
-        if (ret) {
-               gf_log (frame->this->name, GF_LOG_ERROR,
-                       "In GET_VOLUME_INFO cbk, received error: %s",
-                       strerror(errno));
-        }
+    if (ret) {
+        gf_smsg(frame->this->name, GF_LOG_ERROR, errno,
+                API_MSG_GET_VOLINFO_CBK_FAILED, "error=%s", strerror(errno),
+                NULL);
+    }
 
-        if (dict)
-                dict_destroy (dict);
+    if (dict)
+        dict_unref(dict);
 
-        if (rsp.dict.dict_val)
-                free (rsp.dict.dict_val);
+    if (rsp.dict.dict_val)
+        free(rsp.dict.dict_val);
 
-        if (rsp.op_errstr && *rsp.op_errstr)
-                free (rsp.op_errstr);
+    if (rsp.op_errstr)
+        free(rsp.op_errstr);
 
-        gf_log (frame->this->name, GF_LOG_DEBUG, "Returning: %d", ret);
+    gf_msg_debug(frame->this->name, 0, "Returning: %d", ret);
 
-        __wake (args);
+    __wake(args);
 
-        return ret;
+    return ret;
 }
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_get_volumeid, 3.5.0)
 int
-pub_glfs_get_volumeid (struct glfs *fs, char *volid, size_t size)
+pub_glfs_get_volumeid(struct glfs *fs, char *volid, size_t size)
 {
-        /* TODO: Define a global macro to store UUID size */
-        size_t uuid_size = 16;
-
-        pthread_mutex_lock (&fs->mutex);
-        {
-                /* check if the volume uuid is initialized */
-                if (!uuid_is_null (fs->vol_uuid)) {
-                        pthread_mutex_unlock (&fs->mutex);
-                        goto done;
-                }
+    /* TODO: Define a global macro to store UUID size */
+    size_t uuid_size = 16;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    pthread_mutex_lock(&fs->mutex);
+    {
+        /* check if the volume uuid is initialized */
+        if (!gf_uuid_is_null(fs->vol_uuid)) {
+            pthread_mutex_unlock(&fs->mutex);
+            goto done;
         }
-        pthread_mutex_unlock (&fs->mutex);
+    }
+    pthread_mutex_unlock(&fs->mutex);
 
-        /* Need to fetch volume_uuid */
-        glfs_get_volume_info (fs);
+    /* Need to fetch volume_uuid */
+    glfs_get_volume_info(fs);
 
-        if (uuid_is_null (fs->vol_uuid)) {
-                gf_log (THIS->name, GF_LOG_ERROR, "Unable to fetch volume UUID");
-                return -1;
-        }
+    if (gf_uuid_is_null(fs->vol_uuid)) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, EINVAL, API_MSG_FETCH_VOLUUID_FAILED,
+                NULL);
+        goto out;
+    }
 
 done:
-        if (!volid || !size) {
-                gf_log (THIS->name, GF_LOG_DEBUG, "volumeid/size is null");
-                return uuid_size;
-        }
+    if (!volid || !size) {
+        gf_msg_debug(THIS->name, 0, "volumeid/size is null");
+        __GLFS_EXIT_FS;
+        return uuid_size;
+    }
 
-        if (size < uuid_size) {
-                gf_log (THIS->name, GF_LOG_ERROR, "Insufficient size passed");
-                errno = ERANGE;
-                return -1;
-        }
+    if (size < uuid_size) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, ERANGE, API_MSG_INSUFF_SIZE, NULL);
+        errno = ERANGE;
+        goto out;
+    }
 
-        memcpy (volid, fs->vol_uuid, uuid_size);
+    memcpy(volid, fs->vol_uuid, uuid_size);
 
-        gf_log (THIS->name, GF_LOG_INFO, "volume uuid: %s", volid);
+    __GLFS_EXIT_FS;
 
-        return uuid_size;
-}
+    return uuid_size;
+
+out:
+    __GLFS_EXIT_FS;
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_get_volumeid, 3.5.0);
+invalid_fs:
+    return -1;
+}
 
 int
-glfs_get_volume_info (struct glfs *fs)
+glfs_get_volume_info(struct glfs *fs)
 {
-        call_frame_t     *frame = NULL;
-        glusterfs_ctx_t  *ctx   = NULL;
-        struct syncargs  args   = {0, };
-        int              ret    = 0;
+    call_frame_t *frame = NULL;
+    glusterfs_ctx_t *ctx = NULL;
+    struct syncargs args = {
+        0,
+    };
+    int ret = 0;
 
-        ctx = fs->ctx;
-        frame = create_frame (THIS, ctx->pool);
-        frame->local = &args;
+    ctx = fs->ctx;
+    frame = create_frame(THIS, ctx->pool);
+    if (!frame) {
+        gf_smsg("glfs", GF_LOG_ERROR, ENOMEM, API_MSG_FRAME_CREAT_FAILED, NULL);
+        ret = -1;
+        goto out;
+    }
 
-        __yawn ((&args));
+    frame->local = &args;
 
-        ret = glfs_get_volume_info_rpc (frame, THIS, fs);
-        if (ret)
-                goto out;
+    __yawn((&args));
+
+    ret = glfs_get_volume_info_rpc(frame, THIS, fs);
+    if (ret)
+        goto out;
 
-        __yield ((&args));
+    __yield((&args));
 
-        frame->local = NULL;
-        STACK_DESTROY (frame->root);
+    frame->local = NULL;
+    STACK_DESTROY(frame->root);
 
 out:
-        return ret;
+    return ret;
 }
 
 int32_t
-glfs_get_volume_info_rpc (call_frame_t *frame, xlator_t *this,
-                          struct glfs *fs)
+glfs_get_volume_info_rpc(call_frame_t *frame, xlator_t *this, struct glfs *fs)
 {
-        gf_get_volume_info_req  req       = {{0,}};
-        int                     ret       = 0;
-        glusterfs_ctx_t         *ctx      = NULL;
-        dict_t                  *dict     = NULL;
-        int32_t                 flags     = 0;
-
-        if (!frame || !this || !fs) {
-                ret = -1;
-                goto out;
-        }
-
-        ctx = fs->ctx;
-
-        dict = dict_new ();
-        if (!dict) {
-                ret = -1;
-                goto out;
-        }
-
-        if (fs->volname) {
-                ret = dict_set_str (dict, "volname", fs->volname);
-                if (ret)
-                        goto out;
-        }
-
-        // Set the flags for the fields which we are interested in
-        flags = (int32_t)GF_GET_VOLUME_UUID; //ctx->flags;
-        ret = dict_set_int32 (dict, "flags", flags);
-        if (ret) {
-                gf_log (frame->this->name, GF_LOG_ERROR, "failed to set flags");
-                goto out;
-        }
-
-        ret = dict_allocate_and_serialize (dict, &req.dict.dict_val,
-                                           &req.dict.dict_len);
-
-
-        ret = mgmt_submit_request (&req, frame, ctx, &clnt_handshake_prog,
-                                   GF_HNDSK_GET_VOLUME_INFO,
-                                   mgmt_get_volinfo_cbk,
-                                   (xdrproc_t)xdr_gf_get_volume_info_req);
+    gf_get_volume_info_req req = {{
+        0,
+    }};
+    int ret = 0;
+    glusterfs_ctx_t *ctx = NULL;
+    dict_t *dict = NULL;
+    int32_t flags = 0;
+
+    if (!frame || !this || !fs) {
+        ret = -1;
+        goto out;
+    }
+
+    ctx = fs->ctx;
+
+    dict = dict_new();
+    if (!dict) {
+        ret = -1;
+        goto out;
+    }
+
+    if (fs->volname) {
+        ret = dict_set_str(dict, "volname", fs->volname);
+        if (ret)
+            goto out;
+    }
+
+    // Set the flags for the fields which we are interested in
+    flags = (int32_t)GF_GET_VOLUME_UUID;  // ctx->flags;
+    ret = dict_set_int32(dict, "flags", flags);
+    if (ret) {
+        gf_smsg(frame->this->name, GF_LOG_ERROR, EINVAL,
+                API_MSG_DICT_SET_FAILED, "flags", NULL);
+        goto out;
+    }
+
+    ret = dict_allocate_and_serialize(dict, &req.dict.dict_val,
+                                      &req.dict.dict_len);
+
+    ret = mgmt_submit_request(&req, frame, ctx, &clnt_handshake_prog,
+                              GF_HNDSK_GET_VOLUME_INFO, mgmt_get_volinfo_cbk,
+                              (xdrproc_t)xdr_gf_get_volume_info_req);
 out:
-        if (dict) {
-                dict_unref (dict);
-        }
+    if (dict) {
+        dict_unref(dict);
+    }
 
-        GF_FREE (req.dict.dict_val);
+    GF_FREE(req.dict.dict_val);
 
-        return ret;
+    return ret;
 }
 
 static int
-glusterfs_oldvolfile_update (struct glfs *fs, char *volfile, ssize_t size)
+glusterfs_oldvolfile_update(struct glfs *fs, char *volfile, ssize_t size)
 {
-	int ret = -1;
-
-	fs->oldvollen = size;
-	if (!fs->oldvolfile) {
-		fs->oldvolfile = GF_CALLOC (1, size+1, glfs_mt_volfile_t);
-	} else {
-		fs->oldvolfile = GF_REALLOC (fs->oldvolfile, size+1);
-	}
-
-	if (!fs->oldvolfile) {
-		fs->oldvollen = 0;
-	} else {
-		memcpy (fs->oldvolfile, volfile, size);
-		fs->oldvollen = size;
-		ret = 0;
-	}
-
-	return ret;
-}
+    int ret = -1;
+
+    pthread_mutex_lock(&fs->mutex);
+
+    fs->oldvollen = size;
+    if (!fs->oldvolfile) {
+        fs->oldvolfile = CALLOC(1, size + 1);
+    } else {
+        fs->oldvolfile = REALLOC(fs->oldvolfile, size + 1);
+    }
+
+    if (!fs->oldvolfile) {
+        fs->oldvollen = 0;
+    } else {
+        memcpy(fs->oldvolfile, volfile, size);
+        fs->oldvollen = size;
+        ret = 0;
+    }
 
+    pthread_mutex_unlock(&fs->mutex);
+
+    return ret;
+}
 
 int
-mgmt_getspec_cbk (struct rpc_req *req, struct iovec *iov, int count,
-		  void *myframe)
+glfs_mgmt_getspec_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                      void *myframe)
 {
-	gf_getspec_rsp		 rsp   = {0,};
-	call_frame_t		*frame = NULL;
-	glusterfs_ctx_t		*ctx = NULL;
-	int			 ret   = 0;
-	ssize_t			 size = 0;
-	FILE			*tmpfp = NULL;
-	int                      need_retry = 0;
-	struct glfs		*fs = NULL;
-
-	frame = myframe;
-	ctx = frame->this->ctx;
-	fs = ((xlator_t *)ctx->master)->private;
-
-	if (-1 == req->rpc_status) {
-		ret = -1;
-		need_retry = 1;
-		goto out;
-	}
-
-	ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_getspec_rsp);
-	if (ret < 0) {
-		gf_log (frame->this->name, GF_LOG_ERROR, "XDR decoding error");
-		ret   = -1;
-		goto out;
-	}
-
-	if (-1 == rsp.op_ret) {
-		gf_log (frame->this->name, GF_LOG_ERROR,
-			"failed to get the 'volume file' from server");
-		ret = -1;
-                errno = rsp.op_errno;
-		goto out;
-	}
-
-	ret = 0;
-	size = rsp.op_ret;
-
-	if ((size == fs->oldvollen) &&
-	    (memcmp (fs->oldvolfile, rsp.spec, size) == 0)) {
-		gf_log (frame->this->name, GF_LOG_INFO,
-			"No change in volfile, continuing");
-		goto out;
-	}
-
-	tmpfp = tmpfile ();
-	if (!tmpfp) {
-		ret = -1;
-		goto out;
-	}
-
-	fwrite (rsp.spec, size, 1, tmpfp);
-	fflush (tmpfp);
-	if (ferror (tmpfp)) {
-		ret = -1;
-		goto out;
-	}
-
-	/*  Check if only options have changed. No need to reload the
-	*  volfile if topology hasn't changed.
-	*  glusterfs_volfile_reconfigure returns 3 possible return states
-	*  return 0	     =======> reconfiguration of options has succeeded
-	*  return 1	     =======> the graph has to be reconstructed and all the xlators should be inited
-	*  return -1(or -ve) =======> Some Internal Error occurred during the operation
-	*/
-
-	ret = glusterfs_volfile_reconfigure (fs->oldvollen, tmpfp, fs->ctx,
-                                             fs->oldvolfile);
-	if (ret == 0) {
-		gf_log ("glusterfsd-mgmt", GF_LOG_DEBUG,
-			"No need to re-load volfile, reconfigure done");
-		ret = glusterfs_oldvolfile_update (fs, rsp.spec, size);
-		goto out;
-	}
-
-	if (ret < 0) {
-		gf_log ("glusterfsd-mgmt", GF_LOG_DEBUG,
-			"Reconfigure failed !!");
-		goto out;
-	}
-
-	ret = glfs_process_volfp (fs, tmpfp);
-	/* tmpfp closed */
-	tmpfp = NULL;
-	if (ret)
-		goto out;
-
-	ret = glusterfs_oldvolfile_update (fs, rsp.spec, size);
+    gf_getspec_rsp rsp = {
+        0,
+    };
+    call_frame_t *frame = NULL;
+    glusterfs_ctx_t *ctx = NULL;
+    int ret = 0;
+    ssize_t size = 0;
+    FILE *tmpfp = NULL;
+    int need_retry = 0;
+    struct glfs *fs = NULL;
+    dict_t *dict = NULL;
+    char *servers_list = NULL;
+    int tmp_fd = -1;
+    char template[] = "/tmp/gfapi.volfile.XXXXXX";
+
+    frame = myframe;
+    ctx = frame->this->ctx;
+
+    if (!ctx) {
+        gf_smsg(frame->this->name, GF_LOG_ERROR, EINVAL, API_MSG_NULL,
+                "context", NULL);
+        errno = EINVAL;
+        ret = -1;
+        goto out;
+    }
+
+    fs = ((xlator_t *)ctx->master)->private;
+
+    if (-1 == req->rpc_status) {
+        ret = -1;
+        need_retry = 1;
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_getspec_rsp);
+    if (ret < 0) {
+        gf_smsg(frame->this->name, GF_LOG_ERROR, 0, API_MSG_XDR_DECODE_FAILED,
+                NULL);
+        ret = -1;
+        goto out;
+    }
+
+    if (-1 == rsp.op_ret) {
+        gf_smsg(frame->this->name, GF_LOG_ERROR, rsp.op_errno,
+                API_MSG_GET_VOLFILE_FAILED, "from server", NULL);
+        ret = -1;
+        errno = rsp.op_errno;
+        goto out;
+    }
+
+    if (!rsp.xdata.xdata_len) {
+        goto volfile;
+    }
+
+    dict = dict_new();
+    if (!dict) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    ret = dict_unserialize(rsp.xdata.xdata_val, rsp.xdata.xdata_len, &dict);
+    if (ret) {
+        gf_log(frame->this->name, GF_LOG_ERROR,
+               "failed to unserialize xdata to dictionary");
+        goto out;
+    }
+    dict->extra_stdfree = rsp.xdata.xdata_val;
+
+    /* glusterd2 only */
+    ret = dict_get_str(dict, "servers-list", &servers_list);
+    if (ret) {
+        goto volfile;
+    }
+
+    gf_log(frame->this->name, GF_LOG_INFO,
+           "Received list of available volfile servers: %s", servers_list);
+
+    ret = gf_process_getspec_servers_list(&ctx->cmd_args, servers_list);
+    if (ret) {
+        gf_log(frame->this->name, GF_LOG_ERROR,
+               "Failed (%s) to process servers list: %s", strerror(errno),
+               servers_list);
+    }
+
+volfile:
+    ret = 0;
+    size = rsp.op_ret;
+
+    pthread_mutex_lock(&fs->mutex);
+    if ((size == fs->oldvollen) &&
+        (memcmp(fs->oldvolfile, rsp.spec, size) == 0)) {
+        pthread_mutex_unlock(&fs->mutex);
+        gf_smsg(frame->this->name, GF_LOG_INFO, 0, API_MSG_VOLFILE_INFO, NULL);
+        goto out;
+    }
+    pthread_mutex_unlock(&fs->mutex);
+
+    /* coverity[secure_temp] mkstemp uses 0600 as the mode and is safe */
+    tmp_fd = mkstemp(template);
+    if (-1 == tmp_fd) {
+        ret = -1;
+        goto out;
+    }
+
+    /* Calling unlink so that when the file is closed or program
+     * terminates the temporary file is deleted.
+     */
+    ret = sys_unlink(template);
+    if (ret < 0) {
+        gf_smsg(frame->this->name, GF_LOG_INFO, 0, API_MSG_UNABLE_TO_DEL,
+                "template=%s", template, NULL);
+        ret = 0;
+    }
+
+    tmpfp = fdopen(tmp_fd, "w+b");
+    if (!tmpfp) {
+        ret = -1;
+        goto out;
+    }
+
+    fwrite(rsp.spec, size, 1, tmpfp);
+    fflush(tmpfp);
+    if (ferror(tmpfp)) {
+        ret = -1;
+        goto out;
+    }
+
+    /*  Check if only options have changed. No need to reload the
+     *  volfile if topology hasn't changed.
+     *  glusterfs_volfile_reconfigure returns 3 possible return states
+     *  return 0	     =======> reconfiguration of options has succeeded
+     *  return 1	     =======> the graph has to be reconstructed and all
+     * the xlators should be inited return -1(or -ve) =======> Some Internal
+     * Error occurred during the operation
+     */
+
+    pthread_mutex_lock(&fs->mutex);
+    ret = gf_volfile_reconfigure(fs->oldvollen, tmpfp, fs->ctx, fs->oldvolfile);
+    pthread_mutex_unlock(&fs->mutex);
+
+    if (ret == 0) {
+        gf_msg_debug("glusterfsd-mgmt", 0,
+                     "No need to re-load "
+                     "volfile, reconfigure done");
+        ret = glusterfs_oldvolfile_update(fs, rsp.spec, size);
+        goto out;
+    }
+
+    if (ret < 0) {
+        gf_msg_debug("glusterfsd-mgmt", 0, "Reconfigure failed !!");
+        goto out;
+    }
+
+    ret = glfs_process_volfp(fs, tmpfp);
+    /* tmpfp closed */
+    tmpfp = NULL;
+    tmp_fd = -1;
+    if (ret)
+        goto out;
+
+    ret = glusterfs_oldvolfile_update(fs, rsp.spec, size);
 out:
-	STACK_DESTROY (frame->root);
-
-	if (rsp.spec)
-		free (rsp.spec);
-
-        // Stop if server is running at an unsupported op-version
-        if (ENOTSUP == ret) {
-                gf_log ("mgmt", GF_LOG_ERROR, "Server is operating at an "
-                        "op-version which is not supported");
-                errno = ENOTSUP;
-                priv_glfs_init_done (fs, -1);
+    STACK_DESTROY(frame->root);
+
+    if (rsp.spec)
+        free(rsp.spec);
+
+    if (dict)
+        dict_unref(dict);
+
+    // Stop if server is running at an unsupported op-version
+    if (ENOTSUP == ret) {
+        gf_smsg("mgmt", GF_LOG_ERROR, ENOTSUP, API_MSG_WRONG_OPVERSION, NULL);
+        errno = ENOTSUP;
+        glfs_init_done(fs, -1);
+    }
+
+    if (ret && ctx && !ctx->active) {
+        /* Do it only for the first time */
+        /* Failed to get the volume file, something wrong,
+           restart the process */
+        gf_smsg("glfs-mgmt", GF_LOG_ERROR, EINVAL, API_MSG_GET_VOLFILE_FAILED,
+                "key=%s", ctx->cmd_args.volfile_id, NULL);
+        if (!need_retry) {
+            if (!errno)
+                errno = EINVAL;
+            glfs_init_done(fs, -1);
         }
+    }
 
-	if (ret && ctx && !ctx->active) {
-		/* Do it only for the first time */
-		/* Failed to get the volume file, something wrong,
-		   restart the process */
-		gf_log ("glfs-mgmt", GF_LOG_ERROR,
-			"failed to fetch volume file (key:%s)",
-			ctx->cmd_args.volfile_id);
-		if (!need_retry) {
-                        if (!errno)
-                                errno = EINVAL;
-			priv_glfs_init_done (fs, -1);
-                }
-	}
-
-	if (tmpfp)
-		fclose (tmpfp);
+    if (tmpfp)
+        fclose(tmpfp);
+    else if (tmp_fd != -1)
+        sys_close(tmp_fd);
 
-	return 0;
+    return 0;
 }
 
-
 int
-glfs_volfile_fetch (struct glfs *fs)
+glfs_volfile_fetch(struct glfs *fs)
 {
-	cmd_args_t	 *cmd_args = NULL;
-	gf_getspec_req	  req = {0, };
-	int		  ret = 0;
-	call_frame_t	 *frame = NULL;
-	glusterfs_ctx_t	 *ctx = NULL;
-        dict_t           *dict = NULL;
-
-	ctx = fs->ctx;
-	cmd_args = &ctx->cmd_args;
-
-	frame = create_frame (THIS, ctx->pool);
-
-	req.key = cmd_args->volfile_id;
-	req.flags = 0;
-
-        dict = dict_new ();
-        if (!dict) {
-                ret = -1;
-                goto out;
-        }
-
-        // Set the supported min and max op-versions, so glusterd can make a
-        // decision
-        ret = dict_set_int32 (dict, "min-op-version", GD_OP_VERSION_MIN);
-        if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "Failed to set min-op-version"
-                        " in request dict");
-                goto out;
-        }
-
-        ret = dict_set_int32 (dict, "max-op-version", GD_OP_VERSION_MAX);
-        if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "Failed to set max-op-version"
-                        " in request dict");
-                goto out;
-        }
-
-        ret = dict_allocate_and_serialize (dict, &req.xdata.xdata_val,
-                                           &req.xdata.xdata_len);
-        if (ret < 0) {
-                gf_log (THIS->name, GF_LOG_ERROR,
-                        "Failed to serialize dictionary");
-                goto out;
-        }
-
-	ret = mgmt_submit_request (&req, frame, ctx, &clnt_handshake_prog,
-				   GF_HNDSK_GETSPEC, mgmt_getspec_cbk,
-				   (xdrproc_t)xdr_gf_getspec_req);
+    cmd_args_t *cmd_args = NULL;
+    gf_getspec_req req = {
+        0,
+    };
+    int ret = -1;
+    call_frame_t *frame = NULL;
+    glusterfs_ctx_t *ctx = NULL;
+    dict_t *dict = NULL;
+
+    ctx = fs->ctx;
+    cmd_args = &ctx->cmd_args;
+
+    req.key = cmd_args->volfile_id;
+    req.flags = 0;
+
+    dict = dict_new();
+    if (!dict) {
+        goto out;
+    }
+
+    // Set the supported min and max op-versions, so glusterd can make a
+    // decision
+    ret = dict_set_int32(dict, "min-op-version", GD_OP_VERSION_MIN);
+    if (ret) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, EINVAL, API_MSG_DICT_SET_FAILED,
+                "min-op-version", NULL);
+        goto out;
+    }
+
+    ret = dict_set_int32(dict, "max-op-version", GD_OP_VERSION_MAX);
+    if (ret) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, EINVAL, API_MSG_DICT_SET_FAILED,
+                "max-op-version", NULL);
+        goto out;
+    }
+
+    /* Ask for a list of volfile (glusterd2 only) servers */
+    if (GF_CLIENT_PROCESS == ctx->process_mode) {
+        req.flags = req.flags | GF_GETSPEC_FLAG_SERVERS_LIST;
+    }
+
+    ret = dict_allocate_and_serialize(dict, &req.xdata.xdata_val,
+                                      &req.xdata.xdata_len);
+    if (ret < 0) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, 0, API_MSG_DICT_SERIALIZE_FAILED,
+                NULL);
+        goto out;
+    }
+
+    frame = create_frame(THIS, ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = mgmt_submit_request(&req, frame, ctx, &clnt_handshake_prog,
+                              GF_HNDSK_GETSPEC, glfs_mgmt_getspec_cbk,
+                              (xdrproc_t)xdr_gf_getspec_req);
 out:
-        return ret;
-}
+    if (req.xdata.xdata_val)
+        GF_FREE(req.xdata.xdata_val);
+    if (dict)
+        dict_unref(dict);
 
+    return ret;
+}
 
 static int
-mgmt_rpc_notify (struct rpc_clnt *rpc, void *mydata, rpc_clnt_event_t event,
-		 void *data)
+mgmt_rpc_notify(struct rpc_clnt *rpc, void *mydata, rpc_clnt_event_t event,
+                void *data)
 {
-	xlator_t	*this = NULL;
-	glusterfs_ctx_t *ctx = NULL;
-	struct glfs	 *fs = NULL;
-	int		 ret = 0;
-        server_cmdline_t *server = NULL;
-        rpc_transport_t  *rpc_trans = NULL;
+    xlator_t *this = NULL;
+    glusterfs_ctx_t *ctx = NULL;
+    server_cmdline_t *server = NULL;
+    rpc_transport_t *rpc_trans = NULL;
+    struct glfs *fs = NULL;
+    int ret = 0;
+    struct dnscache6 *dnscache = NULL;
+
+    this = mydata;
+    rpc_trans = rpc->conn.trans;
 
-        this = mydata;
-        rpc_trans = rpc->conn.trans;
+    ctx = this->ctx;
+    if (!ctx)
+        goto out;
 
-        ctx = this->ctx;
-        fs = ((xlator_t *)ctx->master)->private;
+    fs = ((xlator_t *)ctx->master)->private;
 
-        switch (event) {
+    switch (event) {
         case RPC_CLNT_DISCONNECT:
-                if (!ctx->active) {
-                        gf_log ("glfs-mgmt", GF_LOG_ERROR,
-                                "failed to connect with remote-host: %s (%s)",
-                                ctx->cmd_args.volfile_server,
-                                strerror (errno));
-                        server = ctx->cmd_args.curr_server;
-                        if (server->list.next == &ctx->cmd_args.volfile_servers) {
-                                errno = ENOTCONN;
-                                gf_log("glfs-mgmt", GF_LOG_INFO,
-                                       "Exhausted all volfile servers");
-                                priv_glfs_init_done (fs, -1);
-                                break;
-                        }
-                        server = list_entry (server->list.next, typeof(*server),
-                                             list);
-                        ctx->cmd_args.curr_server = server;
-                        ctx->cmd_args.volfile_server_port = server->port;
-                        ctx->cmd_args.volfile_server = server->volfile_server;
-                        ctx->cmd_args.volfile_server_transport = server->transport;
-                        ret = dict_set_int32 (rpc_trans->options,
-                                              "remote-port",
-                                              server->port);
-                        if (ret != 0) {
-                                gf_log ("glfs-mgmt", GF_LOG_ERROR,
-                                        "failed to set remote-port: %d",
-                                        server->port);
-                                errno = ENOTCONN;
-                                priv_glfs_init_done (fs, -1);
-                                break;
-                        }
-
-                        ret = dict_set_str (rpc_trans->options,
-                                            "remote-host",
-                                            server->volfile_server);
-                        if (ret != 0) {
-                                gf_log ("glfs-mgmt", GF_LOG_ERROR,
-                                        "failed to set remote-host: %s",
-                                        server->volfile_server);
-                                errno = ENOTCONN;
-                                priv_glfs_init_done (fs, -1);
-                                break;
-                        }
-
-                        ret = dict_set_str (rpc_trans->options,
-                                            "transport-type",
-                                            server->transport);
-                        if (ret != 0) {
-                                gf_log ("glfs-mgmt", GF_LOG_ERROR,
-                                        "failed to set transport-type: %s",
-                                        server->transport);
-                                errno = ENOTCONN;
-                                priv_glfs_init_done (fs, -1);
-                                break;
-                        }
-                        gf_log ("glfs-mgmt", GF_LOG_INFO,
-                                "connecting to next volfile server %s"
-                                " at port %d with transport: %s",
-                                server->volfile_server, server->port,
-                                server->transport);
+            if (!ctx->active) {
+                if (rpc_trans->connect_failed)
+                    gf_smsg("glfs-mgmt", GF_LOG_ERROR, 0,
+                            API_MSG_REMOTE_HOST_CONN_FAILED, "server=%s",
+                            ctx->cmd_args.volfile_server, NULL);
+                else
+                    gf_smsg("glfs-mgmt", GF_LOG_INFO, 0,
+                            API_MSG_REMOTE_HOST_CONN_FAILED, "server=%s",
+                            ctx->cmd_args.volfile_server, NULL);
+
+                if (!rpc->disabled) {
+                    /*
+                     * Check if dnscache is exhausted for current server
+                     * and continue until cache is exhausted
+                     */
+                    dnscache = rpc_trans->dnscache;
+                    if (dnscache && dnscache->next) {
+                        break;
+                    }
+                }
+                server = ctx->cmd_args.curr_server;
+                if (server->list.next == &ctx->cmd_args.volfile_servers) {
+                    errno = ENOTCONN;
+                    gf_smsg("glfs-mgmt", GF_LOG_INFO, ENOTCONN,
+                            API_MSG_VOLFILE_SERVER_EXHAUST, NULL);
+                    glfs_init_done(fs, -1);
+                    break;
+                }
+                server = list_entry(server->list.next, typeof(*server), list);
+                ctx->cmd_args.curr_server = server;
+                ctx->cmd_args.volfile_server_port = server->port;
+                ctx->cmd_args.volfile_server = server->volfile_server;
+                ctx->cmd_args.volfile_server_transport = server->transport;
+
+                ret = dict_set_str(rpc_trans->options, "transport-type",
+                                   server->transport);
+                if (ret != 0) {
+                    gf_smsg("glfs-mgmt", GF_LOG_ERROR, ENOTCONN,
+                            API_MSG_DICT_SET_FAILED, "transport-type=%s",
+                            server->transport, NULL);
+                    errno = ENOTCONN;
+                    glfs_init_done(fs, -1);
+                    break;
                 }
-                break;
-	case RPC_CLNT_CONNECT:
-		rpc_clnt_set_connected (&((struct rpc_clnt*)ctx->mgmt)->conn);
-
-		ret = glfs_volfile_fetch (fs);
-		if (ret && ctx && (ctx->active == NULL)) {
-			/* Do it only for the first time */
-			/* Exit the process.. there are some wrong options */
-			gf_log ("glfs-mgmt", GF_LOG_ERROR,
-				"failed to fetch volume file (key:%s)",
-				ctx->cmd_args.volfile_id);
-                        errno = EINVAL;
-			priv_glfs_init_done (fs, -1);
-		}
-
-                break;
-	default:
-		break;
-	}
-
-	return 0;
-}
 
+                if (strcmp(server->transport, "unix") == 0) {
+                    ret = dict_set_str(rpc_trans->options,
+                                       "transport.socket.connect-path",
+                                       server->volfile_server);
+                    if (ret != 0) {
+                        gf_smsg("glfs-mgmt", GF_LOG_ERROR, ENOTCONN,
+                                API_MSG_DICT_SET_FAILED,
+                                "socket.connect-path=%s",
+                                server->volfile_server, NULL);
+                        errno = ENOTCONN;
+                        glfs_init_done(fs, -1);
+                        break;
+                    }
+                    /* delete the remote-host and remote-port keys
+                     * in case they were set while looping through
+                     * list of volfile servers previously
+                     */
+                    dict_del(rpc_trans->options, "remote-host");
+                    dict_del(rpc_trans->options, "remote-port");
+                } else {
+                    ret = dict_set_int32(rpc_trans->options, "remote-port",
+                                         server->port);
+                    if (ret != 0) {
+                        gf_smsg("glfs-mgmt", GF_LOG_ERROR, ENOTCONN,
+                                API_MSG_DICT_SET_FAILED, "remote-port=%d",
+                                server->port, NULL);
+                        errno = ENOTCONN;
+                        glfs_init_done(fs, -1);
+                        break;
+                    }
+
+                    ret = dict_set_str(rpc_trans->options, "remote-host",
+                                       server->volfile_server);
+                    if (ret != 0) {
+                        gf_smsg("glfs-mgmt", GF_LOG_ERROR, ENOTCONN,
+                                API_MSG_DICT_SET_FAILED, "remote-host=%s",
+                                server->volfile_server, NULL);
+                        errno = ENOTCONN;
+                        glfs_init_done(fs, -1);
+                        break;
+                    }
+                    /* delete the "transport.socket.connect-path"
+                     * key in case if it was set while looping
+                     * through list of volfile servers previously
+                     */
+                    dict_del(rpc_trans->options,
+                             "transport.socket.connect-path");
+                }
+
+                gf_smsg("glfs-mgmt", GF_LOG_INFO, 0, API_MSG_VOLFILE_CONNECTING,
+                        "server=%s", server->volfile_server, "port=%d",
+                        server->port, "transport=%s", server->transport, NULL);
+            }
+            break;
+        case RPC_CLNT_CONNECT:
+            ret = glfs_volfile_fetch(fs);
+            if (ret && (ctx->active == NULL)) {
+                /* Do it only for the first time */
+                /* Exit the process.. there are some wrong options */
+                gf_smsg("glfs-mgmt", GF_LOG_ERROR, EINVAL,
+                        API_MSG_GET_VOLFILE_FAILED, "key=%s",
+                        ctx->cmd_args.volfile_id, NULL);
+                errno = EINVAL;
+                glfs_init_done(fs, -1);
+            }
+
+            break;
+        default:
+            break;
+    }
+out:
+    return 0;
+}
 
 int
-glusterfs_mgmt_notify (int32_t op, void *data, ...)
+glusterfs_mgmt_notify(int32_t op, void *data, ...)
 {
-	int ret = 0;
+    int ret = 0;
 
-	switch (op)
-	{
-		case GF_EN_DEFRAG_STATUS:
-			break;
+    switch (op) {
+        case GF_EN_DEFRAG_STATUS:
+            break;
 
-		default:
-			break;
-	}
+        default:
+            break;
+    }
 
-	return ret;
+    return ret;
 }
 
-
 int
-glfs_mgmt_init (struct glfs *fs)
+glfs_mgmt_init(struct glfs *fs)
 {
-	cmd_args_t		*cmd_args = NULL;
-	struct rpc_clnt		*rpc = NULL;
-	dict_t			*options = NULL;
-	int			ret = -1;
-	int			port = GF_DEFAULT_BASE_PORT;
-	char			*host = NULL;
-	glusterfs_ctx_t		*ctx = NULL;
-
-	ctx = fs->ctx;
-	cmd_args = &ctx->cmd_args;
-
-	if (ctx->mgmt)
-		return 0;
-
-	if (cmd_args->volfile_server_port)
-		port = cmd_args->volfile_server_port;
-
-	host = "localhost";
-	if (cmd_args->volfile_server)
-		host = cmd_args->volfile_server;
-
-	ret = rpc_transport_inet_options_build (&options, host, port);
-	if (ret)
-		goto out;
-
-	rpc = rpc_clnt_new (options, THIS->ctx, THIS->name, 8);
-	if (!rpc) {
-		ret = -1;
-		gf_log (THIS->name, GF_LOG_WARNING,
-			"failed to create rpc clnt");
-		goto out;
-	}
-
-	ret = rpc_clnt_register_notify (rpc, mgmt_rpc_notify, THIS);
-	if (ret) {
-		gf_log (THIS->name, GF_LOG_WARNING,
-			"failed to register notify function");
-		goto out;
-	}
-
-	ret = rpcclnt_cbk_program_register (rpc, &mgmt_cbk_prog, THIS);
-	if (ret) {
-		gf_log (THIS->name, GF_LOG_WARNING,
-			"failed to register callback function");
-		goto out;
-	}
-
-	ctx->notify = glusterfs_mgmt_notify;
-
-	/* This value should be set before doing the 'rpc_clnt_start()' as
-	   the notify function uses this variable */
-	ctx->mgmt = rpc;
-
-	ret = rpc_clnt_start (rpc);
+    cmd_args_t *cmd_args = NULL;
+    struct rpc_clnt *rpc = NULL;
+    dict_t *options = NULL;
+    int ret = -1;
+    int port = GF_DEFAULT_BASE_PORT;
+    char *host = NULL;
+    glusterfs_ctx_t *ctx = NULL;
+
+    ctx = fs->ctx;
+    cmd_args = &ctx->cmd_args;
+
+    if (ctx->mgmt)
+        return 0;
+
+    options = dict_new();
+    if (!options)
+        goto out;
+
+    if (cmd_args->volfile_server_port)
+        port = cmd_args->volfile_server_port;
+
+    if (cmd_args->volfile_server) {
+        host = cmd_args->volfile_server;
+    } else if (cmd_args->volfile_server_transport &&
+               !strcmp(cmd_args->volfile_server_transport, "unix")) {
+        host = DEFAULT_GLUSTERD_SOCKFILE;
+    } else {
+        host = "localhost";
+    }
+
+    if (cmd_args->volfile_server_transport &&
+        !strcmp(cmd_args->volfile_server_transport, "unix")) {
+        ret = rpc_transport_unix_options_build(options, host, 0);
+    } else {
+        xlator_cmdline_option_t *opt = find_xlator_option_in_cmd_args_t(
+            "address-family", cmd_args);
+        ret = rpc_transport_inet_options_build(options, host, port,
+                                               (opt ? opt->value : NULL));
+    }
+
+    if (ret)
+        goto out;
+
+    rpc = rpc_clnt_new(options, THIS, THIS->name, 8);
+    if (!rpc) {
+        ret = -1;
+        gf_smsg(THIS->name, GF_LOG_WARNING, 0, API_MSG_CREATE_RPC_CLIENT_FAILED,
+                NULL);
+        goto out;
+    }
+
+    ret = rpc_clnt_register_notify(rpc, mgmt_rpc_notify, THIS);
+    if (ret) {
+        gf_smsg(THIS->name, GF_LOG_WARNING, 0, API_MSG_REG_NOTIFY_FUNC_FAILED,
+                NULL);
+        goto out;
+    }
+
+    ret = rpcclnt_cbk_program_register(rpc, &mgmt_cbk_prog, THIS);
+    if (ret) {
+        gf_smsg(THIS->name, GF_LOG_WARNING, 0, API_MSG_REG_CBK_FUNC_FAILED,
+                NULL);
+        goto out;
+    }
+
+    ctx->notify = glusterfs_mgmt_notify;
+
+    /* This value should be set before doing the 'rpc_clnt_start()' as
+       the notify function uses this variable */
+    ctx->mgmt = rpc;
+
+    ret = rpc_clnt_start(rpc);
 out:
-	return ret;
+    if (options)
+        dict_unref(options);
+    return ret;
 }
diff --git a/api/src/glfs-resolve.c b/api/src/glfs-resolve.c
index 138ab422c8a..8a393ecb464 100644
--- a/api/src/glfs-resolve.c
+++ b/api/src/glfs-resolve.c
@@ -1,5 +1,5 @@
 /*
-  Copyright (c) 2012 Red Hat, Inc. <http://www.redhat.com>
+  Copyright (c) 2012-2018 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
   This file is licensed to you under your choice of the GNU Lesser
@@ -8,7 +8,6 @@
   cases as published by the Free Software Foundation.
 */
 
-
 #include <unistd.h>
 #include <string.h>
 #include <stdlib.h>
@@ -16,959 +15,1185 @@
 #include <inttypes.h>
 #include <limits.h>
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "glusterfs.h"
-#include "logging.h"
-#include "stack.h"
-#include "event.h"
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/stack.h>
+#include <glusterfs/gf-event.h>
 #include "glfs-mem-types.h"
-#include "common-utils.h"
-#include "syncop.h"
-#include "call-stub.h"
-
+#include <glusterfs/common-utils.h>
+#include <glusterfs/syncop.h>
+#include <glusterfs/call-stub.h>
+#include "gfapi-messages.h"
+#include <glusterfs/inode.h>
 #include "glfs-internal.h"
 
-#define graphid_str(subvol) (uuid_utoa((unsigned char *)subvol->graph->graph_uuid))
-
-
+#define graphid_str(subvol)                                                    \
+    (uuid_utoa((unsigned char *)subvol->graph->graph_uuid))
 int
-glfs_first_lookup_safe (xlator_t *subvol)
+glfs_first_lookup_safe(xlator_t *subvol)
 {
-	loc_t  loc = {0, };
-	int    ret = -1;
+    loc_t loc = {
+        0,
+    };
+    int ret = -1;
 
-	loc.inode = subvol->itable->root;
-	memset (loc.gfid, 0, 16);
-	loc.gfid[15] = 1;
-	loc.path = "/";
-	loc.name = "";
+    loc.inode = subvol->itable->root;
+    memset(loc.gfid, 0, 16);
+    loc.gfid[15] = 1;
+    loc.path = "/";
+    loc.name = "";
 
-	ret = syncop_lookup (subvol, &loc, 0, 0, 0, 0);
+    ret = syncop_lookup(subvol, &loc, 0, 0, 0, 0);
+    DECODE_SYNCOP_ERR(ret);
 
-	gf_log (subvol->name, GF_LOG_DEBUG, "first lookup complete %d", ret);
+    gf_msg_debug(subvol->name, 0, "first lookup complete %d", ret);
 
-	return ret;
+    return ret;
 }
 
-
 int
-__glfs_first_lookup (struct glfs *fs, xlator_t *subvol)
+__glfs_first_lookup(struct glfs *fs, xlator_t *subvol)
 {
-	int ret = -1;
-
-	fs->migration_in_progress = 1;
-	pthread_mutex_unlock (&fs->mutex);
-	{
-		ret = glfs_first_lookup_safe (subvol);
-	}
-	pthread_mutex_lock (&fs->mutex);
-	fs->migration_in_progress = 0;
-	pthread_cond_broadcast (&fs->cond);
-
-	return ret;
-}
+    int ret = -1;
 
+    fs->migration_in_progress = 1;
+    pthread_mutex_unlock(&fs->mutex);
+    {
+        ret = glfs_first_lookup_safe(subvol);
+    }
+    pthread_mutex_lock(&fs->mutex);
+    fs->migration_in_progress = 0;
+    pthread_cond_broadcast(&fs->cond);
 
-inode_t *
-glfs_refresh_inode_safe (xlator_t *subvol, inode_t *oldinode)
-{
-	loc_t        loc = {0, };
-	int          ret = -1;
-	struct iatt  iatt = {0, };
-	inode_t     *newinode = NULL;
-
+    /* wake up other waiting tasks */
+    __GLFS_SYNCTASK_WAKE(fs);
 
-	if (!oldinode)
-		return NULL;
+    return ret;
+}
 
-	if (oldinode->table->xl == subvol)
-		return inode_ref (oldinode);
+/**
+ * We have to check if need_lookup flag is set in both old and the new inodes.
+ * If its set in oldinode, then directly go ahead and do an explicit lookup.
+ * But if its not set in the oldinode, then check if the newinode is linked
+ * via readdirp. If so an explicit lookup is needed on the new inode, so that
+ * below xlators can set their respective contexts.
+ */
+inode_t *
+glfs_refresh_inode_safe(xlator_t *subvol, inode_t *oldinode,
+                        gf_boolean_t need_lookup)
+{
+    loc_t loc = {
+        0,
+    };
+    int ret = -1;
+    struct iatt iatt = {
+        0,
+    };
+    inode_t *newinode = NULL;
+    gf_boolean_t lookup_needed = _gf_false;
+    uint64_t ctx_value = LOOKUP_NOT_NEEDED;
+
+    if (!oldinode)
+        return NULL;
+
+    if (!need_lookup && oldinode->table->xl == subvol)
+        return inode_ref(oldinode);
+
+    newinode = inode_find(subvol->itable, oldinode->gfid);
+    if (!need_lookup && newinode) {
+        lookup_needed = inode_needs_lookup(newinode, THIS);
+        if (!lookup_needed)
+            return newinode;
+    }
+
+    gf_uuid_copy(loc.gfid, oldinode->gfid);
+    if (!newinode)
+        loc.inode = inode_new(subvol->itable);
+    else
+        loc.inode = newinode;
+
+    if (!loc.inode)
+        return NULL;
+
+    ret = syncop_lookup(subvol, &loc, &iatt, 0, 0, 0);
+    DECODE_SYNCOP_ERR(ret);
+
+    if (ret) {
+        gf_smsg(subvol->name, GF_LOG_WARNING, errno,
+                API_MSG_INODE_REFRESH_FAILED, "gfid=%s",
+                uuid_utoa(oldinode->gfid), "err=%s", strerror(errno), NULL);
+        loc_wipe(&loc);
+        return NULL;
+    }
+
+    newinode = inode_link(loc.inode, 0, 0, &iatt);
+    if (newinode) {
+        if (newinode == loc.inode)
+            inode_ctx_set(newinode, THIS, &ctx_value);
+        inode_lookup(newinode);
+    } else {
+        gf_smsg(subvol->name, GF_LOG_WARNING, errno, API_MSG_INODE_LINK_FAILED,
+                "gfid=%s", uuid_utoa((unsigned char *)&iatt.ia_gfid), NULL);
+    }
+
+    loc_wipe(&loc);
+
+    return newinode;
+}
 
-	newinode = inode_find (subvol->itable, oldinode->gfid);
-	if (newinode)
-		return newinode;
+inode_t *
+__glfs_refresh_inode(struct glfs *fs, xlator_t *subvol, inode_t *inode,
+                     gf_boolean_t need_lookup)
+{
+    inode_t *newinode = NULL;
 
-	uuid_copy (loc.gfid, oldinode->gfid);
-	loc.inode = inode_new (subvol->itable);
-	if (!loc.inode)
-		return NULL;
+    fs->migration_in_progress = 1;
+    pthread_mutex_unlock(&fs->mutex);
+    {
+        newinode = glfs_refresh_inode_safe(subvol, inode, need_lookup);
+    }
+    pthread_mutex_lock(&fs->mutex);
+    fs->migration_in_progress = 0;
+    pthread_cond_broadcast(&fs->cond);
 
-	ret = syncop_lookup (subvol, &loc, 0, &iatt, 0, 0);
+    /* wake up other waiting tasks */
+    __GLFS_SYNCTASK_WAKE(fs);
 
-	if (ret) {
-		gf_log (subvol->name, GF_LOG_WARNING,
-			"inode refresh of %s failed: %s",
-			uuid_utoa (oldinode->gfid), strerror (errno));
-		loc_wipe (&loc);
-		return NULL;
-	}
+    return newinode;
+}
 
-	newinode = inode_link (loc.inode, 0, 0, &iatt);
-	if (newinode)
-		inode_lookup (newinode);
+GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_loc_touchup, 3.4.0)
+int
+priv_glfs_loc_touchup(loc_t *loc)
+{
+    int ret = 0;
 
-	loc_wipe (&loc);
+    ret = loc_touchup(loc, loc->name);
+    if (ret < 0) {
+        errno = -ret;
+        ret = -1;
+    }
 
-	return newinode;
+    return ret;
 }
 
-
-inode_t *
-__glfs_refresh_inode (struct glfs *fs, xlator_t *subvol, inode_t *inode)
+int
+glfs_resolve_symlink(struct glfs *fs, xlator_t *subvol, inode_t *inode,
+                     char **lpath)
 {
-	inode_t *newinode = NULL;
-
-	fs->migration_in_progress = 1;
-	pthread_mutex_unlock (&fs->mutex);
-	{
-		newinode = glfs_refresh_inode_safe (subvol, inode);
-	}
-	pthread_mutex_lock (&fs->mutex);
-	fs->migration_in_progress = 0;
-	pthread_cond_broadcast (&fs->cond);
-
-	return newinode;
+    loc_t loc = {
+        0,
+    };
+    char *path = NULL;
+    char *rpath = NULL;
+    int ret = -1;
+
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
+    ret = inode_path(inode, NULL, &rpath);
+    if (ret < 0)
+        goto out;
+    loc.path = rpath;
+
+    ret = syncop_readlink(subvol, &loc, &path, 4096, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    if (ret < 0)
+        goto out;
+
+    if (lpath)
+        *lpath = path;
+out:
+    loc_wipe(&loc);
+    return ret;
 }
 
 int
-priv_glfs_loc_touchup (loc_t *loc)
+glfs_resolve_base(struct glfs *fs, xlator_t *subvol, inode_t *inode,
+                  struct iatt *iatt)
 {
-	char *path = NULL;
-	int   ret = -1;
-	char *bn = NULL;
-
-	if (loc->parent)
-		ret = inode_path (loc->parent, loc->name, &path);
-	else
-		ret = inode_path (loc->inode, 0, &path);
-
-	loc->path = path;
-
-	if (ret < 0 || !path) {
-		ret = -1;
-		errno = ENOMEM;
-		goto out;
-	}
-
-	bn = strrchr (path, '/');
-	if (bn)
-		bn++;
-	loc->name = bn;
-	ret = 0;
+    loc_t loc = {
+        0,
+    };
+    int ret = -1;
+    char *path = NULL;
+
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
+
+    ret = inode_path(loc.inode, NULL, &path);
+    loc.path = path;
+    if (ret < 0)
+        goto out;
+
+    ret = syncop_lookup(subvol, &loc, iatt, NULL, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
 out:
-	return ret;
+    loc_wipe(&loc);
+
+    return ret;
 }
+/*
+ * This function can be used to call named lookup on root.
+ * If you use glfs_resolve_base, that will be a nameless lookup.
+ */
+static int
+glfs_resolve_root(struct glfs *fs, xlator_t *subvol, inode_t *inode,
+                  struct iatt *iatt)
+{
+    loc_t loc = {
+        0,
+    };
+    int ret = -1;
+    char *path = NULL;
+
+    loc.inode = inode_ref(inode);
+
+    ret = inode_path(loc.inode, ".", &path);
+    loc.path = path;
+    loc.name = ".";
+    /* Having a value in loc.name will help to bypass md-cache check for
+     * nameless lookup.
+     * TODO: Re-visit on nameless lookup and md-cache.
+     * Github issue : https://github.com/gluster/glusterfs/issues/232
+     */
+    loc.parent = inode_ref(inode);
+    if (ret < 0)
+        goto out;
+
+    ret = syncop_lookup(subvol, &loc, iatt, NULL, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+out:
+    loc_wipe(&loc);
 
-GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_loc_touchup, 3.4.0);
+    return ret;
+}
 
-int
-glfs_resolve_symlink (struct glfs *fs, xlator_t *subvol, inode_t *inode,
-		      char **lpath)
+inode_t *
+glfs_resolve_component(struct glfs *fs, xlator_t *subvol, inode_t *parent,
+                       const char *component, struct iatt *iatt,
+                       int force_lookup)
 {
-	loc_t  loc = {0, };
-	char  *path = NULL;
-	char  *rpath = NULL;
-	int    ret = -1;
+    loc_t loc = {
+        0,
+    };
+    inode_t *inode = NULL;
+    inode_t *temp_parent = NULL;
+    int reval = 0;
+    int ret = -1;
+    int glret = -1;
+    struct iatt ciatt = {
+        0,
+    };
+    uuid_t gfid;
+    dict_t *xattr_req = NULL;
+    uint64_t ctx_value = LOOKUP_NOT_NEEDED;
+
+    loc.parent = inode_ref(parent);
+    gf_uuid_copy(loc.pargfid, parent->gfid);
+
+    if (__is_root_gfid(parent->gfid) &&
+        ((strcmp(component, ".") == 0) || (strcmp(component, "..") == 0) ||
+         (strcmp(component, "") == 0))) {
+        if (!force_lookup) {
+            inode = inode_ref(parent);
+        } else {
+            ret = glfs_resolve_root(fs, subvol, parent, &ciatt);
+            if (!ret)
+                inode = inode_ref(parent);
+        }
+        goto found;
+    }
+    /* *
+     * if the component name is either "." or "..", it will try to
+     * resolve that if inode has a proper parent (named lookup).
+     *
+     * Below condition works like this
+     *
+     * Example 1 :
+     * Path /out_dir/dir/in_dir/.
+     *     In put values :
+     *         parent = in_dir
+     *         component : "."
+     *
+     *      Out put values:
+     *         parent : dir
+     *         component : "in_dir"
+     *
+     * Example 2 :
+     * Path /out_dir/dir/in_dir/..
+     *     In put values :
+     *         parent = in_dir
+     *         component : ".."
+     *
+     *     Out put values:
+     *         parent : output_dir
+     *         component : "dir"
+     *
+     * In case of nameless lookup, both "." and ".." retained
+     */
+
+    if (strcmp(component, ".") == 0) {
+        loc.inode = inode_ref(parent);
+        temp_parent = inode_parent(loc.inode, 0, 0);
+        if (temp_parent) {
+            inode_unref(loc.parent);
+            loc.parent = temp_parent;
+            gf_uuid_copy(loc.pargfid, temp_parent->gfid);
+            inode_find_directory_name(loc.inode, &loc.name);
+        }
 
-	loc.inode = inode_ref (inode);
-	uuid_copy (loc.gfid, inode->gfid);
-	ret = inode_path (inode, NULL, &rpath);
-	if (ret < 0)
-		goto out;
-	loc.path = rpath;
+    } else if (strcmp(component, "..") == 0) {
+        loc.inode = inode_parent(parent, 0, 0);
+        if (loc.inode) {
+            temp_parent = inode_parent(loc.inode, 0, 0);
+            if (temp_parent) {
+                inode_unref(loc.parent);
+                loc.parent = temp_parent;
+                gf_uuid_copy(loc.pargfid, temp_parent->gfid);
+                inode_find_directory_name(loc.inode, &loc.name);
+            } else if (__is_root_gfid(loc.inode->gfid)) {
+                inode_unref(loc.parent);
+                loc.parent = inode_ref(loc.inode);
+                gf_uuid_copy(loc.pargfid, loc.inode->gfid);
+                loc.name = ".";
+            } else {
+                inode_unref(loc.inode);
+                loc.inode = NULL;
+            }
+        }
+    } else
+        loc.inode = inode_grep(parent->table, parent, component);
 
-	ret = syncop_readlink (subvol, &loc, &path, 4096);
+    if (!loc.name)
+        loc.name = component;
 
-	if (ret < 0)
-		goto out;
+    if (loc.inode) {
+        gf_uuid_copy(loc.gfid, loc.inode->gfid);
+        reval = 1;
 
-	if (lpath)
-		*lpath = path;
-out:
-	loc_wipe (&loc);
-	return ret;
-}
+        if (!(force_lookup || inode_needs_lookup(loc.inode, THIS))) {
+            inode = inode_ref(loc.inode);
+            goto found;
+        }
+    } else {
+        gf_uuid_generate(gfid);
+        loc.inode = inode_new(parent->table);
+        if (!loc.inode) {
+            errno = ENOMEM;
+            goto out;
+        }
 
+        xattr_req = dict_new();
+        if (!xattr_req) {
+            errno = ENOMEM;
+            goto out;
+        }
 
-int
-glfs_resolve_base (struct glfs *fs, xlator_t *subvol, inode_t *inode,
-		   struct iatt *iatt)
-{
-	loc_t       loc = {0, };
-	int         ret = -1;
-	char       *path = NULL;
+        ret = dict_set_gfuuid(xattr_req, "gfid-req", gfid, true);
+        if (ret) {
+            errno = ENOMEM;
+            goto out;
+        }
+    }
+
+    glret = priv_glfs_loc_touchup(&loc);
+    if (glret < 0) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = syncop_lookup(subvol, &loc, &ciatt, NULL, xattr_req, NULL);
+    if (ret && reval) {
+        /*
+         * A stale mapping might exist for a dentry/inode that has been
+         * removed from another client.
+         */
+        if (-ret == ENOENT) {
+            inode_unlink(loc.inode, loc.parent, loc.name);
+            if (!inode_has_dentry(loc.inode))
+                inode_forget(loc.inode, 0);
+        }
+
+        inode_unref(loc.inode);
+        gf_uuid_clear(loc.gfid);
+        loc.inode = inode_new(parent->table);
+        if (!loc.inode) {
+            errno = ENOMEM;
+            goto out;
+        }
 
-	loc.inode = inode_ref (inode);
-	uuid_copy (loc.gfid, inode->gfid);
+        xattr_req = dict_new();
+        if (!xattr_req) {
+            errno = ENOMEM;
+            goto out;
+        }
 
-	ret = inode_path (loc.inode, NULL, &path);
-	loc.path = path;
-	if (ret < 0)
-		goto out;
+        gf_uuid_generate(gfid);
 
-	ret = syncop_lookup (subvol, &loc, NULL, iatt, NULL, NULL);
-out:
-	loc_wipe (&loc);
+        ret = dict_set_gfuuid(xattr_req, "gfid-req", gfid, true);
+        if (ret) {
+            errno = ENOMEM;
+            goto out;
+        }
 
-	return ret;
-}
+        ret = syncop_lookup(subvol, &loc, &ciatt, NULL, xattr_req, NULL);
+    }
+    DECODE_SYNCOP_ERR(ret);
+    if (ret)
+        goto out;
 
+    inode = inode_link(loc.inode, loc.parent, component, &ciatt);
 
-inode_t *
-glfs_resolve_component (struct glfs *fs, xlator_t *subvol, inode_t *parent,
-			const char *component, struct iatt *iatt,
-			int force_lookup)
-{
-	loc_t        loc = {0, };
-	inode_t     *inode = NULL;
-	int          reval = 0;
-	int          ret = -1;
-	int          glret = -1;
-	struct iatt  ciatt = {0, };
-	uuid_t       gfid;
-	dict_t      *xattr_req = NULL;
-
-	loc.name = component;
-
-	loc.parent = inode_ref (parent);
-	uuid_copy (loc.pargfid, parent->gfid);
-
-
-	if (strcmp (component, ".") == 0)
-		loc.inode = inode_ref (parent);
-	else if (strcmp (component, "..") == 0)
-		loc.inode = inode_parent (parent, 0, 0);
-	else
-		loc.inode = inode_grep (parent->table, parent, component);
-
-	if (loc.inode) {
-		uuid_copy (loc.gfid, loc.inode->gfid);
-		reval = 1;
-
-		if (!force_lookup) {
-			inode = inode_ref (loc.inode);
-			ciatt.ia_type = inode->ia_type;
-			goto found;
-		}
-	} else {
-		uuid_generate (gfid);
-		loc.inode = inode_new (parent->table);
-	}
-
-	if (!loc.inode)
-		goto out;
-
-	glret = priv_glfs_loc_touchup (&loc);
-	if (glret < 0) {
-		ret = -1;
-		goto out;
-	}
-
-	ret = syncop_lookup (subvol, &loc, NULL, &ciatt, NULL, NULL);
-	if (ret && reval) {
-		inode_unref (loc.inode);
-		loc.inode = inode_new (parent->table);
-		if (!loc.inode) {
-			errno = ENOMEM;
-			goto out;
-		}
-
-		xattr_req = dict_new ();
-		if (!xattr_req) {
-			errno = ENOMEM;
-			goto out;
-		}
-
-		uuid_generate (gfid);
-
-		ret = dict_set_static_bin (xattr_req, "gfid-req", gfid, 16);
-		if (ret) {
-			errno = ENOMEM;
-			goto out;
-		}
-
-		ret = syncop_lookup (subvol, &loc, xattr_req, &ciatt,
-				     NULL, NULL);
-	}
-	if (ret)
-		goto out;
-
-	inode = inode_link (loc.inode, loc.parent, component, &ciatt);
+    if (!inode) {
+        gf_smsg(subvol->name, GF_LOG_WARNING, errno, API_MSG_INODE_LINK_FAILED,
+                "gfid=%s", uuid_utoa((unsigned char *)&ciatt.ia_gfid), NULL);
+        goto out;
+    } else if (inode == loc.inode)
+        inode_ctx_set(inode, THIS, &ctx_value);
 found:
-	if (inode)
-		inode_lookup (inode);
-	if (iatt)
-		*iatt = ciatt;
+    if (inode) {
+        ciatt.ia_type = inode->ia_type;
+        inode_lookup(inode);
+    }
+    if (iatt)
+        *iatt = ciatt;
 out:
-	if (xattr_req)
-		dict_unref (xattr_req);
+    if (xattr_req)
+        dict_unref(xattr_req);
+    loc_wipe(&loc);
 
-	loc_wipe (&loc);
-
-	return inode;
+    return inode;
 }
 
-
+GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_resolve_at, 3.4.0)
 int
-priv_glfs_resolve_at (struct glfs *fs, xlator_t *subvol, inode_t *at,
-		 const char *origpath, loc_t *loc, struct iatt *iatt,
-		 int follow, int reval)
+priv_glfs_resolve_at(struct glfs *fs, xlator_t *subvol, inode_t *at,
+                     const char *origpath, loc_t *loc, struct iatt *iatt,
+                     int follow, int reval)
 {
-	inode_t    *inode = NULL;
-	inode_t    *parent = NULL;
-	char       *saveptr = NULL;
-	char       *path = NULL;
-	char       *component = NULL;
-	char       *next_component = NULL;
-	int         ret = -1;
-	struct iatt ciatt = {0, };
-
-	path = gf_strdup (origpath);
-	if (!path) {
-		errno = ENOMEM;
-		return -1;
-	}
-
-	parent = NULL;
-	if (at && path[0] != '/') {
-		/* A relative resolution of a path which starts with '/'
-		   is equal to an absolute path resolution.
-		*/
-		inode = inode_ref (at);
-	} else {
-		inode = inode_ref (subvol->itable->root);
-
-		if (strcmp (path, "/") == 0)
-			glfs_resolve_base (fs, subvol, inode, &ciatt);
-	}
-
-	for (component = strtok_r (path, "/", &saveptr);
-	     component; component = next_component) {
-
-		next_component = strtok_r (NULL, "/", &saveptr);
-
-		if (parent)
-			inode_unref (parent);
-
-		parent = inode;
-
-		inode = glfs_resolve_component (fs, subvol, parent,
-						component, &ciatt,
-						/* force hard lookup on the last
-						   component, as the caller
-						   wants proper iatt filled
-						*/
-						(reval || (!next_component &&
-						iatt)));
-		if (!inode)
-			break;
-
-		if (IA_ISLNK (ciatt.ia_type) && (next_component || follow)) {
-			/* If the component is not the last piece,
-			   then following it is necessary even if
-			   not requested by the caller
-			*/
-			char *lpath = NULL;
-			loc_t sym_loc = {0,};
-
-			if (follow > GLFS_SYMLINK_MAX_FOLLOW) {
-				errno = ELOOP;
-				ret = -1;
-				if (inode) {
-					inode_unref (inode);
-					inode = NULL;
-				}
-				break;
-			}
-
-			ret = glfs_resolve_symlink (fs, subvol, inode, &lpath);
-			inode_unref (inode);
-			inode = NULL;
-			if (ret < 0)
-				break;
-
-			ret = priv_glfs_resolve_at (fs, subvol, parent, lpath,
-					       &sym_loc,
-					       /* followed iatt becomes the
-						  component iatt
-					       */
-					       &ciatt,
-					       /* always recurisvely follow while
-						  following symlink
-					       */
-					       follow + 1, reval);
-			if (ret == 0)
-				inode = inode_ref (sym_loc.inode);
-			loc_wipe (&sym_loc);
-			GF_FREE (lpath);
-		}
-
-		if (!next_component)
-			break;
-
-		if (!IA_ISDIR (ciatt.ia_type)) {
-			/* next_component exists and this component is
-			   not a directory
-			*/
-			inode_unref (inode);
-			inode = NULL;
-			ret = -1;
-			errno = ENOTDIR;
-			break;
-		}
-	}
-
-	if (parent && next_component)
-		/* resolution failed mid-way */
-		goto out;
-
-	/* At this point, all components up to the last parent directory
-	   have been resolved successfully (@parent). Resolution of basename
-	   might have failed (@inode) if at all.
-	*/
-
-	loc->parent = parent;
-	if (parent) {
-		uuid_copy (loc->pargfid, parent->gfid);
-		loc->name = component;
-	}
-
-	loc->inode = inode;
-	if (inode) {
-		uuid_copy (loc->gfid, inode->gfid);
-		if (iatt)
-			*iatt = ciatt;
-		ret = 0;
-	}
-
-	priv_glfs_loc_touchup (loc);
-out:
-	GF_FREE (path);
-
-	/* do NOT loc_wipe here as only last component might be missing */
+    inode_t *inode = NULL;
+    inode_t *parent = NULL;
+    char *saveptr = NULL;
+    char *path = NULL;
+    char *component = NULL;
+    char *next_component = NULL;
+    int ret = -1;
+    struct iatt ciatt = {
+        0,
+    };
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    if (origpath[0] == '\0') {
+        errno = EINVAL;
+        goto invalid_fs;
+    }
+
+    parent = NULL;
+    if (at && origpath[0] != '/') {
+        /* A relative resolution of a path which starts with '/'
+           is equal to an absolute path resolution.
+        */
+        inode = inode_ref(at);
+    } else {
+        inode = inode_ref(subvol->itable->root);
+
+        if (strcmp(origpath, "/") == 0)
+            glfs_resolve_root(fs, subvol, inode, &ciatt);
+    }
+
+    path = gf_strdup(origpath);
+    if (!path)
+        goto invalid_fs;
+
+    for (component = strtok_r(path, "/", &saveptr); component;
+         component = next_component) {
+        next_component = strtok_r(NULL, "/", &saveptr);
+
+        if (parent)
+            inode_unref(parent);
+        parent = inode;
+        inode = glfs_resolve_component(fs, subvol, parent, component, &ciatt,
+                                       /* force hard lookup on the last
+                                          component, as the caller
+                                          wants proper iatt filled
+                                       */
+                                       (reval || (!next_component && iatt)));
+        if (!inode) {
+            ret = -1;
+            break;
+        }
 
-	return ret;
-}
+        if (IA_ISLNK(ciatt.ia_type) && (next_component || follow)) {
+            /* If the component is not the last piece,
+               then following it is necessary even if
+               not requested by the caller
+            */
+            char *lpath = NULL;
+            loc_t sym_loc = {
+                0,
+            };
+
+            if (follow > GLFS_SYMLINK_MAX_FOLLOW) {
+                errno = ELOOP;
+                ret = -1;
+                if (inode) {
+                    inode_unref(inode);
+                    inode = NULL;
+                }
+                break;
+            }
+
+            ret = glfs_resolve_symlink(fs, subvol, inode, &lpath);
+            inode_unref(inode);
+            inode = NULL;
+            if (ret < 0)
+                break;
+
+            ret = priv_glfs_resolve_at(fs, subvol, parent, lpath, &sym_loc,
+                                       /* followed iatt becomes the
+                                          component iatt
+                                       */
+                                       &ciatt,
+                                       /* always recurisvely follow while
+                                          following symlink
+                                       */
+                                       follow + 1, reval);
+            if (ret == 0)
+                inode = inode_ref(sym_loc.inode);
+            loc_wipe(&sym_loc);
+            GF_FREE(lpath);
+        }
 
-GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_resolve_at, 3.4.0);
+        if (!next_component)
+            break;
+
+        if (!IA_ISDIR(ciatt.ia_type)) {
+            /* next_component exists and this component is
+               not a directory
+            */
+            inode_unref(inode);
+            inode = NULL;
+            ret = -1;
+            errno = ENOTDIR;
+            break;
+        }
+    }
+
+    if (parent && next_component)
+        /* resolution failed mid-way */
+        goto out;
+
+    /* At this point, all components up to the last parent directory
+       have been resolved successfully (@parent). Resolution of basename
+       might have failed (@inode) if at all.
+    */
+
+    loc->parent = parent;
+    if (parent) {
+        gf_uuid_copy(loc->pargfid, parent->gfid);
+        loc->name = component;
+    }
+
+    loc->inode = inode;
+    if (inode) {
+        gf_uuid_copy(loc->gfid, inode->gfid);
+        if (iatt)
+            *iatt = ciatt;
+        ret = 0;
+    }
+
+    if (priv_glfs_loc_touchup(loc) < 0) {
+        ret = -1;
+    }
+out:
+    GF_FREE(path);
+    __GLFS_EXIT_FS;
 
+    /* do NOT loc_wipe here as only last component might be missing */
+invalid_fs:
+    return ret;
+}
 
 int
-glfs_resolve_path (struct glfs *fs, xlator_t *subvol, const char *origpath,
-		   loc_t *loc, struct iatt *iatt, int follow, int reval)
+glfs_resolve_path(struct glfs *fs, xlator_t *subvol, const char *origpath,
+                  loc_t *loc, struct iatt *iatt, int follow, int reval)
 {
-	int ret = -1;
-	inode_t *cwd = NULL;
+    int ret = -1;
+    inode_t *cwd = NULL;
 
-	if (origpath[0] == '/')
-		return priv_glfs_resolve_at (fs, subvol, NULL, origpath, loc,
-                                             iatt, follow, reval);
+    if (origpath[0] == '/')
+        return priv_glfs_resolve_at(fs, subvol, NULL, origpath, loc, iatt,
+                                    follow, reval);
 
-	cwd = glfs_cwd_get (fs);
+    cwd = glfs_cwd_get(fs);
+    if (NULL == cwd) {
+        gf_smsg(subvol->name, GF_LOG_WARNING, EIO, API_MSG_GET_CWD_FAILED,
+                NULL);
+        errno = EIO;
+        goto out;
+    }
 
-	ret = priv_glfs_resolve_at (fs, subvol, cwd, origpath, loc, iatt,
-                                    follow, reval);
-	if (cwd)
-		inode_unref (cwd);
+    ret = priv_glfs_resolve_at(fs, subvol, cwd, origpath, loc, iatt, follow,
+                               reval);
+    if (cwd)
+        inode_unref(cwd);
 
-	return ret;
+out:
+    return ret;
 }
 
-
+GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_resolve, 3.7.0)
 int
-glfs_resolve (struct glfs *fs, xlator_t *subvol, const char *origpath,
-	      loc_t *loc, struct iatt *iatt, int reval)
+priv_glfs_resolve(struct glfs *fs, xlator_t *subvol, const char *origpath,
+                  loc_t *loc, struct iatt *iatt, int reval)
 {
-	int ret = -1;
+    int ret = -1;
 
-	ret = glfs_resolve_path (fs, subvol, origpath, loc, iatt, 1, reval);
+    ret = glfs_resolve_path(fs, subvol, origpath, loc, iatt, 1, reval);
 
-	return ret;
+    return ret;
 }
 
-
 int
-glfs_lresolve (struct glfs *fs, xlator_t *subvol, const char *origpath,
-	       loc_t *loc, struct iatt *iatt, int reval)
+glfs_lresolve(struct glfs *fs, xlator_t *subvol, const char *origpath,
+              loc_t *loc, struct iatt *iatt, int reval)
 {
-	int ret = -1;
+    int ret = -1;
 
-	ret = glfs_resolve_path (fs, subvol, origpath, loc, iatt, 0, reval);
+    ret = glfs_resolve_path(fs, subvol, origpath, loc, iatt, 0, reval);
 
-	return ret;
+    return ret;
 }
 
-
 int
-glfs_migrate_fd_locks_safe (struct glfs *fs, xlator_t *oldsubvol, fd_t *oldfd,
-			    xlator_t *newsubvol, fd_t *newfd)
+glfs_migrate_fd_locks_safe(struct glfs *fs, xlator_t *oldsubvol, fd_t *oldfd,
+                           xlator_t *newsubvol, fd_t *newfd)
 {
-	dict_t *lockinfo = NULL;
-	int ret = 0;
-	char uuid1[64];
-
-	if (!oldfd->lk_ctx || fd_lk_ctx_empty (oldfd->lk_ctx))
-		return 0;
-
-	newfd->lk_ctx = fd_lk_ctx_ref (oldfd->lk_ctx);
-
-	ret = syncop_fgetxattr (oldsubvol, oldfd, &lockinfo,
-				GF_XATTR_LOCKINFO_KEY);
-	if (ret < 0) {
-		gf_log (fs->volname, GF_LOG_WARNING,
-			"fgetxattr (%s) failed (%s) on graph %s (%d)",
-			uuid_utoa_r (oldfd->inode->gfid, uuid1),
-			strerror (errno),
-			graphid_str (oldsubvol), oldsubvol->graph->id);
-		goto out;
-	}
-
-	if (!dict_get (lockinfo, GF_XATTR_LOCKINFO_KEY)) {
-		gf_log (fs->volname, GF_LOG_WARNING,
-			"missing lokinfo key (%s) on graph %s (%d)",
-			uuid_utoa_r (oldfd->inode->gfid, uuid1),
-			graphid_str (oldsubvol), oldsubvol->graph->id);
-		goto out;
-	}
-
-	ret = syncop_fsetxattr (newsubvol, newfd, lockinfo, 0);
-	if (ret < 0) {
-		gf_log (fs->volname, GF_LOG_WARNING,
-			"fsetxattr (%s) failed (%s) on graph %s (%d)",
-			uuid_utoa_r (newfd->inode->gfid, uuid1),
-			strerror (errno),
-			graphid_str (newsubvol), newsubvol->graph->id);
-		goto out;
-	}
+    dict_t *lockinfo = NULL;
+    int ret = 0;
+    char uuid1[64];
+
+    if (!oldfd->lk_ctx || fd_lk_ctx_empty(oldfd->lk_ctx))
+        return 0;
+
+    newfd->lk_ctx = fd_lk_ctx_ref(oldfd->lk_ctx);
+
+    ret = syncop_fgetxattr(oldsubvol, oldfd, &lockinfo, GF_XATTR_LOCKINFO_KEY,
+                           NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+    if (ret < 0) {
+        gf_smsg(fs->volname, GF_LOG_WARNING, errno, API_MSG_FGETXATTR_FAILED,
+                "gfid=%s", uuid_utoa_r(oldfd->inode->gfid, uuid1), "err=%s",
+                strerror(errno), "subvol=%s", graphid_str(oldsubvol), "id=%d",
+                oldsubvol->graph->id, NULL);
+        goto out;
+    }
+
+    if (!dict_get(lockinfo, GF_XATTR_LOCKINFO_KEY)) {
+        gf_smsg(fs->volname, GF_LOG_WARNING, 0, API_MSG_LOCKINFO_KEY_MISSING,
+                "gfid=%s", uuid_utoa_r(oldfd->inode->gfid, uuid1), "subvol=%s",
+                graphid_str(oldsubvol), "id=%d", oldsubvol->graph->id, NULL);
+        goto out;
+    }
+
+    ret = syncop_fsetxattr(newsubvol, newfd, lockinfo, 0, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+    if (ret < 0) {
+        gf_smsg(fs->volname, GF_LOG_WARNING, 0, API_MSG_FSETXATTR_FAILED,
+                "gfid=%s", uuid_utoa_r(newfd->inode->gfid, uuid1), "err=%s",
+                strerror(errno), "subvol=%s", graphid_str(newsubvol), "id=%d",
+                newsubvol->graph->id, NULL);
+        goto out;
+    }
 out:
-	if (lockinfo)
-		dict_unref (lockinfo);
-	return ret;
+    if (lockinfo)
+        dict_unref(lockinfo);
+    return ret;
 }
 
-
 fd_t *
-glfs_migrate_fd_safe (struct glfs *fs, xlator_t *newsubvol, fd_t *oldfd)
+glfs_migrate_fd_safe(struct glfs *fs, xlator_t *newsubvol, fd_t *oldfd)
 {
-	fd_t *newfd = NULL;
-	inode_t *oldinode = NULL;
-	inode_t *newinode = NULL;
-	xlator_t *oldsubvol = NULL;
-	int ret = -1;
-	loc_t loc = {0, };
-	char uuid1[64];
-
-
-	oldinode = oldfd->inode;
-	oldsubvol = oldinode->table->xl;
-
-	if (oldsubvol == newsubvol)
-		return fd_ref (oldfd);
-
-	if (!oldsubvol->switched) {
-		ret = syncop_fsync (oldsubvol, oldfd, 0);
-		if (ret) {
-			gf_log (fs->volname, GF_LOG_WARNING,
-				"fsync() failed (%s) on %s graph %s (%d)",
-				strerror (errno),
-				uuid_utoa_r (oldfd->inode->gfid, uuid1),
-				graphid_str (oldsubvol), oldsubvol->graph->id);
-		}
-	}
-
-	newinode = glfs_refresh_inode_safe (newsubvol, oldinode);
-	if (!newinode) {
-		gf_log (fs->volname, GF_LOG_WARNING,
-			"inode (%s) refresh failed (%s) on graph %s (%d)",
-			uuid_utoa_r (oldinode->gfid, uuid1),
-			strerror (errno),
-			graphid_str (newsubvol), newsubvol->graph->id);
-		goto out;
-	}
-
-	newfd = fd_create (newinode, getpid());
-	if (!newfd) {
-		gf_log (fs->volname, GF_LOG_WARNING,
-			"fd_create (%s) failed (%s) on graph %s (%d)",
-			uuid_utoa_r (newinode->gfid, uuid1),
-			strerror (errno),
-			graphid_str (newsubvol), newsubvol->graph->id);
-		goto out;
-	}
-
-	loc.inode = inode_ref (newinode);
-
-        ret = inode_path (oldfd->inode, NULL, (char **)&loc.path);
-        if (ret < 0) {
-                gf_log (fs->volname, GF_LOG_INFO, "inode_path failed");
-                goto out;
+    fd_t *newfd = NULL;
+    inode_t *oldinode = NULL;
+    inode_t *newinode = NULL;
+    xlator_t *oldsubvol = NULL;
+    int ret = -1;
+    loc_t loc = {
+        0,
+    };
+    char uuid1[64];
+    dict_t *xdata = NULL;
+
+    oldinode = oldfd->inode;
+    oldsubvol = oldinode->table->xl;
+
+    if (oldsubvol == newsubvol)
+        return fd_ref(oldfd);
+
+    if (!oldsubvol->switched) {
+        xdata = dict_new();
+        if (!xdata || dict_set_int8(xdata, "last-fsync", 1)) {
+            gf_smsg(fs->volname, GF_LOG_WARNING, ENOMEM, API_MSG_FSYNC_FAILED,
+                    "err=%s", "last-fsync set failed", "gfid=%s",
+                    uuid_utoa_r(oldfd->inode->gfid, uuid1), "subvol=%s",
+                    graphid_str(oldsubvol), "id=%d", oldsubvol->graph->id,
+                    NULL);
         }
 
-        uuid_copy (loc.gfid, oldinode->gfid);
-
-
-	if (IA_ISDIR (oldinode->ia_type))
-		ret = syncop_opendir (newsubvol, &loc, newfd);
-	else
-		ret = syncop_open (newsubvol, &loc,
-				   oldfd->flags & ~(O_TRUNC|O_EXCL|O_CREAT),
-				   newfd);
-	loc_wipe (&loc);
-
-	if (ret) {
-		gf_log (fs->volname, GF_LOG_WARNING,
-			"syncop_open%s (%s) failed (%s) on graph %s (%d)",
-			IA_ISDIR (oldinode->ia_type) ? "dir" : "",
-			uuid_utoa_r (newinode->gfid, uuid1),
-			strerror (errno),
-			graphid_str (newsubvol), newsubvol->graph->id);
-		goto out;
-	}
-
-	ret = glfs_migrate_fd_locks_safe (fs, oldsubvol, oldfd, newsubvol,
-					  newfd);
-
-	if (ret) {
-		gf_log (fs->volname, GF_LOG_WARNING,
-			"lock migration (%s) failed (%s) on graph %s (%d)",
-			uuid_utoa_r (newinode->gfid, uuid1),
-			strerror (errno),
-			graphid_str (newsubvol), newsubvol->graph->id);
-		goto out;
-	}
-
-        newfd->flags = oldfd->flags;
-	fd_bind (newfd);
+        ret = syncop_fsync(oldsubvol, oldfd, 0, NULL, NULL, xdata, NULL);
+        DECODE_SYNCOP_ERR(ret);
+        if (ret) {
+            gf_smsg(fs->volname, GF_LOG_WARNING, errno, API_MSG_FSYNC_FAILED,
+                    "err=%s", strerror(errno), "gfid=%s",
+                    uuid_utoa_r(oldfd->inode->gfid, uuid1), "subvol=%s",
+                    graphid_str(oldsubvol), "id=%d", oldsubvol->graph->id,
+                    NULL);
+        }
+    }
+
+    newinode = glfs_refresh_inode_safe(newsubvol, oldinode, _gf_false);
+    if (!newinode) {
+        gf_smsg(fs->volname, GF_LOG_WARNING, errno,
+                API_MSG_INODE_REFRESH_FAILED, "gfid=%s",
+                uuid_utoa_r(oldinode->gfid, uuid1), "err=%s", strerror(errno),
+                "subvol=%s", graphid_str(newsubvol), "id=%d",
+                newsubvol->graph->id, NULL);
+        goto out;
+    }
+
+    newfd = fd_create(newinode, getpid());
+    if (!newfd) {
+        gf_smsg(fs->volname, GF_LOG_WARNING, errno,
+                API_MSG_FDCREATE_FAILED_ON_GRAPH, "gfid=%s",
+                uuid_utoa_r(newinode->gfid, uuid1), "err=%s", strerror(errno),
+                "subvol=%s", graphid_str(newsubvol), "id=%d",
+                newsubvol->graph->id, NULL);
+        goto out;
+    }
+
+    loc.inode = inode_ref(newinode);
+
+    ret = inode_path(oldfd->inode, NULL, (char **)&loc.path);
+    if (ret < 0) {
+        gf_smsg(fs->volname, GF_LOG_INFO, 0, API_MSG_INODE_PATH_FAILED, NULL);
+        goto out;
+    }
+
+    gf_uuid_copy(loc.gfid, oldinode->gfid);
+
+    if (IA_ISDIR(oldinode->ia_type))
+        ret = syncop_opendir(newsubvol, &loc, newfd, NULL, NULL);
+    else
+        ret = syncop_open(newsubvol, &loc,
+                          oldfd->flags & ~(O_TRUNC | O_EXCL | O_CREAT), newfd,
+                          NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+    loc_wipe(&loc);
+
+    if (ret) {
+        gf_smsg(fs->volname, GF_LOG_WARNING, errno, API_MSG_SYNCOP_OPEN_FAILED,
+                "type=%s", IA_ISDIR(oldinode->ia_type) ? "dir" : "", "gfid=%s",
+                uuid_utoa_r(newinode->gfid, uuid1), "err=%s", strerror(errno),
+                "subvol=%s", graphid_str(newsubvol), "id=%d",
+                newsubvol->graph->id, NULL);
+        goto out;
+    }
+
+    ret = glfs_migrate_fd_locks_safe(fs, oldsubvol, oldfd, newsubvol, newfd);
+
+    if (ret) {
+        gf_smsg(fs->volname, GF_LOG_WARNING, errno, API_MSG_LOCK_MIGRATE_FAILED,
+                "gfid=%s", uuid_utoa_r(newinode->gfid, uuid1), "err=%s",
+                strerror(errno), "subvol=%s", graphid_str(newsubvol), "id=%d",
+                newsubvol->graph->id, NULL);
+        goto out;
+    }
+
+    newfd->flags = oldfd->flags;
+    fd_bind(newfd);
 out:
-	if (newinode)
-		inode_unref (newinode);
+    if (newinode)
+        inode_unref(newinode);
 
-	if (ret) {
-		fd_unref (newfd);
-		newfd = NULL;
-	}
+    if (ret) {
+        fd_unref(newfd);
+        newfd = NULL;
+    }
 
-	return newfd;
-}
+    if (xdata)
+        dict_unref(xdata);
 
+    return newfd;
+}
 
 fd_t *
-__glfs_migrate_fd (struct glfs *fs, xlator_t *newsubvol, struct glfs_fd *glfd)
+__glfs_migrate_fd(struct glfs *fs, xlator_t *newsubvol, struct glfs_fd *glfd)
 {
-	fd_t *oldfd = NULL;
-	fd_t *newfd = NULL;
+    fd_t *oldfd = NULL;
+    fd_t *newfd = NULL;
 
-	oldfd = glfd->fd;
+    oldfd = glfd->fd;
 
-	fs->migration_in_progress = 1;
-	pthread_mutex_unlock (&fs->mutex);
-	{
-		newfd = glfs_migrate_fd_safe (fs, newsubvol, oldfd);
-	}
-	pthread_mutex_lock (&fs->mutex);
-	fs->migration_in_progress = 0;
-	pthread_cond_broadcast (&fs->cond);
+    fs->migration_in_progress = 1;
+    pthread_mutex_unlock(&fs->mutex);
+    {
+        newfd = glfs_migrate_fd_safe(fs, newsubvol, oldfd);
+    }
+    pthread_mutex_lock(&fs->mutex);
+    fs->migration_in_progress = 0;
+    pthread_cond_broadcast(&fs->cond);
 
-	return newfd;
-}
+    /* wake up other waiting tasks */
+    __GLFS_SYNCTASK_WAKE(fs);
 
+    return newfd;
+}
 
 fd_t *
-__glfs_resolve_fd (struct glfs *fs, xlator_t *subvol, struct glfs_fd *glfd)
+__glfs_resolve_fd(struct glfs *fs, xlator_t *subvol, struct glfs_fd *glfd)
 {
-	fd_t *fd = NULL;
+    fd_t *fd = NULL;
 
-	if (glfd->fd->inode->table->xl == subvol)
-		return fd_ref (glfd->fd);
+    if (glfd->fd->inode->table->xl == subvol)
+        return fd_ref(glfd->fd);
 
-	fd = __glfs_migrate_fd (fs, subvol, glfd);
-	if (!fd)
-		return NULL;
+    fd = __glfs_migrate_fd(fs, subvol, glfd);
+    if (!fd)
+        return NULL;
 
-	if (subvol == fs->active_subvol) {
-		fd_unref (glfd->fd);
-		glfd->fd = fd_ref (fd);
-	}
+    if (subvol == fs->active_subvol) {
+        fd_unref(glfd->fd);
+        glfd->fd = fd_ref(fd);
+    }
 
-	return fd;
+    return fd;
 }
 
-
 fd_t *
-glfs_resolve_fd (struct glfs *fs, xlator_t *subvol, struct glfs_fd *glfd)
+glfs_resolve_fd(struct glfs *fs, xlator_t *subvol, struct glfs_fd *glfd)
 {
-	fd_t *fd = NULL;
+    fd_t *fd = NULL;
 
-	glfs_lock (fs);
-	{
-		fd = __glfs_resolve_fd (fs, subvol, glfd);
-	}
-	glfs_unlock (fs);
+    glfs_lock(fs, _gf_true);
+    {
+        fd = __glfs_resolve_fd(fs, subvol, glfd);
+    }
+    glfs_unlock(fs);
 
-	return fd;
+    return fd;
 }
 
-
 void
-__glfs_migrate_openfds (struct glfs *fs, xlator_t *subvol)
+__glfs_migrate_openfds(struct glfs *fs, xlator_t *subvol)
 {
-	struct glfs_fd *glfd = NULL;
-	fd_t *fd = NULL;
-
-	list_for_each_entry (glfd, &fs->openfds, openfds) {
-		if (uuid_is_null (glfd->fd->inode->gfid)) {
-			gf_log (fs->volname, GF_LOG_INFO,
-				"skipping openfd %p/%p in graph %s (%d)",
-				glfd, glfd->fd,	graphid_str(subvol),
-				subvol->graph->id);
-			/* create in progress, defer */
-			continue;
-		}
-
-		fd = __glfs_migrate_fd (fs, subvol, glfd);
-		if (fd) {
-			fd_unref (glfd->fd);
-			glfd->fd = fd;
-		}
-	}
-}
-
+    struct glfs_fd *glfd = NULL;
+    fd_t *fd = NULL;
+
+    list_for_each_entry(glfd, &fs->openfds, openfds)
+    {
+        if (gf_uuid_is_null(glfd->fd->inode->gfid)) {
+            gf_smsg(fs->volname, GF_LOG_INFO, 0, API_MSG_OPENFD_SKIPPED,
+                    "glfd=%p", glfd, "glfd->fd=%p", glfd->fd, "subvol=%s",
+                    graphid_str(subvol), "id=%d", subvol->graph->id, NULL);
+            /* create in progress, defer */
+            continue;
+        }
 
-xlator_t *
-__glfs_active_subvol (struct glfs *fs)
-{
-	xlator_t      *new_subvol = NULL;
-	int            ret = -1;
-	inode_t       *new_cwd = NULL;
-
-	if (!fs->next_subvol)
-		return fs->active_subvol;
-
-	new_subvol = fs->next_subvol;
-
-	ret = __glfs_first_lookup (fs, new_subvol);
-	if (ret) {
-		gf_log (fs->volname, GF_LOG_INFO,
-			"first lookup on graph %s (%d) failed (%s)",
-			graphid_str (new_subvol), new_subvol->graph->id,
-			strerror (errno));
-		return NULL;
-	}
-
-	if (fs->cwd) {
-		new_cwd = __glfs_refresh_inode (fs, new_subvol, fs->cwd);
-
-		if (!new_cwd) {
-			char buf1[64];
-			gf_log (fs->volname, GF_LOG_INFO,
-				"cwd refresh of %s graph %s (%d) failed (%s)",
-				uuid_utoa_r (fs->cwd->gfid, buf1),
-				graphid_str (new_subvol),
-				new_subvol->graph->id, strerror (errno));
-			return NULL;
-		}
-	}
-
-	__glfs_migrate_openfds (fs, new_subvol);
-
-	/* switching @active_subvol and @cwd
-	   should be atomic
-	*/
-	fs->old_subvol = fs->active_subvol;
-	fs->active_subvol = fs->next_subvol;
-	fs->next_subvol = NULL;
-
-	if (new_cwd) {
-		__glfs_cwd_set (fs, new_cwd);
-		inode_unref (new_cwd);
-	}
-
-	gf_log (fs->volname, GF_LOG_INFO, "switched to graph %s (%d)",
-		graphid_str (new_subvol), new_subvol->graph->id);
-
-	return new_subvol;
+        fd = __glfs_migrate_fd(fs, subvol, glfd);
+        if (fd) {
+            fd_unref(glfd->fd);
+            glfd->fd = fd;
+        }
+    }
 }
 
+/* Note that though it appears that this function executes under fs->mutex,
+ * it is not fully executed under fs->mutex. i.e. there are functions like
+ * __glfs_first_lookup, __glfs_refresh_inode, __glfs_migrate_openfds which
+ * unlocks fs->mutex before sending any network fop, and reacquire fs->mutex
+ * once the fop is complete. Hence the variable read from fs at the start of the
+ * function need not have the same value by the end of the function.
+ */
 xlator_t *
-priv_glfs_active_subvol (struct glfs *fs)
+__glfs_active_subvol(struct glfs *fs)
 {
-	xlator_t      *subvol = NULL;
-	xlator_t      *old_subvol = NULL;
+    xlator_t *new_subvol = NULL;
+    int ret = -1;
+    inode_t *new_cwd = NULL;
+
+    if (!fs->next_subvol)
+        return fs->active_subvol;
+
+    new_subvol = fs->mip_subvol = fs->next_subvol;
+    fs->next_subvol = NULL;
+
+    ret = __glfs_first_lookup(fs, new_subvol);
+    if (ret) {
+        gf_smsg(fs->volname, GF_LOG_INFO, errno,
+                API_MSG_FIRST_LOOKUP_GRAPH_FAILED, "subvol=%s",
+                graphid_str(new_subvol), "id=%d", new_subvol->graph->id,
+                "err=%s", strerror(errno), NULL);
+        return NULL;
+    }
+
+    if (fs->cwd) {
+        new_cwd = __glfs_refresh_inode(fs, new_subvol, fs->cwd, _gf_false);
+
+        if (!new_cwd) {
+            char buf1[64];
+            gf_smsg(fs->volname, GF_LOG_INFO, errno,
+                    API_MSG_CWD_GRAPH_REF_FAILED, "buf=%s",
+                    uuid_utoa_r(fs->cwd->gfid, buf1), "subvol=%s",
+                    graphid_str(new_subvol), "id=%d", new_subvol->graph->id,
+                    "err=%s", strerror(errno), NULL);
+            return NULL;
+        }
+    }
 
-	glfs_lock (fs);
-	{
-		subvol = __glfs_active_subvol (fs);
+    __glfs_migrate_openfds(fs, new_subvol);
+    /* TODO: Migrate the fds and inodes which have leases to the new graph
+     * (issue #350)*/
 
-		if (subvol)
-			subvol->winds++;
+    /* switching @active_subvol and @cwd
+       should be atomic
+    */
+    fs->old_subvol = fs->active_subvol;
+    fs->active_subvol = fs->mip_subvol;
+    fs->mip_subvol = NULL;
 
-		if (fs->old_subvol) {
-			old_subvol = fs->old_subvol;
-			fs->old_subvol = NULL;
-			old_subvol->switched = 1;
-		}
-	}
-	glfs_unlock (fs);
+    if (new_cwd) {
+        __glfs_cwd_set(fs, new_cwd);
+        inode_unref(new_cwd);
+    }
 
-	if (old_subvol)
-		priv_glfs_subvol_done (fs, old_subvol);
+    gf_smsg(fs->volname, GF_LOG_INFO, 0, API_MSG_SWITCHED_GRAPH, "subvol=%s",
+            graphid_str(new_subvol), "id=%d", new_subvol->graph->id, NULL);
 
-	return subvol;
+    return new_subvol;
 }
 
-GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_active_subvol, 3.4.0);
-
+GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_subvol_done, 3.4.0)
 void
-priv_glfs_subvol_done (struct glfs *fs, xlator_t *subvol)
+priv_glfs_subvol_done(struct glfs *fs, xlator_t *subvol)
 {
-	int ref = 0;
-	xlator_t *active_subvol = NULL;
-
-	if (!subvol)
-		return;
-
-	glfs_lock (fs);
-	{
-		ref = (--subvol->winds);
-		active_subvol = fs->active_subvol;
-	}
-	glfs_unlock (fs);
-
-	if (ref == 0) {
-		assert (subvol != active_subvol);
-		xlator_notify (subvol, GF_EVENT_PARENT_DOWN, subvol, NULL);
-	}
+    int ref = 0;
+    xlator_t *active_subvol = NULL;
+
+    if (!subvol)
+        return;
+
+    /* For decrementing subvol->wind ref count we need not check/wait for
+     * migration-in-progress flag.
+     * Also glfs_subvol_done is called in call-back path therefore waiting
+     * for migration-in-progress flag can lead to dead-lock.
+     */
+    glfs_lock(fs, _gf_false);
+    {
+        ref = (--subvol->winds);
+        active_subvol = fs->active_subvol;
+    }
+    glfs_unlock(fs);
+
+    if (ref == 0) {
+        assert(subvol != active_subvol);
+        xlator_notify(subvol, GF_EVENT_PARENT_DOWN, subvol, NULL);
+    }
 }
 
-GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_subvol_done, 3.4.0);
+GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_active_subvol, 3.4.0)
+xlator_t *
+priv_glfs_active_subvol(struct glfs *fs)
+{
+    xlator_t *subvol = NULL;
+    xlator_t *old_subvol = NULL;
+
+    glfs_lock(fs, _gf_true);
+    {
+        subvol = __glfs_active_subvol(fs);
+
+        if (subvol)
+            subvol->winds++;
+
+        if (fs->old_subvol) {
+            old_subvol = fs->old_subvol;
+            fs->old_subvol = NULL;
+            old_subvol->switched = 1;
+        }
+    }
+    glfs_unlock(fs);
+
+    if (old_subvol)
+        priv_glfs_subvol_done(fs, old_subvol);
+
+    return subvol;
+}
 
 int
-__glfs_cwd_set (struct glfs *fs, inode_t *inode)
+__glfs_cwd_set(struct glfs *fs, inode_t *inode)
 {
-	if (inode->table->xl != fs->active_subvol) {
-		inode = __glfs_refresh_inode (fs, fs->active_subvol, inode);
-		if (!inode)
-			return -1;
-	} else {
-		inode_ref (inode);
-	}
+    if (inode->table->xl != fs->active_subvol) {
+        inode = __glfs_refresh_inode(fs, fs->active_subvol, inode, _gf_false);
+        if (!inode)
+            return -1;
+    } else {
+        inode_ref(inode);
+    }
 
-	if (fs->cwd)
-		inode_unref (fs->cwd);
+    if (fs->cwd)
+        inode_unref(fs->cwd);
 
-	fs->cwd = inode;
+    fs->cwd = inode;
 
-	return 0;
+    return 0;
 }
 
-
 int
-glfs_cwd_set (struct glfs *fs, inode_t *inode)
+glfs_cwd_set(struct glfs *fs, inode_t *inode)
 {
-	int ret = 0;
+    int ret = 0;
 
-	glfs_lock (fs);
-	{
-		ret = __glfs_cwd_set (fs, inode);
-	}
-	glfs_unlock (fs);
+    glfs_lock(fs, _gf_true);
+    {
+        ret = __glfs_cwd_set(fs, inode);
+    }
+    glfs_unlock(fs);
 
-	return ret;
+    return ret;
 }
 
-
 inode_t *
-__glfs_cwd_get (struct glfs *fs)
+__glfs_cwd_get(struct glfs *fs)
 {
-	inode_t *cwd = NULL;
+    inode_t *cwd = NULL;
 
-	if (!fs->cwd)
-		return NULL;
+    if (!fs->cwd)
+        return NULL;
 
-	if (fs->cwd->table->xl == fs->active_subvol) {
-		cwd = inode_ref (fs->cwd);
-		return cwd;
-	}
+    if (fs->cwd->table->xl == fs->active_subvol) {
+        cwd = inode_ref(fs->cwd);
+        return cwd;
+    }
 
-	cwd = __glfs_refresh_inode (fs, fs->active_subvol, fs->cwd);
+    cwd = __glfs_refresh_inode(fs, fs->active_subvol, fs->cwd, _gf_false);
 
-	return cwd;
+    return cwd;
 }
 
 inode_t *
-glfs_cwd_get (struct glfs *fs)
+glfs_cwd_get(struct glfs *fs)
 {
-	inode_t *cwd = NULL;
+    inode_t *cwd = NULL;
 
-	glfs_lock (fs);
-	{
-		cwd = __glfs_cwd_get (fs);
-	}
-	glfs_unlock (fs);
+    glfs_lock(fs, _gf_true);
+    {
+        cwd = __glfs_cwd_get(fs);
+    }
+    glfs_unlock(fs);
 
-	return cwd;
+    return cwd;
 }
 
 inode_t *
-__glfs_resolve_inode (struct glfs *fs, xlator_t *subvol,
-		    struct glfs_object *object)
+__glfs_resolve_inode(struct glfs *fs, xlator_t *subvol,
+                     struct glfs_object *object)
 {
-	inode_t *inode = NULL;
+    inode_t *inode = NULL;
+    gf_boolean_t lookup_needed = _gf_false;
 
-	if (object->inode->table->xl == subvol)
-		return inode_ref (object->inode);
+    lookup_needed = inode_needs_lookup(object->inode, THIS);
 
-	inode = __glfs_refresh_inode (fs, fs->active_subvol,
-					object->inode);
-	if (!inode)
-		return NULL;
+    if (!lookup_needed && object->inode->table->xl == subvol)
+        return inode_ref(object->inode);
 
-	if (subvol == fs->active_subvol) {
-		inode_unref (object->inode);
-		object->inode = inode_ref (inode);
-	}
+    inode = __glfs_refresh_inode(fs, fs->active_subvol, object->inode,
+                                 lookup_needed);
+    if (!inode)
+        return NULL;
 
-	return inode;
+    if (subvol == fs->active_subvol) {
+        inode_unref(object->inode);
+        object->inode = inode_ref(inode);
+    }
+
+    return inode;
 }
 
 inode_t *
-glfs_resolve_inode (struct glfs *fs, xlator_t *subvol,
-		    struct glfs_object *object)
+glfs_resolve_inode(struct glfs *fs, xlator_t *subvol,
+                   struct glfs_object *object)
 {
-	inode_t *inode = NULL;
+    inode_t *inode = NULL;
 
-	glfs_lock (fs);
-	{
-		inode = __glfs_resolve_inode(fs, subvol, object);
-	}
-	glfs_unlock (fs);
+    glfs_lock(fs, _gf_true);
+    {
+        inode = __glfs_resolve_inode(fs, subvol, object);
+    }
+    glfs_unlock(fs);
 
-	return inode;
+    return inode;
 }
 
 int
-glfs_create_object (loc_t *loc, struct glfs_object **retobject)
+glfs_create_object(loc_t *loc, struct glfs_object **retobject)
 {
-	struct glfs_object *object = NULL;
+    struct glfs_object *object = NULL;
+
+    object = GF_CALLOC(1, sizeof(struct glfs_object), glfs_mt_glfs_object_t);
+    if (object == NULL) {
+        errno = ENOMEM;
+        return -1;
+    }
+
+    object->inode = loc->inode;
+    gf_uuid_copy(object->gfid, object->inode->gfid);
 
-	object = GF_CALLOC (1, sizeof(struct glfs_object),
-			    glfs_mt_glfs_object_t);
-	if (object == NULL) {
-		errno = ENOMEM;
-		return -1;
-	}
+    /* we hold the reference */
+    loc->inode = NULL;
 
-	object->inode = loc->inode;
-	uuid_copy (object->gfid, object->inode->gfid);
+    *retobject = object;
 
-	/* we hold the reference */
-	loc->inode = NULL;
+    return 0;
+}
 
-	*retobject = object;
+struct glfs_object *
+glfs_h_resolve_symlink(struct glfs *fs, struct glfs_object *object)
+{
+    xlator_t *subvol = NULL;
+    loc_t sym_loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    char *lpath = NULL;
+    int ret = 0;
+    struct glfs_object *target_object = NULL;
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    ret = glfs_resolve_symlink(fs, subvol, object->inode, &lpath);
+    if (ret < 0)
+        goto out;
+
+    ret = glfs_resolve_at(fs, subvol, NULL, lpath, &sym_loc, &iatt,
+                          /* always recurisvely follow while
+                            following symlink
+                          */
+                          1, 0);
+    if (ret == 0)
+        ret = glfs_create_object(&sym_loc, &target_object);
 
-	return 0;
+out:
+    loc_wipe(&sym_loc);
+    GF_FREE(lpath);
+    return target_object;
 }
diff --git a/api/src/glfs.c b/api/src/glfs.c
index 5c137ef999d..b4bf1423f6d 100644
--- a/api/src/glfs.c
+++ b/api/src/glfs.c
@@ -1,5 +1,5 @@
 /*
-  Copyright (c) 2012 Red Hat, Inc. <http://www.redhat.com>
+  Copyright (c) 2012-2018 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
   This file is licensed to you under your choice of the GNU Lesser
@@ -8,10 +8,8 @@
   cases as published by the Free Software Foundation.
 */
 
-
 /*
   TODO:
-  - merge locks in glfs_posix_lock for lock self-healing
   - set proper pid/lk_owner to call frames (currently buried in syncop)
   - fix logging.c/h to store logfp and loglevel in glusterfs_ctx_t and
     reach it via THIS.
@@ -32,765 +30,1777 @@
 #include <sys/types.h>
 #include <unistd.h>
 #include <limits.h>
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
+#ifdef GF_LINUX_HOST_OS
+#include <sys/prctl.h>
 #endif
 
-#include "glusterfs.h"
-#include "logging.h"
-#include "stack.h"
-#include "event.h"
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/stack.h>
+#include <glusterfs/gf-event.h>
 #include "glfs-mem-types.h"
-#include "common-utils.h"
-#include "syncop.h"
-#include "call-stub.h"
+#include <glusterfs/common-utils.h>
+#include <glusterfs/syncop.h>
+#include <glusterfs/call-stub.h>
+#include <glusterfs/hashfn.h>
+#include "rpc-clnt.h"
+#include <glusterfs/statedump.h>
+#include <glusterfs/syscall.h>
 
+#include "gfapi-messages.h"
 #include "glfs.h"
 #include "glfs-internal.h"
-#include "hashfn.h"
-#include "rpc-clnt.h"
-
 
 static gf_boolean_t
-vol_assigned (cmd_args_t *args)
+vol_assigned(cmd_args_t *args)
 {
-	return args->volfile || args->volfile_server;
+    return args->volfile || args->volfile_server;
 }
 
-
 static int
-glusterfs_ctx_defaults_init (glusterfs_ctx_t *ctx)
-{
-	call_pool_t   *pool = NULL;
-	int	       ret = -1;
-
-	xlator_mem_acct_init (THIS, glfs_mt_end + 1);
-
-	ctx->process_uuid = generate_glusterfs_ctx_id ();
-	if (!ctx->process_uuid) {
-		goto err;
-	}
-
-	ctx->page_size	= 128 * GF_UNIT_KB;
-
-	ctx->iobuf_pool = iobuf_pool_new ();
-	if (!ctx->iobuf_pool) {
-		goto err;
-	}
-
-	ctx->event_pool = event_pool_new (DEFAULT_EVENT_POOL_SIZE);
-	if (!ctx->event_pool) {
-		goto err;
-	}
-
-	ctx->env = syncenv_new (0, 0, 0);
-	if (!ctx->env) {
-		goto err;
-	}
-
-	pool = GF_CALLOC (1, sizeof (call_pool_t),
-			  glfs_mt_call_pool_t);
-	if (!pool) {
-		goto err;
-	}
-
-	/* frame_mem_pool size 112 * 4k */
-	pool->frame_mem_pool = mem_pool_new (call_frame_t, 4096);
-	if (!pool->frame_mem_pool) {
-		goto err;
-	}
-	/* stack_mem_pool size 256 * 1024 */
-	pool->stack_mem_pool = mem_pool_new (call_stack_t, 1024);
-	if (!pool->stack_mem_pool) {
-		goto err;
-	}
-
-	ctx->stub_mem_pool = mem_pool_new (call_stub_t, 1024);
-	if (!ctx->stub_mem_pool) {
-		goto err;
-	}
-
-	ctx->dict_pool = mem_pool_new (dict_t, GF_MEMPOOL_COUNT_OF_DICT_T);
-	if (!ctx->dict_pool)
-		goto err;
-
-	ctx->dict_pair_pool = mem_pool_new (data_pair_t,
-					    GF_MEMPOOL_COUNT_OF_DATA_PAIR_T);
-	if (!ctx->dict_pair_pool)
-		goto err;
-
-	ctx->dict_data_pool = mem_pool_new (data_t, GF_MEMPOOL_COUNT_OF_DATA_T);
-	if (!ctx->dict_data_pool)
-		goto err;
-
-	INIT_LIST_HEAD (&pool->all_frames);
-	INIT_LIST_HEAD (&ctx->cmd_args.xlator_options);
-        INIT_LIST_HEAD (&ctx->cmd_args.volfile_servers);
-
-	LOCK_INIT (&pool->lock);
-	ctx->pool = pool;
-
-	pthread_mutex_init (&(ctx->lock), NULL);
-
-	ret = 0;
-err:
-	if (ret && pool) {
-		if (pool->frame_mem_pool)
-			mem_pool_destroy (pool->frame_mem_pool);
-		if (pool->stack_mem_pool)
-			mem_pool_destroy (pool->stack_mem_pool);
-		GF_FREE (pool);
-	}
+glusterfs_ctx_defaults_init(glusterfs_ctx_t *ctx)
+{
+    call_pool_t *pool = NULL;
+    int ret = -1;
 
-	if (ret && ctx) {
-		if (ctx->stub_mem_pool)
-			mem_pool_destroy (ctx->stub_mem_pool);
-		if (ctx->dict_pool)
-			mem_pool_destroy (ctx->dict_pool);
-		if (ctx->dict_data_pool)
-			mem_pool_destroy (ctx->dict_data_pool);
-		if (ctx->dict_pair_pool)
-			mem_pool_destroy (ctx->dict_pair_pool);
-	}
+    if (!ctx) {
+        goto err;
+    }
 
-	return ret;
+    ret = xlator_mem_acct_init(THIS, glfs_mt_end + 1);
+    if (ret != 0) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, ENOMEM, API_MSG_MEM_ACCT_INIT_FAILED,
+                NULL);
+        return ret;
+    }
+
+    /* reset ret to -1 so that we don't need to explicitly
+     * set it in all error paths before "goto err"
+     */
+
+    ret = -1;
+
+    ctx->process_uuid = generate_glusterfs_ctx_id();
+    if (!ctx->process_uuid) {
+        goto err;
+    }
+
+    ctx->page_size = 128 * GF_UNIT_KB;
+
+    ctx->iobuf_pool = iobuf_pool_new();
+    if (!ctx->iobuf_pool) {
+        goto err;
+    }
+
+    ctx->event_pool = gf_event_pool_new(DEFAULT_EVENT_POOL_SIZE,
+                                        STARTING_EVENT_THREADS);
+    if (!ctx->event_pool) {
+        goto err;
+    }
+
+    ctx->env = syncenv_new(0, 0, 0);
+    if (!ctx->env) {
+        goto err;
+    }
+
+    pool = GF_CALLOC(1, sizeof(call_pool_t), glfs_mt_call_pool_t);
+    if (!pool) {
+        goto err;
+    }
+
+    /* frame_mem_pool size 112 * 4k */
+    pool->frame_mem_pool = mem_pool_new(call_frame_t, 4096);
+    if (!pool->frame_mem_pool) {
+        goto err;
+    }
+    /* stack_mem_pool size 256 * 1024 */
+    pool->stack_mem_pool = mem_pool_new(call_stack_t, 1024);
+    if (!pool->stack_mem_pool) {
+        goto err;
+    }
+
+    ctx->stub_mem_pool = mem_pool_new(call_stub_t, 1024);
+    if (!ctx->stub_mem_pool) {
+        goto err;
+    }
+
+    ctx->dict_pool = mem_pool_new(dict_t, GF_MEMPOOL_COUNT_OF_DICT_T);
+    if (!ctx->dict_pool)
+        goto err;
+
+    ctx->dict_pair_pool = mem_pool_new(data_pair_t,
+                                       GF_MEMPOOL_COUNT_OF_DATA_PAIR_T);
+    if (!ctx->dict_pair_pool)
+        goto err;
+
+    ctx->dict_data_pool = mem_pool_new(data_t, GF_MEMPOOL_COUNT_OF_DATA_T);
+    if (!ctx->dict_data_pool)
+        goto err;
+
+    ctx->logbuf_pool = mem_pool_new(log_buf_t, GF_MEMPOOL_COUNT_OF_LRU_BUF_T);
+    if (!ctx->logbuf_pool)
+        goto err;
+
+    INIT_LIST_HEAD(&pool->all_frames);
+    INIT_LIST_HEAD(&ctx->cmd_args.xlator_options);
+    INIT_LIST_HEAD(&ctx->cmd_args.volfile_servers);
+
+    LOCK_INIT(&pool->lock);
+    ctx->pool = pool;
+
+    ret = 0;
+err:
+    if (ret && pool) {
+        if (pool->frame_mem_pool)
+            mem_pool_destroy(pool->frame_mem_pool);
+        if (pool->stack_mem_pool)
+            mem_pool_destroy(pool->stack_mem_pool);
+        GF_FREE(pool);
+    }
+
+    if (ret && ctx) {
+        if (ctx->stub_mem_pool)
+            mem_pool_destroy(ctx->stub_mem_pool);
+        if (ctx->dict_pool)
+            mem_pool_destroy(ctx->dict_pool);
+        if (ctx->dict_data_pool)
+            mem_pool_destroy(ctx->dict_data_pool);
+        if (ctx->dict_pair_pool)
+            mem_pool_destroy(ctx->dict_pair_pool);
+        if (ctx->logbuf_pool)
+            mem_pool_destroy(ctx->logbuf_pool);
+    }
+
+    return ret;
 }
 
-
 static int
-create_master (struct glfs *fs)
+create_master(struct glfs *fs)
 {
-	int		 ret = 0;
-	xlator_t	*master = NULL;
-
-	master = GF_CALLOC (1, sizeof (*master),
-			    glfs_mt_xlator_t);
-	if (!master)
-		goto err;
+    int ret = 0;
+    xlator_t *master = NULL;
 
-	master->name = gf_strdup ("gfapi");
-	if (!master->name)
-		goto err;
+    master = GF_CALLOC(1, sizeof(*master), glfs_mt_xlator_t);
+    if (!master)
+        goto err;
 
-	if (xlator_set_type (master, "mount/api") == -1) {
-		gf_log ("glfs", GF_LOG_ERROR,
-			"master xlator for %s initialization failed",
-			fs->volname);
-		goto err;
-	}
+    master->name = gf_strdup("gfapi");
+    if (!master->name)
+        goto err;
 
-	master->ctx	 = fs->ctx;
-	master->private	 = fs;
-	master->options	 = get_new_dict ();
-	if (!master->options)
-		goto err;
+    if (xlator_set_type(master, "mount/api") == -1) {
+        gf_smsg("glfs", GF_LOG_ERROR, 0, API_MSG_MASTER_XLATOR_INIT_FAILED,
+                "name=%s", fs->volname, NULL);
+        goto err;
+    }
 
+    master->ctx = fs->ctx;
+    master->private = fs;
+    master->options = dict_new();
+    if (!master->options)
+        goto err;
 
-	ret = xlator_init (master);
-	if (ret) {
-		gf_log ("glfs", GF_LOG_ERROR,
-			"failed to initialize gfapi translator");
-		goto err;
-	}
+    ret = xlator_init(master);
+    if (ret) {
+        gf_smsg("glfs", GF_LOG_ERROR, 0, API_MSG_GFAPI_XLATOR_INIT_FAILED,
+                NULL);
+        goto err;
+    }
 
-	fs->ctx->master = master;
-	THIS = master;
+    fs->ctx->master = master;
+    THIS = master;
 
-	return 0;
+    return 0;
 
 err:
-	if (master) {
-		xlator_destroy (master);
-	}
+    if (master) {
+        xlator_destroy(master);
+    }
 
-	return -1;
+    return -1;
 }
 
-
 static FILE *
-get_volfp (struct glfs *fs)
+get_volfp(struct glfs *fs)
 {
-	int	     ret = 0;
-	cmd_args_t  *cmd_args = NULL;
-	FILE	    *specfp = NULL;
-	struct stat  statbuf;
-
-	cmd_args = &fs->ctx->cmd_args;
+    cmd_args_t *cmd_args = NULL;
+    FILE *specfp = NULL;
 
-	ret = lstat (cmd_args->volfile, &statbuf);
-	if (ret == -1) {
-		gf_log ("glfs", GF_LOG_ERROR,
-			"%s: %s", cmd_args->volfile, strerror (errno));
-		return NULL;
-	}
+    cmd_args = &fs->ctx->cmd_args;
 
-	if ((specfp = fopen (cmd_args->volfile, "r")) == NULL) {
-		gf_log ("glfs", GF_LOG_ERROR,
-			"volume file %s: %s",
-			cmd_args->volfile,
-			strerror (errno));
-		return NULL;
-	}
+    if ((specfp = fopen(cmd_args->volfile, "r")) == NULL) {
+        gf_smsg("glfs", GF_LOG_ERROR, errno, API_MSG_VOLFILE_OPEN_FAILED,
+                "file=%s", cmd_args->volfile, "err=%s", strerror(errno), NULL);
+        return NULL;
+    }
 
-	gf_log ("glfs", GF_LOG_DEBUG,
-		"loading volume file %s", cmd_args->volfile);
+    gf_msg_debug("glfs", 0, "loading volume file %s", cmd_args->volfile);
 
-	return specfp;
+    return specfp;
 }
 
-
 int
-glfs_volumes_init (struct glfs *fs)
+glfs_volumes_init(struct glfs *fs)
 {
-	FILE		   *fp = NULL;
-	cmd_args_t	   *cmd_args = NULL;
-	int		    ret = 0;
+    FILE *fp = NULL;
+    cmd_args_t *cmd_args = NULL;
+    int ret = 0;
+
+    cmd_args = &fs->ctx->cmd_args;
 
-	cmd_args = &fs->ctx->cmd_args;
+    if (!vol_assigned(cmd_args))
+        return -1;
 
-	if (!vol_assigned (cmd_args))
-		return -1;
+    if (sys_access(SECURE_ACCESS_FILE, F_OK) == 0) {
+        fs->ctx->secure_mgmt = 1;
+        fs->ctx->ssl_cert_depth = glusterfs_read_secure_access_file();
+    }
 
-	if (cmd_args->volfile_server) {
-		ret = glfs_mgmt_init (fs);
-		goto out;
-	}
+    if (cmd_args->volfile_server) {
+        ret = glfs_mgmt_init(fs);
+        goto out;
+    }
 
-	fp = get_volfp (fs);
+    fp = get_volfp(fs);
 
-	if (!fp) {
-		gf_log ("glfs", GF_LOG_ERROR,
-			"Cannot reach volume specification file");
-		ret = -1;
-		goto out;
-	}
+    if (!fp) {
+        gf_smsg("glfs", GF_LOG_ERROR, ENOENT, API_MSG_VOL_SPEC_FILE_ERROR,
+                NULL);
+        ret = -1;
+        goto out;
+    }
 
-	ret = glfs_process_volfp (fs, fp);
-	if (ret)
-		goto out;
+    ret = glfs_process_volfp(fs, fp);
+    if (ret)
+        goto out;
 
 out:
-	return ret;
+    return ret;
 }
 
-
 ///////////////////////////////////////////////////////////////////////////////
 
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_set_xlator_option, 3.4.0)
 int
-pub_glfs_set_xlator_option (struct glfs *fs, const char *xlator,
-                            const char *key, const char *value)
+pub_glfs_set_xlator_option(struct glfs *fs, const char *xlator, const char *key,
+                           const char *value)
 {
-	xlator_cmdline_option_t *option = NULL;
+    xlator_cmdline_option_t *option = NULL;
 
-	option = GF_CALLOC (1, sizeof (*option),
-			    glfs_mt_xlator_cmdline_option_t);
-	if (!option)
-		goto enomem;
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
 
-	INIT_LIST_HEAD (&option->cmd_args);
+    option = GF_CALLOC(1, sizeof(*option), glfs_mt_xlator_cmdline_option_t);
+    if (!option)
+        goto enomem;
 
-	option->volume = gf_strdup (xlator);
-	if (!option->volume)
-		goto enomem;
-	option->key = gf_strdup (key);
-	if (!option->key)
-		goto enomem;
-	option->value = gf_strdup (value);
-	if (!option->value)
-		goto enomem;
+    INIT_LIST_HEAD(&option->cmd_args);
 
-	list_add (&option->cmd_args, &fs->ctx->cmd_args.xlator_options);
+    option->volume = gf_strdup(xlator);
+    if (!option->volume)
+        goto enomem;
+    option->key = gf_strdup(key);
+    if (!option->key)
+        goto enomem;
+    option->value = gf_strdup(value);
+    if (!option->value)
+        goto enomem;
 
-	return 0;
-enomem:
-	errno = ENOMEM;
+    list_add(&option->cmd_args, &fs->ctx->cmd_args.xlator_options);
+
+    __GLFS_EXIT_FS;
 
-	if (!option)
-		return -1;
+    return 0;
+enomem:
+    errno = ENOMEM;
 
-	GF_FREE (option->volume);
-	GF_FREE (option->key);
-	GF_FREE (option->value);
-	GF_FREE (option);
+    if (!option) {
+        __GLFS_EXIT_FS;
+        return -1;
+    }
 
-	return -1;
-}
+    GF_FREE(option->volume);
+    GF_FREE(option->key);
+    GF_FREE(option->value);
+    GF_FREE(option);
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_set_xlator_option, 3.4.0);
+    __GLFS_EXIT_FS;
 
+invalid_fs:
+    return -1;
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_unset_volfile_server, 3.5.1)
 int
-pub_glfs_unset_volfile_server (struct glfs *fs, const char *transport,
-                               const char *host, const int port)
+pub_glfs_unset_volfile_server(struct glfs *fs, const char *transport,
+                              const char *host, const int port)
 {
-        cmd_args_t       *cmd_args = NULL;
-        server_cmdline_t *server = NULL;
-        int               ret = -1;
-
-        if (!transport || !host || !port) {
-                errno = EINVAL;
-                return ret;
-        }
-
-        cmd_args = &fs->ctx->cmd_args;
-        list_for_each_entry(server, &cmd_args->curr_server->list, list) {
-                if ((!strcmp(server->volfile_server, host) &&
-                     !strcmp(server->transport, transport) &&
-                     (server->port == port))) {
-                        list_del (&server->list);
-                        ret = 0;
-                        goto out;
-                }
+    cmd_args_t *cmd_args = NULL;
+    server_cmdline_t *server = NULL;
+    server_cmdline_t *tmp = NULL;
+    char *transport_val = NULL;
+    int port_val = 0;
+    int ret = -1;
+
+    if (!fs || !host) {
+        errno = EINVAL;
+        return ret;
+    }
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    cmd_args = &fs->ctx->cmd_args;
+
+    if (transport) {
+        transport_val = gf_strdup(transport);
+    } else {
+        transport_val = gf_strdup(GF_DEFAULT_VOLFILE_TRANSPORT);
+    }
+
+    if (!transport_val) {
+        errno = ENOMEM;
+        goto out;
+    }
+
+    if (port) {
+        port_val = port;
+    } else {
+        port_val = GF_DEFAULT_BASE_PORT;
+    }
+
+    list_for_each_entry_safe(server, tmp, &cmd_args->curr_server->list, list)
+    {
+        if (!server->volfile_server || !server->transport)
+            continue;
+        if ((!strcmp(server->volfile_server, host) &&
+             !strcmp(server->transport, transport_val) &&
+             (server->port == port_val))) {
+            list_del(&server->list);
+            ret = 0;
+            goto out;
         }
+    }
 
 out:
-        return ret;
-}
-
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_unset_volfile_server, 3.5.1);
+    GF_FREE(transport_val);
+    __GLFS_EXIT_FS;
 
+invalid_fs:
+    return ret;
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_set_volfile_server, 3.4.0)
 int
-pub_glfs_set_volfile_server (struct glfs *fs, const char *transport,
-                             const char *host, int port)
+pub_glfs_set_volfile_server(struct glfs *fs, const char *transport,
+                            const char *host, int port)
 {
-        cmd_args_t            *cmd_args = NULL;
-        server_cmdline_t      *server = NULL;
-        server_cmdline_t      *tmp = NULL;
-        int                    ret = -1;
+    cmd_args_t *cmd_args = NULL;
+    int ret = -1;
+    char *server_host = NULL;
+    char *server_transport = NULL;
 
-        if (!transport || !host) {
-                errno = EINVAL;
-                return ret;
-        }
-
-        cmd_args = &fs->ctx->cmd_args;
-
-        cmd_args->max_connect_attempts = 1;
-
-        server = GF_CALLOC (1, sizeof (server_cmdline_t),
-                            glfs_mt_server_cmdline_t);
-
-        if (!server) {
-                errno = ENOMEM;
-                goto out;
-        }
-
-        INIT_LIST_HEAD (&server->list);
-
-        server->volfile_server = gf_strdup (host);
-        if (!server->volfile_server) {
-                errno = ENOMEM;
-                goto out;
-        }
-
-        server->transport = gf_strdup (transport);
-        if (!server->transport) {
-                errno = ENOMEM;
-                goto out;
+    if (!fs || !host) {
+        errno = EINVAL;
+        return ret;
+    }
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    cmd_args = &fs->ctx->cmd_args;
+    cmd_args->max_connect_attempts = 1;
+
+    server_host = gf_strdup(host);
+    if (!server_host) {
+        errno = ENOMEM;
+        goto out;
+    }
+
+    if (transport) {
+        /* volfile fetch support over tcp|unix only */
+        if (!strcmp(transport, "tcp") || !strcmp(transport, "unix")) {
+            server_transport = gf_strdup(transport);
+        } else if (!strcmp(transport, "rdma")) {
+            server_transport = gf_strdup(GF_DEFAULT_VOLFILE_TRANSPORT);
+            gf_smsg("glfs", GF_LOG_WARNING, EINVAL, API_MSG_TRANS_RDMA_DEP,
+                    NULL);
+        } else {
+            gf_smsg("glfs", GF_LOG_TRACE, EINVAL, API_MSG_TRANS_NOT_SUPPORTED,
+                    "transport=%s", transport, NULL);
+            goto out;
         }
+    } else {
+        server_transport = gf_strdup(GF_DEFAULT_VOLFILE_TRANSPORT);
+    }
+
+    if (!server_transport) {
+        errno = ENOMEM;
+        goto out;
+    }
+
+    if (!port) {
+        port = GF_DEFAULT_BASE_PORT;
+    }
+
+    if (!strcmp(server_transport, "unix")) {
+        port = 0;
+    }
+
+    ret = gf_set_volfile_server_common(cmd_args, server_host, server_transport,
+                                       port);
+    if (ret) {
+        gf_log("glfs", GF_LOG_ERROR, "failed to set volfile server: %s",
+               strerror(errno));
+    }
 
-        server->port = port;
+out:
+    if (server_host) {
+        GF_FREE(server_host);
+    }
 
-        if (!cmd_args->volfile_server) {
-                cmd_args->volfile_server = server->volfile_server;
-                cmd_args->volfile_server_transport = server->transport;
-                cmd_args->volfile_server_port = server->port;
-                cmd_args->curr_server = server;
-        }
+    if (server_transport) {
+        GF_FREE(server_transport);
+    }
 
-        list_for_each_entry(tmp, &cmd_args->volfile_servers, list) {
-                if ((!strcmp(tmp->volfile_server, host) &&
-                     !strcmp(tmp->transport, transport) &&
-                     (tmp->port == port))) {
-                        errno = EEXIST;
-                        ret = -1;
-                        goto out;
-                }
-        }
+    __GLFS_EXIT_FS;
 
-        list_add_tail (&server->list, &cmd_args->volfile_servers);
+invalid_fs:
+    return ret;
+}
 
-        ret = 0;
+/* *
+ * Used to free the arguments allocated by glfs_set_volfile_server()
+ */
+static void
+glfs_free_volfile_servers(cmd_args_t *cmd_args)
+{
+    server_cmdline_t *server = NULL;
+    server_cmdline_t *tmp = NULL;
+
+    GF_VALIDATE_OR_GOTO(THIS->name, cmd_args, out);
+
+    list_for_each_entry_safe(server, tmp, &cmd_args->volfile_servers, list)
+    {
+        list_del_init(&server->list);
+        GF_FREE(server->volfile_server);
+        GF_FREE(server->transport);
+        GF_FREE(server);
+    }
+    cmd_args->curr_server = NULL;
 out:
-        if (ret == -1) {
-                if (server) {
-                        GF_FREE (server->volfile_server);
-                        GF_FREE (server->transport);
-                        GF_FREE (server);
-                }
-        }
+    return;
+}
 
-        return ret;
+static void
+glfs_free_xlator_options(cmd_args_t *cmd_args)
+{
+    xlator_cmdline_option_t *xo = NULL;
+    xlator_cmdline_option_t *tmp_xo = NULL;
+
+    if (!&(cmd_args->xlator_options))
+        return;
+
+    list_for_each_entry_safe(xo, tmp_xo, &cmd_args->xlator_options, cmd_args)
+    {
+        list_del_init(&xo->cmd_args);
+        GF_FREE(xo->volume);
+        GF_FREE(xo->key);
+        GF_FREE(xo->value);
+        GF_FREE(xo);
+    }
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_set_volfile_server, 3.4.0);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_setfsuid, 3.4.2)
+int
+pub_glfs_setfsuid(uid_t fsuid)
+{
+    /* TODO:
+     * - Set the THIS and restore it appropriately
+     */
+    return syncopctx_setfsuid(&fsuid);
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_setfsgid, 3.4.2)
+int
+pub_glfs_setfsgid(gid_t fsgid)
+{
+    /* TODO:
+     * - Set the THIS and restore it appropriately
+     */
+    return syncopctx_setfsgid(&fsgid);
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_setfsgroups, 3.4.2)
 int
-pub_glfs_setfsuid (uid_t fsuid)
+pub_glfs_setfsgroups(size_t size, const gid_t *list)
 {
-	return syncopctx_setfsuid (&fsuid);
+    /* TODO:
+     * - Set the THIS and restore it appropriately
+     */
+    return syncopctx_setfsgroups(size, list);
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_setfsuid, 3.4.2);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_setfsleaseid, 4.0.0)
+int
+pub_glfs_setfsleaseid(glfs_leaseid_t leaseid)
+{
+    int ret = -1;
+    char *gleaseid = NULL;
+
+    gleaseid = gf_leaseid_get();
+    if (gleaseid) {
+        if (leaseid)
+            memcpy(gleaseid, leaseid, LEASE_ID_SIZE);
+        else /* reset leaseid */
+            memset(gleaseid, 0, LEASE_ID_SIZE);
+        ret = 0;
+    }
 
+    if (ret)
+        gf_log("glfs", GF_LOG_ERROR, "failed to set leaseid: %s",
+               strerror(errno));
+    return ret;
+}
 
 int
-pub_glfs_setfsgid (gid_t fsgid)
+get_fop_attr_glfd(dict_t **fop_attr, struct glfs_fd *glfd)
 {
-	return syncopctx_setfsgid (&fsgid);
+    char *leaseid = NULL;
+    int ret = 0;
+    gf_boolean_t dict_create = _gf_false;
+
+    leaseid = GF_MALLOC(LEASE_ID_SIZE, gf_common_mt_char);
+    GF_CHECK_ALLOC_AND_LOG("gfapi", leaseid, ret, "lease id alloc failed", out);
+    memcpy(leaseid, glfd->lease_id, LEASE_ID_SIZE);
+    if (*fop_attr == NULL) {
+        *fop_attr = dict_new();
+        dict_create = _gf_true;
+    }
+    GF_CHECK_ALLOC_AND_LOG("gfapi", *fop_attr, ret, "dict_new failed", out);
+    ret = dict_set_bin(*fop_attr, "lease-id", leaseid, LEASE_ID_SIZE);
+out:
+    if (ret) {
+        GF_FREE(leaseid);
+        if (dict_create) {
+            if (*fop_attr)
+                dict_unref(*fop_attr);
+            *fop_attr = NULL;
+        }
+    }
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_setfsgid, 3.4.2);
+int
+set_fop_attr_glfd(struct glfs_fd *glfd)
+{
+    char *lease_id = NULL;
+    int ret = -1;
 
+    lease_id = gf_existing_leaseid();
+    if (lease_id) {
+        memcpy(glfd->lease_id, lease_id, LEASE_ID_SIZE);
+        ret = 0;
+    }
+    return ret;
+}
 
 int
-pub_glfs_setfsgroups (size_t size, const gid_t *list)
+get_fop_attr_thrd_key(dict_t **fop_attr)
 {
-	return syncopctx_setfsgroups(size, list);
-}
+    char *existing_leaseid = NULL, *leaseid = NULL;
+    int ret = 0;
+    gf_boolean_t dict_create = _gf_false;
+
+    existing_leaseid = gf_existing_leaseid();
+    if (existing_leaseid) {
+        leaseid = GF_MALLOC(LEASE_ID_SIZE, gf_common_mt_char);
+        GF_CHECK_ALLOC_AND_LOG("gfapi", leaseid, ret, "lease id alloc failed",
+                               out);
+        memcpy(leaseid, existing_leaseid, LEASE_ID_SIZE);
+        if (*fop_attr == NULL) {
+            *fop_attr = dict_new();
+            dict_create = _gf_true;
+        }
+        GF_CHECK_ALLOC_AND_LOG("gfapi", *fop_attr, ret, "dict_new failed", out);
+        ret = dict_set_bin(*fop_attr, "lease-id", leaseid, LEASE_ID_SIZE);
+    }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_setfsgroups, 3.4.2);
+out:
+    if (ret) {
+        GF_FREE(leaseid);
+        if (dict_create) {
+            if (*fop_attr)
+                dict_unref(*fop_attr);
+            *fop_attr = NULL;
+        }
+    }
+    return ret;
+}
 
+void
+unset_fop_attr(dict_t **fop_attr)
+{
+    char *lease_id = NULL;
+    lease_id = gf_existing_leaseid();
+    if (lease_id)
+        memset(lease_id, 0, LEASE_ID_SIZE);
+    if (*fop_attr) {
+        dict_unref(*fop_attr);
+        *fop_attr = NULL;
+    }
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_from_glfd, 3.4.0)
 struct glfs *
-pub_glfs_from_glfd (struct glfs_fd *glfd)
+pub_glfs_from_glfd(struct glfs_fd *glfd)
 {
-	return glfd->fs;
+    if (glfd == NULL) {
+        errno = EBADF;
+        return NULL;
+    }
+
+    return glfd->fs;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_from_glfd, 3.4.0);
+static void
+glfs_fd_destroy(struct glfs_fd *glfd)
+{
+    if (!glfd)
+        return;
+
+    glfs_lock(glfd->fs, _gf_true);
+    {
+        list_del_init(&glfd->openfds);
+    }
+    glfs_unlock(glfd->fs);
 
+    if (glfd->fd) {
+        fd_unref(glfd->fd);
+        glfd->fd = NULL;
+    }
+
+    GF_FREE(glfd->readdirbuf);
+
+    GF_FREE(glfd);
+}
 
 struct glfs_fd *
-glfs_fd_new (struct glfs *fs)
+glfs_fd_new(struct glfs *fs)
 {
-	struct glfs_fd  *glfd = NULL;
+    struct glfs_fd *glfd = NULL;
 
-	glfd = GF_CALLOC (1, sizeof (*glfd), glfs_mt_glfs_fd_t);
-	if (!glfd)
-		return NULL;
+    glfd = GF_CALLOC(1, sizeof(*glfd), glfs_mt_glfs_fd_t);
+    if (!glfd)
+        return NULL;
 
-	glfd->fs = fs;
+    glfd->fs = fs;
 
-	INIT_LIST_HEAD (&glfd->openfds);
+    INIT_LIST_HEAD(&glfd->openfds);
 
-	return glfd;
-}
+    GF_REF_INIT(glfd, glfs_fd_destroy);
 
+    return glfd;
+}
 
 void
-glfs_fd_bind (struct glfs_fd *glfd)
+glfs_fd_bind(struct glfs_fd *glfd)
 {
-	struct glfs *fs = NULL;
+    struct glfs *fs = NULL;
 
-	fs = glfd->fs;
+    fs = glfd->fs;
 
-	glfs_lock (fs);
-	{
-		list_add_tail (&glfd->openfds, &fs->openfds);
-	}
-	glfs_unlock (fs);
+    glfs_lock(fs, _gf_true);
+    {
+        list_add_tail(&glfd->openfds, &fs->openfds);
+    }
+    glfs_unlock(fs);
 }
 
-void
-glfs_fd_destroy (struct glfs_fd *glfd)
+static void *
+glfs_poller(void *data)
 {
-	if (!glfd)
-		return;
+    struct glfs *fs = NULL;
+
+    fs = data;
 
-	glfs_lock (glfd->fs);
-	{
-		list_del_init (&glfd->openfds);
-	}
-	glfs_unlock (glfd->fs);
+    gf_event_dispatch(fs->ctx->event_pool);
+
+    return NULL;
+}
+
+static struct glfs *
+glfs_new_fs(const char *volname)
+{
+    struct glfs *fs = NULL;
 
-	if (glfd->fd)
-		fd_unref (glfd->fd);
+    fs = CALLOC(1, sizeof(*fs));
+    if (!fs)
+        return NULL;
 
-	GF_FREE (glfd->readdirbuf);
+    INIT_LIST_HEAD(&fs->openfds);
+    INIT_LIST_HEAD(&fs->upcall_list);
+    INIT_LIST_HEAD(&fs->waitq);
 
-	GF_FREE (glfd);
+    PTHREAD_MUTEX_INIT(&fs->mutex, NULL, fs->pthread_flags, GLFS_INIT_MUTEX,
+                       err);
+
+    PTHREAD_COND_INIT(&fs->cond, NULL, fs->pthread_flags, GLFS_INIT_COND, err);
+
+    PTHREAD_COND_INIT(&fs->child_down_cond, NULL, fs->pthread_flags,
+                      GLFS_INIT_COND_CHILD, err);
+
+    PTHREAD_MUTEX_INIT(&fs->upcall_list_mutex, NULL, fs->pthread_flags,
+                       GLFS_INIT_MUTEX_UPCALL, err);
+
+    fs->volname = strdup(volname);
+    if (!fs->volname)
+        goto err;
+
+    fs->pin_refcnt = 0;
+    fs->upcall_events = 0;
+    fs->up_cbk = NULL;
+    fs->up_data = NULL;
+
+    return fs;
+
+err:
+    glfs_free_from_ctx(fs);
+    return NULL;
 }
 
+extern xlator_t global_xlator;
+extern glusterfs_ctx_t *global_ctx;
+extern pthread_mutex_t global_ctx_mutex;
 
-static void *
-glfs_poller (void *data)
+static int
+glfs_init_global_ctx()
 {
-	struct glfs  *fs = NULL;
+    int ret = 0;
+    glusterfs_ctx_t *ctx = NULL;
+
+    pthread_mutex_lock(&global_ctx_mutex);
+    {
+        if (global_xlator.ctx)
+            goto unlock;
+
+        ctx = glusterfs_ctx_new();
+        if (!ctx) {
+            ret = -1;
+            goto unlock;
+        }
 
-	fs = data;
+        gf_log_globals_init(ctx, GF_LOG_NONE);
 
-	event_dispatch (fs->ctx->event_pool);
+        global_ctx = ctx;
+        global_xlator.ctx = global_ctx;
 
-	return NULL;
-}
+        ret = glusterfs_ctx_defaults_init(ctx);
+        if (ret) {
+            global_ctx = NULL;
+            global_xlator.ctx = NULL;
+            goto unlock;
+        }
+    }
+unlock:
+    pthread_mutex_unlock(&global_ctx_mutex);
+
+    if (ret)
+        FREE(ctx);
 
+    return ret;
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_new, 3.4.0)
 struct glfs *
-pub_glfs_new (const char *volname)
+pub_glfs_new(const char *volname)
 {
-	struct glfs     *fs = NULL;
-	int              ret = -1;
-	glusterfs_ctx_t *ctx = NULL;
+    if (!volname) {
+        errno = EINVAL;
+        return NULL;
+    }
+
+    struct glfs *fs = NULL;
+    int i = 0;
+    int ret = -1;
+    glusterfs_ctx_t *ctx = NULL;
+    xlator_t *old_THIS = NULL;
+    char pname[16] = "";
+    char msg[32] = "";
+
+    if (volname[0] == '/' || volname[0] == '-') {
+        if (strncmp(volname, "/snaps/", 7) == 0) {
+            goto label;
+        }
+        errno = EINVAL;
+        return NULL;
+    }
+
+    for (i = 0; i < strlen(volname); i++) {
+        if (!isalnum(volname[i]) && (volname[i] != '_') &&
+            (volname[i] != '-')) {
+            errno = EINVAL;
+            return NULL;
+        }
+    }
 
-	ctx = glusterfs_ctx_new ();
-	if (!ctx) {
-		return NULL;
-	}
+label:
+    /*
+     * Do this as soon as possible in case something else depends on
+     * pool allocations.
+     */
+    mem_pools_init();
 
-#ifdef DEBUG
-        gf_mem_acct_enable_set (ctx);
-#endif
+    fs = glfs_new_fs(volname);
+    if (!fs)
+        goto out;
+
+    ctx = glusterfs_ctx_new();
+    if (!ctx)
+        goto out;
+
+    /* first globals init, for gf_mem_acct_enable_set () */
 
-	/* first globals init, for gf_mem_acct_enable_set () */
-	ret = glusterfs_globals_init (ctx);
-	if (ret)
-		return NULL;
+    ret = glusterfs_globals_init(ctx);
+    if (ret)
+        goto out;
 
-	THIS->ctx = ctx;
+    old_THIS = THIS;
+    ret = glfs_init_global_ctx();
+    if (ret)
+        goto out;
 
-	/* then ctx_defaults_init, for xlator_mem_acct_init(THIS) */
-	ret = glusterfs_ctx_defaults_init (ctx);
-	if (ret)
-		return NULL;
+    /* then ctx_defaults_init, for xlator_mem_acct_init(THIS) */
 
-	fs = GF_CALLOC (1, sizeof (*fs), glfs_mt_glfs_t);
-	if (!fs)
-		return NULL;
-	fs->ctx = ctx;
+    ret = glusterfs_ctx_defaults_init(ctx);
+    if (ret)
+        goto out;
 
-	glfs_set_logging (fs, "/dev/null", 0);
+    fs->ctx = ctx;
+    fs->ctx->process_mode = GF_CLIENT_PROCESS;
 
-	fs->ctx->cmd_args.volfile_id = gf_strdup (volname);
+    ret = glfs_set_logging(fs, "/dev/null", 0);
+    if (ret)
+        goto out;
 
-	fs->volname = gf_strdup (volname);
+    fs->ctx->cmd_args.volfile_id = gf_strdup(volname);
+    if (!(fs->ctx->cmd_args.volfile_id)) {
+        ret = -1;
+        goto out;
+    }
 
-	pthread_mutex_init (&fs->mutex, NULL);
-	pthread_cond_init (&fs->cond, NULL);
+    ret = -1;
+#ifdef GF_LINUX_HOST_OS
+    ret = prctl(PR_GET_NAME, (unsigned long)pname, 0, 0, 0);
+#endif
+    if (ret)
+        fs->ctx->cmd_args.process_name = gf_strdup("gfapi");
+    else {
+        snprintf(msg, sizeof(msg), "gfapi.%s", pname);
+        fs->ctx->cmd_args.process_name = gf_strdup(msg);
+    }
+    ret = 0;
 
-	INIT_LIST_HEAD (&fs->openfds);
+out:
+    if (ret) {
+        if (fs) {
+            glfs_fini(fs);
+            fs = NULL;
+        } else {
+            /* glfs_fini() calls mem_pools_fini() too */
+            mem_pools_fini();
+        }
+    }
 
-	return fs;
+    if (old_THIS)
+        THIS = old_THIS;
+
+    return fs;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_new, 3.4.0);
+GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_new_from_ctx, 3.7.0)
+struct glfs *
+priv_glfs_new_from_ctx(glusterfs_ctx_t *ctx)
+{
+    struct glfs *fs = NULL;
 
+    if (!ctx)
+        goto out;
 
-int
-pub_glfs_set_volfile (struct glfs *fs, const char *volfile)
+    fs = glfs_new_fs("");
+    if (!fs)
+        goto out;
+
+    fs->ctx = ctx;
+
+out:
+    return fs;
+}
+
+GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_free_from_ctx, 3.7.0)
+void
+priv_glfs_free_from_ctx(struct glfs *fs)
 {
-	cmd_args_t  *cmd_args = NULL;
+    upcall_entry *u_list = NULL;
+    upcall_entry *tmp = NULL;
+
+    if (!fs)
+        return;
+
+    /* cleanup upcall structures */
+    list_for_each_entry_safe(u_list, tmp, &fs->upcall_list, upcall_list)
+    {
+        list_del_init(&u_list->upcall_list);
+        GF_FREE(u_list->upcall_data.data);
+        GF_FREE(u_list);
+    }
+
+    PTHREAD_MUTEX_DESTROY(&fs->mutex, fs->pthread_flags, GLFS_INIT_MUTEX);
+
+    PTHREAD_COND_DESTROY(&fs->cond, fs->pthread_flags, GLFS_INIT_COND);
+
+    PTHREAD_COND_DESTROY(&fs->child_down_cond, fs->pthread_flags,
+                         GLFS_INIT_COND_CHILD);
 
-	cmd_args = &fs->ctx->cmd_args;
+    PTHREAD_MUTEX_DESTROY(&fs->upcall_list_mutex, fs->pthread_flags,
+                          GLFS_INIT_MUTEX_UPCALL);
 
-	if (vol_assigned (cmd_args))
-		return -1;
+    if (fs->oldvolfile)
+        FREE(fs->oldvolfile);
 
-	cmd_args->volfile = gf_strdup (volfile);
+    FREE(fs->volname);
 
-	return 0;
+    FREE(fs);
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_set_volfile, 3.4.0);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_set_volfile, 3.4.0)
+int
+pub_glfs_set_volfile(struct glfs *fs, const char *volfile)
+{
+    cmd_args_t *cmd_args = NULL;
+
+    cmd_args = &fs->ctx->cmd_args;
+
+    if (vol_assigned(cmd_args))
+        return -1;
 
+    cmd_args->volfile = gf_strdup(volfile);
+    if (!cmd_args->volfile)
+        return -1;
+    return 0;
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_set_logging, 3.4.0)
 int
-pub_glfs_set_logging (struct glfs *fs, const char *logfile, int loglevel)
+pub_glfs_set_logging(struct glfs *fs, const char *logfile, int loglevel)
 {
-	int  ret = 0;
-        char *tmplog = NULL;
+    int ret = -1;
+    char *tmplog = NULL;
 
-        if (!logfile) {
-                ret = gf_set_log_file_path (&fs->ctx->cmd_args);
-                if (ret)
-                        goto out;
-                tmplog = fs->ctx->cmd_args.log_file;
-        } else {
-                tmplog = (char *)logfile;
-        }
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
 
-        ret = gf_log_init (fs->ctx, tmplog, NULL);
+    if (!logfile) {
+        ret = gf_set_log_file_path(&fs->ctx->cmd_args, fs->ctx);
         if (ret)
-                goto out;
+            goto out;
+        tmplog = fs->ctx->cmd_args.log_file;
+    } else {
+        tmplog = (char *)logfile;
+    }
+
+    /* finish log set parameters before init */
+    if (loglevel >= 0)
+        gf_log_set_loglevel(fs->ctx, loglevel);
 
-	if (loglevel >= 0)
-		gf_log_set_loglevel (loglevel);
+    ret = gf_log_init(fs->ctx, tmplog, NULL);
+    if (ret)
+        goto out;
+
+    ret = gf_log_inject_timer_event(fs->ctx);
+    if (ret)
+        goto out;
 
 out:
-	return ret;
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
+
+int
+glfs_init_wait(struct glfs *fs)
+{
+    int ret = -1;
+
+    /* Always a top-down call, use glfs_lock() */
+    glfs_lock(fs, _gf_true);
+    {
+        while (!fs->init)
+            pthread_cond_wait(&fs->cond, &fs->mutex);
+        ret = fs->ret;
+        errno = fs->err;
+    }
+    glfs_unlock(fs);
+
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_set_logging, 3.4.0);
+GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_init_done, 3.4.0)
+void
+priv_glfs_init_done(struct glfs *fs, int ret)
+{
+    glfs_init_cbk init_cbk;
+
+    if (!fs) {
+        gf_smsg("glfs", GF_LOG_ERROR, EINVAL, API_MSG_GLFS_FSOBJ_NULL, NULL);
+        goto out;
+    }
+
+    init_cbk = fs->init_cbk;
+
+    /* Always a bottom-up call, use mutex_lock() */
+    pthread_mutex_lock(&fs->mutex);
+    {
+        fs->init = 1;
+        fs->ret = ret;
+        fs->err = errno;
+
+        if (!init_cbk)
+            pthread_cond_broadcast(&fs->cond);
+    }
+    pthread_mutex_unlock(&fs->mutex);
 
+    if (init_cbk)
+        init_cbk(fs, ret);
+out:
+    return;
+}
 
 int
-glfs_init_wait (struct glfs *fs)
+glfs_init_common(struct glfs *fs)
 {
-	int   ret = -1;
+    int ret = -1;
 
-	/* Always a top-down call, use glfs_lock() */
-	glfs_lock (fs);
-	{
-		while (!fs->init)
-			pthread_cond_wait (&fs->cond,
-					   &fs->mutex);
-		ret = fs->ret;
-		errno = fs->err;
-	}
-	glfs_unlock (fs);
+    ret = create_master(fs);
+    if (ret)
+        return ret;
 
-	return ret;
+    ret = gf_thread_create(&fs->poller, NULL, glfs_poller, fs, "glfspoll");
+    if (ret)
+        return ret;
+
+    ret = glfs_volumes_init(fs);
+    if (ret)
+        return ret;
+
+    fs->dev_id = gf_dm_hashfn(fs->volname, strlen(fs->volname));
+    return ret;
 }
 
+int
+glfs_init_async(struct glfs *fs, glfs_init_cbk cbk)
+{
+    int ret = -1;
+
+    if (!fs || !fs->ctx) {
+        gf_smsg("glfs", GF_LOG_ERROR, EINVAL, API_MSG_FS_NOT_INIT, NULL);
+        errno = EINVAL;
+        return ret;
+    }
 
-void
-priv_glfs_init_done (struct glfs *fs, int ret)
+    fs->init_cbk = cbk;
+
+    ret = glfs_init_common(fs);
+
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_init, 3.4.0)
+int
+pub_glfs_init(struct glfs *fs)
 {
-	glfs_init_cbk init_cbk;
+    int ret = -1;
 
-	if (!fs) {
-		gf_log ("glfs", GF_LOG_ERROR,
-			"fs is NULL");
-		goto out;
-	}
+    DECLARE_OLD_THIS;
 
-	init_cbk = fs->init_cbk;
+    if (!fs || !fs->ctx) {
+        gf_smsg("glfs", GF_LOG_ERROR, EINVAL, API_MSG_FS_NOT_INIT, NULL);
+        errno = EINVAL;
+        return ret;
+    }
 
-	/* Always a bottom-up call, use mutex_lock() */
-	pthread_mutex_lock (&fs->mutex);
-	{
-		fs->init = 1;
-		fs->ret = ret;
-		fs->err = errno;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
 
-		if (!init_cbk)
-			pthread_cond_broadcast (&fs->cond);
-	}
-	pthread_mutex_unlock (&fs->mutex);
+    ret = glfs_init_common(fs);
+    if (ret)
+        goto out;
 
-	if (init_cbk)
-		init_cbk (fs, ret);
+    ret = glfs_init_wait(fs);
 out:
-	return;
-}
+    __GLFS_EXIT_FS;
 
-GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_init_done, 3.4.0);
+    /* Set the initial current working directory to "/" */
+    if (ret >= 0) {
+        ret = glfs_chdir(fs, "/");
+    }
 
+invalid_fs:
+    return ret;
+}
 
+static int
+glusterfs_ctx_destroy(glusterfs_ctx_t *ctx)
+{
+    call_pool_t *pool = NULL;
+    int ret = 0;
+    glusterfs_graph_t *trav_graph = NULL;
+    glusterfs_graph_t *tmp = NULL;
+
+    if (ctx == NULL)
+        return 0;
+
+    if (ctx->cmd_args.curr_server)
+        glfs_free_volfile_servers(&ctx->cmd_args);
+
+    glfs_free_xlator_options(&ctx->cmd_args);
+
+    /* For all the graphs, crawl through the xlator_t structs and free
+     * all its members except for the mem_acct member,
+     * as GF_FREE will be referencing it.
+     */
+    list_for_each_entry_safe(trav_graph, tmp, &ctx->graphs, list)
+    {
+        xlator_tree_free_members(trav_graph->first);
+    }
+
+    /* Free the memory pool */
+    if (ctx->stub_mem_pool)
+        mem_pool_destroy(ctx->stub_mem_pool);
+    if (ctx->dict_pool)
+        mem_pool_destroy(ctx->dict_pool);
+    if (ctx->dict_data_pool)
+        mem_pool_destroy(ctx->dict_data_pool);
+    if (ctx->dict_pair_pool)
+        mem_pool_destroy(ctx->dict_pair_pool);
+    if (ctx->logbuf_pool)
+        mem_pool_destroy(ctx->logbuf_pool);
+
+    pool = ctx->pool;
+    if (pool) {
+        if (pool->frame_mem_pool)
+            mem_pool_destroy(pool->frame_mem_pool);
+        if (pool->stack_mem_pool)
+            mem_pool_destroy(pool->stack_mem_pool);
+        LOCK_DESTROY(&pool->lock);
+        GF_FREE(pool);
+    }
+
+    /* Free the event pool */
+    ret = gf_event_pool_destroy(ctx->event_pool);
+
+    /* Free the iobuf pool */
+    iobuf_pool_destroy(ctx->iobuf_pool);
+
+    GF_FREE(ctx->process_uuid);
+    GF_FREE(ctx->cmd_args.volfile_id);
+    GF_FREE(ctx->cmd_args.process_name);
+
+    LOCK_DESTROY(&ctx->lock);
+    pthread_mutex_destroy(&ctx->notify_lock);
+    pthread_cond_destroy(&ctx->notify_cond);
+
+    /* Free all the graph structs and its containing xlator_t structs
+     * from this point there should be no reference to GF_FREE/GF_CALLOC
+     * as it will try to access mem_acct and the below function would
+     * have freed the same.
+     */
+    list_for_each_entry_safe(trav_graph, tmp, &ctx->graphs, list)
+    {
+        glusterfs_graph_destroy_residual(trav_graph);
+    }
+
+    GF_FREE(ctx->statedump_path);
+    FREE(ctx);
+
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fini, 3.4.0)
 int
-glfs_init_common (struct glfs *fs)
+pub_glfs_fini(struct glfs *fs)
 {
-	int  ret = -1;
+    int ret = -1;
+    int countdown = 100;
+    xlator_t *subvol = NULL;
+    glusterfs_ctx_t *ctx = NULL;
+    glusterfs_graph_t *graph = NULL;
+    call_pool_t *call_pool = NULL;
+    int fs_init = 0;
+    int err = -1;
+    struct synctask *waittask = NULL;
+
+    DECLARE_OLD_THIS;
+
+    if (!fs) {
+        errno = EINVAL;
+        goto invalid_fs;
+    }
+
+    ctx = fs->ctx;
+    if (!ctx) {
+        goto free_fs;
+    }
+
+    THIS = fs->ctx->master;
+
+    if (ctx->mgmt) {
+        rpc_clnt_disable(ctx->mgmt);
+    }
+
+    call_pool = fs->ctx->pool;
+
+    /* Wake up any suspended synctasks */
+    while (!list_empty(&fs->waitq)) {
+        waittask = list_entry(fs->waitq.next, struct synctask, waitq);
+        list_del_init(&waittask->waitq);
+        synctask_wake(waittask);
+    }
+
+    while (countdown--) {
+        /* give some time for background frames to finish */
+        pthread_mutex_lock(&fs->mutex);
+        {
+            /* Do we need to increase countdown? */
+            if ((!call_pool->cnt) && (!fs->pin_refcnt)) {
+                gf_msg_trace("glfs", 0,
+                             "call_pool_cnt - %" PRId64
+                             ","
+                             "pin_refcnt - %d",
+                             call_pool->cnt, fs->pin_refcnt);
+
+                ctx->cleanup_started = 1;
+                pthread_mutex_unlock(&fs->mutex);
+                break;
+            }
+        }
+        pthread_mutex_unlock(&fs->mutex);
+        gf_nanosleep(100000 * GF_US_IN_NS);
+    }
 
-	ret = create_master (fs);
-	if (ret)
-		return ret;
+    /* leaked frames may exist, we ignore */
 
-	ret = gf_thread_create (&fs->poller, NULL, glfs_poller, fs);
-	if (ret)
-		return ret;
+    /*We deem glfs_fini as successful if there are no pending frames in the call
+     *pool*/
+    ret = (call_pool->cnt == 0) ? 0 : -1;
 
-	ret = glfs_volumes_init (fs);
-	if (ret)
-		return ret;
+    pthread_mutex_lock(&fs->mutex);
+    {
+        fs_init = fs->init;
+    }
+    pthread_mutex_unlock(&fs->mutex);
 
-	fs->dev_id = gf_dm_hashfn (fs->volname, strlen (fs->volname));
-	return ret;
+    if (fs_init != 0) {
+        subvol = glfs_active_subvol(fs);
+        if (subvol) {
+            /* PARENT_DOWN within glfs_subvol_done() is issued
+               only on graph switch (new graph should activiate
+               and decrement the extra @winds count taken in
+               glfs_graph_setup()
+
+               Since we are explicitly destroying,
+               PARENT_DOWN is necessary
+            */
+            xlator_notify(subvol, GF_EVENT_PARENT_DOWN, subvol, 0);
+            /* Here we wait for GF_EVENT_CHILD_DOWN before exiting,
+               in case of asynchrnous cleanup
+            */
+            graph = subvol->graph;
+            err = pthread_mutex_lock(&fs->mutex);
+            if (err != 0) {
+                gf_smsg("glfs", GF_LOG_ERROR, err, API_MSG_FSMUTEX_LOCK_FAILED,
+                        "error=%s", strerror(err), NULL);
+                goto fail;
+            }
+            /* check and wait for CHILD_DOWN for active subvol*/
+            {
+                while (graph->used) {
+                    err = pthread_cond_wait(&fs->child_down_cond, &fs->mutex);
+                    if (err != 0)
+                        gf_smsg("glfs", GF_LOG_INFO, err,
+                                API_MSG_COND_WAIT_FAILED, "name=%s",
+                                subvol->name, "err=%s", strerror(err), NULL);
+                }
+            }
+
+            err = pthread_mutex_unlock(&fs->mutex);
+            if (err != 0) {
+                gf_smsg("glfs", GF_LOG_ERROR, err,
+                        API_MSG_FSMUTEX_UNLOCK_FAILED, "error=%s",
+                        strerror(err), NULL);
+                goto fail;
+            }
+        }
+        glfs_subvol_done(fs, subvol);
+    }
+
+    ctx->cleanup_started = 1;
+
+    if (fs_init != 0) {
+        /* Destroy all the inode tables of all the graphs.
+         * NOTE:
+         * - inode objects should be destroyed before calling fini()
+         *   of each xlator, as fini() and forget() of the xlators
+         *   can share few common locks or data structures, calling
+         *   fini first might destroy those required by forget
+         *   ( eg: in quick-read)
+         * - The call to inode_table_destroy_all is not required when
+         *   the cleanup during graph switch is implemented to perform
+         *   inode table destroy.
+         */
+        inode_table_destroy_all(ctx);
+
+        /* Call fini() of all the xlators in the active graph
+         * NOTE:
+         * - xlator fini() should be called before destroying any of
+         *   the threads. (eg: fini() in protocol-client uses timer
+         *   thread) */
+        glusterfs_graph_deactivate(ctx->active);
+
+        /* Join the syncenv_processor threads and cleanup
+         * syncenv resources*/
+        syncenv_destroy(ctx->env);
+
+        /* Join the poller thread */
+        if (gf_event_dispatch_destroy(ctx->event_pool) < 0)
+            ret = -1;
+    }
+
+    /* Avoid dispatching events to mgmt after freed,
+     * unreference mgmt after the event_dispatch_destroy */
+    if (ctx->mgmt) {
+        rpc_clnt_unref(ctx->mgmt);
+        ctx->mgmt = NULL;
+    }
+
+    /* log infra has to be brought down before destroying
+     * timer registry, as logging uses timer infra
+     */
+    if (gf_log_fini(ctx) != 0)
+        ret = -1;
+
+    /* Join the timer thread */
+    if (fs_init != 0) {
+        gf_timer_registry_destroy(ctx);
+    }
+
+    /* Destroy the context and the global pools */
+    if (glusterfs_ctx_destroy(ctx) != 0)
+        ret = -1;
+
+free_fs:
+    glfs_free_from_ctx(fs);
+
+    /*
+     * Do this as late as possible in case anything else has (or
+     * grows) a dependency on mem-pool allocations.
+     */
+    mem_pools_fini();
+
+fail:
+    if (!ret)
+        ret = err;
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
 }
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_get_volfile, 3.6.0)
+ssize_t
+pub_glfs_get_volfile(struct glfs *fs, void *buf, size_t len)
+{
+    ssize_t res = -1;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    glfs_lock(fs, _gf_true);
+    if (len >= fs->oldvollen) {
+        gf_msg_trace("glfs", 0, "copying %zu to %p", len, buf);
+        memcpy(buf, fs->oldvolfile, len);
+        res = len;
+    } else {
+        res = len - fs->oldvollen;
+        gf_msg_trace("glfs", 0, "buffer is %zd too short", -res);
+    }
+    glfs_unlock(fs);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return res;
+}
 
+GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_ipc, 3.12.0)
 int
-glfs_init_async (struct glfs *fs, glfs_init_cbk cbk)
+priv_glfs_ipc(struct glfs *fs, int opcode, void *xd_in, void **xd_out)
 {
-	int  ret = -1;
+    xlator_t *subvol = NULL;
+    int ret = -1;
 
-	fs->init_cbk = cbk;
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
 
-	ret = glfs_init_common (fs);
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
 
-	return ret;
-}
+    ret = syncop_ipc(subvol, opcode, (dict_t *)xd_in, (dict_t **)xd_out);
+    DECODE_SYNCOP_ERR(ret);
 
+out:
+    glfs_subvol_done(fs, subvol);
+    __GLFS_EXIT_FS;
 
+invalid_fs:
+    return ret;
+}
+
+GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_setfspid, 6.1)
 int
-pub_glfs_init (struct glfs *fs)
+priv_glfs_setfspid(struct glfs *fs, pid_t pid)
 {
-	int  ret = -1;
+    cmd_args_t *cmd_args = NULL;
+    int ret = 0;
 
-	ret = glfs_init_common (fs);
-	if (ret)
-		return ret;
+    cmd_args = &fs->ctx->cmd_args;
+    cmd_args->client_pid = pid;
+    cmd_args->client_pid_set = 1;
+    ret = syncopctx_setfspid(&pid);
 
-	ret = glfs_init_wait (fs);
+    return ret;
+}
 
-	return ret;
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_free, 3.7.16)
+void
+pub_glfs_free(void *ptr)
+{
+    GLFS_FREE(ptr);
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_get_fs, 3.7.16)
+struct glfs *
+pub_glfs_upcall_get_fs(struct glfs_upcall *arg)
+{
+    return arg->fs;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_get_reason, 3.7.16)
+enum glfs_upcall_reason
+pub_glfs_upcall_get_reason(struct glfs_upcall *arg)
+{
+    return arg->reason;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_init, 3.4.0);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_get_event, 3.7.16)
+void *
+pub_glfs_upcall_get_event(struct glfs_upcall *arg)
+{
+    return arg->event;
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_inode_get_object, 3.7.16)
+struct glfs_object *
+pub_glfs_upcall_inode_get_object(struct glfs_upcall_inode *arg)
+{
+    return arg->object;
+}
 
-extern xlator_t *
-priv_glfs_active_subvol (struct glfs *);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_inode_get_flags, 3.7.16)
+uint64_t
+pub_glfs_upcall_inode_get_flags(struct glfs_upcall_inode *arg)
+{
+    return arg->flags;
+}
 
-int
-pub_glfs_fini (struct glfs *fs)
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_inode_get_stat, 3.7.16)
+struct stat *
+pub_glfs_upcall_inode_get_stat(struct glfs_upcall_inode *arg)
+{
+    return &arg->buf;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_inode_get_expire, 3.7.16)
+uint64_t
+pub_glfs_upcall_inode_get_expire(struct glfs_upcall_inode *arg)
+{
+    return arg->expire_time_attr;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_inode_get_pobject, 3.7.16)
+struct glfs_object *
+pub_glfs_upcall_inode_get_pobject(struct glfs_upcall_inode *arg)
+{
+    return arg->p_object;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_inode_get_pstat, 3.7.16)
+struct stat *
+pub_glfs_upcall_inode_get_pstat(struct glfs_upcall_inode *arg)
+{
+    return &arg->p_buf;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_inode_get_oldpobject, 3.7.16)
+struct glfs_object *
+pub_glfs_upcall_inode_get_oldpobject(struct glfs_upcall_inode *arg)
+{
+    return arg->oldp_object;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_inode_get_oldpstat, 3.7.16)
+struct stat *
+pub_glfs_upcall_inode_get_oldpstat(struct glfs_upcall_inode *arg)
 {
-        int             ret = -1;
-        int             countdown = 100;
-        xlator_t        *subvol = NULL;
-        glusterfs_ctx_t *ctx = NULL;
-        call_pool_t     *call_pool = NULL;
+    return &arg->oldp_buf;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_lease_get_object, 4.1.6)
+struct glfs_object *
+pub_glfs_upcall_lease_get_object(struct glfs_upcall_lease *arg)
+{
+    return arg->object;
+}
 
-        ctx = fs->ctx;
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_lease_get_lease_type, 4.1.6)
+uint32_t
+pub_glfs_upcall_lease_get_lease_type(struct glfs_upcall_lease *arg)
+{
+    return arg->lease_type;
+}
 
-        if (ctx->mgmt) {
-                rpc_clnt_disable (ctx->mgmt);
-                ctx->mgmt = NULL;
+/* definitions of the GLFS_SYSRQ_* chars are in glfs.h */
+static struct glfs_sysrq_help {
+    char sysrq;
+    char *msg;
+} glfs_sysrq_help[] = {{GLFS_SYSRQ_HELP, "(H)elp"},
+                       {GLFS_SYSRQ_STATEDUMP, "(S)tatedump"},
+                       {0, NULL}};
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_sysrq, 3.10.0)
+int
+pub_glfs_sysrq(struct glfs *fs, char sysrq)
+{
+    glusterfs_ctx_t *ctx = NULL;
+    int ret = 0;
+    int msg_len;
+    char msg[1024] = {
+        0,
+    }; /* should not exceed 1024 chars */
+
+    if (!fs || !fs->ctx) {
+        ret = -1;
+        errno = EINVAL;
+        goto out;
+    }
+
+    ctx = fs->ctx;
+
+    switch (sysrq) {
+        case GLFS_SYSRQ_HELP: {
+            struct glfs_sysrq_help *usage = NULL;
+
+            for (usage = glfs_sysrq_help; usage->sysrq; usage++) {
+                msg_len = strlen(msg);
+                snprintf(msg + msg_len, /* append to msg */
+                         sizeof(msg) - msg_len - 2,
+                         /* - 2 for the " " + terminating \0 */
+                         " %s", usage->msg);
+            }
+
+            /* not really an 'error', but make sure it gets logged */
+            gf_log("glfs", GF_LOG_ERROR, "available events: %s", msg);
+
+            break;
         }
+        case GLFS_SYSRQ_STATEDUMP:
+            gf_proc_dump_info(SIGUSR1, ctx);
+            break;
+        default:
+            gf_smsg("glfs", GF_LOG_ERROR, ENOTSUP, API_MSG_INVALID_SYSRQ,
+                    "sysrq=%c", sysrq, NULL);
+            errno = ENOTSUP;
+            ret = -1;
+    }
+out:
+    return ret;
+}
 
-        __glfs_entry_fs (fs);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_register, 3.13.0)
+int
+pub_glfs_upcall_register(struct glfs *fs, uint32_t event_list,
+                         glfs_upcall_cbk cbk, void *data)
+{
+    int ret = 0;
+
+    /* list of supported upcall events */
+    uint32_t up_events = (GLFS_EVENT_INODE_INVALIDATE |
+                          GLFS_EVENT_RECALL_LEASE);
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    GF_VALIDATE_OR_GOTO(THIS->name, cbk, out);
+
+    /* Event list should be either GLFS_EVENT_ANY
+     * or list of supported individual events (up_events)
+     */
+    if ((event_list != GLFS_EVENT_ANY) && (event_list & ~up_events)) {
+        errno = EINVAL;
+        ret = -1;
+        gf_smsg(THIS->name, GF_LOG_ERROR, errno, API_MSG_INVALID_ARG,
+                "event_list=(0x%08x)", event_list, NULL);
+        goto out;
+    }
+
+    /* in case other thread does unregister */
+    pthread_mutex_lock(&fs->mutex);
+    {
+        if (event_list & GLFS_EVENT_INODE_INVALIDATE) {
+            /* @todo: Check if features.cache-invalidation is
+             * enabled.
+             */
+            fs->upcall_events |= GF_UPCALL_CACHE_INVALIDATION;
+            ret |= GLFS_EVENT_INODE_INVALIDATE;
+        }
+        if (event_list & GLFS_EVENT_RECALL_LEASE) {
+            /* @todo: Check if features.leases is enabled */
+            fs->upcall_events |= GF_UPCALL_RECALL_LEASE;
+            ret |= GLFS_EVENT_RECALL_LEASE;
+        }
+        /* Override cbk function if existing */
+        fs->up_cbk = cbk;
+        fs->up_data = data;
+        fs->cache_upcalls = _gf_true;
+    }
+    pthread_mutex_unlock(&fs->mutex);
 
-        call_pool = fs->ctx->pool;
+out:
+    __GLFS_EXIT_FS;
 
-        while (countdown--) {
-                /* give some time for background frames to finish */
-                if (!call_pool->cnt)
-                        break;
-                usleep (100000);
+invalid_fs:
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_unregister, 3.13.0)
+int
+pub_glfs_upcall_unregister(struct glfs *fs, uint32_t event_list)
+{
+    int ret = 0;
+    /* list of supported upcall events */
+    uint32_t up_events = (GLFS_EVENT_INODE_INVALIDATE |
+                          GLFS_EVENT_RECALL_LEASE);
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* Event list should be either GLFS_EVENT_ANY
+     * or list of supported individual events (up_events)
+     */
+    if ((event_list != GLFS_EVENT_ANY) && (event_list & ~up_events)) {
+        errno = EINVAL;
+        ret = -1;
+        gf_smsg(THIS->name, GF_LOG_ERROR, errno, API_MSG_INVALID_ARG,
+                "event_list=(0x%08x)", event_list, NULL);
+        goto out;
+    }
+
+    pthread_mutex_lock(&fs->mutex);
+    {
+        /* We already checked if event_list contains list of supported
+         * upcall events. No other specific checks needed as of now for
+         * unregister */
+        fs->upcall_events &= ~(event_list);
+        ret |= ((event_list == GLFS_EVENT_ANY) ? up_events : event_list);
+
+        /* If there are no upcall events registered, reset cbk */
+        if (fs->upcall_events == 0) {
+            fs->up_cbk = NULL;
+            fs->up_data = NULL;
+            fs->cache_upcalls = _gf_false;
         }
-        /* leaked frames may exist, we ignore */
+    }
+    pthread_mutex_unlock(&fs->mutex);
 
-        /*We deem glfs_fini as successful if there are no pending frames in the call
-         *pool*/
-        ret = (call_pool->cnt == 0)? 0: -1;
+out:
+    __GLFS_EXIT_FS;
 
-        subvol = priv_glfs_active_subvol (fs);
-        if (subvol) {
-                /* PARENT_DOWN within priv_glfs_subvol_done() is issued only
-                   on graph switch (new graph should activiate and
-                   decrement the extra @winds count taken in glfs_graph_setup()
-
-                   Since we are explicitly destroying, PARENT_DOWN is necessary
-                */
-                xlator_notify (subvol, GF_EVENT_PARENT_DOWN, subvol, 0);
-                /* TBD: wait for CHILD_DOWN before exiting, in case of
-                   asynchronous cleanup like graceful socket disconnection
-                   in the future.
-                */
+invalid_fs:
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_set_statedump_path, 7.0)
+int
+pub_glfs_set_statedump_path(struct glfs *fs, const char *path)
+{
+    struct stat st;
+    int ret;
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    if (!path) {
+        gf_log("glfs", GF_LOG_ERROR, "path is NULL");
+        errno = EINVAL;
+        goto err;
+    }
+
+    /* If path is not present OR, if it is directory AND has enough permission
+     * to create files, then proceed */
+    ret = sys_stat(path, &st);
+    if (ret && errno != ENOENT) {
+        gf_log("glfs", GF_LOG_ERROR, "%s: not a valid path (%s)", path,
+               strerror(errno));
+        errno = EINVAL;
+        goto err;
+    }
+
+    if (!ret) {
+        /* file is present, now check other things */
+        if (!S_ISDIR(st.st_mode)) {
+            gf_log("glfs", GF_LOG_ERROR, "%s: path is not directory", path);
+            errno = EINVAL;
+            goto err;
         }
+        if (sys_access(path, W_OK | X_OK) < 0) {
+            gf_log("glfs", GF_LOG_ERROR,
+                   "%s: path doesn't have write permission", path);
+            errno = EPERM;
+            goto err;
+        }
+    }
 
-        priv_glfs_subvol_done (fs, subvol);
+    /* If set, it needs to be freed, so we don't have leak */
+    GF_FREE(fs->ctx->statedump_path);
 
-        if (gf_log_fini(ctx) != 0)
-                ret = -1;
+    fs->ctx->statedump_path = gf_strdup(path);
+    if (!fs->ctx->statedump_path) {
+        gf_log("glfs", GF_LOG_ERROR,
+               "%s: failed to set statedump path, no memory", path);
+        errno = ENOMEM;
+        goto err;
+    }
 
-        return ret;
-}
+    __GLFS_EXIT_FS;
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fini, 3.4.0);
+    return 0;
+err:
+    __GLFS_EXIT_FS;
 
+invalid_fs:
+    return -1;
+}
diff --git a/api/src/glfs.h b/api/src/glfs.h
index 41907240dd7..279d11d58ee 100644
--- a/api/src/glfs.h
+++ b/api/src/glfs.h
@@ -1,5 +1,5 @@
 /*
-  Copyright (c) 2012 Red Hat, Inc. <http://www.redhat.com>
+  Copyright (c) 2012-2018 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
   This file is licensed to you under your choice of the GNU Lesser
@@ -8,7 +8,6 @@
   cases as published by the Free Software Foundation.
 */
 
-
 #ifndef _GLFS_H
 #define _GLFS_H
 
@@ -21,6 +20,17 @@
   both the library and the application.
 */
 
+/* Values for valid flags to be used when using XXXsetattr, to set multiple
+ attribute values passed via the related stat structure.
+ */
+
+#define GFAPI_SET_ATTR_MODE 0x1
+#define GFAPI_SET_ATTR_UID 0x2
+#define GFAPI_SET_ATTR_GID 0x4
+#define GFAPI_SET_ATTR_SIZE 0x8
+#define GFAPI_SET_ATTR_ATIME 0x10
+#define GFAPI_SET_ATTR_MTIME 0x20
+
 #ifndef _FILE_OFFSET_BITS
 #define _FILE_OFFSET_BITS 64
 #endif
@@ -41,14 +51,64 @@
 #include <sys/cdefs.h>
 #include <dirent.h>
 #include <sys/statvfs.h>
+#include <stdint.h>
+#include <sys/time.h>
+
+/*
+ * For off64_t to be defined, we need both
+ * __USE_LARGEFILE64 to be true and __off64_t_defnined to be
+ * false. But, making __USE_LARGEFILE64 true causes other issues
+ * such as redinition of stat and fstat to stat64 and fstat64
+ * respectively which again causes compilation issues.
+ * Without off64_t being defined, this will not compile as
+ * copy_file_range uses off64_t. Hence define it here. First
+ * check whether __off64_t_defined is true or not. <unistd.h>
+ * sets that flag when it defines off64_t. If __off64_t_defined
+ * is false and __USE_FILE_OFFSET64 is true, then go on to define
+ * off64_t using __off64_t.
+ */
+#ifndef GF_BSD_HOST_OS
+#if defined(__USE_FILE_OFFSET64) && !defined(__off64_t_defined)
+typedef __off64_t off64_t;
+#endif /* defined(__USE_FILE_OFFSET64) && !defined(__off64_t_defined) */
+#else
+#include <stdio.h>
+#ifndef _OFF64_T_DECLARED
+/*
+ * Including <stdio.h> (done above) should actually define
+ * _OFF64_T_DECLARED with off64_t data type being available
+ * for consumption. But, off64_t data type is not recognizable
+ * for FreeBSD versions less than 11. Hence, int64_t is typedefed
+ * to off64_t.
+ */
+#define _OFF64_T_DECLARED
+typedef int64_t off64_t;
+#endif /* _OFF64_T_DECLARED */
+#endif /* GF_BSD_HOST_OS */
+
+#if defined(HAVE_SYS_ACL_H) || (defined(USE_POSIX_ACLS) && USE_POSIX_ACLS)
+#include <sys/acl.h>
+#else
+typedef void *acl_t;
+typedef int acl_type_t;
+#endif
 
 /* Portability non glibc c++ build systems */
 #ifndef __THROW
-# if defined __cplusplus
-#  define __THROW       throw ()
-# else
-#  define __THROW
-# endif
+#if defined __cplusplus
+#define __THROW throw()
+#else
+#define __THROW
+#endif
+#endif
+
+#ifndef GF_DARWIN_HOST_OS
+#define GFAPI_PUBLIC(sym, ver)  /**/
+#define GFAPI_PRIVATE(sym, ver) /**/
+#else
+#define GFAPI_PUBLIC(sym, ver) __asm("_" __STRING(sym) "$GFAPI_" __STRING(ver))
+#define GFAPI_PRIVATE(sym, ver)                                                \
+    __asm("_" __STRING(sym) "$GFAPI_PRIVATE_" __STRING(ver))
 #endif
 
 __BEGIN_DECLS
@@ -57,7 +117,6 @@ __BEGIN_DECLS
 struct glfs;
 typedef struct glfs glfs_t;
 
-
 /*
   SYNOPSIS
 
@@ -78,8 +137,8 @@ typedef struct glfs glfs_t;
 
   @volname: Name of the volume. This identifies the server-side volume and
             the fetched volfile (equivalent of --volfile-id command line
-	    parameter to glusterfsd). When used with glfs_set_volfile() the
-	    @volname has no effect (except for appearing in log messages).
+            parameter to glusterfsd). When used with glfs_set_volfile() the
+            @volname has no effect (except for appearing in log messages).
 
   RETURN VALUES
 
@@ -88,8 +147,8 @@ typedef struct glfs glfs_t;
 
 */
 
-glfs_t *glfs_new (const char *volname);
-
+glfs_t *
+glfs_new(const char *volname) __THROW GFAPI_PUBLIC(glfs_new, 3.4.0);
 
 /*
   SYNOPSIS
@@ -118,8 +177,9 @@ glfs_t *glfs_new (const char *volname);
 
 */
 
-int glfs_set_volfile (glfs_t *fs, const char *volfile);
-
+int
+glfs_set_volfile(glfs_t *fs, const char *volfile) __THROW
+    GFAPI_PUBLIC(glfs_set_volfile, 3.4.0);
 
 /*
   SYNOPSIS
@@ -144,15 +204,16 @@ int glfs_set_volfile (glfs_t *fs, const char *volfile);
        specification file.
 
   @transport: String specifying the transport used to connect to the
-              management daemon. Specifying NULL will result in the usage
-              of the default (tcp) transport type. Permitted values
-              are those what you specify as transport-type in a volume
-              specification file (e.g "tcp", "rdma" etc.)
+              management daemon. Specifying NULL will result in the
+              usage of the default (tcp) transport type. Permitted
+              values are "tcp" or "unix".
 
   @host:      String specifying the address where to find the management daemon.
+              Socket path, while using Unix domain socket as transport type.
               This would either be
-              - FQDN (e.g: "storage01.company.com") or
-              - ASCII (e.g: "192.168.22.1")
+              - FQDN (e.g : "storage01.company.com") or
+              - ASCII (e.g : "192.168.22.1") or
+              - Socket path (e.g : "/var/run/glusterd.socket")
 
   NOTE: This API is special, multiple calls to this function with different
         volfile servers, port or transport-type would create a list of volfile
@@ -169,12 +230,15 @@ int glfs_set_volfile (glfs_t *fs, const char *volfile);
 
 */
 
-int glfs_set_volfile_server (glfs_t *fs, const char *transport,
-                             const char *host, int port);
-
-int glfs_unset_volfile_server (glfs_t *fs, const char *transport,
-                               const char *host, int port) __THROW;
+int
+glfs_set_volfile_server(glfs_t *fs, const char *transport, const char *host,
+                        int port) __THROW
+    GFAPI_PUBLIC(glfs_set_volfile_server, 3.4.0);
 
+int
+glfs_unset_volfile_server(glfs_t *fs, const char *transport, const char *host,
+                          int port) __THROW
+    GFAPI_PUBLIC(glfs_unset_volfile_server, 3.5.1);
 /*
   SYNOPSIS
 
@@ -204,8 +268,9 @@ int glfs_unset_volfile_server (glfs_t *fs, const char *transport,
 
 */
 
-int glfs_set_logging (glfs_t *fs, const char *logfile, int loglevel);
-
+int
+glfs_set_logging(glfs_t *fs, const char *logfile, int loglevel) __THROW
+    GFAPI_PUBLIC(glfs_set_logging, 3.4.0);
 
 /*
   SYNOPSIS
@@ -231,8 +296,8 @@ int glfs_set_logging (glfs_t *fs, const char *logfile, int loglevel);
 
 */
 
-int glfs_init (glfs_t *fs);
-
+int
+glfs_init(glfs_t *fs) __THROW GFAPI_PUBLIC(glfs_init, 3.4.0);
 
 /*
   SYNOPSIS
@@ -264,8 +329,40 @@ int glfs_init (glfs_t *fs);
    0 : Success.
 */
 
-int glfs_fini (glfs_t *fs);
+int
+glfs_fini(glfs_t *fs) __THROW GFAPI_PUBLIC(glfs_fini, 3.4.0);
+
+/*
+  SYNOPSIS
+
+      glfs_getvol: Get the volfile associated with a 'virtual mount'
+
+  DESCRIPTION
+
+      Sometimes it's useful e.g. for scripts to see the volfile, so that they
+      can parse it and find subvolumes to do things like split-brain resolution
+      or custom layouts.  The API here was specifically intended to make access
+      e.g. from Python as simple as possible.
+
+      Note that the volume must be started (not necessarily mounted) for this
+      to work.
+
+  PARAMETERS
+
+      @fs:  The 'virtual mount' object for which a volfile is desired
+      @buf: Pointer to a place for the volfile length to be stored
+      @len: Length of @buf
+
+  RETURN VALUES
+
+      >0: filled N bytes of buffer
+       0: no volfile available
+      <0: volfile length exceeds @len by N bytes (@buf unchanged)
+*/
 
+ssize_t
+glfs_get_volfile(glfs_t *fs, void *buf, size_t len) __THROW
+    GFAPI_PUBLIC(glfs_get_volfile, 3.6.0);
 
 /*
   SYNOPSIS
@@ -278,9 +375,9 @@ int glfs_fini (glfs_t *fs);
        the management server (glusterd) to fetch volume uuid and stores it
        in the glusterfs_context linked to the glfs object fs which can be used
        in the subsequent calls. Later it parses that UUID to convert it from
-       cannonical string format into an opaque byte array and copy it into
-       the volid array. Incase if either of the input parameters, volid or size,
-       is NULL, number of bytes required to copy the volume UUID is returned.
+       canonical string format into an opaque byte array and copy it into
+       the volid array. In case if either of the input parameters, volid or
+  size, is NULL, number of bytes required to copy the volume UUID is returned.
 
   PARAMETERS
 
@@ -295,8 +392,9 @@ int glfs_fini (glfs_t *fs);
         Others : length of the volume UUID stored.
 */
 
-int glfs_get_volumeid (struct glfs *fs, char *volid, size_t size);
-
+int
+glfs_get_volumeid(glfs_t *fs, char *volid, size_t size) __THROW
+    GFAPI_PUBLIC(glfs_get_volumeid, 3.5.0);
 
 /*
  * FILE OPERATION
@@ -318,6 +416,119 @@ struct glfs_fd;
 typedef struct glfs_fd glfs_fd_t;
 
 /*
+ * Mask for request/result items in the struct glfs_stat.
+ *
+ * Query request/result mask for glfs_stat() (family of functions) and
+ * struct glfs_stat::glfs_st_mask.
+ *
+ * These bits should be set in the mask argument of glfs_stat() (family of
+ * functions) to request particular items when calling glfs_stat().
+ *
+ * NOTE: Lower order 32 bits are used to reflect statx(2) bits. For Gluster
+ * specific attrs/extensions, use higher order 32 bits.
+ *
+ */
+#define GLFS_STAT_TYPE 0x0000000000000001U   /* Want/got stx_mode & S_IFMT */
+#define GLFS_STAT_MODE 0x0000000000000002U   /* Want/got stx_mode & ~S_IFMT */
+#define GLFS_STAT_NLINK 0x0000000000000004U  /* Want/got stx_nlink */
+#define GLFS_STAT_UID 0x0000000000000008U    /* Want/got stx_uid */
+#define GLFS_STAT_GID 0x0000000000000010U    /* Want/got stx_gid */
+#define GLFS_STAT_ATIME 0x0000000000000020U  /* Want/got stx_atime */
+#define GLFS_STAT_MTIME 0x0000000000000040U  /* Want/got stx_mtime */
+#define GLFS_STAT_CTIME 0x0000000000000080U  /* Want/got stx_ctime */
+#define GLFS_STAT_INO 0x0000000000000100U    /* Want/got stx_ino */
+#define GLFS_STAT_SIZE 0x0000000000000200U   /* Want/got stx_size */
+#define GLFS_STAT_BLOCKS 0x0000000000000400U /* Want/got stx_blocks */
+#define GLFS_STAT_BASIC_STATS                                                  \
+    0x00000000000007ffU /* Items in the normal stat struct */
+#define GLFS_STAT_BTIME 0x0000000000000800U /* Want/got stx_btime */
+#define GLFS_STAT_ALL 0x0000000000000fffU   /* All currently supported flags */
+#define GLFS_STAT_RESERVED                                                     \
+    0x8000000000000000U /* Reserved to denote future expansion */
+
+/* Macros for checking validity of struct glfs_stat members.*/
+#define GLFS_STAT_TYPE_VALID(stmask) (stmask & GLFS_STAT_TYPE)
+#define GLFS_STAT_MODE_VALID(stmask) (stmask & GLFS_STAT_MODE)
+#define GLFS_STAT_NLINK_VALID(stmask) (stmask & GLFS_STAT_NLINK)
+#define GLFS_STAT_UID_VALID(stmask) (stmask & GLFS_STAT_UID)
+#define GLFS_STAT_GID_VALID(stmask) (stmask & GLFS_STAT_GID)
+#define GLFS_STAT_ATIME_VALID(stmask) (stmask & GLFS_STAT_ATIME)
+#define GLFS_STAT_MTIME_VALID(stmask) (stmask & GLFS_STAT_MTIME)
+#define GLFS_STAT_CTIME_VALID(stmask) (stmask & GLFS_STAT_CTIME)
+#define GLFS_STAT_INO_VALID(stmask) (stmask & GLFS_STAT_INO)
+#define GLFS_STAT_SIZE_VALID(stmask) (stmask & GLFS_STAT_SIZE)
+#define GLFS_STAT_BLOCKS_VALID(stmask) (stmask & GLFS_STAT_BLOCKS)
+#define GLFS_STAT_BTIME_VALID(stmask) (stmask & GLFS_STAT_BTIME)
+#define GLFS_STAT_GFID_VALID(stmask) (stmask & GLFS_STAT_GFID)
+
+/*
+ * Attributes to be found in glfs_st_attributes and masked in
+ * glfs_st_attributes_mask.
+ *
+ * These give information about the features or the state of a file that might
+ * be of use to programs.
+ *
+ * NOTE: Lower order 32 bits are used to reflect statx(2) attribute bits. For
+ * Gluster specific attrs, use higher order 32 bits.
+ *
+ * NOTE: We do not support any file attributes or state as yet!
+ */
+#define GLFS_STAT_ATTR_RESERVED                                                \
+    0x8000000000000000U /* Reserved to denote future expansion */
+
+/* Extended file attribute structure.
+ *
+ * The caller passes a mask of what they're specifically interested in as a
+ * parameter to glfs_stat().  What glfs_stat() actually got will be indicated
+ * in glfs_st_mask upon return.
+ *
+ * For each bit in the mask argument:
+ *
+ * - if the datum is not supported:
+ *
+ *   - the bit will be cleared, and
+ *
+ *   - the datum value is undefined
+ *
+ * - otherwise, if explicitly requested:
+ *
+ *   - the field will be filled in and the bit will be set;
+ *
+ * - otherwise, if not requested, but available in, it will be filled in
+ * anyway, and the bit will be set upon return;
+ *
+ * - otherwise the field and the bit will be cleared before returning.
+ *
+ */
+
+struct glfs_stat {
+    uint64_t glfs_st_mask;       /* What results were written [uncond] */
+    uint64_t glfs_st_attributes; /* Flags conveying information about the file
+                                    [uncond] */
+    uint64_t glfs_st_attributes_mask; /* Mask to show what's supported in
+                                         st_attributes [ucond] */
+    struct timespec glfs_st_atime;    /* Last access time */
+    struct timespec glfs_st_btime;    /* File creation time */
+    struct timespec glfs_st_ctime;    /* Last attribute change time */
+    struct timespec glfs_st_mtime;    /* Last data modification time */
+    ino_t glfs_st_ino;                /* Inode number */
+    off_t glfs_st_size;               /* File size */
+    blkcnt_t glfs_st_blocks;          /* Number of 512-byte blocks allocated */
+    uint32_t glfs_st_rdev_major; /* Device ID of special file [if bdev/cdev] */
+    uint32_t glfs_st_rdev_minor;
+    uint32_t glfs_st_dev_major; /* ID of device containing file [uncond] */
+    uint32_t glfs_st_dev_minor;
+    blksize_t glfs_st_blksize; /* Preferred general I/O size [uncond] */
+    nlink_t glfs_st_nlink;     /* Number of hard links */
+    uid_t glfs_st_uid;         /* User ID of owner */
+    gid_t glfs_st_gid;         /* Group ID of owner */
+    mode_t glfs_st_mode;       /* File mode */
+};
+
+#define GLFS_LEASE_ID_SIZE 16 /* 128bits */
+typedef char glfs_leaseid_t[GLFS_LEASE_ID_SIZE];
+
+/*
  * PER THREAD IDENTITY MODIFIERS
  *
  * The following operations enable to set a per thread identity context
@@ -334,15 +545,27 @@ typedef struct glfs_fd glfs_fd_t;
  *    caller
  *  - The groups once set, need to be unset by setting the size to 0 (in which
  *    case the list argument is a do not care)
+ *  - In case of leases feature enables, setfsleaseid is used to set and reset
+ *    leaseid before and after every I/O operation.
  *  - Once a process for a thread of operation choses to set the IDs, all glfs
  *    calls made from that thread would default to the IDs set for the thread.
  *    As a result use these APIs with care and ensure that the set IDs are
  *    reverted to global process defaults as required.
  *
  */
-int glfs_setfsuid (uid_t fsuid);
-int glfs_setfsgid (gid_t fsgid);
-int glfs_setfsgroups (size_t size, const gid_t *list);
+int
+glfs_setfsuid(uid_t fsuid) __THROW GFAPI_PUBLIC(glfs_setfsuid, 3.4.2);
+
+int
+glfs_setfsgid(gid_t fsgid) __THROW GFAPI_PUBLIC(glfs_setfsgid, 3.4.2);
+
+int
+glfs_setfsgroups(size_t size, const gid_t *list) __THROW
+    GFAPI_PUBLIC(glfs_setfsgroups, 3.4.2);
+
+int
+glfs_setfsleaseid(glfs_leaseid_t leaseid) __THROW
+    GFAPI_PUBLIC(glfs_setfsleaseid, 4.0.0);
 
 /*
   SYNOPSIS
@@ -369,8 +592,9 @@ int glfs_setfsgroups (size_t size, const gid_t *list);
 
  */
 
-glfs_fd_t *glfs_open (glfs_t *fs, const char *path, int flags);
-
+glfs_fd_t *
+glfs_open(glfs_t *fs, const char *path, int flags) __THROW
+    GFAPI_PUBLIC(glfs_open, 3.4.0);
 
 /*
   SYNOPSIS
@@ -398,15 +622,20 @@ glfs_fd_t *glfs_open (glfs_t *fs, const char *path, int flags);
 
  */
 
-glfs_fd_t *glfs_creat (glfs_t *fs, const char *path, int flags,
-		       mode_t mode);
+glfs_fd_t *
+glfs_creat(glfs_t *fs, const char *path, int flags, mode_t mode) __THROW
+    GFAPI_PUBLIC(glfs_creat, 3.4.0);
 
-int glfs_close (glfs_fd_t *fd);
+int
+glfs_close(glfs_fd_t *fd) __THROW GFAPI_PUBLIC(glfs_close, 3.4.0);
 
-glfs_t *glfs_from_glfd (glfs_fd_t *fd);
+glfs_t *
+glfs_from_glfd(glfs_fd_t *fd) __THROW GFAPI_PUBLIC(glfs_from_glfd, 3.4.0);
 
-int glfs_set_xlator_option (glfs_t *fs, const char *xlator, const char *key,
-			    const char *value);
+int
+glfs_set_xlator_option(glfs_t *fs, const char *xlator, const char *key,
+                       const char *value) __THROW
+    GFAPI_PUBLIC(glfs_set_xlator_option, 3.4.0);
 
 /*
 
@@ -429,199 +658,828 @@ int glfs_set_xlator_option (glfs_t *fs, const char *xlator, const char *key,
   time of issuing the async IO call. This can be used by the
   caller to differentiate different instances of the async requests
   in a common callback function.
+
+  @prestat and @poststat are allocated on the stack, that are auto destroyed
+  post the callback function returns.
 */
 
-typedef void (*glfs_io_cbk) (glfs_fd_t *fd, ssize_t ret, void *data);
+typedef void (*glfs_io_cbk)(glfs_fd_t *fd, ssize_t ret,
+                            struct glfs_stat *prestat,
+                            struct glfs_stat *poststat, void *data);
 
 // glfs_{read,write}[_async]
 
-ssize_t glfs_read (glfs_fd_t *fd, void *buf, size_t count, int flags);
-ssize_t glfs_write (glfs_fd_t *fd, const void *buf, size_t count, int flags);
-int glfs_read_async (glfs_fd_t *fd, void *buf, size_t count, int flags,
-		     glfs_io_cbk fn, void *data);
-int glfs_write_async (glfs_fd_t *fd, const void *buf, size_t count, int flags,
-		      glfs_io_cbk fn, void *data);
+ssize_t
+glfs_read(glfs_fd_t *fd, void *buf, size_t count, int flags) __THROW
+    GFAPI_PUBLIC(glfs_read, 3.4.0);
+
+ssize_t
+glfs_write(glfs_fd_t *fd, const void *buf, size_t count, int flags) __THROW
+    GFAPI_PUBLIC(glfs_write, 3.4.0);
+
+int
+glfs_read_async(glfs_fd_t *fd, void *buf, size_t count, int flags,
+                glfs_io_cbk fn, void *data) __THROW
+    GFAPI_PUBLIC(glfs_read_async, 6.0);
+
+int
+glfs_write_async(glfs_fd_t *fd, const void *buf, size_t count, int flags,
+                 glfs_io_cbk fn, void *data) __THROW
+    GFAPI_PUBLIC(glfs_write_async, 6.0);
 
 // glfs_{read,write}v[_async]
 
-ssize_t glfs_readv (glfs_fd_t *fd, const struct iovec *iov, int iovcnt,
-		    int flags);
-ssize_t glfs_writev (glfs_fd_t *fd, const struct iovec *iov, int iovcnt,
-		     int flags);
-int glfs_readv_async (glfs_fd_t *fd, const struct iovec *iov, int count,
-		      int flags, glfs_io_cbk fn, void *data);
-int glfs_writev_async (glfs_fd_t *fd, const struct iovec *iov, int count,
-		       int flags, glfs_io_cbk fn, void *data);
+ssize_t
+glfs_readv(glfs_fd_t *fd, const struct iovec *iov, int iovcnt,
+           int flags) __THROW GFAPI_PUBLIC(glfs_readv, 3.4.0);
+
+ssize_t
+glfs_writev(glfs_fd_t *fd, const struct iovec *iov, int iovcnt,
+            int flags) __THROW GFAPI_PUBLIC(glfs_writev, 3.4.0);
+
+int
+glfs_readv_async(glfs_fd_t *fd, const struct iovec *iov, int count, int flags,
+                 glfs_io_cbk fn, void *data) __THROW
+    GFAPI_PUBLIC(glfs_readv_async, 6.0);
+
+int
+glfs_writev_async(glfs_fd_t *fd, const struct iovec *iov, int count, int flags,
+                  glfs_io_cbk fn, void *data) __THROW
+    GFAPI_PUBLIC(glfs_writev_async, 6.0);
 
 // glfs_p{read,write}[_async]
 
-ssize_t glfs_pread (glfs_fd_t *fd, void *buf, size_t count, off_t offset,
-		    int flags);
-ssize_t glfs_pwrite (glfs_fd_t *fd, const void *buf, size_t count,
-		     off_t offset, int flags);
-int glfs_pread_async (glfs_fd_t *fd, void *buf, size_t count, off_t offset,
-		      int flags, glfs_io_cbk fn, void *data);
-int glfs_pwrite_async (glfs_fd_t *fd, const void *buf, int count, off_t offset,
-		       int flags, glfs_io_cbk fn, void *data);
+ssize_t
+glfs_pread(glfs_fd_t *fd, void *buf, size_t count, off_t offset, int flags,
+           struct glfs_stat *poststat) __THROW GFAPI_PUBLIC(glfs_pread, 6.0);
+
+ssize_t
+glfs_pwrite(glfs_fd_t *fd, const void *buf, size_t count, off_t offset,
+            int flags, struct glfs_stat *prestat,
+            struct glfs_stat *poststat) __THROW GFAPI_PUBLIC(glfs_pwrite, 6.0);
+
+int
+glfs_pread_async(glfs_fd_t *fd, void *buf, size_t count, off_t offset,
+                 int flags, glfs_io_cbk fn, void *data) __THROW
+    GFAPI_PUBLIC(glfs_pread_async, 6.0);
+
+int
+glfs_pwrite_async(glfs_fd_t *fd, const void *buf, int count, off_t offset,
+                  int flags, glfs_io_cbk fn, void *data) __THROW
+    GFAPI_PUBLIC(glfs_pwrite_async, 6.0);
 
 // glfs_p{read,write}v[_async]
 
-ssize_t glfs_preadv (glfs_fd_t *fd, const struct iovec *iov, int iovcnt,
-		     off_t offset, int flags);
-ssize_t glfs_pwritev (glfs_fd_t *fd, const struct iovec *iov, int iovcnt,
-		      off_t offset, int flags);
-int glfs_preadv_async (glfs_fd_t *fd, const struct iovec *iov, int count,
-		       off_t offset, int flags, glfs_io_cbk fn, void *data);
-int glfs_pwritev_async (glfs_fd_t *fd, const struct iovec *iov, int count,
-			off_t offset, int flags, glfs_io_cbk fn, void *data);
+ssize_t
+glfs_preadv(glfs_fd_t *fd, const struct iovec *iov, int iovcnt, off_t offset,
+            int flags) __THROW GFAPI_PUBLIC(glfs_preadv, 3.4.0);
+
+ssize_t
+glfs_pwritev(glfs_fd_t *fd, const struct iovec *iov, int iovcnt, off_t offset,
+             int flags) __THROW GFAPI_PUBLIC(glfs_pwritev, 3.4.0);
+
+int
+glfs_preadv_async(glfs_fd_t *fd, const struct iovec *iov, int count,
+                  off_t offset, int flags, glfs_io_cbk fn, void *data) __THROW
+    GFAPI_PUBLIC(glfs_preadv_async, 6.0);
+
+int
+glfs_pwritev_async(glfs_fd_t *fd, const struct iovec *iov, int count,
+                   off_t offset, int flags, glfs_io_cbk fn, void *data) __THROW
+    GFAPI_PUBLIC(glfs_pwritev_async, 6.0);
+
+off_t
+glfs_lseek(glfs_fd_t *fd, off_t offset, int whence) __THROW
+    GFAPI_PUBLIC(glfs_lseek, 3.4.0);
+
+ssize_t
+glfs_copy_file_range(struct glfs_fd *glfd_in, off64_t *off_in,
+                     struct glfs_fd *glfd_out, off64_t *off_out, size_t len,
+                     unsigned int flags, struct glfs_stat *statbuf,
+                     struct glfs_stat *prestat,
+                     struct glfs_stat *poststat) __THROW
+    GFAPI_PUBLIC(glfs_copy_file_range, 6.0);
+
+int
+glfs_truncate(glfs_t *fs, const char *path, off_t length) __THROW
+    GFAPI_PUBLIC(glfs_truncate, 3.7.15);
+
+int
+glfs_ftruncate(glfs_fd_t *fd, off_t length, struct glfs_stat *prestat,
+               struct glfs_stat *poststat) __THROW
+    GFAPI_PUBLIC(glfs_ftruncate, 6.0);
+
+int
+glfs_ftruncate_async(glfs_fd_t *fd, off_t length, glfs_io_cbk fn,
+                     void *data) __THROW
+    GFAPI_PUBLIC(glfs_ftruncate_async, 6.0);
+
+int
+glfs_lstat(glfs_t *fs, const char *path, struct stat *buf) __THROW
+    GFAPI_PUBLIC(glfs_lstat, 3.4.0);
+
+int
+glfs_stat(glfs_t *fs, const char *path, struct stat *buf) __THROW
+    GFAPI_PUBLIC(glfs_stat, 3.4.0);
+
+int
+glfs_fstat(glfs_fd_t *fd, struct stat *buf) __THROW
+    GFAPI_PUBLIC(glfs_fstat, 3.4.0);
+
+int
+glfs_fsync(glfs_fd_t *fd, struct glfs_stat *prestat,
+           struct glfs_stat *poststat) __THROW GFAPI_PUBLIC(glfs_fsync, 6.0);
+
+int
+glfs_fsync_async(glfs_fd_t *fd, glfs_io_cbk fn, void *data) __THROW
+    GFAPI_PUBLIC(glfs_fsync_async, 6.0);
+
+int
+glfs_fdatasync(glfs_fd_t *fd, struct glfs_stat *prestat,
+               struct glfs_stat *poststat) __THROW
+    GFAPI_PUBLIC(glfs_fdatasync, 6.0);
+
+int
+glfs_fdatasync_async(glfs_fd_t *fd, glfs_io_cbk fn, void *data) __THROW
+    GFAPI_PUBLIC(glfs_fdatasync_async, 6.0);
+
+int
+glfs_access(glfs_t *fs, const char *path, int mode) __THROW
+    GFAPI_PUBLIC(glfs_access, 3.4.0);
+
+int
+glfs_symlink(glfs_t *fs, const char *oldpath, const char *newpath) __THROW
+    GFAPI_PUBLIC(glfs_symlink, 3.4.0);
+
+int
+glfs_readlink(glfs_t *fs, const char *path, char *buf, size_t bufsiz) __THROW
+    GFAPI_PUBLIC(glfs_readlink, 3.4.0);
+
+int
+glfs_mknod(glfs_t *fs, const char *path, mode_t mode, dev_t dev) __THROW
+    GFAPI_PUBLIC(glfs_mknod, 3.4.0);
+
+int
+glfs_mkdir(glfs_t *fs, const char *path, mode_t mode) __THROW
+    GFAPI_PUBLIC(glfs_mkdir, 3.4.0);
+
+int
+glfs_unlink(glfs_t *fs, const char *path) __THROW
+    GFAPI_PUBLIC(glfs_unlink, 3.4.0);
+
+int
+glfs_rmdir(glfs_t *fs, const char *path) __THROW
+    GFAPI_PUBLIC(glfs_rmdir, 3.4.0);
+
+int
+glfs_rename(glfs_t *fs, const char *oldpath, const char *newpath) __THROW
+    GFAPI_PUBLIC(glfs_rename, 3.4.0);
+
+int
+glfs_link(glfs_t *fs, const char *oldpath, const char *newpath) __THROW
+    GFAPI_PUBLIC(glfs_link, 3.4.0);
+
+glfs_fd_t *
+glfs_opendir(glfs_t *fs, const char *path) __THROW
+    GFAPI_PUBLIC(glfs_opendir, 3.4.0);
+
+/*
+ * @glfs_readdir_r and @glfs_readdirplus_r ARE thread safe AND re-entrant,
+ * but the interface has ambiguity about the size of @dirent to be allocated
+ * before calling the APIs. 512 byte buffer (for @dirent) is sufficient for
+ * all known systems which are tested againt glusterfs/gfapi, but may be
+ * insufficient in the future.
+ */
+
+int
+glfs_readdir_r(glfs_fd_t *fd, struct dirent *dirent,
+               struct dirent **result) __THROW
+    GFAPI_PUBLIC(glfs_readdir_r, 3.4.0);
+
+int
+glfs_readdirplus_r(glfs_fd_t *fd, struct stat *stat, struct dirent *dirent,
+                   struct dirent **result) __THROW
+    GFAPI_PUBLIC(glfs_readdirplus_r, 3.4.0);
+
+/*
+ * @glfs_readdir and @glfs_readdirplus are NEITHER thread safe NOR re-entrant
+ * when called on the same directory handle. However they ARE thread safe
+ * AND re-entrant when called on different directory handles (which may be
+ * referring to the same directory too.)
+ */
+
+struct dirent *
+glfs_readdir(glfs_fd_t *fd) __THROW GFAPI_PUBLIC(glfs_readdir, 3.5.0);
 
+struct dirent *
+glfs_readdirplus(glfs_fd_t *fd, struct stat *stat) __THROW
+    GFAPI_PUBLIC(glfs_readdirplus, 3.5.0);
 
-off_t glfs_lseek (glfs_fd_t *fd, off_t offset, int whence);
+long
+glfs_telldir(glfs_fd_t *fd) __THROW GFAPI_PUBLIC(glfs_telldir, 3.4.0);
 
-int glfs_truncate (glfs_t *fs, const char *path, off_t length);
+void
+glfs_seekdir(glfs_fd_t *fd, long offset) __THROW
+    GFAPI_PUBLIC(glfs_seekdir, 3.4.0);
 
-int glfs_ftruncate (glfs_fd_t *fd, off_t length);
-int glfs_ftruncate_async (glfs_fd_t *fd, off_t length, glfs_io_cbk fn,
-			  void *data);
+int
+glfs_closedir(glfs_fd_t *fd) __THROW GFAPI_PUBLIC(glfs_closedir, 3.4.0);
 
-int glfs_lstat (glfs_t *fs, const char *path, struct stat *buf);
-int glfs_stat (glfs_t *fs, const char *path, struct stat *buf);
-int glfs_fstat (glfs_fd_t *fd, struct stat *buf);
+int
+glfs_statvfs(glfs_t *fs, const char *path, struct statvfs *buf) __THROW
+    GFAPI_PUBLIC(glfs_statvfs, 3.4.0);
 
-int glfs_fsync (glfs_fd_t *fd);
-int glfs_fsync_async (glfs_fd_t *fd, glfs_io_cbk fn, void *data);
+int
+glfs_chmod(glfs_t *fs, const char *path, mode_t mode) __THROW
+    GFAPI_PUBLIC(glfs_chmod, 3.4.0);
 
-int glfs_fdatasync (glfs_fd_t *fd);
-int glfs_fdatasync_async (glfs_fd_t *fd, glfs_io_cbk fn, void *data);
+int
+glfs_fchmod(glfs_fd_t *fd, mode_t mode) __THROW
+    GFAPI_PUBLIC(glfs_fchmod, 3.4.0);
 
-int glfs_access (glfs_t *fs, const char *path, int mode);
+int
+glfs_chown(glfs_t *fs, const char *path, uid_t uid, gid_t gid) __THROW
+    GFAPI_PUBLIC(glfs_chown, 3.4.0);
 
-int glfs_symlink (glfs_t *fs, const char *oldpath, const char *newpath);
+int
+glfs_lchown(glfs_t *fs, const char *path, uid_t uid, gid_t gid) __THROW
+    GFAPI_PUBLIC(glfs_lchown, 3.4.0);
 
-int glfs_readlink (glfs_t *fs, const char *path, char *buf, size_t bufsiz);
+int
+glfs_fchown(glfs_fd_t *fd, uid_t uid, gid_t gid) __THROW
+    GFAPI_PUBLIC(glfs_fchown, 3.4.0);
 
-int glfs_mknod (glfs_t *fs, const char *path, mode_t mode, dev_t dev);
+int
+glfs_utimens(glfs_t *fs, const char *path,
+             const struct timespec times[2]) __THROW
+    GFAPI_PUBLIC(glfs_utimens, 3.4.0);
 
-int glfs_mkdir (glfs_t *fs, const char *path, mode_t mode);
+int
+glfs_lutimens(glfs_t *fs, const char *path,
+              const struct timespec times[2]) __THROW
+    GFAPI_PUBLIC(glfs_lutimens, 3.4.0);
 
-int glfs_unlink (glfs_t *fs, const char *path);
+int
+glfs_futimens(glfs_fd_t *fd, const struct timespec times[2]) __THROW
+    GFAPI_PUBLIC(glfs_futimens, 3.4.0);
 
-int glfs_rmdir (glfs_t *fs, const char *path);
+ssize_t
+glfs_getxattr(glfs_t *fs, const char *path, const char *name, void *value,
+              size_t size) __THROW GFAPI_PUBLIC(glfs_getxattr, 3.4.0);
 
-int glfs_rename (glfs_t *fs, const char *oldpath, const char *newpath);
+ssize_t
+glfs_lgetxattr(glfs_t *fs, const char *path, const char *name, void *value,
+               size_t size) __THROW GFAPI_PUBLIC(glfs_lgetxattr, 3.4.0);
 
-int glfs_link (glfs_t *fs, const char *oldpath, const char *newpath);
+ssize_t
+glfs_fgetxattr(glfs_fd_t *fd, const char *name, void *value,
+               size_t size) __THROW GFAPI_PUBLIC(glfs_fgetxattr, 3.4.0);
 
-glfs_fd_t *glfs_opendir (glfs_t *fs, const char *path);
+ssize_t
+glfs_listxattr(glfs_t *fs, const char *path, void *value, size_t size) __THROW
+    GFAPI_PUBLIC(glfs_listxattr, 3.4.0);
+
+ssize_t
+glfs_llistxattr(glfs_t *fs, const char *path, void *value, size_t size) __THROW
+    GFAPI_PUBLIC(glfs_llistxattr, 3.4.0);
+
+ssize_t
+glfs_flistxattr(glfs_fd_t *fd, void *value, size_t size) __THROW
+    GFAPI_PUBLIC(glfs_flistxattr, 3.4.0);
+
+int
+glfs_setxattr(glfs_t *fs, const char *path, const char *name, const void *value,
+              size_t size, int flags) __THROW
+    GFAPI_PUBLIC(glfs_setxattr, 3.4.0);
+
+int
+glfs_lsetxattr(glfs_t *fs, const char *path, const char *name,
+               const void *value, size_t size, int flags) __THROW
+    GFAPI_PUBLIC(glfs_lsetxattr, 3.4.0);
+
+int
+glfs_fsetxattr(glfs_fd_t *fd, const char *name, const void *value, size_t size,
+               int flags) __THROW GFAPI_PUBLIC(glfs_fsetxattr, 3.4.0);
+
+int
+glfs_removexattr(glfs_t *fs, const char *path, const char *name) __THROW
+    GFAPI_PUBLIC(glfs_removexattr, 3.4.0);
+
+int
+glfs_lremovexattr(glfs_t *fs, const char *path, const char *name) __THROW
+    GFAPI_PUBLIC(glfs_lremovexattr, 3.4.0);
+
+int
+glfs_fremovexattr(glfs_fd_t *fd, const char *name) __THROW
+    GFAPI_PUBLIC(glfs_fremovexattr, 3.4.0);
+
+int
+glfs_fallocate(glfs_fd_t *fd, int keep_size, off_t offset, size_t len) __THROW
+    GFAPI_PUBLIC(glfs_fallocate, 3.5.0);
+
+int
+glfs_discard(glfs_fd_t *fd, off_t offset, size_t len) __THROW
+    GFAPI_PUBLIC(glfs_discard, 3.5.0);
+
+int
+glfs_discard_async(glfs_fd_t *fd, off_t length, size_t lent, glfs_io_cbk fn,
+                   void *data) __THROW GFAPI_PUBLIC(glfs_discard_async, 6.0);
+
+int
+glfs_zerofill(glfs_fd_t *fd, off_t offset, off_t len) __THROW
+    GFAPI_PUBLIC(glfs_zerofill, 3.5.0);
+
+int
+glfs_zerofill_async(glfs_fd_t *fd, off_t length, off_t len, glfs_io_cbk fn,
+                    void *data) __THROW GFAPI_PUBLIC(glfs_zerofill_async, 6.0);
+
+char *
+glfs_getcwd(glfs_t *fs, char *buf, size_t size) __THROW
+    GFAPI_PUBLIC(glfs_getcwd, 3.4.0);
+
+int
+glfs_chdir(glfs_t *fs, const char *path) __THROW
+    GFAPI_PUBLIC(glfs_chdir, 3.4.0);
+
+int
+glfs_fchdir(glfs_fd_t *fd) __THROW GFAPI_PUBLIC(glfs_fchdir, 3.4.0);
+
+char *
+glfs_realpath(glfs_t *fs, const char *path, char *resolved_path) __THROW
+    GFAPI_PUBLIC(glfs_realpath, 3.7.17);
 
 /*
- * @glfs_readdir_r and @glfs_readdirplus_r ARE thread safe AND re-entrant,
- * but the interface has ambiguity about the size of @dirent to be allocated
- * before calling the APIs. 512 byte buffer (for @dirent) is sufficient for
- * all known systems which are tested againt glusterfs/gfapi, but may be
- * insufficient in the future.
+ * @cmd and @flock are as specified in man fcntl(2).
+ */
+int
+glfs_posix_lock(glfs_fd_t *fd, int cmd, struct flock *flock) __THROW
+    GFAPI_PUBLIC(glfs_posix_lock, 3.4.0);
+
+/*
+  SYNOPSIS
+
+  glfs_file_lock: Request extended byte range lock on a file
+
+  DESCRIPTION
+
+  This function is capable of requesting either advisory or mandatory type
+  byte range locks on a file.
+
+  Note: To set a unique owner key for locks based on a particular file
+  descriptor, make use of glfs_fd_set_lkowner() api to do so before
+  requesting lock via this api. This owner key will be further consumed
+  by other incoming data modifying file operations via the same file
+  descriptor.
+
+  PARAMETERS
+
+  @fd: File descriptor
+
+  @cmd: As specified in man fcntl(2).
+
+  @flock: As specified in man fcntl(2).
+
+  @lk_mode: Required lock type from options available with the
+            enum glfs_lock_mode_t defined below.
+
+  RETURN VALUES
+
+  0   : Success. Lock has been granted.
+  -1  : Failure. @errno will be set indicating the type of failure.
+
  */
 
-int glfs_readdir_r (glfs_fd_t *fd, struct dirent *dirent,
-		    struct dirent **result);
+/* Lock modes used by glfs_file_lock() */
+enum glfs_lock_mode { GLFS_LK_ADVISORY = 0, GLFS_LK_MANDATORY };
+typedef enum glfs_lock_mode glfs_lock_mode_t;
 
-int glfs_readdirplus_r (glfs_fd_t *fd, struct stat *stat, struct dirent *dirent,
-			struct dirent **result);
+int
+glfs_file_lock(glfs_fd_t *fd, int cmd, struct flock *flock,
+               glfs_lock_mode_t lk_mode) __THROW
+    GFAPI_PUBLIC(glfs_file_lock, 3.13.0);
+
+glfs_fd_t *
+glfs_dup(glfs_fd_t *fd) __THROW GFAPI_PUBLIC(glfs_dup, 3.4.0);
+
+void
+glfs_free(void *ptr) __THROW GFAPI_PUBLIC(glfs_free, 3.7.16);
 
 /*
- * @glfs_readdir and @glfs_readdirplus are NEITHER thread safe NOR re-entrant
- * when called on the same directory handle. However they ARE thread safe
- * AND re-entrant when called on different directory handles (which may be
- * referring to the same directory too.)
+ * glfs_sysrq: send a system request to the @fs instance
+ *
+ * Different commands for @sysrq are possible, the defines for these are listed
+ * below the function definition.
+ *
+ * This function always returns success if the @sysrq is recognized. The return
+ * value does not way anythin about the result of the @sysrq execution. Not all
+ * @sysrq command will be able to return a success/failure status.
  */
+int
+glfs_sysrq(glfs_t *fs, char sysrq) __THROW GFAPI_PUBLIC(glfs_sysrq, 3.10.0);
 
-struct dirent *glfs_readdir (glfs_fd_t *fd);
+#define GLFS_SYSRQ_HELP 'h' /* log a message with supported sysrq commands */
+#define GLFS_SYSRQ_STATEDUMP 's' /* create a statedump */
 
-struct dirent *glfs_readdirplus (glfs_fd_t *fd, struct stat *stat);
+/*
+ * Structure returned as part of xreaddirplus
+ */
+struct glfs_xreaddirp_stat;
+typedef struct glfs_xreaddirp_stat glfs_xreaddirp_stat_t;
 
-long glfs_telldir (glfs_fd_t *fd);
+/* Request flags to be used in XREADDIRP operation */
+#define GFAPI_XREADDIRP_NULL                                                   \
+    0x00000000 /* by default, no stat will be fetched */
+#define GFAPI_XREADDIRP_STAT 0x00000001   /* Get stat */
+#define GFAPI_XREADDIRP_HANDLE 0x00000002 /* Get object handle */
 
-void glfs_seekdir (glfs_fd_t *fd, long offset);
+/*
+ * This stat structure returned gets freed as part of glfs_free(xstat)
+ */
+struct stat *
+glfs_xreaddirplus_get_stat(glfs_xreaddirp_stat_t *xstat) __THROW
+    GFAPI_PUBLIC(glfs_xreaddirplus_get_stat, 3.11.0);
 
-int glfs_closedir (glfs_fd_t *fd);
+/*
+ * SYNOPSIS
+ *
+ * glfs_xreaddirplus_r: Extended Readirplus operation
+ *
+ * DESCRIPTION
+ *
+ * This API does readdirplus operation, but along with stat it can fetch other
+ * extra information like object handles etc for each of the dirents returned
+ * based on requested flags. On success it returns the set of flags successfully
+ * processed.
+ *
+ * Note that there are chances that some of the requested information may not be
+ * available or returned (for example if reached EOD). Ensure to validate the
+ * returned value to determine what flags have been successfully processed
+ * & set.
+ *
+ * PARAMETERS
+ *
+ * INPUT:
+ * @glfd: GFAPI file descriptor of the directory
+ * @flags: Flags determining xreaddirp_stat requested
+ *         Current available values are:
+ *              GFAPI_XREADDIRP_NULL
+ *              GFAPI_XREADDIRP_STAT
+ *              GFAPI_XREADDIRP_HANDLE
+ * @ext: Dirent struture to copy the values to
+ *       (though optional recommended to be allocated by application
+ *        esp., in multi-threaded environment)
+ *
+ * OUTPUT:
+ * @res: to store the next dirent value. If NULL and return value is '0',
+ *       it means it reached end of the directory.
+ * @xstat_p: Pointer to contain all the requested data returned
+ *           for that dirent. Application should make use of glfs_free() API
+ *           to free this pointer and the variables returned by
+ *           glfs_xreaddirplus_get_*() APIs.
+ *
+ * RETURN VALUE:
+ * >=0: SUCCESS (value contains the flags successfully processed)
+ *  -1: FAILURE
+ */
+int
+glfs_xreaddirplus_r(glfs_fd_t *glfd, uint32_t flags,
+                    glfs_xreaddirp_stat_t **xstat_p, struct dirent *ext,
+                    struct dirent **res) __THROW
+    GFAPI_PUBLIC(glfs_xreaddirplus_r, 3.11.0);
 
-int glfs_statvfs (glfs_t *fs, const char *path, struct statvfs *buf);
+#define GFAPI_MAX_LOCK_OWNER_LEN 255
 
-int glfs_chmod (glfs_t *fs, const char *path, mode_t mode);
+/*
+ *
+ * DESCRIPTION
+ *
+ * This API allows application to set lk_owner on a fd.
+ * A glfd can be associated with only single lk_owner. In case if there
+ * is need to set another lk_owner, applications can make use of
+ * 'glfs_dup' to get duplicate glfd and set new lk_owner on that second
+ * glfd.
+ *
+ * Also its not recommended to override or clear lk_owner value as the
+ * same shall be used to flush any outstanding locks while closing the fd.
+ *
+ * PARAMETERS
+ *
+ * INPUT:
+ * @glfd: GFAPI file descriptor
+ * @len: Size of lk_owner buffer. Max value can be GFAPI_MAX_LOCK_OWNER_LEN
+ * @data: lk_owner data buffer.
+ *
+ * OUTPUT:
+ * 0: SUCCESS
+ * -1: FAILURE
+ */
+int
+glfs_fd_set_lkowner(glfs_fd_t *glfd, void *data, int len) __THROW
+    GFAPI_PUBLIC(glfs_fd_set_lkowner, 3.10.7);
 
-int glfs_fchmod (glfs_fd_t *fd, mode_t mode);
+/*
+ * Applications (currently NFS-Ganesha) can make use of this
+ * structure to read upcall notifications sent by server either
+ * by polling or registering a callback function.
+ *
+ * On success, applications need to check for 'reason' to decide
+ * if any upcall event is received.
+ *
+ * Currently supported upcall_events -
+ *      GLFS_UPCALL_INODE_INVALIDATE -
+ *              'event_arg' - glfs_upcall_inode
+ *
+ * After processing the event, applications need to free 'event_arg' with
+ * glfs_free().
+ *
+ * Also similar to I/Os, the application should ideally stop polling
+ * or unregister upcall_cbk function before calling glfs_fini(..).
+ * Hence making an assumption that 'fs' & ctx structures cannot be
+ * freed while in this routine.
+ */
+struct glfs_upcall;
+typedef struct glfs_upcall glfs_upcall_t;
 
-int glfs_chown (glfs_t *fs, const char *path, uid_t uid, gid_t gid);
+glfs_t *
+glfs_upcall_get_fs(glfs_upcall_t *arg) __THROW
+    GFAPI_PUBLIC(glfs_upcall_get_fs, 3.7.16);
 
-int glfs_lchown (glfs_t *fs, const char *path, uid_t uid, gid_t gid);
+enum glfs_upcall_reason {
+    GLFS_UPCALL_EVENT_NULL = 0,
+    GLFS_UPCALL_INODE_INVALIDATE, /* invalidate cache entry */
+    GLFS_UPCALL_RECALL_LEASE,     /* recall lease */
+};
+typedef enum glfs_upcall_reason glfs_upcall_reason_t;
 
-int glfs_fchown (glfs_fd_t *fd, uid_t uid, gid_t gid);
+glfs_upcall_reason_t
+glfs_upcall_get_reason(glfs_upcall_t *arg) __THROW
+    GFAPI_PUBLIC(glfs_upcall_get_reason, 3.7.16);
 
-int glfs_utimens (glfs_t *fs, const char *path, struct timespec times[2]);
+/*
+ * Applications first need to make use of above API i.e,
+ * "glfs_upcall_get_reason" to determine which upcall event it has
+ * received. Post that below API - "glfs_upcall_get_event" should
+ * be used to get corresponding upcall event object.
+ *
+ * Below are the upcall_reason and corresponding upcall_event objects:
+ *      ==========================================================
+ *      glfs_upcall_reason           -    event_object
+ *      ==========================================================
+ *      GLFS_UPCALL_EVENT_NULL       -    NULL
+ *      GLFS_UPCALL_INODE_INVALIDATE -    struct glfs_upcall_inode
+ *      GLFS_UPCALL_RECALL_LEASE     -    struct glfs_upcall_lease
+ *
+ * After processing upcall event, glfs_free() should be called on the
+ * glfs_upcall.
+ */
+void *
+glfs_upcall_get_event(glfs_upcall_t *arg) __THROW
+    GFAPI_PUBLIC(glfs_upcall_get_event, 3.7.16);
 
-int glfs_lutimens (glfs_t *fs, const char *path, struct timespec times[2]);
+/*
+ * SYNOPSIS
+ *
+ * glfs_upcall_cbk: Upcall callback definition
+ *
+ * This is function type definition of the callback function pointer
+ * which has to be provided by the caller while registering for any
+ * upcall events.
+ *
+ * This function is called whenever any upcall which the application
+ * has registered for is received from the server.
+ *
+ * @up_arg: Upcall structure whose contents need to be interpreted by
+ * making use of glfs_upcall_* helper routines.
+ *
+ * @data: The same context pointer provided by the caller at the time of
+ * registering of upcall events. This may be used by the caller for any
+ * of its internal use while processing upcalls.
+ */
+typedef void (*glfs_upcall_cbk)(glfs_upcall_t *up_arg, void *data);
 
-int glfs_futimens (glfs_fd_t *fd, struct timespec times[2]);
+/*
+ * List of upcall events supported by gluster/gfapi
+ */
+#define GLFS_EVENT_INODE_INVALIDATE 0x00000001 /* invalidate cache entry */
+#define GLFS_EVENT_RECALL_LEASE 0x00000002     /* Recall lease */
+#define GLFS_EVENT_ANY 0xffffffff              /* for all the above events */
 
-ssize_t glfs_getxattr (glfs_t *fs, const char *path, const char *name,
-		       void *value, size_t size);
+/*
+ * SYNOPSIS
+ *
+ * glfs_upcall_register: Register for upcall events
+ *
+ * DESCRIPTION
+ *
+ * This function is used to register for various upcall events application
+ * is interested in and the callback function to be invoked when such
+ * events are triggered.
+ *
+ * Multiple calls of this routine shall override cbk function. That means
+ * only one cbk function can be used for all the upcall events registered
+ * and that shall be the one last updated.
+ *
+ * PARAMETERS:
+ *
+ * INPUT:
+ * @fs: The 'virtual mount' object
+ *
+ * @event_list: List of upcall events to be registered.
+ *              Current available values are:
+ *               - GLFS_EVENT_INODE_INVALIDATE
+ *               - GLFS_EVENT_RECALL_LEASE
+ *
+ * @cbk: The cbk routine to be invoked in case of any upcall received
+ * @data: Any opaque pointer provided by caller which shall be using while
+ * making cbk calls. This pointer may be used by caller for any of its
+ * internal use while processing upcalls. Can be NULL.
+ *
+ * RETURN VALUE:
+ * >0: SUCCESS (value contains the events successfully registered)
+ * -1: FAILURE
+ */
+int
+glfs_upcall_register(glfs_t *fs, uint32_t event_list, glfs_upcall_cbk cbk,
+                     void *data) __THROW
+    GFAPI_PUBLIC(glfs_upcall_register, 3.13.0);
 
-ssize_t glfs_lgetxattr (glfs_t *fs, const char *path, const char *name,
-			void *value, size_t size);
+/*
+ * SYNOPSIS
+ *
+ * glfs_upcall_unregister: Unregister for upcall events
+ *
+ * DESCRIPTION
+ *
+ * This function is used to unregister the upcall events application
+ * is not interested in. In case if the caller unregisters all the events
+ * it has registered for, it shall no more receive any upcall event.
+ *
+ * PARAMETERS:
+ *
+ * INPUT:
+ * @fs: The 'virtual mount' object
+ *
+ * @event_list: List of upcall events to be unregistered.
+ *              Current available values are:
+ *               - GLFS_EVENT_INODE_INVALIDATE
+ *               - GLFS_EVENT_RECALL_LEASE
+ * RETURN VALUE:
+ * >0: SUCCESS (value contains the events successfully unregistered)
+ * -1: FAILURE
+ */
+int
+glfs_upcall_unregister(glfs_t *fs, uint32_t event_list) __THROW
+    GFAPI_PUBLIC(glfs_upcall_unregister, 3.13.0);
+
+/* Lease Types */
+enum glfs_lease_types {
+    GLFS_LEASE_NONE = 0,
+    GLFS_RD_LEASE = 1,
+    GLFS_RW_LEASE = 2,
+};
+typedef enum glfs_lease_types glfs_lease_types_t;
+
+/* Lease cmds */
+enum glfs_lease_cmds {
+    GLFS_GET_LEASE = 1,
+    GLFS_SET_LEASE = 2,
+    GLFS_UNLK_LEASE = 3,
+};
+typedef enum glfs_lease_cmds glfs_lease_cmds_t;
+
+struct glfs_lease {
+    glfs_lease_cmds_t cmd;
+    glfs_lease_types_t lease_type;
+    glfs_leaseid_t lease_id;
+    unsigned int lease_flags;
+};
+typedef struct glfs_lease glfs_lease_t;
+
+typedef void (*glfs_recall_cbk)(glfs_lease_t lease, void *data);
 
-ssize_t glfs_fgetxattr (glfs_fd_t *fd, const char *name,
-			void *value, size_t size);
+/*
+  SYNOPSIS
 
-ssize_t glfs_listxattr (glfs_t *fs, const char *path, void *value, size_t size);
+  glfs_lease: Takes a lease on a file.
 
-ssize_t glfs_llistxattr (glfs_t *fs, const char *path, void *value,
-			 size_t size);
+  DESCRIPTION
 
-ssize_t glfs_flistxattr (glfs_fd_t *fd, void *value, size_t size);
+  This function takes lease on an open file.
 
-int glfs_setxattr (glfs_t *fs, const char *path, const char *name,
-		   const void *value, size_t size, int flags);
+  PARAMETERS
 
-int glfs_lsetxattr (glfs_t *fs, const char *path, const char *name,
-		    const void *value, size_t size, int flags);
+  @glfd: The fd of the file on which lease should be taken,
+   this fd is returned by glfs_open/glfs_create.
 
-int glfs_fsetxattr (glfs_fd_t *fd, const char *name,
-		    const void *value, size_t size, int flags);
+  @lease: Struct that defines the lease operation to be performed
+   on the file.
+      @lease.cmd - Can be one of the following values
+         GF_GET_LEASE:  Get the lease type currently present on the file,
+                        lease.lease_type will contain GF_RD_LEASE
+                        or GF_RW_LEASE or 0 if no leases.
+         GF_SET_LEASE:  Set the lease of given lease.lease_type on the file.
+         GF_UNLK_LEASE: Unlock the lease present on the given fd.
+                        Note that the every lease request should have
+                        a corresponding unlk_lease.
 
-int glfs_removexattr (glfs_t *fs, const char *path, const char *name);
+      @lease.lease_type - Can be one of the following values
+         GF_RD_LEASE:   Read lease on a file, shared lease.
+         GF_RW_LEASE:   Read-Write lease on a file, exclusive lease.
 
-int glfs_lremovexattr (glfs_t *fs, const char *path, const char *name);
+      @lease.lease_id - A unique identification of lease, 128bits.
 
-int glfs_fremovexattr (glfs_fd_t *fd, const char *name);
+  @fn: This is the function that is invoked when the lease has to be recalled
+  @data: It is a cookie, this pointer is returned as a part of recall
 
-int glfs_fallocate(glfs_fd_t *fd, int keep_size, off_t offset, size_t len);
+  fn and data field are stored as a part of glfs_fd, hence if there are multiple
+  glfs_lease calls, each of them updates the fn and data fields. glfs_recall_cbk
+  will be invoked with the last updated fn and data
 
-int glfs_discard(glfs_fd_t *fd, off_t offset, size_t len);
+  RETURN VALUES
+  0:  Successful completion
+  <0: Failure. @errno will be set with the type of failure
+*/
 
+int
+glfs_lease(glfs_fd_t *glfd, glfs_lease_t *lease, glfs_recall_cbk fn,
+           void *data) __THROW GFAPI_PUBLIC(glfs_lease, 4.0.0);
 
-int glfs_discard_async (glfs_fd_t *fd, off_t length, size_t lent,
-			glfs_io_cbk fn, void *data);
+/*
+  SYNOPSIS
 
-int glfs_zerofill(glfs_fd_t *fd, off_t offset, off_t len);
+  glfs_fsetattr: Function to set attributes.
+  glfs_setattr: Function to set attributes
+
+  DESCRIPTION
+
+  The functions are used to set attributes on the file.
+
+  PARAMETERS
 
-int glfs_zerofill_async (glfs_fd_t *fd, off_t length, off_t len,
-                        glfs_io_cbk fn, void *data);
+  @glfs_fsetattr
 
-char *glfs_getcwd (glfs_t *fs, char *buf, size_t size);
+     @glfd: The fd of the file for which the attributes are to be set,
+            this fd is returned by glfs_open/glfs_create.
 
-int glfs_chdir (glfs_t *fs, const char *path);
+  @glfs_setattr
 
-int glfs_fchdir (glfs_fd_t *fd);
+         @fs: File object.
 
-char *glfs_realpath (glfs_t *fs, const char *path, char *resolved_path);
+         @path: The path of the file that is being operated on.
+
+         @follow: Flag used to resolve symlink.
+
+
+  @stat: Struct that has information about the file.
+
+  @valid: This is the mask bit, that accepts GFAPI_SET_ATTR* masks.
+          Refer glfs.h to see the mask definitions.
+
+  Both functions are similar in functionality, just that the
+  func setattr() uses file path whereas the func fsetattr()
+  uses the fd.
+
+  RETURN VALUES
+  0:  Successful completion
+  <0: Failure. @errno will be set with the type of failure
+
+ */
+
+int
+glfs_fsetattr(struct glfs_fd *glfd, struct glfs_stat *stat) __THROW
+    GFAPI_PUBLIC(glfs_fsetattr, 6.0);
+
+int
+glfs_setattr(struct glfs *fs, const char *path, struct glfs_stat *stat,
+             int follow) __THROW GFAPI_PUBLIC(glfs_setattr, 6.0);
 
 /*
- * @cmd and @flock are as specified in man fcntl(2).
+  SYNOPSIS
+
+  glfs_set_statedump_path: Function to set statedump path.
+
+  DESCRIPTION
+
+  This function is used to set statedump directory
+
+  PARAMETERS
+
+  @fs: The 'virtual mount' object to be configured with the volume
+       specification file.
+
+  @path: statedump path. Should be a directory. But the API won't fail if the
+  directory doesn't exist yet, as one may create it later.
+
+  RETURN VALUES
+
+   0 : Success.
+  -1 : Failure. @errno will be set with the type of failure.
+
  */
-int glfs_posix_lock (glfs_fd_t *fd, int cmd, struct flock *flock);
 
-glfs_fd_t *glfs_dup (glfs_fd_t *fd);
+int
+glfs_set_statedump_path(struct glfs *fs, const char *path) __THROW
+    GFAPI_PUBLIC(glfs_set_statedump_path, 7.0);
 
 __END_DECLS
-
 #endif /* !_GLFS_H */
diff --git a/argp-standalone/Makefile.am b/argp-standalone/Makefile.am
deleted file mode 100644
index 4775d4876aa..00000000000
--- a/argp-standalone/Makefile.am
+++ /dev/null
@@ -1,38 +0,0 @@
-# From glibc
-
-# Copyright (C) 1997, 2003, 2004 Free Software Foundation, Inc.
-# This file is part of the GNU C Library.
-
-# The GNU C Library is free software; you can redistribute it and/or
-# modify it under the terms of the GNU Library General Public License as
-# published by the Free Software Foundation; either version 2 of the
-# License, or (at your option) any later version.
-
-# The GNU C Library is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-# Library General Public License for more details.
-
-# You should have received a copy of the GNU Library General Public
-# License along with the GNU C Library; see the file COPYING.LIB.  If
-# not, write to the Free Software Foundation, Inc.,
-# 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
-AUTOMAKE_OPTIONS = foreign
-SUBDIRS = .
-
-LIBOBJS = @LIBOBJS@
-
-noinst_LIBRARIES = libargp.a
-
-noinst_HEADERS = argp.h argp-fmtstream.h argp-namefrob.h
-
-EXTRA_DIST = mempcpy.c strchrnul.c strndup.c strcasecmp.c vsnprintf.c autogen.sh
-
-# Leaves out argp-fs-xinl.c and argp-xinl.c 
-libargp_a_SOURCES = argp-ba.c argp-eexst.c argp-fmtstream.c \
-		    argp-help.c argp-parse.c argp-pv.c \
-		    argp-pvh.c
-
-libargp_a_LIBADD = $(LIBOBJS)
-
-
diff --git a/argp-standalone/acinclude.m4 b/argp-standalone/acinclude.m4
deleted file mode 100644
index fb61e957dfa..00000000000
--- a/argp-standalone/acinclude.m4
+++ /dev/null
@@ -1,1084 +0,0 @@
-dnl Try to detect the type of the third arg to getsockname() et al
-AC_DEFUN([LSH_TYPE_SOCKLEN_T],
-[AH_TEMPLATE([socklen_t], [Length type used by getsockopt])
-AC_CACHE_CHECK([for socklen_t in sys/socket.h], ac_cv_type_socklen_t,
-[AC_EGREP_HEADER(socklen_t, sys/socket.h,
-  [ac_cv_type_socklen_t=yes], [ac_cv_type_socklen_t=no])])
-if test $ac_cv_type_socklen_t = no; then
-        AC_MSG_CHECKING(for AIX)
-        AC_EGREP_CPP(yes, [
-#ifdef _AIX
- yes
-#endif
-],[
-AC_MSG_RESULT(yes)
-AC_DEFINE(socklen_t, size_t)
-],[
-AC_MSG_RESULT(no)
-AC_DEFINE(socklen_t, int)
-])
-fi
-])
-
-dnl Choose cc flags for compiling position independent code
-AC_DEFUN([LSH_CCPIC],
-[AC_MSG_CHECKING(CCPIC)
-AC_CACHE_VAL(lsh_cv_sys_ccpic,[
-  if test -z "$CCPIC" ; then
-    if test "$GCC" = yes ; then
-      case `uname -sr` in
-	BSD/OS*)
-         case `uname -r` in
-           4.*) CCPIC="-fPIC";;
-           *) CCPIC="";;
-         esac
-	;;
-	Darwin*)
-	  CCPIC="-fPIC"
-	;;
-	SunOS\ 5.*)
-	  # Could also use -fPIC, if there are a large number of symbol reference
-	  CCPIC="-fPIC"
-	;;
-	CYGWIN*)
-	  CCPIC=""
-	;;
-	*)
-	  CCPIC="-fpic"
-	;;
-      esac
-    else
-      case `uname -sr` in
-	Darwin*)
-	  CCPIC="-fPIC"
-	;;
-        IRIX*)
-          CCPIC="-share"
-        ;;
-	hp*|HP*) CCPIC="+z"; ;;
-	FreeBSD*) CCPIC="-fpic";;
-	SCO_SV*) CCPIC="-KPIC -dy -Bdynamic";;
-        UnixWare*|OpenUNIX*) CCPIC="-KPIC -dy -Bdynamic";;
-	Solaris*) CCPIC="-KPIC -Bdynamic";;
-	Windows_NT*) CCPIC="-shared" ;;
-      esac
-    fi
-  fi
-  OLD_CFLAGS="$CFLAGS"
-  CFLAGS="$CFLAGS $CCPIC"
-  AC_TRY_COMPILE([], [exit(0);],
-    lsh_cv_sys_ccpic="$CCPIC", lsh_cv_sys_ccpic='')
-  CFLAGS="$OLD_CFLAGS"
-])
-CCPIC="$lsh_cv_sys_ccpic"
-AC_MSG_RESULT($CCPIC)
-AC_SUBST([CCPIC])])
-
-dnl LSH_PATH_ADD(path-id, directory)
-AC_DEFUN([LSH_PATH_ADD],
-[AC_MSG_CHECKING($2)
-ac_exists=no
-if test -d "$2/." ; then
-  ac_real_dir=`cd $2 && pwd`
-  if test -n "$ac_real_dir" ; then
-    ac_exists=yes
-    for old in $1_REAL_DIRS ; do
-      ac_found=no
-      if test x$ac_real_dir = x$old ; then
-        ac_found=yes;
-	break;
-      fi
-    done
-    if test $ac_found = yes ; then
-      AC_MSG_RESULT(already added)
-    else
-      AC_MSG_RESULT(added)
-      # LDFLAGS="$LDFLAGS -L $2"
-      $1_REAL_DIRS="$ac_real_dir [$]$1_REAL_DIRS"
-      $1_DIRS="$2 [$]$1_DIRS"
-    fi
-  fi
-fi
-if test $ac_exists = no ; then
-  AC_MSG_RESULT(not found)
-fi
-])
-
-dnl LSH_RPATH_ADD(dir)
-AC_DEFUN([LSH_RPATH_ADD], [LSH_PATH_ADD(RPATH_CANDIDATE, $1)])
-
-dnl LSH_RPATH_INIT(candidates)
-AC_DEFUN([LSH_RPATH_INIT],
-[AC_MSG_CHECKING([for -R flag])
-RPATHFLAG=''
-case `uname -sr` in
-  OSF1\ V4.*)
-    RPATHFLAG="-rpath "
-    ;;
-  IRIX\ 6.*)
-    RPATHFLAG="-rpath "
-    ;;
-  IRIX\ 5.*)
-    RPATHFLAG="-rpath "
-    ;;
-  SunOS\ 5.*)
-    if test "$TCC" = "yes"; then
-      # tcc doesn't know about -R
-      RPATHFLAG="-Wl,-R,"
-    else
-      RPATHFLAG=-R
-    fi
-    ;;
-  Linux\ 2.*)
-    RPATHFLAG="-Wl,-rpath,"
-    ;;
-  *)
-    :
-    ;;
-esac
-
-if test x$RPATHFLAG = x ; then
-  AC_MSG_RESULT(none)
-else
-  AC_MSG_RESULT([using $RPATHFLAG])
-fi
-
-RPATH_CANDIDATE_REAL_DIRS=''
-RPATH_CANDIDATE_DIRS=''
-
-AC_MSG_RESULT([Searching for libraries])
-
-for d in $1 ; do
-  LSH_RPATH_ADD($d)
-done
-])    
-
-dnl Try to execute a main program, and if it fails, try adding some
-dnl -R flag.
-dnl LSH_RPATH_FIX
-AC_DEFUN([LSH_RPATH_FIX],
-[if test $cross_compiling = no -a "x$RPATHFLAG" != x ; then
-  ac_success=no
-  AC_TRY_RUN([int main(int argc, char **argv) { return 0; }],
-    ac_success=yes, ac_success=no, :)
-  
-  if test $ac_success = no ; then
-    AC_MSG_CHECKING([Running simple test program failed. Trying -R flags])
-dnl echo RPATH_CANDIDATE_DIRS = $RPATH_CANDIDATE_DIRS
-    ac_remaining_dirs=''
-    ac_rpath_save_LDFLAGS="$LDFLAGS"
-    for d in $RPATH_CANDIDATE_DIRS ; do
-      if test $ac_success = yes ; then
-  	ac_remaining_dirs="$ac_remaining_dirs $d"
-      else
-  	LDFLAGS="$RPATHFLAG$d $LDFLAGS"
-dnl echo LDFLAGS = $LDFLAGS
-  	AC_TRY_RUN([int main(int argc, char **argv) { return 0; }],
-  	  [ac_success=yes
-  	  ac_rpath_save_LDFLAGS="$LDFLAGS"
-  	  AC_MSG_RESULT([adding $RPATHFLAG$d])
-  	  ],
-  	  [ac_remaining_dirs="$ac_remaining_dirs $d"], :)
-  	LDFLAGS="$ac_rpath_save_LDFLAGS"
-      fi
-    done
-    RPATH_CANDIDATE_DIRS=$ac_remaining_dirs
-  fi
-  if test $ac_success = no ; then
-    AC_MSG_RESULT(failed)
-  fi
-fi
-])
-
-dnl Like AC_CHECK_LIB, but uses $KRB_LIBS rather than $LIBS.
-dnl LSH_CHECK_KRB_LIB(LIBRARY, FUNCTION, [, ACTION-IF-FOUND [,
-dnl                  ACTION-IF-NOT-FOUND [, OTHER-LIBRARIES]]])
-
-AC_DEFUN([LSH_CHECK_KRB_LIB],
-[AC_CHECK_LIB([$1], [$2],
-  ifelse([$3], ,
-      [[ac_tr_lib=HAVE_LIB`echo $1 | sed -e 's/[^a-zA-Z0-9_]/_/g' \
-     	    -e 'y/abcdefghijklmnopqrstuvwxyz/ABCDEFGHIJKLMNOPQRSTUVWXYZ/'`
-        AC_DEFINE_UNQUOTED($ac_tr_lib)
-        KRB_LIBS="-l$1 $KRB_LIBS"
-      ]], [$3]),
-  ifelse([$4], , , [$4
-])dnl
-, [$5 $KRB_LIBS])
-])
-
-dnl LSH_LIB_ARGP(ACTION-IF-OK, ACTION-IF-BAD)
-AC_DEFUN([LSH_LIB_ARGP],
-[ ac_argp_save_LIBS="$LIBS"
-  ac_argp_save_LDFLAGS="$LDFLAGS"
-  ac_argp_ok=no
-  # First check if we can link with argp.
-  AC_SEARCH_LIBS(argp_parse, argp,
-  [ LSH_RPATH_FIX
-    AC_CACHE_CHECK([for working argp],
-      lsh_cv_lib_argp_works,
-      [ AC_TRY_RUN(
-[#include <argp.h>
-#include <stdlib.h>
-
-static const struct argp_option
-options[] =
-{
-  { NULL, 0, NULL, 0, NULL, 0 }
-};
-
-struct child_state
-{
-  int n;
-};
-
-static error_t
-child_parser(int key, char *arg, struct argp_state *state)
-{
-  struct child_state *input = (struct child_state *) state->input;
-  
-  switch(key)
-    {
-    default:
-      return ARGP_ERR_UNKNOWN;
-    case ARGP_KEY_END:
-      if (!input->n)
-	input->n = 1;
-      break;
-    }
-  return 0;
-}
-
-const struct argp child_argp =
-{
-  options,
-  child_parser,
-  NULL, NULL, NULL, NULL, NULL
-};
-
-struct main_state
-{
-  struct child_state child;
-  int m;
-};
-
-static error_t
-main_parser(int key, char *arg, struct argp_state *state)
-{
-  struct main_state *input = (struct main_state *) state->input;
-
-  switch(key)
-    {
-    default:
-      return ARGP_ERR_UNKNOWN;
-    case ARGP_KEY_INIT:
-      state->child_inputs[0] = &input->child;
-      break;
-    case ARGP_KEY_END:
-      if (!input->m)
-	input->m = input->child.n;
-      
-      break;
-    }
-  return 0;
-}
-
-static const struct argp_child
-main_children[] =
-{
-  { &child_argp, 0, "", 0 },
-  { NULL, 0, NULL, 0}
-};
-
-static const struct argp
-main_argp =
-{ options, main_parser, 
-  NULL,
-  NULL,
-  main_children,
-  NULL, NULL
-};
-
-int main(int argc, char **argv)
-{
-  struct main_state input = { { 0 }, 0 };
-  char *v[2] = { "foo", NULL };
-
-  argp_parse(&main_argp, 1, v, 0, NULL, &input);
-
-  if ( (input.m == 1) && (input.child.n == 1) )
-    return 0;
-  else
-    return 1;
-}
-], lsh_cv_lib_argp_works=yes,
-   lsh_cv_lib_argp_works=no,
-   lsh_cv_lib_argp_works=no)])
-
-  if test x$lsh_cv_lib_argp_works = xyes ; then
-    ac_argp_ok=yes
-  else
-    # Reset link flags
-    LIBS="$ac_argp_save_LIBS"
-    LDFLAGS="$ac_argp_save_LDFLAGS"
-  fi])
-
-  if test x$ac_argp_ok = xyes ; then
-    ifelse([$1],, true, [$1])
-  else
-    ifelse([$2],, true, [$2])
-  fi   
-])
-
-dnl LSH_GCC_ATTRIBUTES
-dnl Check for gcc's __attribute__ construction
-
-AC_DEFUN([LSH_GCC_ATTRIBUTES],
-[AC_CACHE_CHECK(for __attribute__,
-	       lsh_cv_c_attribute,
-[ AC_TRY_COMPILE([
-#include <stdlib.h>
-],
-[
-static void foo(void) __attribute__ ((noreturn));
-
-static void __attribute__ ((noreturn))
-foo(void)
-{
-  exit(1);
-}
-],
-lsh_cv_c_attribute=yes,
-lsh_cv_c_attribute=no)])
-
-AH_TEMPLATE([HAVE_GCC_ATTRIBUTE], [Define if the compiler understands __attribute__])
-if test "x$lsh_cv_c_attribute" = "xyes"; then
-  AC_DEFINE(HAVE_GCC_ATTRIBUTE)
-fi
-
-AH_BOTTOM(
-[#if __GNUC__ || HAVE_GCC_ATTRIBUTE
-# define NORETURN __attribute__ ((__noreturn__))
-# define PRINTF_STYLE(f, a) __attribute__ ((__format__ (__printf__, f, a)))
-# define UNUSED __attribute__ ((__unused__))
-#else
-# define NORETURN
-# define PRINTF_STYLE(f, a)
-# define UNUSED
-#endif
-])])
-
-AC_DEFUN([LSH_GCC_FUNCTION_NAME],
-[# Check for gcc's __FUNCTION__ variable
-AH_TEMPLATE([HAVE_GCC_FUNCTION],
-	    [Define if the compiler understands __FUNCTION__])
-AH_BOTTOM(
-[#if HAVE_GCC_FUNCTION
-# define FUNCTION_NAME __FUNCTION__
-#else
-# define FUNCTION_NAME "Unknown"
-#endif
-])
-
-AC_CACHE_CHECK(for __FUNCTION__,
-	       lsh_cv_c_FUNCTION,
-  [ AC_TRY_COMPILE(,
-      [ #if __GNUC__ == 3
-	#  error __FUNCTION__ is broken in gcc-3
-	#endif
-        void foo(void) { char c = __FUNCTION__[0]; } ],
-      lsh_cv_c_FUNCTION=yes,
-      lsh_cv_c_FUNCTION=no)])
-
-if test "x$lsh_cv_c_FUNCTION" = "xyes"; then
-  AC_DEFINE(HAVE_GCC_FUNCTION)
-fi
-])
-
-# Check for alloca, and include the standard blurb in config.h
-AC_DEFUN([LSH_FUNC_ALLOCA],
-[AC_FUNC_ALLOCA
-AC_CHECK_HEADERS([malloc.h])
-AH_BOTTOM(
-[/* AIX requires this to be the first thing in the file.  */
-#ifndef __GNUC__
-# if HAVE_ALLOCA_H
-#  include <alloca.h>
-# else
-#  ifdef _AIX
- #pragma alloca
-#  else
-#   ifndef alloca /* predefined by HP cc +Olibcalls */
-char *alloca ();
-#   endif
-#  endif
-# endif
-#else /* defined __GNUC__ */
-# if HAVE_ALLOCA_H
-#  include <alloca.h>
-# endif
-#endif
-/* Needed for alloca on windows */
-#if HAVE_MALLOC_H
-# include <malloc.h>
-#endif
-])])
-
-AC_DEFUN([LSH_FUNC_STRERROR],
-[AC_CHECK_FUNCS(strerror)
-AH_BOTTOM(
-[#if HAVE_STRERROR
-#define STRERROR strerror
-#else
-#define STRERROR(x) (sys_errlist[x])
-#endif
-])])
-
-AC_DEFUN([LSH_FUNC_STRSIGNAL],
-[AC_CHECK_FUNCS(strsignal)
-AC_CHECK_DECLS([sys_siglist, _sys_siglist])
-AH_BOTTOM(
-[#if HAVE_STRSIGNAL
-# define STRSIGNAL strsignal
-#else /* !HAVE_STRSIGNAL */
-# if HAVE_DECL_SYS_SIGLIST
-#  define STRSIGNAL(x) (sys_siglist[x])
-# else
-#  if HAVE_DECL__SYS_SIGLIST
-#   define STRSIGNAL(x) (_sys_siglist[x])
-#  else
-#   define STRSIGNAL(x) "Unknown signal"
-#   if __GNUC__
-#    warning Using dummy STRSIGNAL
-#   endif
-#  endif
-# endif
-#endif /* !HAVE_STRSIGNAL */
-])])
-
-dnl LSH_MAKE_CONDITIONAL(symbol, test)
-AC_DEFUN([LSH_MAKE_CONDITIONAL],
-[if $2 ; then
-  IF_$1=''
-  UNLESS_$1='# '
-else
-  IF_$1='# '
-  UNLESS_$1=''
-fi 
-AC_SUBST(IF_$1)
-AC_SUBST(UNLESS_$1)])
-
-dnl LSH_DEPENDENCY_TRACKING
-
-dnl Defines compiler flags DEP_FLAGS to generate dependency
-dnl information, and DEP_PROCESS that is any shell commands needed for
-dnl massaging the dependency information further. Dependencies are
-dnl generated as a side effect of compilation. Dependency files
-dnl themselves are not treated as targets.
-
-AC_DEFUN([LSH_DEPENDENCY_TRACKING],
-[AC_ARG_ENABLE(dependency_tracking,
-  AC_HELP_STRING([--disable-dependency-tracking],
-    [Disable dependency tracking. Dependency tracking doesn't work with BSD make]),,
-  [enable_dependency_tracking=yes])
-
-DEP_FLAGS=''
-DEP_PROCESS='true'
-if test x$enable_dependency_tracking = xyes ; then
-  if test x$GCC = xyes ; then
-    gcc_version=`gcc --version | head -1`
-    case "$gcc_version" in
-      2.*|*[[!0-9.]]2.*)
-        enable_dependency_tracking=no
-        AC_MSG_WARN([Dependency tracking disabled, gcc-3.x is needed])
-      ;;
-      *)
-        DEP_FLAGS='-MT $[]@ -MD -MP -MF $[]@.d'
-        DEP_PROCESS='true'
-      ;;
-    esac
-  else
-    enable_dependency_tracking=no
-    AC_MSG_WARN([Dependency tracking disabled])
-  fi
-fi
-
-if test x$enable_dependency_tracking = xyes ; then
-  DEP_INCLUDE='include '
-else
-  DEP_INCLUDE='# '
-fi
-
-AC_SUBST([DEP_INCLUDE])
-AC_SUBST([DEP_FLAGS])
-AC_SUBST([DEP_PROCESS])])
-
-dnl @synopsis AX_CREATE_STDINT_H [( HEADER-TO-GENERATE [, HEADERS-TO-CHECK])]
-dnl
-dnl the "ISO C9X: 7.18 Integer types <stdint.h>" section requires the
-dnl existence of an include file <stdint.h> that defines a set of 
-dnl typedefs, especially uint8_t,int32_t,uintptr_t.
-dnl Many older installations will not provide this file, but some will
-dnl have the very same definitions in <inttypes.h>. In other enviroments
-dnl we can use the inet-types in <sys/types.h> which would define the
-dnl typedefs int8_t and u_int8_t respectivly.
-dnl
-dnl This macros will create a local "_stdint.h" or the headerfile given as 
-dnl an argument. In many cases that file will just "#include <stdint.h>" 
-dnl or "#include <inttypes.h>", while in other environments it will provide 
-dnl the set of basic 'stdint's definitions/typedefs: 
-dnl   int8_t,uint8_t,int16_t,uint16_t,int32_t,uint32_t,intptr_t,uintptr_t
-dnl   int_least32_t.. int_fast32_t.. intmax_t
-dnl which may or may not rely on the definitions of other files,
-dnl or using the AC_CHECK_SIZEOF macro to determine the actual
-dnl sizeof each type.
-dnl
-dnl if your header files require the stdint-types you will want to create an
-dnl installable file mylib-int.h that all your other installable header
-dnl may include. So if you have a library package named "mylib", just use
-dnl      AX_CREATE_STDINT_H(mylib-int.h) 
-dnl in configure.ac and go to install that very header file in Makefile.am
-dnl along with the other headers (mylib.h) - and the mylib-specific headers
-dnl can simply use "#include <mylib-int.h>" to obtain the stdint-types.
-dnl
-dnl Remember, if the system already had a valid <stdint.h>, the generated
-dnl file will include it directly. No need for fuzzy HAVE_STDINT_H things...
-dnl
-dnl @, (status: used on new platforms) (see http://ac-archive.sf.net/gstdint/)
-dnl @version $Id: acinclude.m4,v 1.27 2004/11/23 21:27:35 nisse Exp $
-dnl @author  Guido Draheim <guidod@gmx.de> 
-
-AC_DEFUN([AX_CREATE_STDINT_H],
-[# ------ AX CREATE STDINT H -------------------------------------
-AC_MSG_CHECKING([for stdint types])
-ac_stdint_h=`echo ifelse($1, , _stdint.h, $1)`
-# try to shortcircuit - if the default include path of the compiler
-# can find a "stdint.h" header then we assume that all compilers can.
-AC_CACHE_VAL([ac_cv_header_stdint_t],[
-old_CXXFLAGS="$CXXFLAGS" ; CXXFLAGS=""
-old_CPPFLAGS="$CPPFLAGS" ; CPPFLAGS=""
-old_CFLAGS="$CFLAGS"     ; CFLAGS=""
-AC_TRY_COMPILE([#include <stdint.h>],[int_least32_t v = 0;],
-[ac_cv_stdint_result="(assuming C99 compatible system)"
- ac_cv_header_stdint_t="stdint.h"; ],
-[ac_cv_header_stdint_t=""])
-CXXFLAGS="$old_CXXFLAGS"
-CPPFLAGS="$old_CPPFLAGS"
-CFLAGS="$old_CFLAGS" ])
-
-v="... $ac_cv_header_stdint_h"
-if test "$ac_stdint_h" = "stdint.h" ; then
- AC_MSG_RESULT([(are you sure you want them in ./stdint.h?)])
-elif test "$ac_stdint_h" = "inttypes.h" ; then
- AC_MSG_RESULT([(are you sure you want them in ./inttypes.h?)])
-elif test "_$ac_cv_header_stdint_t" = "_" ; then
- AC_MSG_RESULT([(putting them into $ac_stdint_h)$v])
-else
- ac_cv_header_stdint="$ac_cv_header_stdint_t"
- AC_MSG_RESULT([$ac_cv_header_stdint (shortcircuit)])
-fi
-
-if test "_$ac_cv_header_stdint_t" = "_" ; then # can not shortcircuit..
-
-dnl .....intro message done, now do a few system checks.....
-dnl btw, all CHECK_TYPE macros do automatically "DEFINE" a type, therefore
-dnl we use the autoconf implementation detail _AC CHECK_TYPE_NEW instead
-
-inttype_headers=`echo $2 | sed -e 's/,/ /g'`
-
-ac_cv_stdint_result="(no helpful system typedefs seen)"
-AC_CACHE_CHECK([for stdint uintptr_t], [ac_cv_header_stdint_x],[
- ac_cv_header_stdint_x="" # the 1997 typedefs (inttypes.h)
-  AC_MSG_RESULT([(..)])
-  for i in stdint.h inttypes.h sys/inttypes.h $inttype_headers ; do
-   unset ac_cv_type_uintptr_t 
-   unset ac_cv_type_uint64_t
-   _AC_CHECK_TYPE_NEW(uintptr_t,[ac_cv_header_stdint_x=$i],dnl
-     continue,[#include <$i>])
-   AC_CHECK_TYPE(uint64_t,[and64="/uint64_t"],[and64=""],[#include<$i>])
-   ac_cv_stdint_result="(seen uintptr_t$and64 in $i)"
-   break;
-  done
-  AC_MSG_CHECKING([for stdint uintptr_t])
- ])
-
-if test "_$ac_cv_header_stdint_x" = "_" ; then
-AC_CACHE_CHECK([for stdint uint32_t], [ac_cv_header_stdint_o],[
- ac_cv_header_stdint_o="" # the 1995 typedefs (sys/inttypes.h)
-  AC_MSG_RESULT([(..)])
-  for i in inttypes.h sys/inttypes.h stdint.h $inttype_headers ; do
-   unset ac_cv_type_uint32_t
-   unset ac_cv_type_uint64_t
-   AC_CHECK_TYPE(uint32_t,[ac_cv_header_stdint_o=$i],dnl
-     continue,[#include <$i>])
-   AC_CHECK_TYPE(uint64_t,[and64="/uint64_t"],[and64=""],[#include<$i>])
-   ac_cv_stdint_result="(seen uint32_t$and64 in $i)"
-   break;
-  done
-  AC_MSG_CHECKING([for stdint uint32_t])
- ])
-fi
-
-if test "_$ac_cv_header_stdint_x" = "_" ; then
-if test "_$ac_cv_header_stdint_o" = "_" ; then
-AC_CACHE_CHECK([for stdint u_int32_t], [ac_cv_header_stdint_u],[
- ac_cv_header_stdint_u="" # the BSD typedefs (sys/types.h)
-  AC_MSG_RESULT([(..)])
-  for i in sys/types.h inttypes.h sys/inttypes.h $inttype_headers ; do
-   unset ac_cv_type_u_int32_t
-   unset ac_cv_type_u_int64_t
-   AC_CHECK_TYPE(u_int32_t,[ac_cv_header_stdint_u=$i],dnl
-     continue,[#include <$i>])
-   AC_CHECK_TYPE(u_int64_t,[and64="/u_int64_t"],[and64=""],[#include<$i>])
-   ac_cv_stdint_result="(seen u_int32_t$and64 in $i)"
-   break;
-  done
-  AC_MSG_CHECKING([for stdint u_int32_t])
- ])
-fi fi
-
-dnl if there was no good C99 header file, do some typedef checks...
-if test "_$ac_cv_header_stdint_x" = "_" ; then
-   AC_MSG_CHECKING([for stdint datatype model])
-   AC_MSG_RESULT([(..)])
-   AC_CHECK_SIZEOF(char)
-   AC_CHECK_SIZEOF(short)
-   AC_CHECK_SIZEOF(int)
-   AC_CHECK_SIZEOF(long)
-   AC_CHECK_SIZEOF(void*)
-   ac_cv_stdint_char_model=""
-   ac_cv_stdint_char_model="$ac_cv_stdint_char_model$ac_cv_sizeof_char"
-   ac_cv_stdint_char_model="$ac_cv_stdint_char_model$ac_cv_sizeof_short"
-   ac_cv_stdint_char_model="$ac_cv_stdint_char_model$ac_cv_sizeof_int"
-   ac_cv_stdint_long_model=""
-   ac_cv_stdint_long_model="$ac_cv_stdint_long_model$ac_cv_sizeof_int"
-   ac_cv_stdint_long_model="$ac_cv_stdint_long_model$ac_cv_sizeof_long"
-   ac_cv_stdint_long_model="$ac_cv_stdint_long_model$ac_cv_sizeof_voidp"
-   name="$ac_cv_stdint_long_model"
-   case "$ac_cv_stdint_char_model/$ac_cv_stdint_long_model" in
-    122/242)     name="$name,  IP16 (standard 16bit machine)" ;;
-    122/244)     name="$name,  LP32 (standard 32bit mac/win)" ;;
-    122/*)       name="$name        (unusual int16 model)" ;; 
-    124/444)     name="$name, ILP32 (standard 32bit unixish)" ;;
-    124/488)     name="$name,  LP64 (standard 64bit unixish)" ;;
-    124/448)     name="$name, LLP64 (unusual  64bit unixish)" ;;
-    124/*)       name="$name        (unusual int32 model)" ;; 
-    128/888)     name="$name, ILP64 (unusual  64bit numeric)" ;;
-    128/*)       name="$name        (unusual int64 model)" ;; 
-    222/*|444/*) name="$name        (unusual dsptype)" ;;
-     *)          name="$name        (very unusal model)" ;;
-   esac
-   AC_MSG_RESULT([combined for stdint datatype model...  $name])
-fi
-
-if test "_$ac_cv_header_stdint_x" != "_" ; then
-   ac_cv_header_stdint="$ac_cv_header_stdint_x"
-elif  test "_$ac_cv_header_stdint_o" != "_" ; then
-   ac_cv_header_stdint="$ac_cv_header_stdint_o"
-elif  test "_$ac_cv_header_stdint_u" != "_" ; then
-   ac_cv_header_stdint="$ac_cv_header_stdint_u"
-else
-   ac_cv_header_stdint="stddef.h"
-fi
-
-AC_MSG_CHECKING([for extra inttypes in chosen header])
-AC_MSG_RESULT([($ac_cv_header_stdint)])
-dnl see if int_least and int_fast types are present in _this_ header.
-unset ac_cv_type_int_least32_t
-unset ac_cv_type_int_fast32_t
-AC_CHECK_TYPE(int_least32_t,,,[#include <$ac_cv_header_stdint>])
-AC_CHECK_TYPE(int_fast32_t,,,[#include<$ac_cv_header_stdint>])
-AC_CHECK_TYPE(intmax_t,,,[#include <$ac_cv_header_stdint>])
-
-fi # shortcircut to system "stdint.h"
-# ------------------ PREPARE VARIABLES ------------------------------
-if test "$GCC" = "yes" ; then
-ac_cv_stdint_message="using gnu compiler "`$CC --version | head -1` 
-else
-ac_cv_stdint_message="using $CC"
-fi
-
-AC_MSG_RESULT([make use of $ac_cv_header_stdint in $ac_stdint_h dnl
-$ac_cv_stdint_result])
-
-# ----------------- DONE inttypes.h checks START header -------------
-AC_CONFIG_COMMANDS([$ac_stdint_h],[
-AC_MSG_NOTICE(creating $ac_stdint_h : $_ac_stdint_h)
-ac_stdint=$tmp/_stdint.h
-
-echo "#ifndef" $_ac_stdint_h >$ac_stdint
-echo "#define" $_ac_stdint_h "1" >>$ac_stdint
-echo "#ifndef" _GENERATED_STDINT_H >>$ac_stdint
-echo "#define" _GENERATED_STDINT_H '"'$PACKAGE $VERSION'"' >>$ac_stdint
-echo "/* generated $ac_cv_stdint_message */" >>$ac_stdint
-if test "_$ac_cv_header_stdint_t" != "_" ; then 
-echo "#define _STDINT_HAVE_STDINT_H" "1" >>$ac_stdint
-fi
-
-cat >>$ac_stdint <<STDINT_EOF
-
-/* ................... shortcircuit part ........................... */
-
-#if defined HAVE_STDINT_H || defined _STDINT_HAVE_STDINT_H
-#include <stdint.h>
-#else
-#include <stddef.h>
-
-/* .................... configured part ............................ */
-
-STDINT_EOF
-
-echo "/* whether we have a C99 compatible stdint header file */" >>$ac_stdint
-if test "_$ac_cv_header_stdint_x" != "_" ; then
-  ac_header="$ac_cv_header_stdint_x"
-  echo "#define _STDINT_HEADER_INTPTR" '"'"$ac_header"'"' >>$ac_stdint
-else
-  echo "/* #undef _STDINT_HEADER_INTPTR */" >>$ac_stdint
-fi
-
-echo "/* whether we have a C96 compatible inttypes header file */" >>$ac_stdint
-if  test "_$ac_cv_header_stdint_o" != "_" ; then
-  ac_header="$ac_cv_header_stdint_o"
-  echo "#define _STDINT_HEADER_UINT32" '"'"$ac_header"'"' >>$ac_stdint
-else
-  echo "/* #undef _STDINT_HEADER_UINT32 */" >>$ac_stdint
-fi
-
-echo "/* whether we have a BSD compatible inet types header */" >>$ac_stdint
-if  test "_$ac_cv_header_stdint_u" != "_" ; then
-  ac_header="$ac_cv_header_stdint_u"
-  echo "#define _STDINT_HEADER_U_INT32" '"'"$ac_header"'"' >>$ac_stdint
-else
-  echo "/* #undef _STDINT_HEADER_U_INT32 */" >>$ac_stdint
-fi
-
-echo "" >>$ac_stdint
-
-if test "_$ac_header" != "_" ; then if test "$ac_header" != "stddef.h" ; then
-  echo "#include <$ac_header>" >>$ac_stdint
-  echo "" >>$ac_stdint
-fi fi
-
-echo "/* which 64bit typedef has been found */" >>$ac_stdint
-if test "$ac_cv_type_uint64_t" = "yes" ; then
-echo "#define   _STDINT_HAVE_UINT64_T" "1"  >>$ac_stdint
-else
-echo "/* #undef _STDINT_HAVE_UINT64_T */" >>$ac_stdint
-fi
-if test "$ac_cv_type_u_int64_t" = "yes" ; then
-echo "#define   _STDINT_HAVE_U_INT64_T" "1"  >>$ac_stdint
-else
-echo "/* #undef _STDINT_HAVE_U_INT64_T */" >>$ac_stdint
-fi
-echo "" >>$ac_stdint
-
-echo "/* which type model has been detected */" >>$ac_stdint
-if test "_$ac_cv_stdint_char_model" != "_" ; then
-echo "#define   _STDINT_CHAR_MODEL" "$ac_cv_stdint_char_model" >>$ac_stdint
-echo "#define   _STDINT_LONG_MODEL" "$ac_cv_stdint_long_model" >>$ac_stdint
-else
-echo "/* #undef _STDINT_CHAR_MODEL // skipped */" >>$ac_stdint
-echo "/* #undef _STDINT_LONG_MODEL // skipped */" >>$ac_stdint
-fi
-echo "" >>$ac_stdint
-
-echo "/* whether int_least types were detected */" >>$ac_stdint
-if test "$ac_cv_type_int_least32_t" = "yes"; then
-echo "#define   _STDINT_HAVE_INT_LEAST32_T" "1"  >>$ac_stdint
-else
-echo "/* #undef _STDINT_HAVE_INT_LEAST32_T */" >>$ac_stdint
-fi
-echo "/* whether int_fast types were detected */" >>$ac_stdint
-if test "$ac_cv_type_int_fast32_t" = "yes"; then
-echo "#define   _STDINT_HAVE_INT_FAST32_T" "1" >>$ac_stdint
-else
-echo "/* #undef _STDINT_HAVE_INT_FAST32_T */" >>$ac_stdint
-fi
-echo "/* whether intmax_t type was detected */" >>$ac_stdint
-if test "$ac_cv_type_intmax_t" = "yes"; then
-echo "#define   _STDINT_HAVE_INTMAX_T" "1" >>$ac_stdint
-else
-echo "/* #undef _STDINT_HAVE_INTMAX_T */" >>$ac_stdint
-fi
-echo "" >>$ac_stdint
-
-  cat >>$ac_stdint <<STDINT_EOF
-/* .................... detections part ............................ */
-
-/* whether we need to define bitspecific types from compiler base types */
-#ifndef _STDINT_HEADER_INTPTR
-#ifndef _STDINT_HEADER_UINT32
-#ifndef _STDINT_HEADER_U_INT32
-#define _STDINT_NEED_INT_MODEL_T
-#else
-#define _STDINT_HAVE_U_INT_TYPES
-#endif
-#endif
-#endif
-
-#ifdef _STDINT_HAVE_U_INT_TYPES
-#undef _STDINT_NEED_INT_MODEL_T
-#endif
-
-#ifdef  _STDINT_CHAR_MODEL
-#if     _STDINT_CHAR_MODEL+0 == 122 || _STDINT_CHAR_MODEL+0 == 124
-#ifndef _STDINT_BYTE_MODEL
-#define _STDINT_BYTE_MODEL 12
-#endif
-#endif
-#endif
-
-#ifndef _STDINT_HAVE_INT_LEAST32_T
-#define _STDINT_NEED_INT_LEAST_T
-#endif
-
-#ifndef _STDINT_HAVE_INT_FAST32_T
-#define _STDINT_NEED_INT_FAST_T
-#endif
-
-#ifndef _STDINT_HEADER_INTPTR
-#define _STDINT_NEED_INTPTR_T
-#ifndef _STDINT_HAVE_INTMAX_T
-#define _STDINT_NEED_INTMAX_T
-#endif
-#endif
-
-
-/* .................... definition part ............................ */
-
-/* some system headers have good uint64_t */
-#ifndef _HAVE_UINT64_T
-#if     defined _STDINT_HAVE_UINT64_T  || defined HAVE_UINT64_T
-#define _HAVE_UINT64_T
-#elif   defined _STDINT_HAVE_U_INT64_T || defined HAVE_U_INT64_T
-#define _HAVE_UINT64_T
-typedef u_int64_t uint64_t;
-#endif
-#endif
-
-#ifndef _HAVE_UINT64_T
-/* .. here are some common heuristics using compiler runtime specifics */
-#if defined __STDC_VERSION__ && defined __STDC_VERSION__ >= 199901L
-#define _HAVE_UINT64_T
-typedef long long int64_t;
-typedef unsigned long long uint64_t;
-
-#elif !defined __STRICT_ANSI__
-#if defined _MSC_VER || defined __WATCOMC__ || defined __BORLANDC__
-#define _HAVE_UINT64_T
-typedef __int64 int64_t;
-typedef unsigned __int64 uint64_t;
-
-#elif defined __GNUC__ || defined __MWERKS__ || defined __ELF__
-/* note: all ELF-systems seem to have loff-support which needs 64-bit */
-#if !defined _NO_LONGLONG
-#define _HAVE_UINT64_T
-typedef long long int64_t;
-typedef unsigned long long uint64_t;
-#endif
-
-#elif defined __alpha || (defined __mips && defined _ABIN32)
-#if !defined _NO_LONGLONG
-typedef long int64_t;
-typedef unsigned long uint64_t;
-#endif
-  /* compiler/cpu type to define int64_t */
-#endif
-#endif
-#endif
-
-#if defined _STDINT_HAVE_U_INT_TYPES
-/* int8_t int16_t int32_t defined by inet code, redeclare the u_intXX types */
-typedef u_int8_t uint8_t;
-typedef u_int16_t uint16_t;
-typedef u_int32_t uint32_t;
-
-/* glibc compatibility */
-#ifndef __int8_t_defined
-#define __int8_t_defined
-#endif
-#endif
-
-#ifdef _STDINT_NEED_INT_MODEL_T
-/* we must guess all the basic types. Apart from byte-adressable system, */
-/* there a few 32-bit-only dsp-systems that we guard with BYTE_MODEL 8-} */
-/* (btw, those nibble-addressable systems are way off, or so we assume) */
-
-dnl   /* have a look at "64bit and data size neutrality" at */
-dnl   /* http://unix.org/version2/whatsnew/login_64bit.html */
-dnl   /* (the shorthand "ILP" types always have a "P" part) */
-
-#if defined _STDINT_BYTE_MODEL
-#if _STDINT_LONG_MODEL+0 == 242
-/* 2:4:2 =  IP16 = a normal 16-bit system                */
-typedef unsigned char   uint8_t;
-typedef unsigned short  uint16_t;
-typedef unsigned long   uint32_t;
-#ifndef __int8_t_defined
-#define __int8_t_defined
-typedef          char    int8_t;
-typedef          short   int16_t;
-typedef          long    int32_t;
-#endif
-#elif _STDINT_LONG_MODEL+0 == 244 || _STDINT_LONG_MODEL == 444
-/* 2:4:4 =  LP32 = a 32-bit system derived from a 16-bit */
-/* 4:4:4 = ILP32 = a normal 32-bit system                */
-typedef unsigned char   uint8_t;
-typedef unsigned short  uint16_t;
-typedef unsigned int    uint32_t;
-#ifndef __int8_t_defined
-#define __int8_t_defined
-typedef          char    int8_t;
-typedef          short   int16_t;
-typedef          int     int32_t;
-#endif
-#elif _STDINT_LONG_MODEL+0 == 484 || _STDINT_LONG_MODEL+0 == 488
-/* 4:8:4 =  IP32 = a 32-bit system prepared for 64-bit    */
-/* 4:8:8 =  LP64 = a normal 64-bit system                 */
-typedef unsigned char   uint8_t;
-typedef unsigned short  uint16_t;
-typedef unsigned int    uint32_t;
-#ifndef __int8_t_defined
-#define __int8_t_defined
-typedef          char    int8_t;
-typedef          short   int16_t;
-typedef          int     int32_t;
-#endif
-/* this system has a "long" of 64bit */
-#ifndef _HAVE_UINT64_T
-#define _HAVE_UINT64_T
-typedef unsigned long   uint64_t;
-typedef          long    int64_t;
-#endif
-#elif _STDINT_LONG_MODEL+0 == 448
-/*      LLP64   a 64-bit system derived from a 32-bit system */
-typedef unsigned char   uint8_t;
-typedef unsigned short  uint16_t;
-typedef unsigned int    uint32_t;
-#ifndef __int8_t_defined
-#define __int8_t_defined
-typedef          char    int8_t;
-typedef          short   int16_t;
-typedef          int     int32_t;
-#endif
-/* assuming the system has a "long long" */
-#ifndef _HAVE_UINT64_T
-#define _HAVE_UINT64_T
-typedef unsigned long long uint64_t;
-typedef          long long  int64_t;
-#endif
-#else
-#define _STDINT_NO_INT32_T
-#endif
-#else
-#define _STDINT_NO_INT8_T
-#define _STDINT_NO_INT32_T
-#endif
-#endif
-
-/*
- * quote from SunOS-5.8 sys/inttypes.h:
- * Use at your own risk.  As of February 1996, the committee is squarely
- * behind the fixed sized types; the "least" and "fast" types are still being
- * discussed.  The probability that the "fast" types may be removed before
- * the standard is finalized is high enough that they are not currently
- * implemented.
- */
-
-#if defined _STDINT_NEED_INT_LEAST_T
-typedef  int8_t    int_least8_t;
-typedef  int16_t   int_least16_t;
-typedef  int32_t   int_least32_t;
-#ifdef _HAVE_UINT64_T
-typedef  int64_t   int_least64_t;
-#endif
-
-typedef uint8_t   uint_least8_t;
-typedef uint16_t  uint_least16_t;
-typedef uint32_t  uint_least32_t;
-#ifdef _HAVE_UINT64_T
-typedef uint64_t  uint_least64_t;
-#endif
-  /* least types */
-#endif
-
-#if defined _STDINT_NEED_INT_FAST_T
-typedef  int8_t    int_fast8_t; 
-typedef  int       int_fast16_t;
-typedef  int32_t   int_fast32_t;
-#ifdef _HAVE_UINT64_T
-typedef  int64_t   int_fast64_t;
-#endif
-
-typedef uint8_t   uint_fast8_t; 
-typedef unsigned  uint_fast16_t;
-typedef uint32_t  uint_fast32_t;
-#ifdef _HAVE_UINT64_T
-typedef uint64_t  uint_fast64_t;
-#endif
-  /* fast types */
-#endif
-
-#ifdef _STDINT_NEED_INTMAX_T
-#ifdef _HAVE_UINT64_T
-typedef  int64_t       intmax_t;
-typedef uint64_t      uintmax_t;
-#else
-typedef          long  intmax_t;
-typedef unsigned long uintmax_t;
-#endif
-#endif
-
-#ifdef _STDINT_NEED_INTPTR_T
-#ifndef __intptr_t_defined
-#define __intptr_t_defined
-/* we encourage using "long" to store pointer values, never use "int" ! */
-#if   _STDINT_LONG_MODEL+0 == 242 || _STDINT_LONG_MODEL+0 == 484
-typedef  unsinged int   uintptr_t;
-typedef           int    intptr_t;
-#elif _STDINT_LONG_MODEL+0 == 244 || _STDINT_LONG_MODEL+0 == 444
-typedef  unsigned long  uintptr_t;
-typedef           long   intptr_t;
-#elif _STDINT_LONG_MODEL+0 == 448 && defined _HAVE_UINT64_T
-typedef        uint64_t uintptr_t;
-typedef         int64_t  intptr_t;
-#else /* matches typical system types ILP32 and LP64 - but not IP16 or LLP64 */
-typedef  unsigned long  uintptr_t;
-typedef           long   intptr_t;
-#endif
-#endif
-#endif
-
-  /* shortcircuit*/
-#endif
-  /* once */
-#endif
-#endif
-STDINT_EOF
-    if cmp -s $ac_stdint_h $ac_stdint 2>/dev/null; then
-      AC_MSG_NOTICE([$ac_stdint_h is unchanged])
-    else
-      ac_dir=`AS_DIRNAME(["$ac_stdint_h"])`
-      AS_MKDIR_P(["$ac_dir"])
-      rm -f $ac_stdint_h
-      mv $ac_stdint $ac_stdint_h
-    fi
-],[# variables for create stdint.h replacement
-PACKAGE="$PACKAGE"
-VERSION="$VERSION"
-ac_stdint_h="$ac_stdint_h"
-_ac_stdint_h=AS_TR_CPP(_$PACKAGE-$ac_stdint_h)
-ac_cv_stdint_message="$ac_cv_stdint_message"
-ac_cv_header_stdint_t="$ac_cv_header_stdint_t"
-ac_cv_header_stdint_x="$ac_cv_header_stdint_x"
-ac_cv_header_stdint_o="$ac_cv_header_stdint_o"
-ac_cv_header_stdint_u="$ac_cv_header_stdint_u"
-ac_cv_type_uint64_t="$ac_cv_type_uint64_t"
-ac_cv_type_u_int64_t="$ac_cv_type_u_int64_t"
-ac_cv_stdint_char_model="$ac_cv_stdint_char_model"
-ac_cv_stdint_long_model="$ac_cv_stdint_long_model"
-ac_cv_type_int_least32_t="$ac_cv_type_int_least32_t"
-ac_cv_type_int_fast32_t="$ac_cv_type_int_fast32_t"
-ac_cv_type_intmax_t="$ac_cv_type_intmax_t"
-])
-])
diff --git a/argp-standalone/argp-ba.c b/argp-standalone/argp-ba.c
deleted file mode 100644
index 0d3958c1151..00000000000
--- a/argp-standalone/argp-ba.c
+++ /dev/null
@@ -1,26 +0,0 @@
-/* Default definition for ARGP_PROGRAM_BUG_ADDRESS.
-   Copyright (C) 1996, 1997, 1999, 2004 Free Software Foundation, Inc.
-   This file is part of the GNU C Library.
-   Written by Miles Bader <miles@gnu.ai.mit.edu>.
-
-   The GNU C Library is free software; you can redistribute it and/or
-   modify it under the terms of the GNU Library General Public License as
-   published by the Free Software Foundation; either version 2 of the
-   License, or (at your option) any later version.
-
-   The GNU C Library is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Library General Public License for more details.
-
-   You should have received a copy of the GNU Library General Public
-   License along with the GNU C Library; see the file COPYING.LIB.  If not,
-   write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330,
-   Boston, MA 02111-1307, USA.  */
-
-/* If set by the user program, it should point to string that is the
-   bug-reporting address for the program.  It will be printed by argp_help if
-   the ARGP_HELP_BUG_ADDR flag is set (as it is by various standard help
-   messages), embedded in a sentence that says something like `Report bugs to
-   ADDR.'.  */
-const char *argp_program_bug_address = 0;
diff --git a/argp-standalone/argp-eexst.c b/argp-standalone/argp-eexst.c
deleted file mode 100644
index 46b27847ad4..00000000000
--- a/argp-standalone/argp-eexst.c
+++ /dev/null
@@ -1,36 +0,0 @@
-/* Default definition for ARGP_ERR_EXIT_STATUS
-   Copyright (C) 1997 Free Software Foundation, Inc.
-   This file is part of the GNU C Library.
-   Written by Miles Bader <miles@gnu.ai.mit.edu>.
-
-   The GNU C Library is free software; you can redistribute it and/or
-   modify it under the terms of the GNU Library General Public License as
-   published by the Free Software Foundation; either version 2 of the
-   License, or (at your option) any later version.
-
-   The GNU C Library is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Library General Public License for more details.
-
-   You should have received a copy of the GNU Library General Public
-   License along with the GNU C Library; see the file COPYING.LIB.  If not,
-   write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330,
-   Boston, MA 02111-1307, USA.  */
-
-#ifdef HAVE_CONFIG_H
-#include <config.h>
-#endif
-
-#if HAVE_SYSEXITS_H
-# include <sysexits.h>
-#else
-# define EX_USAGE 64
-#endif
-
-#include "argp.h"
-
-/* The exit status that argp will use when exiting due to a parsing error.
-   If not defined or set by the user program, this defaults to EX_USAGE from
-   <sysexits.h>.  */
-error_t argp_err_exit_status = EX_USAGE;
diff --git a/argp-standalone/argp-fmtstream.c b/argp-standalone/argp-fmtstream.c
deleted file mode 100644
index 7f792854fc3..00000000000
--- a/argp-standalone/argp-fmtstream.c
+++ /dev/null
@@ -1,475 +0,0 @@
-/* Word-wrapping and line-truncating streams
-   Copyright (C) 1997, 1998, 1999, 2001 Free Software Foundation, Inc.
-   This file is part of the GNU C Library.
-   Written by Miles Bader <miles@gnu.ai.mit.edu>.
-
-   The GNU C Library is free software; you can redistribute it and/or
-   modify it under the terms of the GNU Library General Public License as
-   published by the Free Software Foundation; either version 2 of the
-   License, or (at your option) any later version.
-
-   The GNU C Library is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Library General Public License for more details.
-
-   You should have received a copy of the GNU Library General Public
-   License along with the GNU C Library; see the file COPYING.LIB.  If not,
-   write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330,
-   Boston, MA 02111-1307, USA.  */
-
-/* This package emulates glibc `line_wrap_stream' semantics for systems that
-   don't have that.  */
-
-#ifdef HAVE_CONFIG_H
-#include <config.h>
-#endif
-
-#include <stdlib.h>
-#include <string.h>
-#include <errno.h>
-#include <stdarg.h>
-#include <ctype.h>
-
-#include "argp-fmtstream.h"
-#include "argp-namefrob.h"
-
-#ifndef ARGP_FMTSTREAM_USE_LINEWRAP
-
-#ifndef isblank
-#define isblank(ch) ((ch)==' ' || (ch)=='\t')
-#endif
-
-#if defined _LIBC && defined USE_IN_LIBIO
-# include <libio/libioP.h>
-# define __vsnprintf(s, l, f, a) _IO_vsnprintf (s, l, f, a)
-#endif
-
-#define INIT_BUF_SIZE 200
-#define PRINTF_SIZE_GUESS 150
-
-/* Return an argp_fmtstream that outputs to STREAM, and which prefixes lines
-   written on it with LMARGIN spaces and limits them to RMARGIN columns
-   total.  If WMARGIN >= 0, words that extend past RMARGIN are wrapped by
-   replacing the whitespace before them with a newline and WMARGIN spaces.
-   Otherwise, chars beyond RMARGIN are simply dropped until a newline.
-   Returns NULL if there was an error.  */
-argp_fmtstream_t
-__argp_make_fmtstream (FILE *stream,
-		       size_t lmargin, size_t rmargin, ssize_t wmargin)
-{
-  argp_fmtstream_t fs = malloc (sizeof (struct argp_fmtstream));
-  if (fs)
-    {
-      fs->stream = stream;
-
-      fs->lmargin = lmargin;
-      fs->rmargin = rmargin;
-      fs->wmargin = wmargin;
-      fs->point_col = 0;
-      fs->point_offs = 0;
-
-      fs->buf = malloc (INIT_BUF_SIZE);
-      if (! fs->buf)
-	{
-	  free (fs);
-	  fs = 0;
-	}
-      else
-	{
-	  fs->p = fs->buf;
-	  fs->end = fs->buf + INIT_BUF_SIZE;
-	}
-    }
-
-  return fs;
-}
-#ifdef weak_alias
-weak_alias (__argp_make_fmtstream, argp_make_fmtstream)
-#endif
-
-/* Flush FS to its stream, and free it (but don't close the stream).  */
-void
-__argp_fmtstream_free (argp_fmtstream_t fs)
-{
-  __argp_fmtstream_update (fs);
-  if (fs->p > fs->buf)
-    FWRITE_UNLOCKED (fs->buf, 1, fs->p - fs->buf, fs->stream);
-  free (fs->buf);
-  free (fs);
-}
-#ifdef weak_alias
-weak_alias (__argp_fmtstream_free, argp_fmtstream_free)
-#endif
-
-/* Process FS's buffer so that line wrapping is done from POINT_OFFS to the
-   end of its buffer.  This code is mostly from glibc stdio/linewrap.c.  */
-void
-__argp_fmtstream_update (argp_fmtstream_t fs)
-{
-  char *buf, *nl;
-  size_t len;
-
-  /* Scan the buffer for newlines.  */
-  buf = fs->buf + fs->point_offs;
-  while (buf < fs->p)
-    {
-      size_t r;
-
-      if (fs->point_col == 0 && fs->lmargin != 0)
-	{
-	  /* We are starting a new line.  Print spaces to the left margin.  */
-	  const size_t pad = fs->lmargin;
-	  if (fs->p + pad < fs->end)
-	    {
-	      /* We can fit in them in the buffer by moving the
-		 buffer text up and filling in the beginning.  */
-	      memmove (buf + pad, buf, fs->p - buf);
-	      fs->p += pad; /* Compensate for bigger buffer. */
-	      memset (buf, ' ', pad); /* Fill in the spaces.  */
-	      buf += pad; /* Don't bother searching them.  */
-	    }
-	  else
-	    {
-	      /* No buffer space for spaces.  Must flush.  */
-	      size_t i;
-	      for (i = 0; i < pad; i++)
-		PUTC_UNLOCKED (' ', fs->stream);
-	    }
-	  fs->point_col = pad;
-	}
-
-      len = fs->p - buf;
-      nl = memchr (buf, '\n', len);
-
-      if (fs->point_col < 0)
-	fs->point_col = 0;
-
-      if (!nl)
-	{
-	  /* The buffer ends in a partial line.  */
-
-	  if (fs->point_col + len < fs->rmargin)
-	    {
-	      /* The remaining buffer text is a partial line and fits
-		 within the maximum line width.  Advance point for the
-		 characters to be written and stop scanning.  */
-	      fs->point_col += len;
-	      break;
-	    }
-	  else
-	    /* Set the end-of-line pointer for the code below to
-	       the end of the buffer.  */
-	    nl = fs->p;
-	}
-      else if (fs->point_col + (nl - buf) < (ssize_t) fs->rmargin)
-	{
-	  /* The buffer contains a full line that fits within the maximum
-	     line width.  Reset point and scan the next line.  */
-	  fs->point_col = 0;
-	  buf = nl + 1;
-	  continue;
-	}
-
-      /* This line is too long.  */
-      r = fs->rmargin - 1;
-
-      if (fs->wmargin < 0)
-	{
-	  /* Truncate the line by overwriting the excess with the
-	     newline and anything after it in the buffer.  */
-	  if (nl < fs->p)
-	    {
-	      memmove (buf + (r - fs->point_col), nl, fs->p - nl);
-	      fs->p -= buf + (r - fs->point_col) - nl;
-	      /* Reset point for the next line and start scanning it.  */
-	      fs->point_col = 0;
-	      buf += r + 1; /* Skip full line plus \n. */
-	    }
-	  else
-	    {
-	      /* The buffer ends with a partial line that is beyond the
-		 maximum line width.  Advance point for the characters
-		 written, and discard those past the max from the buffer.  */
-	      fs->point_col += len;
-	      fs->p -= fs->point_col - r;
-	      break;
-	    }
-	}
-      else
-	{
-	  /* Do word wrap.  Go to the column just past the maximum line
-	     width and scan back for the beginning of the word there.
-	     Then insert a line break.  */
-
-	  char *p, *nextline;
-	  int i;
-
-	  p = buf + (r + 1 - fs->point_col);
-	  while (p >= buf && !isblank (*p))
-	    --p;
-	  nextline = p + 1;	/* This will begin the next line.  */
-
-	  if (nextline > buf)
-	    {
-	      /* Swallow separating blanks.  */
-	      if (p >= buf)
-		do
-		  --p;
-		while (p >= buf && isblank (*p));
-	      nl = p + 1;	/* The newline will replace the first blank. */
-	    }
-	  else
-	    {
-	      /* A single word that is greater than the maximum line width.
-		 Oh well.  Put it on an overlong line by itself.  */
-	      p = buf + (r + 1 - fs->point_col);
-	      /* Find the end of the long word.  */
-	      do
-		++p;
-	      while (p < nl && !isblank (*p));
-	      if (p == nl)
-		{
-		  /* It already ends a line.  No fussing required.  */
-		  fs->point_col = 0;
-		  buf = nl + 1;
-		  continue;
-		}
-	      /* We will move the newline to replace the first blank.  */
-	      nl = p;
-	      /* Swallow separating blanks.  */
-	      do
-		++p;
-	      while (isblank (*p));
-	      /* The next line will start here.  */
-	      nextline = p;
-	    }
-
-	  /* Note: There are a bunch of tests below for
-	     NEXTLINE == BUF + LEN + 1; this case is where NL happens to fall
-	     at the end of the buffer, and NEXTLINE is in fact empty (and so
-	     we need not be careful to maintain its contents).  */
-
-	  if (nextline == buf + len + 1
-	      ? fs->end - nl < fs->wmargin + 1
-	      : nextline - (nl + 1) < fs->wmargin)
-	    {
-	      /* The margin needs more blanks than we removed.  */
-	      if (fs->end - fs->p > fs->wmargin + 1)
-		/* Make some space for them.  */
-		{
-		  size_t mv = fs->p - nextline;
-		  memmove (nl + 1 + fs->wmargin, nextline, mv);
-		  nextline = nl + 1 + fs->wmargin;
-		  len = nextline + mv - buf;
-		  *nl++ = '\n';
-		}
-	      else
-		/* Output the first line so we can use the space.  */
-		{
-		  if (nl > fs->buf)
-		    FWRITE_UNLOCKED (fs->buf, 1, nl - fs->buf, fs->stream);
-		  PUTC_UNLOCKED ('\n', fs->stream);
-		  len += buf - fs->buf;
-		  nl = buf = fs->buf;
-		}
-	    }
-	  else
-	    /* We can fit the newline and blanks in before
-	       the next word.  */
-	    *nl++ = '\n';
-
-	  if (nextline - nl >= fs->wmargin
-	      || (nextline == buf + len + 1 && fs->end - nextline >= fs->wmargin))
-	    /* Add blanks up to the wrap margin column.  */
-	    for (i = 0; i < fs->wmargin; ++i)
-	      *nl++ = ' ';
-	  else
-	    for (i = 0; i < fs->wmargin; ++i)
-	      PUTC_UNLOCKED (' ', fs->stream);
-
-	  /* Copy the tail of the original buffer into the current buffer
-	     position.  */
-	  if (nl < nextline)
-	    memmove (nl, nextline, buf + len - nextline);
-	  len -= nextline - buf;
-
-	  /* Continue the scan on the remaining lines in the buffer.  */
-	  buf = nl;
-
-	  /* Restore bufp to include all the remaining text.  */
-	  fs->p = nl + len;
-
-	  /* Reset the counter of what has been output this line.  If wmargin
-	     is 0, we want to avoid the lmargin getting added, so we set
-	     point_col to a magic value of -1 in that case.  */
-	  fs->point_col = fs->wmargin ? fs->wmargin : -1;
-	}
-    }
-
-  /* Remember that we've scanned as far as the end of the buffer.  */
-  fs->point_offs = fs->p - fs->buf;
-}
-
-/* Ensure that FS has space for AMOUNT more bytes in its buffer, either by
-   growing the buffer, or by flushing it.  True is returned iff we succeed. */
-int
-__argp_fmtstream_ensure (struct argp_fmtstream *fs, size_t amount)
-{
-  if ((size_t) (fs->end - fs->p) < amount)
-    {
-      ssize_t wrote;
-
-      /* Flush FS's buffer.  */
-      __argp_fmtstream_update (fs);
-
-      wrote = FWRITE_UNLOCKED (fs->buf, 1, fs->p - fs->buf, fs->stream);
-      if (wrote == fs->p - fs->buf)
-	{
-	  fs->p = fs->buf;
-	  fs->point_offs = 0;
-	}
-      else
-	{
-	  fs->p -= wrote;
-	  fs->point_offs -= wrote;
-	  memmove (fs->buf, fs->buf + wrote, fs->p - fs->buf);
-	  return 0;
-	}
-
-      if ((size_t) (fs->end - fs->buf) < amount)
-	/* Gotta grow the buffer.  */
-	{
-	  size_t new_size = fs->end - fs->buf + amount;
-	  char *new_buf = realloc (fs->buf, new_size);
-
-	  if (! new_buf)
-	    {
-	      __set_errno (ENOMEM);
-	      return 0;
-	    }
-
-	  fs->buf = new_buf;
-	  fs->end = new_buf + new_size;
-	  fs->p = fs->buf;
-	}
-    }
-
-  return 1;
-}
-
-ssize_t
-__argp_fmtstream_printf (struct argp_fmtstream *fs, const char *fmt, ...)
-{
-  size_t out;
-  size_t avail;
-  size_t size_guess = PRINTF_SIZE_GUESS; /* How much space to reserve. */
-
-  do
-    {
-      va_list args;
-
-      if (! __argp_fmtstream_ensure (fs, size_guess))
-	return -1;
-
-      va_start (args, fmt);
-      avail = fs->end - fs->p;
-      out = __vsnprintf (fs->p, avail, fmt, args);
-      va_end (args);
-      if (out >= avail)
-	size_guess = out + 1;
-    }
-  while (out >= avail);
-
-  fs->p += out;
-
-  return out;
-}
-#ifdef weak_alias
-weak_alias (__argp_fmtstream_printf, argp_fmtstream_printf)
-#endif
-
-/* Duplicate the inline definitions in argp-fmtstream.h, for compilers
- * that don't do inlining. */
-size_t
-__argp_fmtstream_write (argp_fmtstream_t __fs,
-			__const char *__str, size_t __len)
-{
-  if (__fs->p + __len <= __fs->end || __argp_fmtstream_ensure (__fs, __len))
-    {
-      memcpy (__fs->p, __str, __len);
-      __fs->p += __len;
-      return __len;
-    }
-  else
-    return 0;
-}
-
-int
-__argp_fmtstream_puts (argp_fmtstream_t __fs, __const char *__str)
-{
-  size_t __len = strlen (__str);
-  if (__len)
-    {
-      size_t __wrote = __argp_fmtstream_write (__fs, __str, __len);
-      return __wrote == __len ? 0 : -1;
-    }
-  else
-    return 0;
-}
-
-int
-__argp_fmtstream_putc (argp_fmtstream_t __fs, int __ch)
-{
-  if (__fs->p < __fs->end || __argp_fmtstream_ensure (__fs, 1))
-    return *__fs->p++ = __ch;
-  else
-    return EOF;
-}
-
-/* Set __FS's left margin to __LMARGIN and return the old value.  */
-size_t
-__argp_fmtstream_set_lmargin (argp_fmtstream_t __fs, size_t __lmargin)
-{
-  size_t __old;
-  if ((size_t) (__fs->p - __fs->buf) > __fs->point_offs)
-    __argp_fmtstream_update (__fs);
-  __old = __fs->lmargin;
-  __fs->lmargin = __lmargin;
-  return __old;
-}
-
-/* Set __FS's right margin to __RMARGIN and return the old value.  */
-size_t
-__argp_fmtstream_set_rmargin (argp_fmtstream_t __fs, size_t __rmargin)
-{
-  size_t __old;
-  if ((size_t) (__fs->p - __fs->buf) > __fs->point_offs)
-    __argp_fmtstream_update (__fs);
-  __old = __fs->rmargin;
-  __fs->rmargin = __rmargin;
-  return __old;
-}
-
-/* Set FS's wrap margin to __WMARGIN and return the old value.  */
-size_t
-__argp_fmtstream_set_wmargin (argp_fmtstream_t __fs, size_t __wmargin)
-{
-  size_t __old;
-  if ((size_t) (__fs->p - __fs->buf) > __fs->point_offs)
-    __argp_fmtstream_update (__fs);
-  __old = __fs->wmargin;
-  __fs->wmargin = __wmargin;
-  return __old;
-}
-
-/* Return the column number of the current output point in __FS.  */
-size_t
-__argp_fmtstream_point (argp_fmtstream_t __fs)
-{
-  if ((size_t) (__fs->p - __fs->buf) > __fs->point_offs)
-    __argp_fmtstream_update (__fs);
-  return __fs->point_col >= 0 ? __fs->point_col : 0;
-}
-
-#endif /* !ARGP_FMTSTREAM_USE_LINEWRAP */
diff --git a/argp-standalone/argp-fmtstream.h b/argp-standalone/argp-fmtstream.h
deleted file mode 100644
index e797b119ebd..00000000000
--- a/argp-standalone/argp-fmtstream.h
+++ /dev/null
@@ -1,319 +0,0 @@
-/* Word-wrapping and line-truncating streams.
-   Copyright (C) 1997, 2003 Free Software Foundation, Inc.
-   This file is part of the GNU C Library.
-   Written by Miles Bader <miles@gnu.ai.mit.edu>.
-
-   The GNU C Library is free software; you can redistribute it and/or
-   modify it under the terms of the GNU Library General Public License as
-   published by the Free Software Foundation; either version 2 of the
-   License, or (at your option) any later version.
-
-   The GNU C Library is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Library General Public License for more details.
-
-   You should have received a copy of the GNU Library General Public
-   License along with the GNU C Library; see the file COPYING.LIB.  If not,
-   write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330,
-   Boston, MA 02111-1307, USA.  */
-
-/* This package emulates glibc `line_wrap_stream' semantics for systems that
-   don't have that.  If the system does have it, it is just a wrapper for
-   that.  This header file is only used internally while compiling argp, and
-   shouldn't be installed.  */
-
-#ifndef _ARGP_FMTSTREAM_H
-#define _ARGP_FMTSTREAM_H
-
-#include <stdio.h>
-#include <string.h>
-
-#if HAVE_UNISTD_H
-# include <unistd.h>
-#else
-/* This is a kludge to make the code compile on windows. Perhaps it
-   would be better to just replace ssize_t with int through out the
-   code. */
-# define ssize_t int
-#endif
-
-#if _LIBC || (defined (HAVE_FLOCKFILE) && defined(HAVE_PUTC_UNLOCKED) \
-     && defined (HAVE_FPUTS_UNLOCKED) && defined (HAVE_FWRITE_UNLOCKED) )
-/* Use locking funxtions */
-# define FLOCKFILE(f) flockfile(f)
-# define FUNLOCKFILE(f) funlockfile(f)
-# define PUTC_UNLOCKED(c, f) putc_unlocked((c), (f))
-# define FPUTS_UNLOCKED(s, f) fputs_unlocked((s), (f))
-# define FWRITE_UNLOCKED(b, s, n, f) fwrite_unlocked((b), (s), (n), (f))
-#else
-/* Disable stdio locking */
-# define FLOCKFILE(f)
-# define FUNLOCKFILE(f)
-# define PUTC_UNLOCKED(c, f) putc((c), (f))
-# define FPUTS_UNLOCKED(s, f) fputs((s), (f))
-# define FWRITE_UNLOCKED(b, s, n, f) fwrite((b), (s), (n), (f))
-#endif /* No thread safe i/o */
-
-#if    (_LIBC - 0 && !defined (USE_IN_LIBIO)) \
-    || (defined (__GNU_LIBRARY__) && defined (HAVE_LINEWRAP_H))
-/* line_wrap_stream is available, so use that.  */
-#define ARGP_FMTSTREAM_USE_LINEWRAP
-#endif
-
-#ifdef ARGP_FMTSTREAM_USE_LINEWRAP
-/* Just be a simple wrapper for line_wrap_stream; the semantics are
-   *slightly* different, as line_wrap_stream doesn't actually make a new
-   object, it just modifies the given stream (reversibly) to do
-   line-wrapping.  Since we control who uses this code, it doesn't matter.  */
-
-#include <linewrap.h>
-
-typedef FILE *argp_fmtstream_t;
-
-#define argp_make_fmtstream line_wrap_stream
-#define __argp_make_fmtstream line_wrap_stream
-#define argp_fmtstream_free line_unwrap_stream
-#define __argp_fmtstream_free line_unwrap_stream
-
-#define __argp_fmtstream_putc(fs,ch) putc(ch,fs)
-#define argp_fmtstream_putc(fs,ch) putc(ch,fs)
-#define __argp_fmtstream_puts(fs,str) fputs(str,fs)
-#define argp_fmtstream_puts(fs,str) fputs(str,fs)
-#define __argp_fmtstream_write(fs,str,len) fwrite(str,1,len,fs)
-#define argp_fmtstream_write(fs,str,len) fwrite(str,1,len,fs)
-#define __argp_fmtstream_printf fprintf
-#define argp_fmtstream_printf fprintf
-
-#define __argp_fmtstream_lmargin line_wrap_lmargin
-#define argp_fmtstream_lmargin line_wrap_lmargin
-#define __argp_fmtstream_set_lmargin line_wrap_set_lmargin
-#define argp_fmtstream_set_lmargin line_wrap_set_lmargin
-#define __argp_fmtstream_rmargin line_wrap_rmargin
-#define argp_fmtstream_rmargin line_wrap_rmargin
-#define __argp_fmtstream_set_rmargin line_wrap_set_rmargin
-#define argp_fmtstream_set_rmargin line_wrap_set_rmargin
-#define __argp_fmtstream_wmargin line_wrap_wmargin
-#define argp_fmtstream_wmargin line_wrap_wmargin
-#define __argp_fmtstream_set_wmargin line_wrap_set_wmargin
-#define argp_fmtstream_set_wmargin line_wrap_set_wmargin
-#define __argp_fmtstream_point line_wrap_point
-#define argp_fmtstream_point line_wrap_point
-
-#else /* !ARGP_FMTSTREAM_USE_LINEWRAP */
-/* Guess we have to define our own version.  */
-
-#ifndef __const
-#define __const const
-#endif
-
-
-struct argp_fmtstream
-{
-  FILE *stream;			/* The stream we're outputting to.  */
-
-  size_t lmargin, rmargin;	/* Left and right margins.  */
-  ssize_t wmargin;		/* Margin to wrap to, or -1 to truncate.  */
-
-  /* Point in buffer to which we've processed for wrapping, but not output.  */
-  size_t point_offs;
-  /* Output column at POINT_OFFS, or -1 meaning 0 but don't add lmargin.  */
-  ssize_t point_col;
-
-  char *buf;			/* Output buffer.  */
-  char *p;			/* Current end of text in BUF. */
-  char *end;			/* Absolute end of BUF.  */
-};
-
-typedef struct argp_fmtstream *argp_fmtstream_t;
-
-/* Return an argp_fmtstream that outputs to STREAM, and which prefixes lines
-   written on it with LMARGIN spaces and limits them to RMARGIN columns
-   total.  If WMARGIN >= 0, words that extend past RMARGIN are wrapped by
-   replacing the whitespace before them with a newline and WMARGIN spaces.
-   Otherwise, chars beyond RMARGIN are simply dropped until a newline.
-   Returns NULL if there was an error.  */
-extern argp_fmtstream_t __argp_make_fmtstream (FILE *__stream,
-					       size_t __lmargin,
-					       size_t __rmargin,
-					       ssize_t __wmargin);
-extern argp_fmtstream_t argp_make_fmtstream (FILE *__stream,
-					     size_t __lmargin,
-					     size_t __rmargin,
-					     ssize_t __wmargin);
-
-/* Flush __FS to its stream, and free it (but don't close the stream).  */
-extern void __argp_fmtstream_free (argp_fmtstream_t __fs);
-extern void argp_fmtstream_free (argp_fmtstream_t __fs);
-
-extern ssize_t __argp_fmtstream_printf (argp_fmtstream_t __fs,
-				       __const char *__fmt, ...)
-     PRINTF_STYLE(2,3);
-extern ssize_t argp_fmtstream_printf (argp_fmtstream_t __fs,
-				      __const char *__fmt, ...)
-     PRINTF_STYLE(2,3);
-
-extern int __argp_fmtstream_putc (argp_fmtstream_t __fs, int __ch);
-extern int argp_fmtstream_putc (argp_fmtstream_t __fs, int __ch);
-
-extern int __argp_fmtstream_puts (argp_fmtstream_t __fs, __const char *__str);
-extern int argp_fmtstream_puts (argp_fmtstream_t __fs, __const char *__str);
-
-extern size_t __argp_fmtstream_write (argp_fmtstream_t __fs,
-				      __const char *__str, size_t __len);
-extern size_t argp_fmtstream_write (argp_fmtstream_t __fs,
-				    __const char *__str, size_t __len);
-
-/* Access macros for various bits of state.  */
-#define argp_fmtstream_lmargin(__fs) ((__fs)->lmargin)
-#define argp_fmtstream_rmargin(__fs) ((__fs)->rmargin)
-#define argp_fmtstream_wmargin(__fs) ((__fs)->wmargin)
-#define __argp_fmtstream_lmargin argp_fmtstream_lmargin
-#define __argp_fmtstream_rmargin argp_fmtstream_rmargin
-#define __argp_fmtstream_wmargin argp_fmtstream_wmargin
-
-/* Set __FS's left margin to LMARGIN and return the old value.  */
-extern size_t argp_fmtstream_set_lmargin (argp_fmtstream_t __fs,
-					  size_t __lmargin);
-extern size_t __argp_fmtstream_set_lmargin (argp_fmtstream_t __fs,
-					    size_t __lmargin);
-
-/* Set __FS's right margin to __RMARGIN and return the old value.  */
-extern size_t argp_fmtstream_set_rmargin (argp_fmtstream_t __fs,
-					  size_t __rmargin);
-extern size_t __argp_fmtstream_set_rmargin (argp_fmtstream_t __fs,
-					    size_t __rmargin);
-
-/* Set __FS's wrap margin to __WMARGIN and return the old value.  */
-extern size_t argp_fmtstream_set_wmargin (argp_fmtstream_t __fs,
-					  size_t __wmargin);
-extern size_t __argp_fmtstream_set_wmargin (argp_fmtstream_t __fs,
-					    size_t __wmargin);
-
-/* Return the column number of the current output point in __FS.  */
-extern size_t argp_fmtstream_point (argp_fmtstream_t __fs);
-extern size_t __argp_fmtstream_point (argp_fmtstream_t __fs);
-
-/* Internal routines.  */
-extern void _argp_fmtstream_update (argp_fmtstream_t __fs);
-extern void __argp_fmtstream_update (argp_fmtstream_t __fs);
-extern int _argp_fmtstream_ensure (argp_fmtstream_t __fs, size_t __amount);
-extern int __argp_fmtstream_ensure (argp_fmtstream_t __fs, size_t __amount);
-
-#ifdef __OPTIMIZE__
-/* Inline versions of above routines.  */
-
-#if !_LIBC
-#define __argp_fmtstream_putc argp_fmtstream_putc
-#define __argp_fmtstream_puts argp_fmtstream_puts
-#define __argp_fmtstream_write argp_fmtstream_write
-#define __argp_fmtstream_set_lmargin argp_fmtstream_set_lmargin
-#define __argp_fmtstream_set_rmargin argp_fmtstream_set_rmargin
-#define __argp_fmtstream_set_wmargin argp_fmtstream_set_wmargin
-#define __argp_fmtstream_point argp_fmtstream_point
-#define __argp_fmtstream_update _argp_fmtstream_update
-#define __argp_fmtstream_ensure _argp_fmtstream_ensure
-#endif
-
-#ifndef ARGP_FS_EI
-#define ARGP_FS_EI extern inline
-#endif
-
-ARGP_FS_EI size_t
-__argp_fmtstream_write (argp_fmtstream_t __fs,
-			__const char *__str, size_t __len)
-{
-  if (__fs->p + __len <= __fs->end || __argp_fmtstream_ensure (__fs, __len))
-    {
-      memcpy (__fs->p, __str, __len);
-      __fs->p += __len;
-      return __len;
-    }
-  else
-    return 0;
-}
-
-ARGP_FS_EI int
-__argp_fmtstream_puts (argp_fmtstream_t __fs, __const char *__str)
-{
-  size_t __len = strlen (__str);
-  if (__len)
-    {
-      size_t __wrote = __argp_fmtstream_write (__fs, __str, __len);
-      return __wrote == __len ? 0 : -1;
-    }
-  else
-    return 0;
-}
-
-ARGP_FS_EI int
-__argp_fmtstream_putc (argp_fmtstream_t __fs, int __ch)
-{
-  if (__fs->p < __fs->end || __argp_fmtstream_ensure (__fs, 1))
-    return *__fs->p++ = __ch;
-  else
-    return EOF;
-}
-
-/* Set __FS's left margin to __LMARGIN and return the old value.  */
-ARGP_FS_EI size_t
-__argp_fmtstream_set_lmargin (argp_fmtstream_t __fs, size_t __lmargin)
-{
-  size_t __old;
-  if ((size_t) (__fs->p - __fs->buf) > __fs->point_offs)
-    __argp_fmtstream_update (__fs);
-  __old = __fs->lmargin;
-  __fs->lmargin = __lmargin;
-  return __old;
-}
-
-/* Set __FS's right margin to __RMARGIN and return the old value.  */
-ARGP_FS_EI size_t
-__argp_fmtstream_set_rmargin (argp_fmtstream_t __fs, size_t __rmargin)
-{
-  size_t __old;
-  if ((size_t) (__fs->p - __fs->buf) > __fs->point_offs)
-    __argp_fmtstream_update (__fs);
-  __old = __fs->rmargin;
-  __fs->rmargin = __rmargin;
-  return __old;
-}
-
-/* Set FS's wrap margin to __WMARGIN and return the old value.  */
-ARGP_FS_EI size_t
-__argp_fmtstream_set_wmargin (argp_fmtstream_t __fs, size_t __wmargin)
-{
-  size_t __old;
-  if ((size_t) (__fs->p - __fs->buf) > __fs->point_offs)
-    __argp_fmtstream_update (__fs);
-  __old = __fs->wmargin;
-  __fs->wmargin = __wmargin;
-  return __old;
-}
-
-/* Return the column number of the current output point in __FS.  */
-ARGP_FS_EI size_t
-__argp_fmtstream_point (argp_fmtstream_t __fs)
-{
-  if ((size_t) (__fs->p - __fs->buf) > __fs->point_offs)
-    __argp_fmtstream_update (__fs);
-  return __fs->point_col >= 0 ? __fs->point_col : 0;
-}
-
-#if !_LIBC
-#undef __argp_fmtstream_putc
-#undef __argp_fmtstream_puts
-#undef __argp_fmtstream_write
-#undef __argp_fmtstream_set_lmargin
-#undef __argp_fmtstream_set_rmargin
-#undef __argp_fmtstream_set_wmargin
-#undef __argp_fmtstream_point
-#undef __argp_fmtstream_update
-#undef __argp_fmtstream_ensure
-#endif
-
-#endif /* __OPTIMIZE__ */
-
-#endif /* ARGP_FMTSTREAM_USE_LINEWRAP */
-
-#endif /* argp-fmtstream.h */
diff --git a/argp-standalone/argp-help.c b/argp-standalone/argp-help.c
deleted file mode 100644
index ced78c4cb26..00000000000
--- a/argp-standalone/argp-help.c
+++ /dev/null
@@ -1,1849 +0,0 @@
-/* Hierarchial argument parsing help output
-   Copyright (C) 1995,96,97,98,99,2000, 2003 Free Software Foundation, Inc.
-   This file is part of the GNU C Library.
-   Written by Miles Bader <miles@gnu.ai.mit.edu>.
-
-   The GNU C Library is free software; you can redistribute it and/or
-   modify it under the terms of the GNU Library General Public License as
-   published by the Free Software Foundation; either version 2 of the
-   License, or (at your option) any later version.
-
-   The GNU C Library is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Library General Public License for more details.
-
-   You should have received a copy of the GNU Library General Public
-   License along with the GNU C Library; see the file COPYING.LIB.  If not,
-   write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330,
-   Boston, MA 02111-1307, USA.  */
-
-#ifndef _GNU_SOURCE
-# define _GNU_SOURCE	1
-#endif
-
-#ifdef HAVE_CONFIG_H
-#include <config.h>
-#endif
-
-#if HAVE_ALLOCA_H
-#include <alloca.h>
-#endif
-
-#include <stddef.h>
-#include <stdlib.h>
-#include <string.h>
-#include <assert.h>
-#include <stdarg.h>
-#include <ctype.h>
-#if HAVE_MALLOC_H
-/* Needed, for alloca on windows */
-# include <malloc.h>
-#endif
-
-#ifndef _
-/* This is for other GNU distributions with internationalized messages.  */
-# if defined HAVE_LIBINTL_H || defined _LIBC
-#  include <libintl.h>
-#  ifdef _LIBC
-#   undef dgettext
-#   define dgettext(domain, msgid) __dcgettext (domain, msgid, LC_MESSAGES)
-#  endif
-# else
-#  define dgettext(domain, msgid) (msgid)
-# endif
-#endif
-
-#include "argp.h"
-#include "argp-fmtstream.h"
-#include "argp-namefrob.h"
-
-
-#ifndef _LIBC
-# ifndef __strchrnul
-#  define __strchrnul strchrnul
-# endif
-# ifndef __mempcpy
-#  define __mempcpy mempcpy
-# endif
-/* We need to use a different name, as __strndup is likely a macro. */
-# define STRNDUP strndup
-# if HAVE_STRERROR
-#  define STRERROR strerror
-# else
-#  define STRERROR(x) (sys_errlist[x])
-# endif
-#else /* _LIBC */
-# define FLOCKFILE __flockfile
-# define FUNLOCKFILE __funlockfile
-# define STRNDUP __strndup
-# define STRERROR strerror
-#endif
-
-#if !_LIBC
-# if !HAVE_STRNDUP
-char *strndup (const char *s, size_t size);
-# endif /* !HAVE_STRNDUP */
-
-# if !HAVE_MEMPCPY
-void *mempcpy (void *to, const void *from, size_t size);
-# endif /* !HAVE_MEMPCPY */
-
-# if !HAVE_STRCHRNUL
-char *strchrnul(const char *s, int c);
-# endif /* !HAVE_STRCHRNUL */
-
-# if !HAVE_STRCASECMP
-int strcasecmp(const char *s1, const char *s2);
-#endif
-
-#endif /* !_LIBC */
-
-
-/* User-selectable (using an environment variable) formatting parameters.
-
-   These may be specified in an environment variable called `ARGP_HELP_FMT',
-   with a contents like:  VAR1=VAL1,VAR2=VAL2,BOOLVAR2,no-BOOLVAR2
-   Where VALn must be a positive integer.  The list of variables is in the
-   UPARAM_NAMES vector, below.  */
-
-/* Default parameters.  */
-#define DUP_ARGS      0		/* True if option argument can be duplicated. */
-#define DUP_ARGS_NOTE 1		/* True to print a note about duplicate args. */
-#define SHORT_OPT_COL 2		/* column in which short options start */
-#define LONG_OPT_COL  6		/* column in which long options start */
-#define DOC_OPT_COL   2		/* column in which doc options start */
-#define OPT_DOC_COL  29		/* column in which option text starts */
-#define HEADER_COL    1		/* column in which group headers are printed */
-#define USAGE_INDENT 12		/* indentation of wrapped usage lines */
-#define RMARGIN      79		/* right margin used for wrapping */
-
-/* User-selectable (using an environment variable) formatting parameters.
-   They must all be of type `int' for the parsing code to work.  */
-struct uparams
-{
-  /* If true, arguments for an option are shown with both short and long
-     options, even when a given option has both, e.g. `-x ARG, --longx=ARG'.
-     If false, then if an option has both, the argument is only shown with
-     the long one, e.g., `-x, --longx=ARG', and a message indicating that
-     this really means both is printed below the options.  */
-  int dup_args;
-
-  /* This is true if when DUP_ARGS is false, and some duplicate arguments have
-     been suppressed, an explanatory message should be printed.  */
-  int dup_args_note;
-
-  /* Various output columns.  */
-  int short_opt_col;
-  int long_opt_col;
-  int doc_opt_col;
-  int opt_doc_col;
-  int header_col;
-  int usage_indent;
-  int rmargin;
-
-  int valid;			/* True when the values in here are valid.  */
-};
-
-/* This is a global variable, as user options are only ever read once.  */
-static struct uparams uparams = {
-  DUP_ARGS, DUP_ARGS_NOTE,
-  SHORT_OPT_COL, LONG_OPT_COL, DOC_OPT_COL, OPT_DOC_COL, HEADER_COL,
-  USAGE_INDENT, RMARGIN,
-  0
-};
-
-/* A particular uparam, and what the user name is.  */
-struct uparam_name
-{
-  const char *name;		/* User name.  */
-  int is_bool;			/* Whether it's `boolean'.  */
-  size_t uparams_offs;		/* Location of the (int) field in UPARAMS.  */
-};
-
-/* The name-field mappings we know about.  */
-static const struct uparam_name uparam_names[] =
-{
-  { "dup-args",       1, offsetof (struct uparams, dup_args) },
-  { "dup-args-note",  1, offsetof (struct uparams, dup_args_note) },
-  { "short-opt-col",  0, offsetof (struct uparams, short_opt_col) },
-  { "long-opt-col",   0, offsetof (struct uparams, long_opt_col) },
-  { "doc-opt-col",    0, offsetof (struct uparams, doc_opt_col) },
-  { "opt-doc-col",    0, offsetof (struct uparams, opt_doc_col) },
-  { "header-col",     0, offsetof (struct uparams, header_col) },
-  { "usage-indent",   0, offsetof (struct uparams, usage_indent) },
-  { "rmargin",        0, offsetof (struct uparams, rmargin) },
-  { 0, 0, 0 }
-};
-
-/* Read user options from the environment, and fill in UPARAMS appropiately.  */
-static void
-fill_in_uparams (const struct argp_state *state)
-{
-
-  const char *var = getenv ("ARGP_HELP_FMT");
-
-#define SKIPWS(p) do { while (isspace (*p)) p++; } while (0);
-
-  if (var)
-    /* Parse var. */
-    while (*var)
-      {
-	SKIPWS (var);
-
-	if (isalpha (*var))
-	  {
-	    size_t var_len;
-	    const struct uparam_name *un;
-	    int unspec = 0, val = 0;
-	    const char *arg = var;
-
-	    while (isalnum (*arg) || *arg == '-' || *arg == '_')
-	      arg++;
-	    var_len = arg - var;
-
-	    SKIPWS (arg);
-
-	    if (*arg == '\0' || *arg == ',')
-	      unspec = 1;
-	    else if (*arg == '=')
-	      {
-		arg++;
-		SKIPWS (arg);
-	      }
-
-	    if (unspec)
-	      {
-		if (var[0] == 'n' && var[1] == 'o' && var[2] == '-')
-		  {
-		    val = 0;
-		    var += 3;
-		    var_len -= 3;
-		  }
-		else
-		  val = 1;
-	      }
-	    else if (isdigit (*arg))
-	      {
-		val = atoi (arg);
-		while (isdigit (*arg))
-		  arg++;
-		SKIPWS (arg);
-	      }
-
-	    for (un = uparam_names; un->name; un++)
-	      if (strlen (un->name) == var_len
-		  && strncmp (var, un->name, var_len) == 0)
-		{
-		  if (unspec && !un->is_bool)
-		    __argp_failure (state, 0, 0,
-				    dgettext (state->root_argp->argp_domain, "\
-%.*s: ARGP_HELP_FMT parameter requires a value"),
-				    (int) var_len, var);
-		  else
-		    *(int *)((char *)&uparams + un->uparams_offs) = val;
-		  break;
-		}
-	    if (! un->name)
-	      __argp_failure (state, 0, 0,
-			      dgettext (state->root_argp->argp_domain, "\
-%.*s: Unknown ARGP_HELP_FMT parameter"),
-			      (int) var_len, var);
-
-	    var = arg;
-	    if (*var == ',')
-	      var++;
-	  }
-	else if (*var)
-	  {
-	    __argp_failure (state, 0, 0,
-			    dgettext (state->root_argp->argp_domain,
-				      "Garbage in ARGP_HELP_FMT: %s"), var);
-	    break;
-	  }
-      }
-}
-
-/* Returns true if OPT hasn't been marked invisible.  Visibility only affects
-   whether OPT is displayed or used in sorting, not option shadowing.  */
-#define ovisible(opt) (! ((opt)->flags & OPTION_HIDDEN))
-
-/* Returns true if OPT is an alias for an earlier option.  */
-#define oalias(opt) ((opt)->flags & OPTION_ALIAS)
-
-/* Returns true if OPT is an documentation-only entry.  */
-#define odoc(opt) ((opt)->flags & OPTION_DOC)
-
-/* Returns true if OPT is the end-of-list marker for a list of options.  */
-#define oend(opt) __option_is_end (opt)
-
-/* Returns true if OPT has a short option.  */
-#define oshort(opt) __option_is_short (opt)
-
-/*
-   The help format for a particular option is like:
-
-     -xARG, -yARG, --long1=ARG, --long2=ARG        Documentation...
-
-   Where ARG will be omitted if there's no argument, for this option, or
-   will be surrounded by "[" and "]" appropiately if the argument is
-   optional.  The documentation string is word-wrapped appropiately, and if
-   the list of options is long enough, it will be started on a separate line.
-   If there are no short options for a given option, the first long option is
-   indented slighly in a way that's supposed to make most long options appear
-   to be in a separate column.
-
-   For example, the following output (from ps):
-
-     -p PID, --pid=PID          List the process PID
-	 --pgrp=PGRP            List processes in the process group PGRP
-     -P, -x, --no-parent        Include processes without parents
-     -Q, --all-fields           Don't elide unusable fields (normally if there's
-				some reason ps can't print a field for any
-				process, it's removed from the output entirely)
-     -r, --reverse, --gratuitously-long-reverse-option
-				Reverse the order of any sort
-	 --session[=SID]        Add the processes from the session SID (which
-				defaults to the sid of the current process)
-
-    Here are some more options:
-     -f ZOT, --foonly=ZOT       Glork a foonly
-     -z, --zaza                 Snit a zar
-
-     -?, --help                 Give this help list
-	 --usage                Give a short usage message
-     -V, --version              Print program version
-
-   The struct argp_option array for the above could look like:
-
-   {
-     {"pid",       'p',      "PID",  0, "List the process PID"},
-     {"pgrp",      OPT_PGRP, "PGRP", 0, "List processes in the process group PGRP"},
-     {"no-parent", 'P',	      0,     0, "Include processes without parents"},
-     {0,           'x',       0,     OPTION_ALIAS},
-     {"all-fields",'Q',       0,     0, "Don't elide unusable fields (normally"
-                                        " if there's some reason ps can't"
-					" print a field for any process, it's"
-                                        " removed from the output entirely)" },
-     {"reverse",   'r',       0,     0, "Reverse the order of any sort"},
-     {"gratuitously-long-reverse-option", 0, 0, OPTION_ALIAS},
-     {"session",   OPT_SESS,  "SID", OPTION_ARG_OPTIONAL,
-                                        "Add the processes from the session"
-					" SID (which defaults to the sid of"
-					" the current process)" },
-
-     {0,0,0,0, "Here are some more options:"},
-     {"foonly", 'f', "ZOT", 0, "Glork a foonly"},
-     {"zaza", 'z', 0, 0, "Snit a zar"},
-
-     {0}
-   }
-
-   Note that the last three options are automatically supplied by argp_parse,
-   unless you tell it not to with ARGP_NO_HELP.
-
-*/
-
-/* Returns true if CH occurs between BEG and END.  */
-static int
-find_char (char ch, char *beg, char *end)
-{
-  while (beg < end)
-    if (*beg == ch)
-      return 1;
-    else
-      beg++;
-  return 0;
-}
-
-struct hol_cluster;		/* fwd decl */
-
-struct hol_entry
-{
-  /* First option.  */
-  const struct argp_option *opt;
-  /* Number of options (including aliases).  */
-  unsigned num;
-
-  /* A pointers into the HOL's short_options field, to the first short option
-     letter for this entry.  The order of the characters following this point
-     corresponds to the order of options pointed to by OPT, and there are at
-     most NUM.  A short option recorded in a option following OPT is only
-     valid if it occurs in the right place in SHORT_OPTIONS (otherwise it's
-     probably been shadowed by some other entry).  */
-  char *short_options;
-
-  /* Entries are sorted by their group first, in the order:
-       1, 2, ..., n, 0, -m, ..., -2, -1
-     and then alphabetically within each group.  The default is 0.  */
-  int group;
-
-  /* The cluster of options this entry belongs to, or 0 if none.  */
-  struct hol_cluster *cluster;
-
-  /* The argp from which this option came.  */
-  const struct argp *argp;
-};
-
-/* A cluster of entries to reflect the argp tree structure.  */
-struct hol_cluster
-{
-  /* A descriptive header printed before options in this cluster.  */
-  const char *header;
-
-  /* Used to order clusters within the same group with the same parent,
-     according to the order in which they occurred in the parent argp's child
-     list.  */
-  int index;
-
-  /* How to sort this cluster with respect to options and other clusters at the
-     same depth (clusters always follow options in the same group).  */
-  int group;
-
-  /* The cluster to which this cluster belongs, or 0 if it's at the base
-     level.  */
-  struct hol_cluster *parent;
-
-  /* The argp from which this cluster is (eventually) derived.  */
-  const struct argp *argp;
-
-  /* The distance this cluster is from the root.  */
-  int depth;
-
-  /* Clusters in a given hol are kept in a linked list, to make freeing them
-     possible.  */
-  struct hol_cluster *next;
-};
-
-/* A list of options for help.  */
-struct hol
-{
-  /* An array of hol_entry's.  */
-  struct hol_entry *entries;
-  /* The number of entries in this hol.  If this field is zero, the others
-     are undefined.  */
-  unsigned num_entries;
-
-  /* A string containing all short options in this HOL.  Each entry contains
-     pointers into this string, so the order can't be messed with blindly.  */
-  char *short_options;
-
-  /* Clusters of entries in this hol.  */
-  struct hol_cluster *clusters;
-};
-
-/* Create a struct hol from the options in ARGP.  CLUSTER is the
-   hol_cluster in which these entries occur, or 0, if at the root.  */
-static struct hol *
-make_hol (const struct argp *argp, struct hol_cluster *cluster)
-{
-  char *so;
-  const struct argp_option *o;
-  const struct argp_option *opts = argp->options;
-  struct hol_entry *entry;
-  unsigned num_short_options = 0;
-  struct hol *hol = malloc (sizeof (struct hol));
-
-  assert (hol);
-
-  hol->num_entries = 0;
-  hol->clusters = 0;
-
-  if (opts)
-    {
-      int cur_group = 0;
-
-      /* The first option must not be an alias.  */
-      assert (! oalias (opts));
-
-      /* Calculate the space needed.  */
-      for (o = opts; ! oend (o); o++)
-	{
-	  if (! oalias (o))
-	    hol->num_entries++;
-	  if (oshort (o))
-	    num_short_options++;	/* This is an upper bound.  */
-	}
-
-      hol->entries = malloc (sizeof (struct hol_entry) * hol->num_entries);
-      hol->short_options = malloc (num_short_options + 1);
-
-      assert (hol->entries && hol->short_options);
-
-      /* Fill in the entries.  */
-      so = hol->short_options;
-      for (o = opts, entry = hol->entries; ! oend (o); entry++)
-	{
-	  entry->opt = o;
-	  entry->num = 0;
-	  entry->short_options = so;
-	  entry->group = cur_group =
-	    o->group
-	    ? o->group
-	    : ((!o->name && !o->key)
-	       ? cur_group + 1
-	       : cur_group);
-	  entry->cluster = cluster;
-	  entry->argp = argp;
-
-	  do
-	    {
-	      entry->num++;
-	      if (oshort (o) && ! find_char (o->key, hol->short_options, so))
-		/* O has a valid short option which hasn't already been used.*/
-		*so++ = o->key;
-	      o++;
-	    }
-	  while (! oend (o) && oalias (o));
-	}
-      *so = '\0';		/* null terminated so we can find the length */
-    }
-
-  return hol;
-}
-
-/* Add a new cluster to HOL, with the given GROUP and HEADER (taken from the
-   associated argp child list entry), INDEX, and PARENT, and return a pointer
-   to it.  ARGP is the argp that this cluster results from.  */
-static struct hol_cluster *
-hol_add_cluster (struct hol *hol, int group, const char *header, int index,
-		 struct hol_cluster *parent, const struct argp *argp)
-{
-  struct hol_cluster *cl = malloc (sizeof (struct hol_cluster));
-  if (cl)
-    {
-      cl->group = group;
-      cl->header = header;
-
-      cl->index = index;
-      cl->parent = parent;
-      cl->argp = argp;
-      cl->depth = parent ? parent->depth + 1 : 0;
-
-      cl->next = hol->clusters;
-      hol->clusters = cl;
-    }
-  return cl;
-}
-
-/* Free HOL and any resources it uses.  */
-static void
-hol_free (struct hol *hol)
-{
-  struct hol_cluster *cl = hol->clusters;
-
-  while (cl)
-    {
-      struct hol_cluster *next = cl->next;
-      free (cl);
-      cl = next;
-    }
-
-  if (hol->num_entries > 0)
-    {
-      free (hol->entries);
-      free (hol->short_options);
-    }
-
-  free (hol);
-}
-
-static inline int
-hol_entry_short_iterate (const struct hol_entry *entry,
-			 int (*func)(const struct argp_option *opt,
-				     const struct argp_option *real,
-				     const char *domain, void *cookie),
-			 const char *domain, void *cookie)
-{
-  unsigned nopts;
-  int val = 0;
-  const struct argp_option *opt, *real = entry->opt;
-  char *so = entry->short_options;
-
-  for (opt = real, nopts = entry->num; nopts > 0 && !val; opt++, nopts--)
-    if (oshort (opt) && *so == opt->key)
-      {
-	if (!oalias (opt))
-	  real = opt;
-	if (ovisible (opt))
-	  val = (*func)(opt, real, domain, cookie);
-	so++;
-      }
-
-  return val;
-}
-
-static inline int
-hol_entry_long_iterate (const struct hol_entry *entry,
-			int (*func)(const struct argp_option *opt,
-				    const struct argp_option *real,
-				    const char *domain, void *cookie),
-			const char *domain, void *cookie)
-{
-  unsigned nopts;
-  int val = 0;
-  const struct argp_option *opt, *real = entry->opt;
-
-  for (opt = real, nopts = entry->num; nopts > 0 && !val; opt++, nopts--)
-    if (opt->name)
-      {
-	if (!oalias (opt))
-	  real = opt;
-	if (ovisible (opt))
-	  val = (*func)(opt, real, domain, cookie);
-      }
-
-  return val;
-}
-
-/* Iterator that returns true for the first short option.  */
-static inline int
-until_short (const struct argp_option *opt, const struct argp_option *real UNUSED,
-	     const char *domain UNUSED, void *cookie UNUSED)
-{
-  return oshort (opt) ? opt->key : 0;
-}
-
-/* Returns the first valid short option in ENTRY, or 0 if there is none.  */
-static char
-hol_entry_first_short (const struct hol_entry *entry)
-{
-  return hol_entry_short_iterate (entry, until_short,
-				  entry->argp->argp_domain, 0);
-}
-
-/* Returns the first valid long option in ENTRY, or 0 if there is none.  */
-static const char *
-hol_entry_first_long (const struct hol_entry *entry)
-{
-  const struct argp_option *opt;
-  unsigned num;
-  for (opt = entry->opt, num = entry->num; num > 0; opt++, num--)
-    if (opt->name && ovisible (opt))
-      return opt->name;
-  return 0;
-}
-
-/* Returns the entry in HOL with the long option name NAME, or 0 if there is
-   none.  */
-static struct hol_entry *
-hol_find_entry (struct hol *hol, const char *name)
-{
-  struct hol_entry *entry = hol->entries;
-  unsigned num_entries = hol->num_entries;
-
-  while (num_entries-- > 0)
-    {
-      const struct argp_option *opt = entry->opt;
-      unsigned num_opts = entry->num;
-
-      while (num_opts-- > 0)
-	if (opt->name && ovisible (opt) && strcmp (opt->name, name) == 0)
-	  return entry;
-	else
-	  opt++;
-
-      entry++;
-    }
-
-  return 0;
-}
-
-/* If an entry with the long option NAME occurs in HOL, set it's special
-   sort position to GROUP.  */
-static void
-hol_set_group (struct hol *hol, const char *name, int group)
-{
-  struct hol_entry *entry = hol_find_entry (hol, name);
-  if (entry)
-    entry->group = group;
-}
-
-/* Order by group:  0, 1, 2, ..., n, -m, ..., -2, -1.
-   EQ is what to return if GROUP1 and GROUP2 are the same.  */
-static int
-group_cmp (int group1, int group2, int eq)
-{
-  if (group1 == group2)
-    return eq;
-  else if ((group1 < 0 && group2 < 0) || (group1 >= 0 && group2 >= 0))
-    return group1 - group2;
-  else
-    return group2 - group1;
-}
-
-/* Compare clusters CL1 & CL2 by the order that they should appear in
-   output.  */
-static int
-hol_cluster_cmp (const struct hol_cluster *cl1, const struct hol_cluster *cl2)
-{
-  /* If one cluster is deeper than the other, use its ancestor at the same
-     level, so that finding the common ancestor is straightforward.  */
-  while (cl1->depth < cl2->depth)
-    cl1 = cl1->parent;
-  while (cl2->depth < cl1->depth)
-    cl2 = cl2->parent;
-
-  /* Now reduce both clusters to their ancestors at the point where both have
-     a common parent; these can be directly compared.  */
-  while (cl1->parent != cl2->parent)
-    cl1 = cl1->parent, cl2 = cl2->parent;
-
-  return group_cmp (cl1->group, cl2->group, cl2->index - cl1->index);
-}
-
-/* Return the ancestor of CL that's just below the root (i.e., has a parent
-   of 0).  */
-static struct hol_cluster *
-hol_cluster_base (struct hol_cluster *cl)
-{
-  while (cl->parent)
-    cl = cl->parent;
-  return cl;
-}
-
-/* Return true if CL1 is a child of CL2.  */
-static int
-hol_cluster_is_child (const struct hol_cluster *cl1,
-		      const struct hol_cluster *cl2)
-{
-  while (cl1 && cl1 != cl2)
-    cl1 = cl1->parent;
-  return cl1 == cl2;
-}
-
-/* Given the name of a OPTION_DOC option, modifies NAME to start at the tail
-   that should be used for comparisons, and returns true iff it should be
-   treated as a non-option.  */
-
-/* FIXME: Can we use unsigned char * for the argument? */
-static int
-canon_doc_option (const char **name)
-{
-  int non_opt;
-  /* Skip initial whitespace.  */
-  while (isspace ( (unsigned char) **name))
-    (*name)++;
-  /* Decide whether this looks like an option (leading `-') or not.  */
-  non_opt = (**name != '-');
-  /* Skip until part of name used for sorting.  */
-  while (**name && !isalnum ( (unsigned char) **name))
-    (*name)++;
-  return non_opt;
-}
-
-/* Order ENTRY1 & ENTRY2 by the order which they should appear in a help
-   listing.  */
-static int
-hol_entry_cmp (const struct hol_entry *entry1,
-	       const struct hol_entry *entry2)
-{
-  /* The group numbers by which the entries should be ordered; if either is
-     in a cluster, then this is just the group within the cluster.  */
-  int group1 = entry1->group, group2 = entry2->group;
-
-  if (entry1->cluster != entry2->cluster)
-    {
-      /* The entries are not within the same cluster, so we can't compare them
-	 directly, we have to use the appropiate clustering level too.  */
-      if (! entry1->cluster)
-	/* ENTRY1 is at the `base level', not in a cluster, so we have to
-	   compare it's group number with that of the base cluster in which
-	   ENTRY2 resides.  Note that if they're in the same group, the
-	   clustered option always comes laster.  */
-	return group_cmp (group1, hol_cluster_base (entry2->cluster)->group, -1);
-      else if (! entry2->cluster)
-	/* Likewise, but ENTRY2's not in a cluster.  */
-	return group_cmp (hol_cluster_base (entry1->cluster)->group, group2, 1);
-      else
-	/* Both entries are in clusters, we can just compare the clusters.  */
-	return hol_cluster_cmp (entry1->cluster, entry2->cluster);
-    }
-  else if (group1 == group2)
-    /* The entries are both in the same cluster and group, so compare them
-       alphabetically.  */
-    {
-      int short1 = hol_entry_first_short (entry1);
-      int short2 = hol_entry_first_short (entry2);
-      int doc1 = odoc (entry1->opt);
-      int doc2 = odoc (entry2->opt);
-      /* FIXME: Can we use unsigned char * instead? */
-      const char *long1 = hol_entry_first_long (entry1);
-      const char *long2 = hol_entry_first_long (entry2);
-
-      if (doc1)
-	doc1 = canon_doc_option (&long1);
-      if (doc2)
-	doc2 = canon_doc_option (&long2);
-
-      if (doc1 != doc2)
-	/* `documentation' options always follow normal options (or
-	   documentation options that *look* like normal options).  */
-	return doc1 - doc2;
-      else if (!short1 && !short2 && long1 && long2)
-	/* Only long options.  */
-	return __strcasecmp (long1, long2);
-      else
-	/* Compare short/short, long/short, short/long, using the first
-	   character of long options.  Entries without *any* valid
-	   options (such as options with OPTION_HIDDEN set) will be put
-	   first, but as they're not displayed, it doesn't matter where
-	   they are.  */
-	{
-	  unsigned char first1 = short1 ? short1 : long1 ? *long1 : 0;
-	  unsigned char first2 = short2 ? short2 : long2 ? *long2 : 0;
-#ifdef _tolower
-	  int lower_cmp = _tolower (first1) - _tolower (first2);
-#else
-	  int lower_cmp = tolower (first1) - tolower (first2);
-#endif
-	  /* Compare ignoring case, except when the options are both the
-	     same letter, in which case lower-case always comes first.  */
-	  /* NOTE: The subtraction below does the right thing
-	     even with eight-bit chars: first1 and first2 are
-	     converted to int *before* the subtraction. */
-	  return lower_cmp ? lower_cmp : first2 - first1;
-	}
-    }
-  else
-    /* Within the same cluster, but not the same group, so just compare
-       groups.  */
-    return group_cmp (group1, group2, 0);
-}
-
-/* Version of hol_entry_cmp with correct signature for qsort.  */
-static int
-hol_entry_qcmp (const void *entry1_v, const void *entry2_v)
-{
-  return hol_entry_cmp (entry1_v, entry2_v);
-}
-
-/* Sort HOL by group and alphabetically by option name (with short options
-   taking precedence over long).  Since the sorting is for display purposes
-   only, the shadowing of options isn't effected.  */
-static void
-hol_sort (struct hol *hol)
-{
-  if (hol->num_entries > 0)
-    qsort (hol->entries, hol->num_entries, sizeof (struct hol_entry),
-	   hol_entry_qcmp);
-}
-
-/* Append MORE to HOL, destroying MORE in the process.  Options in HOL shadow
-   any in MORE with the same name.  */
-static void
-hol_append (struct hol *hol, struct hol *more)
-{
-  struct hol_cluster **cl_end = &hol->clusters;
-
-  /* Steal MORE's cluster list, and add it to the end of HOL's.  */
-  while (*cl_end)
-    cl_end = &(*cl_end)->next;
-  *cl_end = more->clusters;
-  more->clusters = 0;
-
-  /* Merge entries.  */
-  if (more->num_entries > 0)
-    {
-      if (hol->num_entries == 0)
-	{
-	  hol->num_entries = more->num_entries;
-	  hol->entries = more->entries;
-	  hol->short_options = more->short_options;
-	  more->num_entries = 0;	/* Mark MORE's fields as invalid.  */
-	}
-      else
-	/* Append the entries in MORE to those in HOL, taking care to only add
-	   non-shadowed SHORT_OPTIONS values.  */
-	{
-	  unsigned left;
-	  char *so, *more_so;
-	  struct hol_entry *e;
-	  unsigned num_entries = hol->num_entries + more->num_entries;
-	  struct hol_entry *entries =
-	    malloc (num_entries * sizeof (struct hol_entry));
-	  unsigned hol_so_len = strlen (hol->short_options);
-	  char *short_options =
-	    malloc (hol_so_len + strlen (more->short_options) + 1);
-
-	  __mempcpy (__mempcpy (entries, hol->entries,
-				hol->num_entries * sizeof (struct hol_entry)),
-		     more->entries,
-		     more->num_entries * sizeof (struct hol_entry));
-
-	  __mempcpy (short_options, hol->short_options, hol_so_len);
-
-	  /* Fix up the short options pointers from HOL.  */
-	  for (e = entries, left = hol->num_entries; left > 0; e++, left--)
-	    e->short_options += (short_options - hol->short_options);
-
-	  /* Now add the short options from MORE, fixing up its entries
-	     too.  */
-	  so = short_options + hol_so_len;
-	  more_so = more->short_options;
-	  for (left = more->num_entries; left > 0; e++, left--)
-	    {
-	      int opts_left;
-	      const struct argp_option *opt;
-
-	      e->short_options = so;
-
-	      for (opts_left = e->num, opt = e->opt; opts_left; opt++, opts_left--)
-		{
-		  int ch = *more_so;
-		  if (oshort (opt) && ch == opt->key)
-		    /* The next short option in MORE_SO, CH, is from OPT.  */
-		    {
-		      if (! find_char (ch, short_options,
-				       short_options + hol_so_len))
-			/* The short option CH isn't shadowed by HOL's options,
-			   so add it to the sum.  */
-			*so++ = ch;
-		      more_so++;
-		    }
-		}
-	    }
-
-	  *so = '\0';
-
-	  free (hol->entries);
-	  free (hol->short_options);
-
-	  hol->entries = entries;
-	  hol->num_entries = num_entries;
-	  hol->short_options = short_options;
-	}
-    }
-
-  hol_free (more);
-}
-
-/* Inserts enough spaces to make sure STREAM is at column COL.  */
-static void
-indent_to (argp_fmtstream_t stream, unsigned col)
-{
-  int needed = col - __argp_fmtstream_point (stream);
-  while (needed-- > 0)
-    __argp_fmtstream_putc (stream, ' ');
-}
-
-/* Output to STREAM either a space, or a newline if there isn't room for at
-   least ENSURE characters before the right margin.  */
-static void
-space (argp_fmtstream_t stream, size_t ensure)
-{
-  if (__argp_fmtstream_point (stream) + ensure
-      >= __argp_fmtstream_rmargin (stream))
-    __argp_fmtstream_putc (stream, '\n');
-  else
-    __argp_fmtstream_putc (stream, ' ');
-}
-
-/* If the option REAL has an argument, we print it in using the printf
-   format REQ_FMT or OPT_FMT depending on whether it's a required or
-   optional argument.  */
-static void
-arg (const struct argp_option *real, const char *req_fmt, const char *opt_fmt,
-     const char *domain UNUSED, argp_fmtstream_t stream)
-{
-  if (real->arg)
-    {
-      if (real->flags & OPTION_ARG_OPTIONAL)
-	__argp_fmtstream_printf (stream, opt_fmt,
-				 dgettext (domain, real->arg));
-      else
-	__argp_fmtstream_printf (stream, req_fmt,
-				 dgettext (domain, real->arg));
-    }
-}
-
-/* Helper functions for hol_entry_help.  */
-
-/* State used during the execution of hol_help.  */
-struct hol_help_state
-{
-  /* PREV_ENTRY should contain the previous entry printed, or 0.  */
-  struct hol_entry *prev_entry;
-
-  /* If an entry is in a different group from the previous one, and SEP_GROUPS
-     is true, then a blank line will be printed before any output. */
-  int sep_groups;
-
-  /* True if a duplicate option argument was suppressed (only ever set if
-     UPARAMS.dup_args is false).  */
-  int suppressed_dup_arg;
-};
-
-/* Some state used while printing a help entry (used to communicate with
-   helper functions).  See the doc for hol_entry_help for more info, as most
-   of the fields are copied from its arguments.  */
-struct pentry_state
-{
-  const struct hol_entry *entry;
-  argp_fmtstream_t stream;
-  struct hol_help_state *hhstate;
-
-  /* True if nothing's been printed so far.  */
-  int first;
-
-  /* If non-zero, the state that was used to print this help.  */
-  const struct argp_state *state;
-};
-
-/* If a user doc filter should be applied to DOC, do so.  */
-static const char *
-filter_doc (const char *doc, int key, const struct argp *argp,
-	    const struct argp_state *state)
-{
-  if (argp->help_filter)
-    /* We must apply a user filter to this output.  */
-    {
-      void *input = __argp_input (argp, state);
-      return (*argp->help_filter) (key, doc, input);
-    }
-  else
-    /* No filter.  */
-    return doc;
-}
-
-/* Prints STR as a header line, with the margin lines set appropiately, and
-   notes the fact that groups should be separated with a blank line.  ARGP is
-   the argp that should dictate any user doc filtering to take place.  Note
-   that the previous wrap margin isn't restored, but the left margin is reset
-   to 0.  */
-static void
-print_header (const char *str, const struct argp *argp,
-	      struct pentry_state *pest)
-{
-  const char *tstr = dgettext (argp->argp_domain, str);
-  const char *fstr = filter_doc (tstr, ARGP_KEY_HELP_HEADER, argp, pest->state);
-
-  if (fstr)
-    {
-      if (*fstr)
-	{
-	  if (pest->hhstate->prev_entry)
-	    /* Precede with a blank line.  */
-	    __argp_fmtstream_putc (pest->stream, '\n');
-	  indent_to (pest->stream, uparams.header_col);
-	  __argp_fmtstream_set_lmargin (pest->stream, uparams.header_col);
-	  __argp_fmtstream_set_wmargin (pest->stream, uparams.header_col);
-	  __argp_fmtstream_puts (pest->stream, fstr);
-	  __argp_fmtstream_set_lmargin (pest->stream, 0);
-	  __argp_fmtstream_putc (pest->stream, '\n');
-	}
-
-      pest->hhstate->sep_groups = 1; /* Separate subsequent groups. */
-    }
-
-  if (fstr != tstr)
-    free ((char *) fstr);
-}
-
-/* Inserts a comma if this isn't the first item on the line, and then makes
-   sure we're at least to column COL.  If this *is* the first item on a line,
-   prints any pending whitespace/headers that should precede this line. Also
-   clears FIRST.  */
-static void
-comma (unsigned col, struct pentry_state *pest)
-{
-  if (pest->first)
-    {
-      const struct hol_entry *pe = pest->hhstate->prev_entry;
-      const struct hol_cluster *cl = pest->entry->cluster;
-
-      if (pest->hhstate->sep_groups && pe && pest->entry->group != pe->group)
-	__argp_fmtstream_putc (pest->stream, '\n');
-
-      if (cl && cl->header && *cl->header
-	  && (!pe
-	      || (pe->cluster != cl
-		  && !hol_cluster_is_child (pe->cluster, cl))))
-	/* If we're changing clusters, then this must be the start of the
-	   ENTRY's cluster unless that is an ancestor of the previous one
-	   (in which case we had just popped into a sub-cluster for a bit).
-	   If so, then print the cluster's header line.  */
-	{
-	  int old_wm = __argp_fmtstream_wmargin (pest->stream);
-	  print_header (cl->header, cl->argp, pest);
-	  __argp_fmtstream_set_wmargin (pest->stream, old_wm);
-	}
-
-      pest->first = 0;
-    }
-  else
-    __argp_fmtstream_puts (pest->stream, ", ");
-
-  indent_to (pest->stream, col);
-}
-
-/* Print help for ENTRY to STREAM.  */
-static void
-hol_entry_help (struct hol_entry *entry, const struct argp_state *state,
-		argp_fmtstream_t stream, struct hol_help_state *hhstate)
-{
-  unsigned num;
-  const struct argp_option *real = entry->opt, *opt;
-  char *so = entry->short_options;
-  int have_long_opt = 0;	/* We have any long options.  */
-  /* Saved margins.  */
-  int old_lm = __argp_fmtstream_set_lmargin (stream, 0);
-  int old_wm = __argp_fmtstream_wmargin (stream);
-  /* PEST is a state block holding some of our variables that we'd like to
-     share with helper functions.  */
-
-  /* Decent initializers are a GNU extension, so don't use it here. */
-  struct pentry_state pest;
-  pest.entry = entry;
-  pest.stream = stream;
-  pest.hhstate = hhstate;
-  pest.first = 1;
-  pest.state = state;
-
-  if (! odoc (real))
-    for (opt = real, num = entry->num; num > 0; opt++, num--)
-      if (opt->name && ovisible (opt))
-	{
-	  have_long_opt = 1;
-	  break;
-	}
-
-  /* First emit short options.  */
-  __argp_fmtstream_set_wmargin (stream, uparams.short_opt_col); /* For truly bizarre cases. */
-  for (opt = real, num = entry->num; num > 0; opt++, num--)
-    if (oshort (opt) && opt->key == *so)
-      /* OPT has a valid (non shadowed) short option.  */
-      {
-	if (ovisible (opt))
-	  {
-	    comma (uparams.short_opt_col, &pest);
-	    __argp_fmtstream_putc (stream, '-');
-	    __argp_fmtstream_putc (stream, *so);
-	    if (!have_long_opt || uparams.dup_args)
-	      arg (real, " %s", "[%s]", state->root_argp->argp_domain, stream);
-	    else if (real->arg)
-	      hhstate->suppressed_dup_arg = 1;
-	  }
-	so++;
-      }
-
-  /* Now, long options.  */
-  if (odoc (real))
-    /* A `documentation' option.  */
-    {
-      __argp_fmtstream_set_wmargin (stream, uparams.doc_opt_col);
-      for (opt = real, num = entry->num; num > 0; opt++, num--)
-	if (opt->name && ovisible (opt))
-	  {
-	    comma (uparams.doc_opt_col, &pest);
-	    /* Calling gettext here isn't quite right, since sorting will
-	       have been done on the original; but documentation options
-	       should be pretty rare anyway...  */
-	    __argp_fmtstream_puts (stream,
-				   dgettext (state->root_argp->argp_domain,
-					     opt->name));
-	  }
-    }
-  else
-    /* A real long option.  */
-    {
-      int first_long_opt = 1;
-
-      __argp_fmtstream_set_wmargin (stream, uparams.long_opt_col);
-      for (opt = real, num = entry->num; num > 0; opt++, num--)
-	if (opt->name && ovisible (opt))
-	  {
-	    comma (uparams.long_opt_col, &pest);
-	    __argp_fmtstream_printf (stream, "--%s", opt->name);
-	    if (first_long_opt || uparams.dup_args)
-	      arg (real, "=%s", "[=%s]", state->root_argp->argp_domain,
-		   stream);
-	    else if (real->arg)
-	      hhstate->suppressed_dup_arg = 1;
-	  }
-    }
-
-  /* Next, documentation strings.  */
-  __argp_fmtstream_set_lmargin (stream, 0);
-
-  if (pest.first)
-    {
-      /* Didn't print any switches, what's up?  */
-      if (!oshort (real) && !real->name)
-	/* This is a group header, print it nicely.  */
-	print_header (real->doc, entry->argp, &pest);
-      else
-	/* Just a totally shadowed option or null header; print nothing.  */
-	goto cleanup;		/* Just return, after cleaning up.  */
-    }
-  else
-    {
-      const char *tstr = real->doc ? dgettext (state->root_argp->argp_domain,
-					       real->doc) : 0;
-      const char *fstr = filter_doc (tstr, real->key, entry->argp, state);
-      if (fstr && *fstr)
-	{
-	  unsigned int col = __argp_fmtstream_point (stream);
-
-	  __argp_fmtstream_set_lmargin (stream, uparams.opt_doc_col);
-	  __argp_fmtstream_set_wmargin (stream, uparams.opt_doc_col);
-
-	  if (col > (unsigned int) (uparams.opt_doc_col + 3))
-	    __argp_fmtstream_putc (stream, '\n');
-	  else if (col >= (unsigned int) uparams.opt_doc_col)
-	    __argp_fmtstream_puts (stream, "   ");
-	  else
-	    indent_to (stream, uparams.opt_doc_col);
-
-	  __argp_fmtstream_puts (stream, fstr);
-	}
-      if (fstr && fstr != tstr)
-	free ((char *) fstr);
-
-      /* Reset the left margin.  */
-      __argp_fmtstream_set_lmargin (stream, 0);
-      __argp_fmtstream_putc (stream, '\n');
-    }
-
-  hhstate->prev_entry = entry;
-
-cleanup:
-  __argp_fmtstream_set_lmargin (stream, old_lm);
-  __argp_fmtstream_set_wmargin (stream, old_wm);
-}
-
-/* Output a long help message about the options in HOL to STREAM.  */
-static void
-hol_help (struct hol *hol, const struct argp_state *state,
-	  argp_fmtstream_t stream)
-{
-  unsigned num;
-  struct hol_entry *entry;
-  struct hol_help_state hhstate = { 0, 0, 0 };
-
-  for (entry = hol->entries, num = hol->num_entries; num > 0; entry++, num--)
-    hol_entry_help (entry, state, stream, &hhstate);
-
-  if (hhstate.suppressed_dup_arg && uparams.dup_args_note)
-    {
-      const char *tstr = dgettext (state->root_argp->argp_domain, "\
-Mandatory or optional arguments to long options are also mandatory or \
-optional for any corresponding short options.");
-      const char *fstr = filter_doc (tstr, ARGP_KEY_HELP_DUP_ARGS_NOTE,
-				     state ? state->root_argp : 0, state);
-      if (fstr && *fstr)
-	{
-	  __argp_fmtstream_putc (stream, '\n');
-	  __argp_fmtstream_puts (stream, fstr);
-	  __argp_fmtstream_putc (stream, '\n');
-	}
-      if (fstr && fstr != tstr)
-	free ((char *) fstr);
-    }
-}
-
-/* Helper functions for hol_usage.  */
-
-/* If OPT is a short option without an arg, append its key to the string
-   pointer pointer to by COOKIE, and advance the pointer.  */
-static int
-add_argless_short_opt (const struct argp_option *opt,
-		       const struct argp_option *real,
-		       const char *domain UNUSED, void *cookie)
-{
-  char **snao_end = cookie;
-  if (!(opt->arg || real->arg)
-      && !((opt->flags | real->flags) & OPTION_NO_USAGE))
-    *(*snao_end)++ = opt->key;
-  return 0;
-}
-
-/* If OPT is a short option with an arg, output a usage entry for it to the
-   stream pointed at by COOKIE.  */
-static int
-usage_argful_short_opt (const struct argp_option *opt,
-			const struct argp_option *real,
-			const char *domain UNUSED, void *cookie)
-{
-  argp_fmtstream_t stream = cookie;
-  const char *arg = opt->arg;
-  int flags = opt->flags | real->flags;
-
-  if (! arg)
-    arg = real->arg;
-
-  if (arg && !(flags & OPTION_NO_USAGE))
-    {
-      arg = dgettext (domain, arg);
-
-      if (flags & OPTION_ARG_OPTIONAL)
-	__argp_fmtstream_printf (stream, " [-%c[%s]]", opt->key, arg);
-      else
-	{
-	  /* Manually do line wrapping so that it (probably) won't
-	     get wrapped at the embedded space.  */
-	  space (stream, 6 + strlen (arg));
-	  __argp_fmtstream_printf (stream, "[-%c %s]", opt->key, arg);
-	}
-    }
-
-  return 0;
-}
-
-/* Output a usage entry for the long option opt to the stream pointed at by
-   COOKIE.  */
-static int
-usage_long_opt (const struct argp_option *opt,
-		const struct argp_option *real,
-		const char *domain UNUSED, void *cookie)
-{
-  argp_fmtstream_t stream = cookie;
-  const char *arg = opt->arg;
-  int flags = opt->flags | real->flags;
-
-  if (! arg)
-    arg = real->arg;
-
-  if (! (flags & OPTION_NO_USAGE))
-    {
-      if (arg)
-	{
-	  arg = dgettext (domain, arg);
-	  if (flags & OPTION_ARG_OPTIONAL)
-	    __argp_fmtstream_printf (stream, " [--%s[=%s]]", opt->name, arg);
-	  else
-	    __argp_fmtstream_printf (stream, " [--%s=%s]", opt->name, arg);
-	}
-      else
-	__argp_fmtstream_printf (stream, " [--%s]", opt->name);
-    }
-
-  return 0;
-}
-
-/* Print a short usage description for the arguments in HOL to STREAM.  */
-static void
-hol_usage (struct hol *hol, argp_fmtstream_t stream)
-{
-  if (hol->num_entries > 0)
-    {
-      unsigned nentries;
-      struct hol_entry *entry;
-      char *short_no_arg_opts = alloca (strlen (hol->short_options) + 1);
-      char *snao_end = short_no_arg_opts;
-
-      /* First we put a list of short options without arguments.  */
-      for (entry = hol->entries, nentries = hol->num_entries
-	   ; nentries > 0
-	   ; entry++, nentries--)
-	hol_entry_short_iterate (entry, add_argless_short_opt,
-				 entry->argp->argp_domain, &snao_end);
-      if (snao_end > short_no_arg_opts)
-	{
-	  *snao_end++ = 0;
-	  __argp_fmtstream_printf (stream, " [-%s]", short_no_arg_opts);
-	}
-
-      /* Now a list of short options *with* arguments.  */
-      for (entry = hol->entries, nentries = hol->num_entries
-	   ; nentries > 0
-	   ; entry++, nentries--)
-	hol_entry_short_iterate (entry, usage_argful_short_opt,
-				 entry->argp->argp_domain, stream);
-
-      /* Finally, a list of long options (whew!).  */
-      for (entry = hol->entries, nentries = hol->num_entries
-	   ; nentries > 0
-	   ; entry++, nentries--)
-	hol_entry_long_iterate (entry, usage_long_opt,
-				entry->argp->argp_domain, stream);
-    }
-}
-
-/* Make a HOL containing all levels of options in ARGP.  CLUSTER is the
-   cluster in which ARGP's entries should be clustered, or 0.  */
-static struct hol *
-argp_hol (const struct argp *argp, struct hol_cluster *cluster)
-{
-  const struct argp_child *child = argp->children;
-  struct hol *hol = make_hol (argp, cluster);
-  if (child)
-    while (child->argp)
-      {
-	struct hol_cluster *child_cluster =
-	  ((child->group || child->header)
-	   /* Put CHILD->argp within its own cluster.  */
-	   ? hol_add_cluster (hol, child->group, child->header,
-			      child - argp->children, cluster, argp)
-	   /* Just merge it into the parent's cluster.  */
-	   : cluster);
-	hol_append (hol, argp_hol (child->argp, child_cluster)) ;
-	child++;
-      }
-  return hol;
-}
-
-/* Calculate how many different levels with alternative args strings exist in
-   ARGP.  */
-static size_t
-argp_args_levels (const struct argp *argp)
-{
-  size_t levels = 0;
-  const struct argp_child *child = argp->children;
-
-  if (argp->args_doc && strchr (argp->args_doc, '\n'))
-    levels++;
-
-  if (child)
-    while (child->argp)
-      levels += argp_args_levels ((child++)->argp);
-
-  return levels;
-}
-
-/* Print all the non-option args documented in ARGP to STREAM.  Any output is
-   preceded by a space.  LEVELS is a pointer to a byte vector the length
-   returned by argp_args_levels; it should be initialized to zero, and
-   updated by this routine for the next call if ADVANCE is true.  True is
-   returned as long as there are more patterns to output.  */
-static int
-argp_args_usage (const struct argp *argp, const struct argp_state *state,
-		 char **levels, int advance, argp_fmtstream_t stream)
-{
-  char *our_level = *levels;
-  int multiple = 0;
-  const struct argp_child *child = argp->children;
-  const char *tdoc = dgettext (argp->argp_domain, argp->args_doc), *nl = 0;
-  const char *fdoc = filter_doc (tdoc, ARGP_KEY_HELP_ARGS_DOC, argp, state);
-
-  if (fdoc)
-    {
-      const char *cp = fdoc;
-      nl = __strchrnul (cp, '\n');
-      if (*nl != '\0')
-	/* This is a `multi-level' args doc; advance to the correct position
-	   as determined by our state in LEVELS, and update LEVELS.  */
-	{
-	  int i;
-	  multiple = 1;
-	  for (i = 0; i < *our_level; i++)
-	    cp = nl + 1, nl = __strchrnul (cp, '\n');
-	  (*levels)++;
-	}
-
-      /* Manually do line wrapping so that it (probably) won't get wrapped at
-	 any embedded spaces.  */
-      space (stream, 1 + nl - cp);
-
-      __argp_fmtstream_write (stream, cp, nl - cp);
-    }
-  if (fdoc && fdoc != tdoc)
-    free ((char *)fdoc);	/* Free user's modified doc string.  */
-
-  if (child)
-    while (child->argp)
-      advance = !argp_args_usage ((child++)->argp, state, levels, advance, stream);
-
-  if (advance && multiple)
-    {
-      /* Need to increment our level.  */
-      if (*nl)
-	/* There's more we can do here.  */
-	{
-	  (*our_level)++;
-	  advance = 0;		/* Our parent shouldn't advance also. */
-	}
-      else if (*our_level > 0)
-	/* We had multiple levels, but used them up; reset to zero.  */
-	*our_level = 0;
-    }
-
-  return !advance;
-}
-
-/* Print the documentation for ARGP to STREAM; if POST is false, then
-   everything preceeding a `\v' character in the documentation strings (or
-   the whole string, for those with none) is printed, otherwise, everything
-   following the `\v' character (nothing for strings without).  Each separate
-   bit of documentation is separated a blank line, and if PRE_BLANK is true,
-   then the first is as well.  If FIRST_ONLY is true, only the first
-   occurrence is output.  Returns true if anything was output.  */
-static int
-argp_doc (const struct argp *argp, const struct argp_state *state,
-	  int post, int pre_blank, int first_only,
-	  argp_fmtstream_t stream)
-{
-  const char *text;
-  const char *inp_text;
-  void *input = 0;
-  int anything = 0;
-  size_t inp_text_limit = 0;
-  const char *doc = dgettext (argp->argp_domain, argp->doc);
-  const struct argp_child *child = argp->children;
-
-  if (doc)
-    {
-      char *vt = strchr (doc, '\v');
-      inp_text = post ? (vt ? vt + 1 : 0) : doc;
-      inp_text_limit = (!post && vt) ? (vt - doc) : 0;
-    }
-  else
-    inp_text = 0;
-
-  if (argp->help_filter)
-    /* We have to filter the doc strings.  */
-    {
-      if (inp_text_limit)
-	/* Copy INP_TEXT so that it's nul-terminated.  */
-	inp_text = STRNDUP (inp_text, inp_text_limit);
-      input = __argp_input (argp, state);
-      text =
-	(*argp->help_filter) (post
-			      ? ARGP_KEY_HELP_POST_DOC
-			      : ARGP_KEY_HELP_PRE_DOC,
-			      inp_text, input);
-    }
-  else
-    text = (const char *) inp_text;
-
-  if (text)
-    {
-      if (pre_blank)
-	__argp_fmtstream_putc (stream, '\n');
-
-      if (text == inp_text && inp_text_limit)
-	__argp_fmtstream_write (stream, inp_text, inp_text_limit);
-      else
-	__argp_fmtstream_puts (stream, text);
-
-      if (__argp_fmtstream_point (stream) > __argp_fmtstream_lmargin (stream))
-	__argp_fmtstream_putc (stream, '\n');
-
-      anything = 1;
-    }
-
-  if (text && text != inp_text)
-    free ((char *) text);	/* Free TEXT returned from the help filter.  */
-  if (inp_text && inp_text_limit && argp->help_filter)
-    free ((char *) inp_text);	/* We copied INP_TEXT, so free it now.  */
-
-  if (post && argp->help_filter)
-    /* Now see if we have to output a ARGP_KEY_HELP_EXTRA text.  */
-    {
-      text = (*argp->help_filter) (ARGP_KEY_HELP_EXTRA, 0, input);
-      if (text)
-	{
-	  if (anything || pre_blank)
-	    __argp_fmtstream_putc (stream, '\n');
-	  __argp_fmtstream_puts (stream, text);
-	  free ((char *) text);
-	  if (__argp_fmtstream_point (stream)
-	      > __argp_fmtstream_lmargin (stream))
-	    __argp_fmtstream_putc (stream, '\n');
-	  anything = 1;
-	}
-    }
-
-  if (child)
-    while (child->argp && !(first_only && anything))
-      anything |=
-	argp_doc ((child++)->argp, state,
-		  post, anything || pre_blank, first_only,
-		  stream);
-
-  return anything;
-}
-
-/* Output a usage message for ARGP to STREAM.  If called from
-   argp_state_help, STATE is the relevent parsing state.  FLAGS are from the
-   set ARGP_HELP_*.  NAME is what to use wherever a `program name' is
-   needed. */
-
-static void
-_help (const struct argp *argp, const struct argp_state *state, FILE *stream,
-       unsigned flags, const char *name)
-{
-  int anything = 0;		/* Whether we've output anything.  */
-  struct hol *hol = 0;
-  argp_fmtstream_t fs;
-
-  if (! stream)
-    return;
-
-  FLOCKFILE (stream);
-
-  if (! uparams.valid)
-    fill_in_uparams (state);
-
-  fs = __argp_make_fmtstream (stream, 0, uparams.rmargin, 0);
-  if (! fs)
-    {
-      FUNLOCKFILE (stream);
-      return;
-    }
-
-  if (flags & (ARGP_HELP_USAGE | ARGP_HELP_SHORT_USAGE | ARGP_HELP_LONG))
-    {
-      hol = argp_hol (argp, 0);
-
-      /* If present, these options always come last.  */
-      hol_set_group (hol, "help", -1);
-      hol_set_group (hol, "version", -1);
-
-      hol_sort (hol);
-    }
-
-  if (flags & (ARGP_HELP_USAGE | ARGP_HELP_SHORT_USAGE))
-    /* Print a short `Usage:' message.  */
-    {
-      int first_pattern = 1, more_patterns;
-      size_t num_pattern_levels = argp_args_levels (argp);
-      char *pattern_levels = alloca (num_pattern_levels);
-
-      memset (pattern_levels, 0, num_pattern_levels);
-
-      do
-	{
-	  int old_lm;
-	  int old_wm = __argp_fmtstream_set_wmargin (fs, uparams.usage_indent);
-	  char *levels = pattern_levels;
-
-	  if (first_pattern)
-	    __argp_fmtstream_printf (fs, "%s %s",
-				     dgettext (argp->argp_domain, "Usage:"),
-				     name);
-	  else
-	    __argp_fmtstream_printf (fs, "%s %s",
-				     dgettext (argp->argp_domain, "  or: "),
-				     name);
-
-	  /* We set the lmargin as well as the wmargin, because hol_usage
-	     manually wraps options with newline to avoid annoying breaks.  */
-	  old_lm = __argp_fmtstream_set_lmargin (fs, uparams.usage_indent);
-
-	  if (flags & ARGP_HELP_SHORT_USAGE)
-	    /* Just show where the options go.  */
-	    {
-	      if (hol->num_entries > 0)
-		__argp_fmtstream_puts (fs, dgettext (argp->argp_domain,
-						     " [OPTION...]"));
-	    }
-	  else
-	    /* Actually print the options.  */
-	    {
-	      hol_usage (hol, fs);
-	      flags |= ARGP_HELP_SHORT_USAGE; /* But only do so once.  */
-	    }
-
-	  more_patterns = argp_args_usage (argp, state, &levels, 1, fs);
-
-	  __argp_fmtstream_set_wmargin (fs, old_wm);
-	  __argp_fmtstream_set_lmargin (fs, old_lm);
-
-	  __argp_fmtstream_putc (fs, '\n');
-	  anything = 1;
-
-	  first_pattern = 0;
-	}
-      while (more_patterns);
-    }
-
-  if (flags & ARGP_HELP_PRE_DOC)
-    anything |= argp_doc (argp, state, 0, 0, 1, fs);
-
-  if (flags & ARGP_HELP_SEE)
-    {
-      __argp_fmtstream_printf (fs, dgettext (argp->argp_domain, "\
-Try `%s --help' or `%s --usage' for more information.\n"),
-			       name, name);
-      anything = 1;
-    }
-
-  if (flags & ARGP_HELP_LONG)
-    /* Print a long, detailed help message.  */
-    {
-      /* Print info about all the options.  */
-      if (hol->num_entries > 0)
-	{
-	  if (anything)
-	    __argp_fmtstream_putc (fs, '\n');
-	  hol_help (hol, state, fs);
-	  anything = 1;
-	}
-    }
-
-  if (flags & ARGP_HELP_POST_DOC)
-    /* Print any documentation strings at the end.  */
-    anything |= argp_doc (argp, state, 1, anything, 0, fs);
-
-  if ((flags & ARGP_HELP_BUG_ADDR) && argp_program_bug_address)
-    {
-      if (anything)
-	__argp_fmtstream_putc (fs, '\n');
-      __argp_fmtstream_printf (fs, dgettext (argp->argp_domain,
-					     "Report bugs to %s.\n"),
- 			       argp_program_bug_address);
-      anything = 1;
-    }
-
-  FUNLOCKFILE (stream);
-
-  if (hol)
-    hol_free (hol);
-
-  __argp_fmtstream_free (fs);
-}
-
-/* Output a usage message for ARGP to STREAM.  FLAGS are from the set
-   ARGP_HELP_*.  NAME is what to use wherever a `program name' is needed. */
-void __argp_help (const struct argp *argp, FILE *stream,
-		  unsigned flags, char *name)
-{
-  _help (argp, 0, stream, flags, name);
-}
-#ifdef weak_alias
-weak_alias (__argp_help, argp_help)
-#endif
-
-char *__argp_basename(char *name)
-{
-  char *short_name = strrchr(name, '/');
-  return short_name ? short_name + 1 : name;
-}
-
-char *
-__argp_short_program_name(const struct argp_state *state)
-{
-  if (state)
-    return state->name;
-#if HAVE_DECL_PROGRAM_INVOCATION_SHORT_NAME
-  return program_invocation_short_name;
-#elif HAVE_DECL_PROGRAM_INVOCATION_NAME
-  return __argp_basename(program_invocation_name);
-#else /* !HAVE_DECL_PROGRAM_INVOCATION_NAME */
-  /* FIXME: What now? Miles suggests that it is better to use NULL,
-     but currently the value is passed on directly to fputs_unlocked,
-     so that requires more changes. */
-# if __GNUC__
-  return "";
-# endif /* __GNUC__ */
-#endif /* !HAVE_DECL_PROGRAM_INVOCATION_NAME */
-}
-
-/* Output, if appropriate, a usage message for STATE to STREAM.  FLAGS are
-   from the set ARGP_HELP_*.  */
-void
-__argp_state_help (const struct argp_state *state, FILE *stream, unsigned flags)
-{
-  if ((!state || ! (state->flags & ARGP_NO_ERRS)) && stream)
-    {
-      if (state && (state->flags & ARGP_LONG_ONLY))
-	flags |= ARGP_HELP_LONG_ONLY;
-
-      _help (state ? state->root_argp : 0, state, stream, flags,
-	     __argp_short_program_name(state));
-
-      if (!state || ! (state->flags & ARGP_NO_EXIT))
-	{
-	  if (flags & ARGP_HELP_EXIT_ERR)
-	    exit (argp_err_exit_status);
-	  if (flags & ARGP_HELP_EXIT_OK)
-	    exit (0);
-	}
-  }
-}
-#ifdef weak_alias
-weak_alias (__argp_state_help, argp_state_help)
-#endif
-
-/* If appropriate, print the printf string FMT and following args, preceded
-   by the program name and `:', to stderr, and followed by a `Try ... --help'
-   message, then exit (1).  */
-void
-__argp_error (const struct argp_state *state, const char *fmt, ...)
-{
-  if (!state || !(state->flags & ARGP_NO_ERRS))
-    {
-      FILE *stream = state ? state->err_stream : stderr;
-
-      if (stream)
-	{
-	  va_list ap;
-
-	  FLOCKFILE (stream);
-
-	  FPUTS_UNLOCKED (__argp_short_program_name(state),
-			  stream);
-	  PUTC_UNLOCKED (':', stream);
-	  PUTC_UNLOCKED (' ', stream);
-
-	  va_start (ap, fmt);
-	  vfprintf (stream, fmt, ap);
-	  va_end (ap);
-
-	  PUTC_UNLOCKED ('\n', stream);
-
-	  __argp_state_help (state, stream, ARGP_HELP_STD_ERR);
-
-	  FUNLOCKFILE (stream);
-	}
-    }
-}
-#ifdef weak_alias
-weak_alias (__argp_error, argp_error)
-#endif
-
-/* Similar to the standard gnu error-reporting function error(), but will
-   respect the ARGP_NO_EXIT and ARGP_NO_ERRS flags in STATE, and will print
-   to STATE->err_stream.  This is useful for argument parsing code that is
-   shared between program startup (when exiting is desired) and runtime
-   option parsing (when typically an error code is returned instead).  The
-   difference between this function and argp_error is that the latter is for
-   *parsing errors*, and the former is for other problems that occur during
-   parsing but don't reflect a (syntactic) problem with the input.  */
-void
-__argp_failure (const struct argp_state *state, int status, int errnum,
-		const char *fmt, ...)
-{
-  if (!state || !(state->flags & ARGP_NO_ERRS))
-    {
-      FILE *stream = state ? state->err_stream : stderr;
-
-      if (stream)
-	{
-	  FLOCKFILE (stream);
-
-	  FPUTS_UNLOCKED (__argp_short_program_name(state),
-			  stream);
-
-	  if (fmt)
-	    {
-	      va_list ap;
-
-	      PUTC_UNLOCKED (':', stream);
-	      PUTC_UNLOCKED (' ', stream);
-
-	      va_start (ap, fmt);
-	      vfprintf (stream, fmt, ap);
-	      va_end (ap);
-	    }
-
-	  if (errnum)
-	    {
-	      PUTC_UNLOCKED (':', stream);
-	      PUTC_UNLOCKED (' ', stream);
-	      fputs (STRERROR (errnum), stream);
-	    }
-
-	  PUTC_UNLOCKED ('\n', stream);
-
-	  FUNLOCKFILE (stream);
-
-	  if (status && (!state || !(state->flags & ARGP_NO_EXIT)))
-	    exit (status);
-	}
-    }
-}
-#ifdef weak_alias
-weak_alias (__argp_failure, argp_failure)
-#endif
diff --git a/argp-standalone/argp-namefrob.h b/argp-standalone/argp-namefrob.h
deleted file mode 100644
index 0ce11481a7b..00000000000
--- a/argp-standalone/argp-namefrob.h
+++ /dev/null
@@ -1,96 +0,0 @@
-/* Name frobnication for compiling argp outside of glibc
-   Copyright (C) 1997 Free Software Foundation, Inc.
-   This file is part of the GNU C Library.
-   Written by Miles Bader <miles@gnu.ai.mit.edu>.
-
-   The GNU C Library is free software; you can redistribute it and/or
-   modify it under the terms of the GNU Library General Public License as
-   published by the Free Software Foundation; either version 2 of the
-   License, or (at your option) any later version.
-
-   The GNU C Library is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Library General Public License for more details.
-
-   You should have received a copy of the GNU Library General Public
-   License along with the GNU C Library; see the file COPYING.LIB.  If not,
-   write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330,
-   Boston, MA 02111-1307, USA.  */
-
-#if !_LIBC
-/* This code is written for inclusion in gnu-libc, and uses names in the
-   namespace reserved for libc.  If we're not compiling in libc, define those
-   names to be the normal ones instead.  */
-
-/* argp-parse functions */
-#undef __argp_parse
-#define __argp_parse argp_parse
-#undef __option_is_end
-#define __option_is_end _option_is_end
-#undef __option_is_short
-#define __option_is_short _option_is_short
-#undef __argp_input
-#define __argp_input _argp_input
-
-/* argp-help functions */
-#undef __argp_help
-#define __argp_help argp_help
-#undef __argp_error
-#define __argp_error argp_error
-#undef __argp_failure
-#define __argp_failure argp_failure
-#undef __argp_state_help
-#define __argp_state_help argp_state_help
-#undef __argp_usage
-#define __argp_usage argp_usage
-#undef __argp_basename
-#define __argp_basename _argp_basename
-#undef __argp_short_program_name
-#define __argp_short_program_name _argp_short_program_name
-
-/* argp-fmtstream functions */
-#undef __argp_make_fmtstream
-#define __argp_make_fmtstream argp_make_fmtstream
-#undef __argp_fmtstream_free
-#define __argp_fmtstream_free argp_fmtstream_free
-#undef __argp_fmtstream_putc
-#define __argp_fmtstream_putc argp_fmtstream_putc
-#undef __argp_fmtstream_puts
-#define __argp_fmtstream_puts argp_fmtstream_puts
-#undef __argp_fmtstream_write
-#define __argp_fmtstream_write argp_fmtstream_write
-#undef __argp_fmtstream_printf
-#define __argp_fmtstream_printf argp_fmtstream_printf
-#undef __argp_fmtstream_set_lmargin
-#define __argp_fmtstream_set_lmargin argp_fmtstream_set_lmargin
-#undef __argp_fmtstream_set_rmargin
-#define __argp_fmtstream_set_rmargin argp_fmtstream_set_rmargin
-#undef __argp_fmtstream_set_wmargin
-#define __argp_fmtstream_set_wmargin argp_fmtstream_set_wmargin
-#undef __argp_fmtstream_point
-#define __argp_fmtstream_point argp_fmtstream_point
-#undef __argp_fmtstream_update
-#define __argp_fmtstream_update _argp_fmtstream_update
-#undef __argp_fmtstream_ensure
-#define __argp_fmtstream_ensure _argp_fmtstream_ensure
-#undef __argp_fmtstream_lmargin
-#define __argp_fmtstream_lmargin argp_fmtstream_lmargin
-#undef __argp_fmtstream_rmargin
-#define __argp_fmtstream_rmargin argp_fmtstream_rmargin
-#undef __argp_fmtstream_wmargin
-#define __argp_fmtstream_wmargin argp_fmtstream_wmargin
-
-/* normal libc functions we call */
-#undef __sleep
-#define __sleep sleep
-#undef __strcasecmp
-#define __strcasecmp strcasecmp
-#undef __vsnprintf
-#define __vsnprintf vsnprintf
-
-#endif /* !_LIBC */
-
-#ifndef __set_errno
-#define __set_errno(e) (errno = (e))
-#endif
diff --git a/argp-standalone/argp-parse.c b/argp-standalone/argp-parse.c
deleted file mode 100644
index 78f7bf139b6..00000000000
--- a/argp-standalone/argp-parse.c
+++ /dev/null
@@ -1,1305 +0,0 @@
-/* Hierarchial argument parsing
-   Copyright (C) 1995, 96, 97, 98, 99, 2000,2003 Free Software Foundation, Inc.
-   This file is part of the GNU C Library.
-   Written by Miles Bader <miles@gnu.ai.mit.edu>.
-
-   The GNU C Library is free software; you can redistribute it and/or
-   modify it under the terms of the GNU Library General Public License as
-   published by the Free Software Foundation; either version 2 of the
-   License, or (at your option) any later version.
-
-   The GNU C Library is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Library General Public License for more details.
-
-   You should have received a copy of the GNU Library General Public
-   License along with the GNU C Library; see the file COPYING.LIB.  If not,
-   write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330,
-   Boston, MA 02111-1307, USA.  */
-
-#ifndef _GNU_SOURCE
-# define _GNU_SOURCE	1
-#endif
-
-#ifdef HAVE_CONFIG_H
-#include <config.h>
-#endif
-
-#if HAVE_ALLOCA_H
-#include <alloca.h>
-#endif
-
-#include <stdlib.h>
-#include <string.h>
-#if HAVE_UNISTD_H
-# include <unistd.h>
-#endif
-#include <limits.h>
-#include <assert.h>
-
-#if HAVE_MALLOC_H
-/* Needed, for alloca on windows */
-# include <malloc.h>
-#endif
-
-#ifndef _
-/* This is for other GNU distributions with internationalized messages.
-   When compiling libc, the _ macro is predefined.  */
-# if defined HAVE_LIBINTL_H || defined _LIBC
-#  include <libintl.h>
-#  ifdef _LIBC
-#   undef dgettext
-#   define dgettext(domain, msgid) __dcgettext (domain, msgid, LC_MESSAGES)
-#  endif
-# else
-#  define dgettext(domain, msgid) (msgid)
-#  define gettext(msgid) (msgid)
-# endif
-#endif
-#ifndef N_
-# define N_(msgid) (msgid)
-#endif
-
-#if _LIBC - 0
-#include <bits/libc-lock.h>
-#else
-#ifdef HAVE_CTHREADS_H
-#include <cthreads.h>
-#endif
-#endif /* _LIBC */
-
-#include "argp.h"
-#include "argp-namefrob.h"
-
-
-/* The meta-argument used to prevent any further arguments being interpreted
-   as options.  */
-#define QUOTE "--"
-
-/* EZ alias for ARGP_ERR_UNKNOWN.  */
-#define EBADKEY ARGP_ERR_UNKNOWN
-
-
-/* Default options.  */
-
-/* When argp is given the --HANG switch, _ARGP_HANG is set and argp will sleep
-   for one second intervals, decrementing _ARGP_HANG until it's zero.  Thus
-   you can force the program to continue by attaching a debugger and setting
-   it to 0 yourself.  */
-volatile int _argp_hang;
-
-#define OPT_PROGNAME	-2
-#define OPT_USAGE	-3
-#if HAVE_SLEEP && HAVE_GETPID
-#define OPT_HANG	-4
-#endif
-
-static const struct argp_option argp_default_options[] =
-{
-  {"help",	  '?',    	0, 0,  N_("Give this help list"), -1},
-  {"usage",	  OPT_USAGE,	0, 0,  N_("Give a short usage message"), 0 },
-  {"program-name",OPT_PROGNAME,"NAME", OPTION_HIDDEN,
-     N_("Set the program name"), 0},
-#if OPT_HANG
-  {"HANG",	  OPT_HANG,    "SECS", OPTION_ARG_OPTIONAL | OPTION_HIDDEN,
-     N_("Hang for SECS seconds (default 3600)"), 0 },
-#endif
-  {0, 0, 0, 0, 0, 0}
-};
-
-static error_t
-argp_default_parser (int key, char *arg, struct argp_state *state)
-{
-  switch (key)
-    {
-    case '?':
-      __argp_state_help (state, state->out_stream, ARGP_HELP_STD_HELP);
-      break;
-    case OPT_USAGE:
-      __argp_state_help (state, state->out_stream,
-		       ARGP_HELP_USAGE | ARGP_HELP_EXIT_OK);
-      break;
-
-    case OPT_PROGNAME:		/* Set the program name.  */
-#if HAVE_DECL_PROGRAM_INVOCATION_NAME
-      program_invocation_name = arg;
-#endif
-      /* [Note that some systems only have PROGRAM_INVOCATION_SHORT_NAME (aka
-	 __PROGNAME), in which case, PROGRAM_INVOCATION_NAME is just defined
-	 to be that, so we have to be a bit careful here.]  */
-
-      /* Update what we use for messages.  */
-
-      state->name = __argp_basename(arg);
-      
-#if HAVE_DECL_PROGRAM_INVOCATION_SHORT_NAME
-      program_invocation_short_name = state->name;
-#endif
-
-      if ((state->flags & (ARGP_PARSE_ARGV0 | ARGP_NO_ERRS))
-	  == ARGP_PARSE_ARGV0)
-	/* Update what getopt uses too.  */
-	state->argv[0] = arg;
-
-      break;
-      
-#if OPT_HANG
-    case OPT_HANG:
-      _argp_hang = atoi (arg ? arg : "3600");
-      fprintf(state->err_stream, "%s: pid = %ld\n",
-	      state->name, (long) getpid());
-      while (_argp_hang-- > 0)
-	__sleep (1);
-      break;
-#endif
-      
-    default:
-      return EBADKEY;
-    }
-  return 0;
-}
-
-static const struct argp argp_default_argp =
-  {argp_default_options, &argp_default_parser, NULL, NULL, NULL, NULL, "libc"};
-
-
-static const struct argp_option argp_version_options[] =
-{
-  {"version",	  'V',    	0, 0,  N_("Print program version"), -1},
-  {0, 0, 0, 0, 0, 0 }
-};
-
-static error_t
-argp_version_parser (int key, char *arg UNUSED, struct argp_state *state)
-{
-  switch (key)
-    {
-    case 'V':
-      if (argp_program_version_hook)
-	(*argp_program_version_hook) (state->out_stream, state);
-      else if (argp_program_version)
-	fprintf (state->out_stream, "%s\n", argp_program_version);
-      else
-	__argp_error (state, dgettext (state->root_argp->argp_domain,
-				       "(PROGRAM ERROR) No version known!?"));
-      if (! (state->flags & ARGP_NO_EXIT))
-	exit (0);
-      break;
-    default:
-      return EBADKEY;
-    }
-  return 0;
-}
-
-static const struct argp argp_version_argp =
-  {argp_version_options, &argp_version_parser, NULL, NULL, NULL, NULL, "libc"};
-
-
-
-/* The state of a `group' during parsing.  Each group corresponds to a
-   particular argp structure from the tree of such descending from the top
-   level argp passed to argp_parse.  */
-struct group
-{
-  /* This group's parsing function.  */
-  argp_parser_t parser;
-
-  /* Which argp this group is from.  */
-  const struct argp *argp;
-
-  /* The number of non-option args sucessfully handled by this parser.  */
-  unsigned args_processed;
-
-  /* This group's parser's parent's group.  */
-  struct group *parent;
-  unsigned parent_index;	/* And the our position in the parent.   */
-
-  /* These fields are swapped into and out of the state structure when
-     calling this group's parser.  */
-  void *input, **child_inputs;
-  void *hook;
-};
-
-/* Call GROUP's parser with KEY and ARG, swapping any group-specific info
-   from STATE before calling, and back into state afterwards.  If GROUP has
-   no parser, EBADKEY is returned.  */
-static error_t
-group_parse (struct group *group, struct argp_state *state, int key, char *arg)
-{
-  if (group->parser)
-    {
-      error_t err;
-      state->hook = group->hook;
-      state->input = group->input;
-      state->child_inputs = group->child_inputs;
-      state->arg_num = group->args_processed;
-      err = (*group->parser)(key, arg, state);
-      group->hook = state->hook;
-      return err;
-    }
-  else
-    return EBADKEY;
-}
-
-struct parser
-{
-  const struct argp *argp;
-
-  const char *posixly_correct;
-  
-  /* True if there are only no-option arguments left, which are just
-     passed verbatim with ARGP_KEY_ARG. This is set if we encounter a
-     quote, or the end of the proper options, but may be cleared again
-     if the user moves the next argument pointer backwards. */
-  int args_only;
-
-  /* Describe how to deal with options that follow non-option ARGV-elements.
-
-     If the caller did not specify anything, the default is
-     REQUIRE_ORDER if the environment variable POSIXLY_CORRECT is
-     defined, PERMUTE otherwise.
-
-     REQUIRE_ORDER means don't recognize them as options; stop option
-     processing when the first non-option is seen. This is what Unix
-     does. This mode of operation is selected by either setting the
-     environment variable POSIXLY_CORRECT, or using `+' as the first
-     character of the list of option characters.
-
-     PERMUTE is the default. We permute the contents of ARGV as we
-     scan, so that eventually all the non-options are at the end. This
-     allows options to be given in any order, even with programs that
-     were not written to expect this.
-
-     RETURN_IN_ORDER is an option available to programs that were
-     written to expect options and other ARGV-elements in any order
-     and that care about the ordering of the two. We describe each
-     non-option ARGV-element as if it were the argument of an option
-     with character code 1. Using `-' as the first character of the
-     list of option characters selects this mode of operation.
-
-  */
-  enum { REQUIRE_ORDER, PERMUTE, RETURN_IN_ORDER } ordering;
-
-  /* A segment of non-option arguments that have been skipped for
-     later processing, after all options. `first_nonopt' is the index
-     in ARGV of the first of them; `last_nonopt' is the index after
-     the last of them.
-
-     If quoted or args_only is non-zero, this segment should be empty. */
-
-  /* FIXME: I'd prefer to use unsigned, but it's more consistent to
-     use the same type as for state.next. */
-  int first_nonopt;
-  int last_nonopt;
-  
-  /* String of all recognized short options. Needed for ARGP_LONG_ONLY. */
-  /* FIXME: Perhaps change to a pointer to a suitable bitmap instead? */
-  char *short_opts;
-
-  /* For parsing combined short options. */
-  char *nextchar;
-  
-  /* States of the various parsing groups.  */
-  struct group *groups;
-  /* The end of the GROUPS array.  */
-  struct group *egroup;
-  /* An vector containing storage for the CHILD_INPUTS field in all groups.  */
-  void **child_inputs;
-  
-  /* State block supplied to parsing routines.  */
-  struct argp_state state;
-
-  /* Memory used by this parser.  */
-  void *storage;
-};
-
-/* Search for a group defining a short option. */
-static const struct argp_option *
-find_short_option(struct parser *parser, int key, struct group **p)
-{
-  struct group *group;
-  
-  assert(key >= 0);
-  assert(isascii(key));
-
-  for (group = parser->groups; group < parser->egroup; group++)
-    {
-      const struct argp_option *opts;
-
-      for (opts = group->argp->options; !__option_is_end(opts); opts++)
-	if (opts->key == key)
-	  {
-	    *p = group;
-	    return opts;
-	  }
-    }
-  return NULL;
-}
-
-enum match_result { MATCH_EXACT, MATCH_PARTIAL, MATCH_NO };
-
-/* If defined, allow complete.el-like abbreviations of long options. */
-#ifndef ARGP_COMPLETE
-#define ARGP_COMPLETE 0
-#endif
-
-/* Matches an encountern long-option argument ARG against an option NAME.
- * ARG is terminated by NUL or '='. */
-static enum match_result
-match_option(const char *arg, const char *name)
-{
-  unsigned i, j;
-  for (i = j = 0;; i++, j++)
-    {
-      switch(arg[i])
-	{
-	case '\0':
-	case '=':
-	  return name[j] ? MATCH_PARTIAL : MATCH_EXACT;
-#if ARGP_COMPLETE
-	case '-':
-	  while (name[j] != '-')
-	    if (!name[j++])
-	      return MATCH_NO;
-	  break;
-#endif
-	default:
-	  if (arg[i] != name[j])
-	    return MATCH_NO;
-	}
-    }
-}
-
-static const struct argp_option *
-find_long_option(struct parser *parser,
-		 const char *arg,
-		 struct group **p)
-{
-  struct group *group;
-
-  /* Partial match found so far. */
-  struct group *matched_group = NULL;
-  const struct argp_option *matched_option = NULL;
-
-  /* Number of partial matches. */
-  int num_partial = 0;
-
-  for (group = parser->groups; group < parser->egroup; group++)
-    {
-      const struct argp_option *opts;
-
-      for (opts = group->argp->options; !__option_is_end(opts); opts++)
-	{
-	  if (!opts->name)
-	    continue;
-	  switch (match_option(arg, opts->name))
-	    {
-	    case MATCH_NO:
-	      break;
-	    case MATCH_PARTIAL:
-	      num_partial++;
-
-	      matched_group = group;
-	      matched_option = opts;
-
-	      break;
-	    case MATCH_EXACT:
-	      /* Exact match. */
-	      *p = group;
-	      return opts;
-	    }
-	}
-    }
-  if (num_partial == 1)
-    {
-      *p = matched_group;
-      return matched_option;
-    }
-
-  return NULL;
-}
-
-
-/* The next usable entries in the various parser tables being filled in by
-   convert_options.  */
-struct parser_convert_state
-{
-  struct parser *parser;
-  char *short_end;
-  void **child_inputs_end;
-};
-
-/* Initialize GROUP from ARGP. If CVT->SHORT_END is non-NULL, short
-   options are recorded in the short options string. Returns the next
-   unused group entry. CVT holds state used during the conversion. */
-static struct group *
-convert_options (const struct argp *argp,
-		 struct group *parent, unsigned parent_index,
-		 struct group *group, struct parser_convert_state *cvt)
-{
-  const struct argp_option *opt = argp->options;
-  const struct argp_child *children = argp->children;
-
-  if (opt || argp->parser)
-    {
-      /* This parser needs a group. */
-      if (cvt->short_end)
-	{
-	  /* Record any short options. */
-	  for ( ; !__option_is_end (opt); opt++)
-	    if (__option_is_short(opt))
-	      *cvt->short_end++ = opt->key;
-	}
-      
-      group->parser = argp->parser;
-      group->argp = argp;
-      group->args_processed = 0;
-      group->parent = parent;
-      group->parent_index = parent_index;
-      group->input = 0;
-      group->hook = 0;
-      group->child_inputs = 0;
-      
-      if (children)
-	/* Assign GROUP's CHILD_INPUTS field some space from
-	   CVT->child_inputs_end.*/
-	{
-	  unsigned num_children = 0;
-	  while (children[num_children].argp)
-	    num_children++;
-	  group->child_inputs = cvt->child_inputs_end;
-	  cvt->child_inputs_end += num_children;
-	}
-      parent = group++;
-    }
-  else
-    parent = 0;
-
-  if (children)
-    {
-      unsigned index = 0;
-      while (children->argp)
-	group =
-	  convert_options (children++->argp, parent, index++, group, cvt);
-    }
-
-  return group;
-}
-/* Allocate and initialize the group structures, so that they are
-   ordered as if by traversing the corresponding argp parser tree in
-   pre-order. Also build the list of short options, if that is needed. */
-static void
-parser_convert (struct parser *parser, const struct argp *argp)
-{
-  struct parser_convert_state cvt;
-
-  cvt.parser = parser;
-  cvt.short_end = parser->short_opts;
-  cvt.child_inputs_end = parser->child_inputs;
-
-  parser->argp = argp;
-
-  if (argp)
-    parser->egroup = convert_options (argp, 0, 0, parser->groups, &cvt);
-  else
-    parser->egroup = parser->groups; /* No parsers at all! */
-
-  if (parser->short_opts)
-    *cvt.short_end ='\0';
-}
-
-/* Lengths of various parser fields which we will allocated.  */
-struct parser_sizes
-{
-  /* Needed only ARGP_LONG_ONLY */
-  size_t short_len;		/* Number of short options.  */
-
-  size_t num_groups;		/* Group structures we allocate.  */
-  size_t num_child_inputs;	/* Child input slots.  */
-};
-
-/* For ARGP, increments the NUM_GROUPS field in SZS by the total
-   number of argp structures descended from it, and the SHORT_LEN by
-   the total number of short options. */
-static void
-calc_sizes (const struct argp *argp,  struct parser_sizes *szs)
-{
-  const struct argp_child *child = argp->children;
-  const struct argp_option *opt = argp->options;
-
-  if (opt || argp->parser)
-    {
-      /* This parser needs a group. */
-      szs->num_groups++;
-      if (opt)
-	{
-	  while (__option_is_short (opt++))
-	    szs->short_len++;
-	}
-    }
-
-  if (child)
-    while (child->argp)
-      {
-	calc_sizes ((child++)->argp, szs);
-	szs->num_child_inputs++;
-      }
-}
-
-/* Initializes PARSER to parse ARGP in a manner described by FLAGS.  */
-static error_t
-parser_init (struct parser *parser, const struct argp *argp,
-	     int argc, char **argv, int flags, void *input)
-{
-  error_t err = 0;
-  struct group *group;
-  struct parser_sizes szs;
-
-  parser->posixly_correct = getenv ("POSIXLY_CORRECT");
-
-  if (flags & ARGP_IN_ORDER)
-    parser->ordering = RETURN_IN_ORDER;
-  else if (flags & ARGP_NO_ARGS)
-    parser->ordering = REQUIRE_ORDER;
-  else if (parser->posixly_correct)
-    parser->ordering = REQUIRE_ORDER;
-  else
-    parser->ordering = PERMUTE;
-  
-  szs.short_len = 0;
-  szs.num_groups = 0;
-  szs.num_child_inputs = 0;
-
-  if (argp)
-    calc_sizes (argp, &szs);
-
-  if (!(flags & ARGP_LONG_ONLY))
-    /* We have no use for the short option array. */
-    szs.short_len = 0;
-  
-  /* Lengths of the various bits of storage used by PARSER.  */
-#define GLEN (szs.num_groups + 1) * sizeof (struct group)
-#define CLEN (szs.num_child_inputs * sizeof (void *))
-#define SLEN (szs.short_len + 1)
-#define STORAGE(offset) ((void *) (((char *) parser->storage) + (offset)))
-
-  parser->storage = malloc (GLEN + CLEN + SLEN);
-  if (! parser->storage)
-    return ENOMEM;
-
-  parser->groups = parser->storage;
-
-  parser->child_inputs = STORAGE(GLEN);
-  memset (parser->child_inputs, 0, szs.num_child_inputs * sizeof (void *));
-
-  if (flags & ARGP_LONG_ONLY)
-    parser->short_opts = STORAGE(GLEN + CLEN);
-  else
-    parser->short_opts = NULL;
-
-  parser_convert (parser, argp);
-
-  memset (&parser->state, 0, sizeof (struct argp_state));
-  
-  parser->state.root_argp = parser->argp;
-  parser->state.argc = argc;
-  parser->state.argv = argv;
-  parser->state.flags = flags;
-  parser->state.err_stream = stderr;
-  parser->state.out_stream = stdout;
-  parser->state.pstate = parser;
-
-  parser->args_only = 0;
-  parser->nextchar = NULL;
-  parser->first_nonopt = parser->last_nonopt = 0;
-    
-  /* Call each parser for the first time, giving it a chance to propagate
-     values to child parsers.  */
-  if (parser->groups < parser->egroup)
-    parser->groups->input = input;
-  for (group = parser->groups;
-       group < parser->egroup && (!err || err == EBADKEY);
-       group++)
-    {
-      if (group->parent)
-	/* If a child parser, get the initial input value from the parent. */
-	group->input = group->parent->child_inputs[group->parent_index];
-
-      if (!group->parser
-	  && group->argp->children && group->argp->children->argp)
-	/* For the special case where no parsing function is supplied for an
-	   argp, propagate its input to its first child, if any (this just
-	   makes very simple wrapper argps more convenient).  */
-	group->child_inputs[0] = group->input;
-
-      err = group_parse (group, &parser->state, ARGP_KEY_INIT, 0);
-    }
-  if (err == EBADKEY)
-    err = 0;			/* Some parser didn't understand.  */
-
-  if (err)
-    return err;
-
-  if (argv[0] && !(parser->state.flags & ARGP_PARSE_ARGV0))
-    /* There's an argv[0]; use it for messages.  */
-    {
-      parser->state.name = __argp_basename(argv[0]);
-
-      /* Don't parse it as an argument. */
-      parser->state.next = 1;
-    }
-  else
-    parser->state.name = __argp_short_program_name(NULL);
-  
-  return 0;
-}
-
-/* Free any storage consumed by PARSER (but not PARSER itself).  */
-static error_t
-parser_finalize (struct parser *parser,
-		 error_t err, int arg_ebadkey, int *end_index)
-{
-  struct group *group;
-
-  if (err == EBADKEY && arg_ebadkey)
-    /* Suppress errors generated by unparsed arguments.  */
-    err = 0;
-
-  if (! err)
-    {
-      if (parser->state.next == parser->state.argc)
-	/* We successfully parsed all arguments!  Call all the parsers again,
-	   just a few more times... */
-	{
-	  for (group = parser->groups;
-	       group < parser->egroup && (!err || err==EBADKEY);
-	       group++)
-	    if (group->args_processed == 0)
-	      err = group_parse (group, &parser->state, ARGP_KEY_NO_ARGS, 0);
-	  for (group = parser->egroup - 1;
-	       group >= parser->groups && (!err || err==EBADKEY);
-	       group--)
-	    err = group_parse (group, &parser->state, ARGP_KEY_END, 0);
-
-	  if (err == EBADKEY)
-	    err = 0;		/* Some parser didn't understand.  */
-
-	  /* Tell the user that all arguments are parsed.  */
-	  if (end_index)
-	    *end_index = parser->state.next;
-	}
-      else if (end_index)
-	/* Return any remaining arguments to the user.  */
-	*end_index = parser->state.next;
-      else
-	/* No way to return the remaining arguments, they must be bogus. */
-	{
-	  if (!(parser->state.flags & ARGP_NO_ERRS)
-	      && parser->state.err_stream)
-	    fprintf (parser->state.err_stream,
-		     dgettext (parser->argp->argp_domain,
-			       "%s: Too many arguments\n"),
-		     parser->state.name);
-	  err = EBADKEY;
-	}
-    }
-
-  /* Okay, we're all done, with either an error or success; call the parsers
-     to indicate which one.  */
-
-  if (err)
-    {
-      /* Maybe print an error message.  */
-      if (err == EBADKEY)
-	/* An appropriate message describing what the error was should have
-	   been printed earlier.  */
-	__argp_state_help (&parser->state, parser->state.err_stream,
-			   ARGP_HELP_STD_ERR);
-
-      /* Since we didn't exit, give each parser an error indication.  */
-      for (group = parser->groups; group < parser->egroup; group++)
-	group_parse (group, &parser->state, ARGP_KEY_ERROR, 0);
-    }
-  else
-    /* Notify parsers of success, and propagate back values from parsers.  */
-    {
-      /* We pass over the groups in reverse order so that child groups are
-	 given a chance to do there processing before passing back a value to
-	 the parent.  */
-      for (group = parser->egroup - 1
-	   ; group >= parser->groups && (!err || err == EBADKEY)
-	   ; group--)
-	err = group_parse (group, &parser->state, ARGP_KEY_SUCCESS, 0);
-      if (err == EBADKEY)
-	err = 0;		/* Some parser didn't understand.  */
-    }
-
-  /* Call parsers once more, to do any final cleanup.  Errors are ignored.  */
-  for (group = parser->egroup - 1; group >= parser->groups; group--)
-    group_parse (group, &parser->state, ARGP_KEY_FINI, 0);
-
-  if (err == EBADKEY)
-    err = EINVAL;
-
-  free (parser->storage);
-
-  return err;
-}
-
-/* Call the user parsers to parse the non-option argument VAL, at the
-   current position, returning any error. The state NEXT pointer
-   should point to the argument; this function will adjust it
-   correctly to reflect however many args actually end up being
-   consumed. */
-static error_t
-parser_parse_arg (struct parser *parser, char *val)
-{
-  /* Save the starting value of NEXT */
-  int index = parser->state.next;
-  error_t err = EBADKEY;
-  struct group *group;
-  int key = 0;			/* Which of ARGP_KEY_ARG[S] we used.  */
-
-  /* Try to parse the argument in each parser.  */
-  for (group = parser->groups
-       ; group < parser->egroup && err == EBADKEY
-       ; group++)
-    {
-      parser->state.next++;	/* For ARGP_KEY_ARG, consume the arg.  */
-      key = ARGP_KEY_ARG;
-      err = group_parse (group, &parser->state, key, val);
-
-      if (err == EBADKEY)
-	/* This parser doesn't like ARGP_KEY_ARG; try ARGP_KEY_ARGS instead. */
-	{
-	  parser->state.next--;	/* For ARGP_KEY_ARGS, put back the arg.  */
-	  key = ARGP_KEY_ARGS;
-	  err = group_parse (group, &parser->state, key, 0);
-	}
-    }
-
-  if (! err)
-    {
-      if (key == ARGP_KEY_ARGS)
-	/* The default for ARGP_KEY_ARGS is to assume that if NEXT isn't
-	   changed by the user, *all* arguments should be considered
-	   consumed.  */
-	parser->state.next = parser->state.argc;
-
-      if (parser->state.next > index)
-	/* Remember that we successfully processed a non-option
-	   argument -- but only if the user hasn't gotten tricky and set
-	   the clock back.  */
-	(--group)->args_processed += (parser->state.next - index);
-      else
-	/* The user wants to reparse some args, so try looking for options again.  */
-	parser->args_only = 0;
-    }
-
-  return err;
-}
-
-/* Exchange two adjacent subsequences of ARGV.
-   One subsequence is elements [first_nonopt,last_nonopt)
-   which contains all the non-options that have been skipped so far.
-   The other is elements [last_nonopt,next), which contains all
-   the options processed since those non-options were skipped.
-
-   `first_nonopt' and `last_nonopt' are relocated so that they describe
-   the new indices of the non-options in ARGV after they are moved.  */
-
-static void
-exchange (struct parser *parser)
-{
-  int bottom = parser->first_nonopt;
-  int middle = parser->last_nonopt;
-  int top = parser->state.next;
-  char **argv = parser->state.argv;
-  
-  char *tem;
-
-  /* Exchange the shorter segment with the far end of the longer segment.
-     That puts the shorter segment into the right place.
-     It leaves the longer segment in the right place overall,
-     but it consists of two parts that need to be swapped next.  */
-
-  while (top > middle && middle > bottom)
-    {
-      if (top - middle > middle - bottom)
-	{
-	  /* Bottom segment is the short one.  */
-	  int len = middle - bottom;
-	  register int i;
-
-	  /* Swap it with the top part of the top segment.  */
-	  for (i = 0; i < len; i++)
-	    {
-	      tem = argv[bottom + i];
-	      argv[bottom + i] = argv[top - (middle - bottom) + i];
-	      argv[top - (middle - bottom) + i] = tem;
-	    }
-	  /* Exclude the moved bottom segment from further swapping.  */
-	  top -= len;
-	}
-      else
-	{
-	  /* Top segment is the short one.  */
-	  int len = top - middle;
-	  register int i;
-
-	  /* Swap it with the bottom part of the bottom segment.  */
-	  for (i = 0; i < len; i++)
-	    {
-	      tem = argv[bottom + i];
-	      argv[bottom + i] = argv[middle + i];
-	      argv[middle + i] = tem;
-	    }
-	  /* Exclude the moved top segment from further swapping.  */
-	  bottom += len;
-	}
-    }
-
-  /* Update records for the slots the non-options now occupy.  */
-
-  parser->first_nonopt += (parser->state.next - parser->last_nonopt);
-  parser->last_nonopt = parser->state.next;
-}
-
-
-
-enum arg_type { ARG_ARG, ARG_SHORT_OPTION,
-		ARG_LONG_OPTION, ARG_LONG_ONLY_OPTION,
-		ARG_QUOTE };
-
-static enum arg_type
-classify_arg(struct parser *parser, char *arg, char **opt)
-{
-  if (arg[0] == '-')
-    /* Looks like an option... */
-    switch (arg[1])
-      {
-      case '\0':
-	/* "-" is not an option. */
-	return ARG_ARG;
-      case '-':
-	/* Long option, or quote. */
-	if (!arg[2])
-	  return ARG_QUOTE;
-	  
-	/* A long option. */
-	if (opt)
-	  *opt = arg + 2;
-	return ARG_LONG_OPTION;
-
-      default:
-	/* Short option. But if ARGP_LONG_ONLY, it can also be a long option. */
-
-	if (opt)
-	  *opt = arg + 1;
-
-	if (parser->state.flags & ARGP_LONG_ONLY)
-	  {
-	    /* Rules from getopt.c:
-
-	       If long_only and the ARGV-element has the form "-f",
-	       where f is a valid short option, don't consider it an
-	       abbreviated form of a long option that starts with f.
-	       Otherwise there would be no way to give the -f short
-	       option.
-
-	       On the other hand, if there's a long option "fubar" and
-	       the ARGV-element is "-fu", do consider that an
-	       abbreviation of the long option, just like "--fu", and
-	       not "-f" with arg "u".
-
-	       This distinction seems to be the most useful approach. */
-
-	    assert(parser->short_opts);
-	    
-	    if (arg[2] || !strchr(parser->short_opts, arg[1]))
-	      return ARG_LONG_ONLY_OPTION;
-	  }
-
-	return ARG_SHORT_OPTION;
-      }
-  
-  else
-    return ARG_ARG;
-}
-
-/* Parse the next argument in PARSER (as indicated by PARSER->state.next).
-   Any error from the parsers is returned, and *ARGP_EBADKEY indicates
-   whether a value of EBADKEY is due to an unrecognized argument (which is
-   generally not fatal).  */
-static error_t
-parser_parse_next (struct parser *parser, int *arg_ebadkey)
-{
-  if (parser->state.quoted && parser->state.next < parser->state.quoted)
-    /* The next argument pointer has been moved to before the quoted
-       region, so pretend we never saw the quoting `--', and start
-       looking for options again. If the `--' is still there we'll just
-       process it one more time. */
-    parser->state.quoted = parser->args_only = 0;
-
-  /* Give FIRST_NONOPT & LAST_NONOPT rational values if NEXT has been
-     moved back by the user (who may also have changed the arguments).  */
-  if (parser->last_nonopt > parser->state.next)
-    parser->last_nonopt = parser->state.next;
-  if (parser->first_nonopt > parser->state.next)
-    parser->first_nonopt = parser->state.next;
-
-  if (parser->nextchar)
-    /* Deal with short options. */
-    {
-      struct group *group;
-      char c;
-      const struct argp_option *option;
-      char *value = NULL;;
-
-      assert(!parser->args_only);
-
-      c = *parser->nextchar++;
-      
-      option = find_short_option(parser, c, &group);
-      if (!option)
-	{
-	  if (parser->posixly_correct)
-	    /* 1003.2 specifies the format of this message.  */
-	    fprintf (parser->state.err_stream,
-		     dgettext(parser->state.root_argp->argp_domain,
-			      "%s: illegal option -- %c\n"),
-		     parser->state.name, c);
-	  else
-	    fprintf (parser->state.err_stream,
-		     dgettext(parser->state.root_argp->argp_domain,
-			      "%s: invalid option -- %c\n"),
-		     parser->state.name, c);
-
-	  *arg_ebadkey = 0;
-	  return EBADKEY;
-	}
-
-      if (!*parser->nextchar)
-	parser->nextchar = NULL;
-
-      if (option->arg)
-	{
-	  value = parser->nextchar;
-	  parser->nextchar = NULL;
-	      
-	  if (!value
-	      && !(option->flags & OPTION_ARG_OPTIONAL))
-	    /* We need an mandatory argument. */
-	    {
-	      if (parser->state.next == parser->state.argc)
-		/* Missing argument */
-		{
-		  /* 1003.2 specifies the format of this message.  */
-		  fprintf (parser->state.err_stream,
-			   dgettext(parser->state.root_argp->argp_domain,
-				    "%s: option requires an argument -- %c\n"),
-			   parser->state.name, c);
-
-		  *arg_ebadkey = 0;
-		  return EBADKEY;
-		}
-	      value = parser->state.argv[parser->state.next++];
-	    }
-	}
-      return group_parse(group, &parser->state,
-			 option->key, value);
-    }
-  else
-    /* Advance to the next ARGV-element.  */
-    {
-      if (parser->args_only)
-	{
-	  *arg_ebadkey = 1;
-	  if (parser->state.next >= parser->state.argc)
-	    /* We're done. */
-	    return EBADKEY;
-	  else
-	    return parser_parse_arg(parser,
-				    parser->state.argv[parser->state.next]);
-	}
-      
-      if (parser->state.next >= parser->state.argc)
-	/* Almost done. If there are non-options that we skipped
-	   previously, we should process them now. */
-	{
-	  *arg_ebadkey = 1;
-	  if (parser->first_nonopt != parser->last_nonopt)
-	    {
-	      exchange(parser);
-	      
-	      /* Start processing the arguments we skipped previously. */
-	      parser->state.next = parser->first_nonopt;
-	      
-	      parser->first_nonopt = parser->last_nonopt = 0;
-
-	      parser->args_only = 1;
-	      return 0;
-	    }
-	  else
-	    /* Indicate that we're really done. */
-	    return EBADKEY;
-	}
-      else
-	/* Look for options. */
-	{
-	  char *arg = parser->state.argv[parser->state.next];
-
-	  char *optstart;
-	  enum arg_type token = classify_arg(parser, arg, &optstart);
-	  
-	  switch (token)
-	    {
-	    case ARG_ARG:
-	      switch (parser->ordering)
-		{
-		case PERMUTE:
-		  if (parser->first_nonopt == parser->last_nonopt)
-		    /* Skipped sequence is empty; start a new one. */
-		    parser->first_nonopt = parser->last_nonopt = parser->state.next;
-
-		  else if (parser->last_nonopt != parser->state.next)
-		    /* We have a non-empty skipped sequence, and
-		       we're not at the end-point, so move it. */
-		    exchange(parser);
-
-		  assert(parser->last_nonopt == parser->state.next);
-		  
-		  /* Skip this argument for now. */
-		  parser->state.next++;
-		  parser->last_nonopt = parser->state.next; 
-		  
-		  return 0;
-
-		case REQUIRE_ORDER:
-		  /* Implicit quote before the first argument. */
-		   parser->args_only = 1;
-		   return 0;
-		   
-		case RETURN_IN_ORDER:
-		  *arg_ebadkey = 1;
-		  return parser_parse_arg(parser, arg);
-
-		default:
-		  abort();
-		}
-	    case ARG_QUOTE:
-	      /* Skip it, then exchange with any previous non-options. */
-	      parser->state.next++;
-	      assert (parser->last_nonopt != parser->state.next);
-
-	      if (parser->first_nonopt != parser->last_nonopt)
-		{
-		  exchange(parser);
-		  
-		  /* Start processing the skipped and the quoted
-		     arguments. */
-
-		  parser->state.quoted = parser->state.next = parser->first_nonopt;
-
-		  /* Also empty the skipped-list, to avoid confusion
-		     if the user resets the next pointer. */
-		  parser->first_nonopt = parser->last_nonopt = 0;
-		}
-	      else
-		parser->state.quoted = parser->state.next;
-
-	      parser->args_only = 1;	      
-	      return 0;
-
-	    case ARG_LONG_ONLY_OPTION:
-	    case ARG_LONG_OPTION:
-	      {
-		struct group *group;
-		const struct argp_option *option;
-		char *value;
-
-		parser->state.next++;
-		option = find_long_option(parser, optstart, &group);
-		
-		if (!option)
-		  {
-		    /* NOTE: This includes any "=something" in the output. */
-		    fprintf (parser->state.err_stream,
-			     dgettext(parser->state.root_argp->argp_domain,
-				      "%s: unrecognized option `%s'\n"),
-			     parser->state.name, arg);
-		    *arg_ebadkey = 0;
-		    return EBADKEY;
-		  }
-
-		value = strchr(optstart, '=');
-		if (value)
-		  value++;
-		
-		if (value && !option->arg)
-		  /* Unexpected argument. */
-		  {
-		    if (token == ARG_LONG_OPTION)
-		      /* --option */
-		      fprintf (parser->state.err_stream,
-			       dgettext(parser->state.root_argp->argp_domain,
-					"%s: option `--%s' doesn't allow an argument\n"),
-			       parser->state.name, option->name);
-		    else
-		      /* +option or -option */
-		      fprintf (parser->state.err_stream,
-			       dgettext(parser->state.root_argp->argp_domain,
-					"%s: option `%c%s' doesn't allow an argument\n"),
-			       parser->state.name, arg[0], option->name);
-
-		    *arg_ebadkey = 0;
-		    return EBADKEY;
-		  }
-		
-		if (option->arg && !value
-		    && !(option->flags & OPTION_ARG_OPTIONAL))
-		  /* We need an mandatory argument. */
-		  {
-		    if (parser->state.next == parser->state.argc)
-		      /* Missing argument */
-		      {
-			if (token == ARG_LONG_OPTION)
-			  /* --option */
-			  fprintf (parser->state.err_stream,
-				   dgettext(parser->state.root_argp->argp_domain,
-					    "%s: option `--%s' requires an argument\n"),
-				 parser->state.name, option->name);
-			else
-			  /* +option or -option */
-			  fprintf (parser->state.err_stream,
-				   dgettext(parser->state.root_argp->argp_domain,
-					    "%s: option `%c%s' requires an argument\n"),
-				   parser->state.name, arg[0], option->name);
-
-			*arg_ebadkey = 0;
-			return EBADKEY;
-		      }
-
-		    value = parser->state.argv[parser->state.next++];
-		  }
-		*arg_ebadkey = 0;
-		return group_parse(group, &parser->state,
-				   option->key, value);
-	      }
-	    case ARG_SHORT_OPTION:
-	      parser->state.next++;
-	      parser->nextchar = optstart;
-	      return 0;
-
-	    default:
-	      abort();
-	    }
-	}
-    }
-}
-
-/* Parse the options strings in ARGC & ARGV according to the argp in ARGP.
-   FLAGS is one of the ARGP_ flags above.  If END_INDEX is non-NULL, the
-   index in ARGV of the first unparsed option is returned in it.  If an
-   unknown option is present, EINVAL is returned; if some parser routine
-   returned a non-zero value, it is returned; otherwise 0 is returned.  */
-error_t
-__argp_parse (const struct argp *argp, int argc, char **argv, unsigned flags,
-	      int *end_index, void *input)
-{
-  error_t err;
-  struct parser parser;
-
-  /* If true, then err == EBADKEY is a result of a non-option argument failing
-     to be parsed (which in some cases isn't actually an error).  */
-  int arg_ebadkey = 0;
-
-  if (! (flags & ARGP_NO_HELP))
-    /* Add our own options.  */
-    {
-      struct argp_child *child = alloca (4 * sizeof (struct argp_child));
-      struct argp *top_argp = alloca (sizeof (struct argp));
-
-      /* TOP_ARGP has no options, it just serves to group the user & default
-	 argps.  */
-      memset (top_argp, 0, sizeof (*top_argp));
-      top_argp->children = child;
-
-      memset (child, 0, 4 * sizeof (struct argp_child));
-
-      if (argp)
-	(child++)->argp = argp;
-      (child++)->argp = &argp_default_argp;
-      if (argp_program_version || argp_program_version_hook)
-	(child++)->argp = &argp_version_argp;
-      child->argp = 0;
-
-      argp = top_argp;
-    }
-
-  /* Construct a parser for these arguments.  */
-  err = parser_init (&parser, argp, argc, argv, flags, input);
-
-  if (! err)
-    /* Parse! */
-    {
-      while (! err)
-	err = parser_parse_next (&parser, &arg_ebadkey);
-      err = parser_finalize (&parser, err, arg_ebadkey, end_index);
-    }
-
-  return err;
-}
-#ifdef weak_alias
-weak_alias (__argp_parse, argp_parse)
-#endif
-
-/* Return the input field for ARGP in the parser corresponding to STATE; used
-   by the help routines.  */
-void *
-__argp_input (const struct argp *argp, const struct argp_state *state)
-{
-  if (state)
-    {
-      struct group *group;
-      struct parser *parser = state->pstate;
-
-      for (group = parser->groups; group < parser->egroup; group++)
-	if (group->argp == argp)
-	  return group->input;
-    }
-
-  return 0;
-}
-#ifdef weak_alias
-weak_alias (__argp_input, _argp_input)
-#endif
-
-/* Defined here, in case a user is not inlining the definitions in
- * argp.h */
-void
-__argp_usage (__const struct argp_state *__state)
-{
-  __argp_state_help (__state, stderr, ARGP_HELP_STD_USAGE);
-}
-
-int
-__option_is_short (__const struct argp_option *__opt)
-{
-  if (__opt->flags & OPTION_DOC)
-    return 0;
-  else
-    {
-      int __key = __opt->key;
-      /* FIXME: whether or not a particular key implies a short option
-       * ought not to be locale dependent. */
-      return __key > 0 && isprint (__key);
-    }
-}
-
-int
-__option_is_end (__const struct argp_option *__opt)
-{
-  return !__opt->key && !__opt->name && !__opt->doc && !__opt->group;
-}
diff --git a/argp-standalone/argp-pv.c b/argp-standalone/argp-pv.c
deleted file mode 100644
index d7d374a66bd..00000000000
--- a/argp-standalone/argp-pv.c
+++ /dev/null
@@ -1,25 +0,0 @@
-/* Default definition for ARGP_PROGRAM_VERSION.
-   Copyright (C) 1996, 1997, 1999, 2004 Free Software Foundation, Inc.
-   This file is part of the GNU C Library.
-   Written by Miles Bader <miles@gnu.ai.mit.edu>.
-
-   The GNU C Library is free software; you can redistribute it and/or
-   modify it under the terms of the GNU Library General Public License as
-   published by the Free Software Foundation; either version 2 of the
-   License, or (at your option) any later version.
-
-   The GNU C Library is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Library General Public License for more details.
-
-   You should have received a copy of the GNU Library General Public
-   License along with the GNU C Library; see the file COPYING.LIB.  If not,
-   write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330,
-   Boston, MA 02111-1307, USA.  */
-
-/* If set by the user program to a non-zero value, then a default option
-   --version is added (unless the ARGP_NO_HELP flag is used), which will
-   print this this string followed by a newline and exit (unless the
-   ARGP_NO_EXIT flag is used).  Overridden by ARGP_PROGRAM_VERSION_HOOK.  */
-const char *argp_program_version = 0;
diff --git a/argp-standalone/argp-pvh.c b/argp-standalone/argp-pvh.c
deleted file mode 100644
index 829a1cda80d..00000000000
--- a/argp-standalone/argp-pvh.c
+++ /dev/null
@@ -1,32 +0,0 @@
-/* Default definition for ARGP_PROGRAM_VERSION_HOOK.
-   Copyright (C) 1996, 1997, 1999, 2004 Free Software Foundation, Inc.
-   This file is part of the GNU C Library.
-   Written by Miles Bader <miles@gnu.ai.mit.edu>.
-
-   The GNU C Library is free software; you can redistribute it and/or
-   modify it under the terms of the GNU Library General Public License as
-   published by the Free Software Foundation; either version 2 of the
-   License, or (at your option) any later version.
-
-   The GNU C Library is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Library General Public License for more details.
-
-   You should have received a copy of the GNU Library General Public
-   License along with the GNU C Library; see the file COPYING.LIB.  If not,
-   write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330,
-   Boston, MA 02111-1307, USA.  */
-
-#ifdef HAVE_CONFIG_H
-#include <config.h>
-#endif
-
-#include "argp.h"
-
-/* If set by the user program to a non-zero value, then a default option
-   --version is added (unless the ARGP_NO_HELP flag is used), which calls
-   this function with a stream to print the version to and a pointer to the
-   current parsing state, and then exits (unless the ARGP_NO_EXIT flag is
-   used).  This variable takes precedent over ARGP_PROGRAM_VERSION.  */
-void (*argp_program_version_hook) (FILE *stream, struct argp_state *state) = 0;
diff --git a/argp-standalone/argp.h b/argp-standalone/argp.h
deleted file mode 100644
index 29d3dfe9720..00000000000
--- a/argp-standalone/argp.h
+++ /dev/null
@@ -1,602 +0,0 @@
-/* Hierarchial argument parsing.
-   Copyright (C) 1995, 96, 97, 98, 99, 2003 Free Software Foundation, Inc.
-   This file is part of the GNU C Library.
-   Written by Miles Bader <miles@gnu.ai.mit.edu>.
-
-   The GNU C Library is free software; you can redistribute it and/or
-   modify it under the terms of the GNU Library General Public License as
-   published by the Free Software Foundation; either version 2 of the
-   License, or (at your option) any later version.
-
-   The GNU C Library is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Library General Public License for more details.
-
-   You should have received a copy of the GNU Library General Public
-   License along with the GNU C Library; see the file COPYING.LIB.  If not,
-   write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330,
-   Boston, MA 02111-1307, USA.  */
-
-#ifndef _ARGP_H
-#define _ARGP_H
-
-#include <stdio.h>
-#include <ctype.h>
-
-#define __need_error_t
-#include <errno.h>
-
-#ifndef __THROW
-# define __THROW
-#endif
-
-#ifndef __const
-# define __const const
-#endif
-
-#ifndef __error_t_defined
-typedef int error_t;
-# define __error_t_defined
-#endif
-
-/* FIXME: What's the right way to check for __restrict? Sun's cc seems
-   not to have it. Perhaps it's easiest to just delete the use of
-   __restrict from the prototypes. */
-#ifndef __restrict
-# ifndef __GNUC___
-#  define __restrict
-# endif
-#endif
-
-/* NOTE: We can't use the autoconf tests, since this is supposed to be
-   an installed header file and argp's config.h is of course not
-   installed. */
-#ifndef PRINTF_STYLE
-# if __GNUC__ >= 2
-#  define PRINTF_STYLE(f, a) __attribute__ ((__format__ (__printf__, f, a)))
-# else
-#  define PRINTF_STYLE(f, a)
-# endif
-#endif
-
-
-#ifdef  __cplusplus
-extern "C" {
-#endif
-
-/* A description of a particular option.  A pointer to an array of
-   these is passed in the OPTIONS field of an argp structure.  Each option
-   entry can correspond to one long option and/or one short option; more
-   names for the same option can be added by following an entry in an option
-   array with options having the OPTION_ALIAS flag set.  */
-struct argp_option
-{
-  /* The long option name.  For more than one name for the same option, you
-     can use following options with the OPTION_ALIAS flag set.  */
-  __const char *name;
-
-  /* What key is returned for this option.  If > 0 and printable, then it's
-     also accepted as a short option.  */
-  int key;
-
-  /* If non-NULL, this is the name of the argument associated with this
-     option, which is required unless the OPTION_ARG_OPTIONAL flag is set. */
-  __const char *arg;
-
-  /* OPTION_ flags.  */
-  int flags;
-
-  /* The doc string for this option.  If both NAME and KEY are 0, This string
-     will be printed outdented from the normal option column, making it
-     useful as a group header (it will be the first thing printed in its
-     group); in this usage, it's conventional to end the string with a `:'.  */
-  __const char *doc;
-
-  /* The group this option is in.  In a long help message, options are sorted
-     alphabetically within each group, and the groups presented in the order
-     0, 1, 2, ..., n, -m, ..., -2, -1.  Every entry in an options array with
-     if this field 0 will inherit the group number of the previous entry, or
-     zero if it's the first one, unless its a group header (NAME and KEY both
-     0), in which case, the previous entry + 1 is the default.  Automagic
-     options such as --help are put into group -1.  */
-  int group;
-};
-
-/* The argument associated with this option is optional.  */
-#define OPTION_ARG_OPTIONAL	0x1
-
-/* This option isn't displayed in any help messages.  */
-#define OPTION_HIDDEN	       	0x2
-
-/* This option is an alias for the closest previous non-alias option.  This
-   means that it will be displayed in the same help entry, and will inherit
-   fields other than NAME and KEY from the aliased option.  */
-#define OPTION_ALIAS		0x4
-
-/* This option isn't actually an option (and so should be ignored by the
-   actual option parser), but rather an arbitrary piece of documentation that
-   should be displayed in much the same manner as the options.  If this flag
-   is set, then the option NAME field is displayed unmodified (e.g., no `--'
-   prefix is added) at the left-margin (where a *short* option would normally
-   be displayed), and the documentation string in the normal place.  For
-   purposes of sorting, any leading whitespace and puncuation is ignored,
-   except that if the first non-whitespace character is not `-', this entry
-   is displayed after all options (and OPTION_DOC entries with a leading `-')
-   in the same group.  */
-#define OPTION_DOC		0x8
-
-/* This option shouldn't be included in `long' usage messages (but is still
-   included in help messages).  This is mainly intended for options that are
-   completely documented in an argp's ARGS_DOC field, in which case including
-   the option in the generic usage list would be redundant.  For instance,
-   if ARGS_DOC is "FOO BAR\n-x BLAH", and the `-x' option's purpose is to
-   distinguish these two cases, -x should probably be marked
-   OPTION_NO_USAGE.  */
-#define OPTION_NO_USAGE		0x10
-
-struct argp;			/* fwd declare this type */
-struct argp_state;		/* " */
-struct argp_child;		/* " */
-
-/* The type of a pointer to an argp parsing function.  */
-typedef error_t (*argp_parser_t) (int key, char *arg,
-				  struct argp_state *state);
-
-/* What to return for unrecognized keys.  For special ARGP_KEY_ keys, such
-   returns will simply be ignored.  For user keys, this error will be turned
-   into EINVAL (if the call to argp_parse is such that errors are propagated
-   back to the user instead of exiting); returning EINVAL itself would result
-   in an immediate stop to parsing in *all* cases.  */
-#define ARGP_ERR_UNKNOWN	E2BIG /* Hurd should never need E2BIG.  XXX */
-
-/* Special values for the KEY argument to an argument parsing function.
-   ARGP_ERR_UNKNOWN should be returned if they aren't understood.
-
-   The sequence of keys to a parsing function is either (where each
-   uppercased word should be prefixed by `ARGP_KEY_' and opt is a user key):
-
-       INIT opt... NO_ARGS END SUCCESS  -- No non-option arguments at all
-   or  INIT (opt | ARG)... END SUCCESS  -- All non-option args parsed
-   or  INIT (opt | ARG)... SUCCESS      -- Some non-option arg unrecognized
-
-   The third case is where every parser returned ARGP_KEY_UNKNOWN for an
-   argument, in which case parsing stops at that argument (returning the
-   unparsed arguments to the caller of argp_parse if requested, or stopping
-   with an error message if not).
-
-   If an error occurs (either detected by argp, or because the parsing
-   function returned an error value), then the parser is called with
-   ARGP_KEY_ERROR, and no further calls are made.  */
-
-/* This is not an option at all, but rather a command line argument.  If a
-   parser receiving this key returns success, the fact is recorded, and the
-   ARGP_KEY_NO_ARGS case won't be used.  HOWEVER, if while processing the
-   argument, a parser function decrements the NEXT field of the state it's
-   passed, the option won't be considered processed; this is to allow you to
-   actually modify the argument (perhaps into an option), and have it
-   processed again.  */
-#define ARGP_KEY_ARG		0
-/* There are remaining arguments not parsed by any parser, which may be found
-   starting at (STATE->argv + STATE->next).  If success is returned, but
-   STATE->next left untouched, it's assumed that all arguments were consume,
-   otherwise, the parser should adjust STATE->next to reflect any arguments
-   consumed.  */
-#define ARGP_KEY_ARGS		0x1000006
-/* There are no more command line arguments at all.  */
-#define ARGP_KEY_END		0x1000001
-/* Because it's common to want to do some special processing if there aren't
-   any non-option args, user parsers are called with this key if they didn't
-   successfully process any non-option arguments.  Called just before
-   ARGP_KEY_END (where more general validity checks on previously parsed
-   arguments can take place).  */
-#define ARGP_KEY_NO_ARGS	0x1000002
-/* Passed in before any parsing is done.  Afterwards, the values of each
-   element of the CHILD_INPUT field, if any, in the state structure is
-   copied to each child's state to be the initial value of the INPUT field.  */
-#define ARGP_KEY_INIT		0x1000003
-/* Use after all other keys, including SUCCESS & END.  */
-#define ARGP_KEY_FINI		0x1000007
-/* Passed in when parsing has successfully been completed (even if there are
-   still arguments remaining).  */
-#define ARGP_KEY_SUCCESS	0x1000004
-/* Passed in if an error occurs.  */
-#define ARGP_KEY_ERROR		0x1000005
-
-/* An argp structure contains a set of options declarations, a function to
-   deal with parsing one, documentation string, a possible vector of child
-   argp's, and perhaps a function to filter help output.  When actually
-   parsing options, getopt is called with the union of all the argp
-   structures chained together through their CHILD pointers, with conflicts
-   being resolved in favor of the first occurrence in the chain.  */
-struct argp
-{
-  /* An array of argp_option structures, terminated by an entry with both
-     NAME and KEY having a value of 0.  */
-  __const struct argp_option *options;
-
-  /* What to do with an option from this structure.  KEY is the key
-     associated with the option, and ARG is any associated argument (NULL if
-     none was supplied).  If KEY isn't understood, ARGP_ERR_UNKNOWN should be
-     returned.  If a non-zero, non-ARGP_ERR_UNKNOWN value is returned, then
-     parsing is stopped immediately, and that value is returned from
-     argp_parse().  For special (non-user-supplied) values of KEY, see the
-     ARGP_KEY_ definitions below.  */
-  argp_parser_t parser;
-
-  /* A string describing what other arguments are wanted by this program.  It
-     is only used by argp_usage to print the `Usage:' message.  If it
-     contains newlines, the strings separated by them are considered
-     alternative usage patterns, and printed on separate lines (lines after
-     the first are prefix by `  or: ' instead of `Usage:').  */
-  __const char *args_doc;
-
-  /* If non-NULL, a string containing extra text to be printed before and
-     after the options in a long help message (separated by a vertical tab
-     `\v' character).  */
-  __const char *doc;
-
-  /* A vector of argp_children structures, terminated by a member with a 0
-     argp field, pointing to child argps should be parsed with this one.  Any
-     conflicts are resolved in favor of this argp, or early argps in the
-     CHILDREN list.  This field is useful if you use libraries that supply
-     their own argp structure, which you want to use in conjunction with your
-     own.  */
-  __const struct argp_child *children;
-
-  /* If non-zero, this should be a function to filter the output of help
-     messages.  KEY is either a key from an option, in which case TEXT is
-     that option's help text, or a special key from the ARGP_KEY_HELP_
-     defines, below, describing which other help text TEXT is.  The function
-     should return either TEXT, if it should be used as-is, a replacement
-     string, which should be malloced, and will be freed by argp, or NULL,
-     meaning `print nothing'.  The value for TEXT is *after* any translation
-     has been done, so if any of the replacement text also needs translation,
-     that should be done by the filter function.  INPUT is either the input
-     supplied to argp_parse, or NULL, if argp_help was called directly.  */
-  char *(*help_filter) (int __key, __const char *__text, void *__input);
-
-  /* If non-zero the strings used in the argp library are translated using
-     the domain described by this string.  Otherwise the currently installed
-     default domain is used.  */
-  const char *argp_domain;
-};
-
-/* Possible KEY arguments to a help filter function.  */
-#define ARGP_KEY_HELP_PRE_DOC	0x2000001 /* Help text preceeding options. */
-#define ARGP_KEY_HELP_POST_DOC	0x2000002 /* Help text following options. */
-#define ARGP_KEY_HELP_HEADER	0x2000003 /* Option header string. */
-#define ARGP_KEY_HELP_EXTRA	0x2000004 /* After all other documentation;
-					     TEXT is NULL for this key.  */
-/* Explanatory note emitted when duplicate option arguments have been
-   suppressed.  */
-#define ARGP_KEY_HELP_DUP_ARGS_NOTE 0x2000005
-#define ARGP_KEY_HELP_ARGS_DOC	0x2000006 /* Argument doc string.  */
-
-/* When an argp has a non-zero CHILDREN field, it should point to a vector of
-   argp_child structures, each of which describes a subsidiary argp.  */
-struct argp_child
-{
-  /* The child parser.  */
-  __const struct argp *argp;
-
-  /* Flags for this child.  */
-  int flags;
-
-  /* If non-zero, an optional header to be printed in help output before the
-     child options.  As a side-effect, a non-zero value forces the child
-     options to be grouped together; to achieve this effect without actually
-     printing a header string, use a value of "".  */
-  __const char *header;
-
-  /* Where to group the child options relative to the other (`consolidated')
-     options in the parent argp; the values are the same as the GROUP field
-     in argp_option structs, but all child-groupings follow parent options at
-     a particular group level.  If both this field and HEADER are zero, then
-     they aren't grouped at all, but rather merged with the parent options
-     (merging the child's grouping levels with the parents).  */
-  int group;
-};
-
-/* Parsing state.  This is provided to parsing functions called by argp,
-   which may examine and, as noted, modify fields.  */
-struct argp_state
-{
-  /* The top level ARGP being parsed.  */
-  __const struct argp *root_argp;
-
-  /* The argument vector being parsed.  May be modified.  */
-  int argc;
-  char **argv;
-
-  /* The index in ARGV of the next arg that to be parsed.  May be modified. */
-  int next;
-
-  /* The flags supplied to argp_parse.  May be modified.  */
-  unsigned flags;
-
-  /* While calling a parsing function with a key of ARGP_KEY_ARG, this is the
-     number of the current arg, starting at zero, and incremented after each
-     such call returns.  At all other times, this is the number of such
-     arguments that have been processed.  */
-  unsigned arg_num;
-
-  /* If non-zero, the index in ARGV of the first argument following a special
-     `--' argument (which prevents anything following being interpreted as an
-     option).  Only set once argument parsing has proceeded past this point. */
-  int quoted;
-
-  /* An arbitrary pointer passed in from the user.  */
-  void *input;
-  /* Values to pass to child parsers.  This vector will be the same length as
-     the number of children for the current parser.  */
-  void **child_inputs;
-
-  /* For the parser's use.  Initialized to 0.  */
-  void *hook;
-
-  /* The name used when printing messages.  This is initialized to ARGV[0],
-     or PROGRAM_INVOCATION_NAME if that is unavailable.  */
-  char *name;
-
-  /* Streams used when argp prints something.  */
-  FILE *err_stream;		/* For errors; initialized to stderr. */
-  FILE *out_stream;		/* For information; initialized to stdout. */
-
-  void *pstate;			/* Private, for use by argp.  */
-};
-
-/* Flags for argp_parse (note that the defaults are those that are
-   convenient for program command line parsing): */
-
-/* Don't ignore the first element of ARGV.  Normally (and always unless
-   ARGP_NO_ERRS is set) the first element of the argument vector is
-   skipped for option parsing purposes, as it corresponds to the program name
-   in a command line.  */
-#define ARGP_PARSE_ARGV0  0x01
-
-/* Don't print error messages for unknown options to stderr; unless this flag
-   is set, ARGP_PARSE_ARGV0 is ignored, as ARGV[0] is used as the program
-   name in the error messages.  This flag implies ARGP_NO_EXIT (on the
-   assumption that silent exiting upon errors is bad behaviour).  */
-#define ARGP_NO_ERRS	0x02
-
-/* Don't parse any non-option args.  Normally non-option args are parsed by
-   calling the parse functions with a key of ARGP_KEY_ARG, and the actual arg
-   as the value.  Since it's impossible to know which parse function wants to
-   handle it, each one is called in turn, until one returns 0 or an error
-   other than ARGP_ERR_UNKNOWN; if an argument is handled by no one, the
-   argp_parse returns prematurely (but with a return value of 0).  If all
-   args have been parsed without error, all parsing functions are called one
-   last time with a key of ARGP_KEY_END.  This flag needn't normally be set,
-   as the normal behavior is to stop parsing as soon as some argument can't
-   be handled.  */
-#define ARGP_NO_ARGS	0x04
-
-/* Parse options and arguments in the same order they occur on the command
-   line -- normally they're rearranged so that all options come first. */
-#define ARGP_IN_ORDER	0x08
-
-/* Don't provide the standard long option --help, which causes usage and
-      option help information to be output to stdout, and exit (0) called. */
-#define ARGP_NO_HELP	0x10
-
-/* Don't exit on errors (they may still result in error messages).  */
-#define ARGP_NO_EXIT	0x20
-
-/* Use the gnu getopt `long-only' rules for parsing arguments.  */
-#define ARGP_LONG_ONLY	0x40
-
-/* Turns off any message-printing/exiting options.  */
-#define ARGP_SILENT    (ARGP_NO_EXIT | ARGP_NO_ERRS | ARGP_NO_HELP)
-
-/* Parse the options strings in ARGC & ARGV according to the options in ARGP.
-   FLAGS is one of the ARGP_ flags above.  If ARG_INDEX is non-NULL, the
-   index in ARGV of the first unparsed option is returned in it.  If an
-   unknown option is present, ARGP_ERR_UNKNOWN is returned; if some parser
-   routine returned a non-zero value, it is returned; otherwise 0 is
-   returned.  This function may also call exit unless the ARGP_NO_HELP flag
-   is set.  INPUT is a pointer to a value to be passed in to the parser.  */
-extern error_t argp_parse (__const struct argp *__restrict argp,
-			   int argc, char **__restrict argv,
-			   unsigned flags, int *__restrict arg_index,
-			   void *__restrict input) __THROW;
-extern error_t __argp_parse (__const struct argp *__restrict argp,
-			     int argc, char **__restrict argv,
-			     unsigned flags, int *__restrict arg_index,
-			     void *__restrict input) __THROW;
-
-/* Global variables.  */
-
-/* If defined or set by the user program to a non-zero value, then a default
-   option --version is added (unless the ARGP_NO_HELP flag is used), which
-   will print this string followed by a newline and exit (unless the
-   ARGP_NO_EXIT flag is used).  Overridden by ARGP_PROGRAM_VERSION_HOOK.  */
-extern __const char *argp_program_version;
-
-/* If defined or set by the user program to a non-zero value, then a default
-   option --version is added (unless the ARGP_NO_HELP flag is used), which
-   calls this function with a stream to print the version to and a pointer to
-   the current parsing state, and then exits (unless the ARGP_NO_EXIT flag is
-   used).  This variable takes precedent over ARGP_PROGRAM_VERSION.  */
-extern void (*argp_program_version_hook) (FILE *__restrict __stream,
-					  struct argp_state *__restrict
-					  __state);
-
-/* If defined or set by the user program, it should point to string that is
-   the bug-reporting address for the program.  It will be printed by
-   argp_help if the ARGP_HELP_BUG_ADDR flag is set (as it is by various
-   standard help messages), embedded in a sentence that says something like
-   `Report bugs to ADDR.'.  */
-extern __const char *argp_program_bug_address;
-
-/* The exit status that argp will use when exiting due to a parsing error.
-   If not defined or set by the user program, this defaults to EX_USAGE from
-   <sysexits.h>.  */
-extern error_t argp_err_exit_status;
-
-/* Flags for argp_help.  */
-#define ARGP_HELP_USAGE		0x01 /* a Usage: message. */
-#define ARGP_HELP_SHORT_USAGE	0x02 /*  " but don't actually print options. */
-#define ARGP_HELP_SEE		0x04 /* a `Try ... for more help' message. */
-#define ARGP_HELP_LONG		0x08 /* a long help message. */
-#define ARGP_HELP_PRE_DOC	0x10 /* doc string preceding long help.  */
-#define ARGP_HELP_POST_DOC	0x20 /* doc string following long help.  */
-#define ARGP_HELP_DOC		(ARGP_HELP_PRE_DOC | ARGP_HELP_POST_DOC)
-#define ARGP_HELP_BUG_ADDR	0x40 /* bug report address */
-#define ARGP_HELP_LONG_ONLY	0x80 /* modify output appropriately to
-					reflect ARGP_LONG_ONLY mode.  */
-
-/* These ARGP_HELP flags are only understood by argp_state_help.  */
-#define ARGP_HELP_EXIT_ERR	0x100 /* Call exit(1) instead of returning.  */
-#define ARGP_HELP_EXIT_OK	0x200 /* Call exit(0) instead of returning.  */
-
-/* The standard thing to do after a program command line parsing error, if an
-   error message has already been printed.  */
-#define ARGP_HELP_STD_ERR \
-  (ARGP_HELP_SEE | ARGP_HELP_EXIT_ERR)
-/* The standard thing to do after a program command line parsing error, if no
-   more specific error message has been printed.  */
-#define ARGP_HELP_STD_USAGE \
-  (ARGP_HELP_SHORT_USAGE | ARGP_HELP_SEE | ARGP_HELP_EXIT_ERR)
-/* The standard thing to do in response to a --help option.  */
-#define ARGP_HELP_STD_HELP \
-  (ARGP_HELP_SHORT_USAGE | ARGP_HELP_LONG | ARGP_HELP_EXIT_OK \
-   | ARGP_HELP_DOC | ARGP_HELP_BUG_ADDR)
-
-/* Output a usage message for ARGP to STREAM.  FLAGS are from the set
-   ARGP_HELP_*.  */
-extern void argp_help (__const struct argp *__restrict __argp,
-		       FILE *__restrict __stream,
-		       unsigned __flags, char *__restrict __name) __THROW;
-extern void __argp_help (__const struct argp *__restrict __argp,
-			 FILE *__restrict __stream, unsigned __flags,
-			 char *__name) __THROW;
-
-/* The following routines are intended to be called from within an argp
-   parsing routine (thus taking an argp_state structure as the first
-   argument).  They may or may not print an error message and exit, depending
-   on the flags in STATE -- in any case, the caller should be prepared for
-   them *not* to exit, and should return an appropiate error after calling
-   them.  [argp_usage & argp_error should probably be called argp_state_...,
-   but they're used often enough that they should be short]  */
-
-/* Output, if appropriate, a usage message for STATE to STREAM.  FLAGS are
-   from the set ARGP_HELP_*.  */
-extern void argp_state_help (__const struct argp_state *__restrict __state,
-			     FILE *__restrict __stream,
-			     unsigned int __flags) __THROW;
-extern void __argp_state_help (__const struct argp_state *__restrict __state,
-			       FILE *__restrict __stream,
-			       unsigned int __flags) __THROW;
-
-/* Possibly output the standard usage message for ARGP to stderr and exit.  */
-extern void argp_usage (__const struct argp_state *__state) __THROW;
-extern void __argp_usage (__const struct argp_state *__state) __THROW;
-
-/* If appropriate, print the printf string FMT and following args, preceded
-   by the program name and `:', to stderr, and followed by a `Try ... --help'
-   message, then exit (1).  */
-extern void argp_error (__const struct argp_state *__restrict __state,
-			__const char *__restrict __fmt, ...) __THROW
-     PRINTF_STYLE(2,3);
-extern void __argp_error (__const struct argp_state *__restrict __state,
-			  __const char *__restrict __fmt, ...) __THROW
-     PRINTF_STYLE(2,3);
-
-/* Similar to the standard gnu error-reporting function error(), but will
-   respect the ARGP_NO_EXIT and ARGP_NO_ERRS flags in STATE, and will print
-   to STATE->err_stream.  This is useful for argument parsing code that is
-   shared between program startup (when exiting is desired) and runtime
-   option parsing (when typically an error code is returned instead).  The
-   difference between this function and argp_error is that the latter is for
-   *parsing errors*, and the former is for other problems that occur during
-   parsing but don't reflect a (syntactic) problem with the input.  */
-extern void argp_failure (__const struct argp_state *__restrict __state,
-			  int __status, int __errnum,
-			  __const char *__restrict __fmt, ...) __THROW
-     PRINTF_STYLE(4,5);
-extern void __argp_failure (__const struct argp_state *__restrict __state,
-			    int __status, int __errnum,
-			    __const char *__restrict __fmt, ...) __THROW
-     PRINTF_STYLE(4,5);
-
-/* Returns true if the option OPT is a valid short option.  */
-extern int _option_is_short (__const struct argp_option *__opt) __THROW;
-extern int __option_is_short (__const struct argp_option *__opt) __THROW;
-
-/* Returns true if the option OPT is in fact the last (unused) entry in an
-   options array.  */
-extern int _option_is_end (__const struct argp_option *__opt) __THROW;
-extern int __option_is_end (__const struct argp_option *__opt) __THROW;
-
-/* Return the input field for ARGP in the parser corresponding to STATE; used
-   by the help routines.  */
-extern void *_argp_input (__const struct argp *__restrict __argp,
-			  __const struct argp_state *__restrict __state)
-     __THROW;
-extern void *__argp_input (__const struct argp *__restrict __argp,
-			   __const struct argp_state *__restrict __state)
-     __THROW;
-
-/* Used for extracting the program name from argv[0] */
-extern char *_argp_basename(char *name) __THROW;
-extern char *__argp_basename(char *name) __THROW;
-
-/* Getting the program name given an argp state */
-extern char *
-_argp_short_program_name(const struct argp_state *state) __THROW;
-extern char *
-__argp_short_program_name(const struct argp_state *state) __THROW;
-
-
-#ifdef __USE_EXTERN_INLINES
-
-# if !_LIBC
-#  define __argp_usage argp_usage
-#  define __argp_state_help argp_state_help
-#  define __option_is_short _option_is_short
-#  define __option_is_end _option_is_end
-# endif
-
-# ifndef ARGP_EI
-#  define ARGP_EI extern __inline__
-# endif
-
-ARGP_EI void
-__argp_usage (__const struct argp_state *__state)
-{
-  __argp_state_help (__state, stderr, ARGP_HELP_STD_USAGE);
-}
-
-ARGP_EI int
-__option_is_short (__const struct argp_option *__opt)
-{
-  if (__opt->flags & OPTION_DOC)
-    return 0;
-  else
-    {
-      int __key = __opt->key;
-      return __key > 0 && isprint (__key);
-    }
-}
-
-ARGP_EI int
-__option_is_end (__const struct argp_option *__opt)
-{
-  return !__opt->key && !__opt->name && !__opt->doc && !__opt->group;
-}
-
-# if !_LIBC
-#  undef __argp_usage
-#  undef __argp_state_help
-#  undef __option_is_short
-#  undef __option_is_end
-# endif
-#endif /* Use extern inlines.  */
-
-#ifdef  __cplusplus
-}
-#endif
-
-#endif /* argp.h */
diff --git a/argp-standalone/autogen.sh b/argp-standalone/autogen.sh
deleted file mode 100755
index 8337353b5ae..00000000000
--- a/argp-standalone/autogen.sh
+++ /dev/null
@@ -1,6 +0,0 @@
-#!/bin/sh
-
-aclocal -I .
-autoheader
-autoconf
-automake --add-missing --copy --foreign
diff --git a/argp-standalone/configure.ac b/argp-standalone/configure.ac
deleted file mode 100644
index 8a892fc3cac..00000000000
--- a/argp-standalone/configure.ac
+++ /dev/null
@@ -1,102 +0,0 @@
-dnl Process this file with autoconf to produce a configure script.
-
-dnl This configure.ac is only for building a standalone argp library.
-AC_INIT([argp], [standalone-1.3])
-AC_PREREQ(2.54)
-AC_CONFIG_SRCDIR([argp-ba.c])
-# Needed to stop autoconf from looking for files in parent directories.
-AC_CONFIG_AUX_DIR([.])
-
-AM_INIT_AUTOMAKE
-AC_CONFIG_HEADERS(config.h)
-
-m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES(yes)])
-
-# GNU libc defaults to supplying the ISO C library functions only. The
-# _GNU_SOURCE define enables these extensions, in particular we want
-# errno.h to declare program_invocation_name. Enable it on all
-# systems; no problems have been reported with it so far.
-AC_GNU_SOURCE
-	
-# Checks for programs.
-AC_PROG_CC
-AC_PROG_MAKE_SET
-AC_PROG_RANLIB
-AC_PROG_CC
-
-if test "x$am_cv_prog_cc_stdc" = xno ; then
-  AC_ERROR([the C compiler doesn't handle ANSI-C])
-fi
-
-# Checks for libraries.
-
-# Checks for header files.
-AC_HEADER_STDC
-AC_CHECK_HEADERS(libintl.h limits.h malloc.h unistd.h sysexits.h stdarg.h)
-
-# Checks for typedefs, structures, and compiler characteristics.
-AC_C_CONST
-AC_C_INLINE
-AC_TYPE_SIZE_T
-
-LSH_GCC_ATTRIBUTES
-
-# Checks for library functions.
-AC_FUNC_ALLOCA
-AC_FUNC_VPRINTF
-AC_CHECK_FUNCS(strerror sleep getpid snprintf)
-
-AC_REPLACE_FUNCS(mempcpy strndup strchrnul strcasecmp vsnprintf)
-
-dnl ARGP_CHECK_FUNC(includes, function-call [, if-found [, if-not-found]])
-AC_DEFUN([ARGP_CHECK_FUNC],
-  [AS_VAR_PUSHDEF([ac_func], m4_substr([$2], 0, m4_index([$2], [(])))
-   AS_VAR_PUSHDEF([ac_var], [ac_cv_func_call_]ac_func)
-   AH_TEMPLATE(AS_TR_CPP(HAVE_[]ac_func),
-               [Define to 1 if you have the `]ac_func[' function.])
-   AC_CACHE_CHECK([for $2], ac_var,
-     [AC_TRY_LINK([$1], [$2],
-		  [AS_VAR_SET(ac_var, yes)],
-		  [AS_VAR_SET(ac_var, no)])])
-   if test AS_VAR_GET(ac_var) = yes ; then
-     ifelse([$3],,
-	    [AC_DEFINE_UNQUOTED(AS_TR_CPP(HAVE_[]ac_func))],
-	    [$3
-])
-   else
-     ifelse([$4],, true, [$4])
-   fi
-   AS_VAR_POPDEF([ac_var])
-   AS_VAR_POPDEF([ac_func])
-   ])
-
-# At least on freebsd, putc_unlocked is a macro, so the standard
-# AC_CHECK_FUNCS doesn't work well.
-ARGP_CHECK_FUNC([#include <stdio.h>], [putc_unlocked('x', stdout)])
-
-AC_CHECK_FUNCS(flockfile)
-AC_CHECK_FUNCS(fputs_unlocked fwrite_unlocked)
-
-# Used only by argp-test.c, so don't use AC_REPLACE_FUNCS.
-AC_CHECK_FUNCS(strdup asprintf)
-
-AC_CHECK_DECLS([program_invocation_name, program_invocation_short_name],
-               [], [], [[#include <errno.h>]])
-
-# Set these flags *last*, or else the test programs won't compile
-if test x$GCC = xyes ; then
-  # Using -ggdb3 makes (some versions of) Redhat's gcc-2.96 dump core
-  if "$CC" --version | grep '^2\.96$' 1>/dev/null 2>&1; then
-    true
-  else
-    CFLAGS="$CFLAGS -ggdb3"
-  fi
-  CFLAGS="$CFLAGS -Wall -W \
- -Wmissing-prototypes -Wmissing-declarations -Wstrict-prototypes \
- -Waggregate-return \
- -Wpointer-arith -Wbad-function-cast -Wnested-externs"
-fi
-
-CPPFLAGS="$CPPFLAGS -I$srcdir"
-
-AC_OUTPUT(Makefile)
diff --git a/argp-standalone/mempcpy.c b/argp-standalone/mempcpy.c
deleted file mode 100644
index 21d8bd2ed94..00000000000
--- a/argp-standalone/mempcpy.c
+++ /dev/null
@@ -1,21 +0,0 @@
-/* strndup.c
- *
- */
-
-/* Written by Niels Möller <nisse@lysator.liu.se>
- *
- * This file is hereby placed in the public domain.
- */
-
-#include <string.h>
-
-void *
-mempcpy (void *, const void *, size_t) ;
-
-void *
-mempcpy (void *to, const void *from, size_t size)
-{
-  memcpy(to, from, size);
-  return (char *) to + size;
-}
-
diff --git a/argp-standalone/strcasecmp.c b/argp-standalone/strcasecmp.c
deleted file mode 100644
index 9c1637232fd..00000000000
--- a/argp-standalone/strcasecmp.c
+++ /dev/null
@@ -1,29 +0,0 @@
-/* strcasecmp.c
- *
- */
-
-/* Written by Niels Möller <nisse@lysator.liu.se>
- *
- * This file is hereby placed in the public domain.
- */
-
-#include <ctype.h>
-int strcasecmp(const char *, const char *);
-
-int strcasecmp(const char *s1, const char *s2)
-{
-  unsigned i;
-  
-  for (i = 0; s1[i] && s2[i]; i++)
-    {
-      unsigned char c1 = tolower( (unsigned char) s1[i]);
-      unsigned char c2 = tolower( (unsigned char) s2[i]);
-
-      if (c1 < c2)
-	return -1;
-      else if (c1 > c2)
-	return 1;
-    }
-
-  return !s2[i] - !s1[i];	
-}
diff --git a/argp-standalone/strchrnul.c b/argp-standalone/strchrnul.c
deleted file mode 100644
index ee4145e4eda..00000000000
--- a/argp-standalone/strchrnul.c
+++ /dev/null
@@ -1,23 +0,0 @@
-/* strchrnul.c
- *
- */
-
-/* Written by Niels Möller <nisse@lysator.liu.se>
- *
- * This file is hereby placed in the public domain.
- */
-
-/* FIXME: What is this function supposed to do? My guess is that it is
- * like strchr, but returns a pointer to the NUL character, not a NULL
- * pointer, if the character isn't found. */
-
-char *strchrnul(const char *, int );
-
-char *strchrnul(const char *s, int c)
-{
-  const char *p = s;
-  while (*p && (*p != c))
-    p++;
-
-  return (char *) p;
-}
diff --git a/argp-standalone/strndup.c b/argp-standalone/strndup.c
deleted file mode 100644
index 4147b7a2051..00000000000
--- a/argp-standalone/strndup.c
+++ /dev/null
@@ -1,34 +0,0 @@
-/* strndup.c
- *
- */
-
-/* Written by Niels Möller <nisse@lysator.liu.se>
- *
- * This file is hereby placed in the public domain.
- */
-
-#include <stdlib.h>
-#include <string.h>
-
-char *
-strndup (const char *, size_t);
-
-char *
-strndup (const char *s, size_t size)
-{
-  char *r;
-  char *end = memchr(s, 0, size);
-  
-  if (end)
-    /* Length + 1 */
-    size = end - s + 1;
-  
-  r = malloc(size);
-
-  if (size)
-    {
-      memcpy(r, s, size-1);
-      r[size-1] = '\0';
-    }
-  return r;
-}
diff --git a/argp-standalone/vsnprintf.c b/argp-standalone/vsnprintf.c
deleted file mode 100644
index 33c9a5d0042..00000000000
--- a/argp-standalone/vsnprintf.c
+++ /dev/null
@@ -1,839 +0,0 @@
-/* Copied from http://www.fiction.net/blong/programs/snprintf.c */
-
-/*
- * Copyright Patrick Powell 1995
- * This code is based on code written by Patrick Powell (papowell@astart.com)
- * It may be used for any purpose as long as this notice remains intact
- * on all source code distributions
- */
-
-/**************************************************************
- * Original:
- * Patrick Powell Tue Apr 11 09:48:21 PDT 1995
- * A bombproof version of doprnt (dopr) included.
- * Sigh.  This sort of thing is always nasty do deal with.  Note that
- * the version here does not include floating point...
- *
- * snprintf() is used instead of sprintf() as it does limit checks
- * for string length.  This covers a nasty loophole.
- *
- * The other functions are there to prevent NULL pointers from
- * causing nast effects.
- *
- * More Recently:
- *  Brandon Long <blong@fiction.net> 9/15/96 for mutt 0.43
- *  This was ugly.  It is still ugly.  I opted out of floating point
- *  numbers, but the formatter understands just about everything
- *  from the normal C string format, at least as far as I can tell from
- *  the Solaris 2.5 printf(3S) man page.
- *
- *  Brandon Long <blong@fiction.net> 10/22/97 for mutt 0.87.1
- *    Ok, added some minimal floating point support, which means this
- *    probably requires libm on most operating systems.  Don't yet
- *    support the exponent (e,E) and sigfig (g,G).  Also, fmtint()
- *    was pretty badly broken, it just wasn't being exercised in ways
- *    which showed it, so that's been fixed.  Also, formated the code
- *    to mutt conventions, and removed dead code left over from the
- *    original.  Also, there is now a builtin-test, just compile with:
- *           gcc -DTEST_SNPRINTF -o snprintf snprintf.c -lm
- *    and run snprintf for results.
- * 
- *  Thomas Roessler <roessler@guug.de> 01/27/98 for mutt 0.89i
- *    The PGP code was using unsigned hexadecimal formats. 
- *    Unfortunately, unsigned formats simply didn't work.
- *
- *  Michael Elkins <me@cs.hmc.edu> 03/05/98 for mutt 0.90.8
- *    The original code assumed that both snprintf() and vsnprintf() were
- *    missing.  Some systems only have snprintf() but not vsnprintf(), so
- *    the code is now broken down under HAVE_SNPRINTF and HAVE_VSNPRINTF.
- *
- *  Andrew Tridgell (tridge@samba.org) Oct 1998
- *    fixed handling of %.0f
- *    added test for HAVE_LONG_DOUBLE
- *
- *  Russ Allbery <rra@stanford.edu> 2000-08-26
- *    fixed return value to comply with C99
- *    fixed handling of snprintf(NULL, ...)
- *
- * Niels Möller <nisse@lysator.liu.se> 2004-03-05
- *    fixed calls to isdigit to use unsigned char.
- *    fixed calls to va_arg; short arguments are always passed as int.
- *
- **************************************************************/
-
-#if HAVE_CONFIG_H
-# include "config.h"
-#endif
-
-#if !defined(HAVE_SNPRINTF) || !defined(HAVE_VSNPRINTF)
-
-#include <string.h>
-#include <ctype.h>
-#include <sys/types.h>
-
-/* Define this as a fall through, HAVE_STDARG_H is probably already set */
-
-#define HAVE_VARARGS_H
-
-
-/* varargs declarations: */
-
-#if defined(HAVE_STDARG_H)
-# include <stdarg.h>
-# define HAVE_STDARGS    /* let's hope that works everywhere (mj) */
-# define VA_LOCAL_DECL   va_list ap
-# define VA_START(f)     va_start(ap, f)
-# define VA_SHIFT(v,t)  ;   /* no-op for ANSI */
-# define VA_END          va_end(ap)
-#else
-# if defined(HAVE_VARARGS_H)
-#  include <varargs.h>
-#  undef HAVE_STDARGS
-#  define VA_LOCAL_DECL   va_list ap
-#  define VA_START(f)     va_start(ap)      /* f is ignored! */
-#  define VA_SHIFT(v,t) v = va_arg(ap,t)
-#  define VA_END        va_end(ap)
-# else
-/*XX ** NO VARARGS ** XX*/
-# endif
-#endif
-
-#ifdef HAVE_LONG_DOUBLE
-#define LDOUBLE long double
-#else
-#define LDOUBLE double
-#endif
-
-int snprintf (char *str, size_t count, const char *fmt, ...);
-int vsnprintf (char *str, size_t count, const char *fmt, va_list arg);
-
-static int dopr (char *buffer, size_t maxlen, const char *format, 
-                 va_list args);
-static int fmtstr (char *buffer, size_t *currlen, size_t maxlen,
-		   char *value, int flags, int min, int max);
-static int fmtint (char *buffer, size_t *currlen, size_t maxlen,
-		   long value, int base, int min, int max, int flags);
-static int fmtfp (char *buffer, size_t *currlen, size_t maxlen,
-		  LDOUBLE fvalue, int min, int max, int flags);
-static int dopr_outch (char *buffer, size_t *currlen, size_t maxlen, char c );
-
-/*
- * dopr(): poor man's version of doprintf
- */
-
-/* format read states */
-#define DP_S_DEFAULT 0
-#define DP_S_FLAGS   1
-#define DP_S_MIN     2
-#define DP_S_DOT     3
-#define DP_S_MAX     4
-#define DP_S_MOD     5
-#define DP_S_CONV    6
-#define DP_S_DONE    7
-
-/* format flags - Bits */
-#define DP_F_MINUS 	(1 << 0)
-#define DP_F_PLUS  	(1 << 1)
-#define DP_F_SPACE 	(1 << 2)
-#define DP_F_NUM   	(1 << 3)
-#define DP_F_ZERO  	(1 << 4)
-#define DP_F_UP    	(1 << 5)
-#define DP_F_UNSIGNED 	(1 << 6)
-
-/* Conversion Flags */
-#define DP_C_SHORT   1
-#define DP_C_LONG    2
-#define DP_C_LDOUBLE 3
-
-#define char_to_int(p) (p - '0')
-#define MAX(p,q) ((p >= q) ? p : q)
-#define MIN(p,q) ((p <= q) ? p : q)
-
-static int dopr (char *buffer, size_t maxlen, const char *format, va_list args)
-{
-  unsigned char ch;
-  long value;
-  LDOUBLE fvalue;
-  char *strvalue;
-  int min;
-  int max;
-  int state;
-  int flags;
-  int cflags;
-  int total;
-  size_t currlen;
-  
-  state = DP_S_DEFAULT;
-  currlen = flags = cflags = min = 0;
-  max = -1;
-  ch = *format++;
-  total = 0;
-
-  while (state != DP_S_DONE)
-  {
-    if (ch == '\0')
-      state = DP_S_DONE;
-
-    switch(state) 
-    {
-    case DP_S_DEFAULT:
-      if (ch == '%') 
-	state = DP_S_FLAGS;
-      else 
-	total += dopr_outch (buffer, &currlen, maxlen, ch);
-      ch = *format++;
-      break;
-    case DP_S_FLAGS:
-      switch (ch) 
-      {
-      case '-':
-	flags |= DP_F_MINUS;
-        ch = *format++;
-	break;
-      case '+':
-	flags |= DP_F_PLUS;
-        ch = *format++;
-	break;
-      case ' ':
-	flags |= DP_F_SPACE;
-        ch = *format++;
-	break;
-      case '#':
-	flags |= DP_F_NUM;
-        ch = *format++;
-	break;
-      case '0':
-	flags |= DP_F_ZERO;
-        ch = *format++;
-	break;
-      default:
-	state = DP_S_MIN;
-	break;
-      }
-      break;
-    case DP_S_MIN:
-      if (isdigit(ch)) 
-      {
-	min = 10*min + char_to_int (ch);
-	ch = *format++;
-      } 
-      else if (ch == '*') 
-      {
-	min = va_arg (args, int);
-	ch = *format++;
-	state = DP_S_DOT;
-      } 
-      else 
-	state = DP_S_DOT;
-      break;
-    case DP_S_DOT:
-      if (ch == '.') 
-      {
-	state = DP_S_MAX;
-	ch = *format++;
-      } 
-      else 
-	state = DP_S_MOD;
-      break;
-    case DP_S_MAX:
-      if (isdigit(ch)) 
-      {
-	if (max < 0)
-	  max = 0;
-	max = 10*max + char_to_int (ch);
-	ch = *format++;
-      } 
-      else if (ch == '*') 
-      {
-	max = va_arg (args, int);
-	ch = *format++;
-	state = DP_S_MOD;
-      } 
-      else 
-	state = DP_S_MOD;
-      break;
-    case DP_S_MOD:
-      /* Currently, we don't support Long Long, bummer */
-      switch (ch) 
-      {
-      case 'h':
-	cflags = DP_C_SHORT;
-	ch = *format++;
-	break;
-      case 'l':
-	cflags = DP_C_LONG;
-	ch = *format++;
-	break;
-      case 'L':
-	cflags = DP_C_LDOUBLE;
-	ch = *format++;
-	break;
-      default:
-	break;
-      }
-      state = DP_S_CONV;
-      break;
-    case DP_S_CONV:
-      switch (ch) 
-      {
-      case 'd':
-      case 'i':
-	if (cflags == DP_C_SHORT) 
-	  value = (short) va_arg (args, int);
-	else if (cflags == DP_C_LONG)
-	  value = va_arg (args, long int);
-	else
-	  value = va_arg (args, int);
-	total += fmtint (buffer, &currlen, maxlen, value, 10, min, max, flags);
-	break;
-      case 'o':
-	flags |= DP_F_UNSIGNED;
-	if (cflags == DP_C_SHORT)
-	  value = (unsigned short) va_arg (args, unsigned);
-	else if (cflags == DP_C_LONG)
-	  value = va_arg (args, unsigned long int);
-	else
-	  value = va_arg (args, unsigned int);
-	total += fmtint (buffer, &currlen, maxlen, value, 8, min, max, flags);
-	break;
-      case 'u':
-	flags |= DP_F_UNSIGNED;
-	if (cflags == DP_C_SHORT)
-	  value = (unsigned short) va_arg (args, unsigned);
-	else if (cflags == DP_C_LONG)
-	  value = va_arg (args, unsigned long int);
-	else
-	  value = va_arg (args, unsigned int);
-	total += fmtint (buffer, &currlen, maxlen, value, 10, min, max, flags);
-	break;
-      case 'X':
-	flags |= DP_F_UP;
-      case 'x':
-	flags |= DP_F_UNSIGNED;
-	if (cflags == DP_C_SHORT)
-	  value = (unsigned short) va_arg (args, unsigned);
-	else if (cflags == DP_C_LONG)
-	  value = va_arg (args, unsigned long int);
-	else
-	  value = va_arg (args, unsigned int);
-	total += fmtint (buffer, &currlen, maxlen, value, 16, min, max, flags);
-	break;
-      case 'f':
-	if (cflags == DP_C_LDOUBLE)
-	  fvalue = va_arg (args, LDOUBLE);
-	else
-	  fvalue = va_arg (args, double);
-	/* um, floating point? */
-	total += fmtfp (buffer, &currlen, maxlen, fvalue, min, max, flags);
-	break;
-      case 'E':
-	flags |= DP_F_UP;
-      case 'e':
-	if (cflags == DP_C_LDOUBLE)
-	  fvalue = va_arg (args, LDOUBLE);
-	else
-	  fvalue = va_arg (args, double);
-	break;
-      case 'G':
-	flags |= DP_F_UP;
-      case 'g':
-	if (cflags == DP_C_LDOUBLE)
-	  fvalue = va_arg (args, LDOUBLE);
-	else
-	  fvalue = va_arg (args, double);
-	break;
-      case 'c':
-	total += dopr_outch (buffer, &currlen, maxlen, va_arg (args, int));
-	break;
-      case 's':
-	strvalue = va_arg (args, char *);
-	total += fmtstr (buffer, &currlen, maxlen, strvalue, flags, min, max);
-	break;
-      case 'p':
-	strvalue = va_arg (args, void *);
-	total += fmtint (buffer, &currlen, maxlen, (long) strvalue, 16, min,
-                         max, flags);
-	break;
-      case 'n':
-	if (cflags == DP_C_SHORT) 
-	{
-	  short int *num;
-	  num = va_arg (args, short int *);
-	  *num = currlen;
-        } 
-	else if (cflags == DP_C_LONG) 
-	{
-	  long int *num;
-	  num = va_arg (args, long int *);
-	  *num = currlen;
-        } 
-	else 
-	{
-	  int *num;
-	  num = va_arg (args, int *);
-	  *num = currlen;
-        }
-	break;
-      case '%':
-	total += dopr_outch (buffer, &currlen, maxlen, ch);
-	break;
-      case 'w':
-	/* not supported yet, treat as next char */
-	ch = *format++;
-	break;
-      default:
-	/* Unknown, skip */
-	break;
-      }
-      ch = *format++;
-      state = DP_S_DEFAULT;
-      flags = cflags = min = 0;
-      max = -1;
-      break;
-    case DP_S_DONE:
-      break;
-    default:
-      /* hmm? */
-      break; /* some picky compilers need this */
-    }
-  }
-  if (buffer != NULL)
-  {
-    if (currlen < maxlen - 1) 
-      buffer[currlen] = '\0';
-    else 
-      buffer[maxlen - 1] = '\0';
-  }
-  return total;
-}
-
-static int fmtstr (char *buffer, size_t *currlen, size_t maxlen,
-                   char *value, int flags, int min, int max)
-{
-  int padlen, strln;     /* amount to pad */
-  int cnt = 0;
-  int total = 0;
-  
-  if (value == 0)
-  {
-    value = "<NULL>";
-  }
-
-  for (strln = 0; value[strln]; ++strln); /* strlen */
-  if (max >= 0 && max < strln)
-    strln = max;
-  padlen = min - strln;
-  if (padlen < 0) 
-    padlen = 0;
-  if (flags & DP_F_MINUS) 
-    padlen = -padlen; /* Left Justify */
-
-  while (padlen > 0)
-  {
-    total += dopr_outch (buffer, currlen, maxlen, ' ');
-    --padlen;
-  }
-  while (*value && ((max < 0) || (cnt < max)))
-  {
-    total += dopr_outch (buffer, currlen, maxlen, *value++);
-    ++cnt;
-  }
-  while (padlen < 0)
-  {
-    total += dopr_outch (buffer, currlen, maxlen, ' ');
-    ++padlen;
-  }
-  return total;
-}
-
-/* Have to handle DP_F_NUM (ie 0x and 0 alternates) */
-
-static int fmtint (char *buffer, size_t *currlen, size_t maxlen,
-		   long value, int base, int min, int max, int flags)
-{
-  int signvalue = 0;
-  unsigned long uvalue;
-  char convert[20];
-  int place = 0;
-  int spadlen = 0; /* amount to space pad */
-  int zpadlen = 0; /* amount to zero pad */
-  int caps = 0;
-  int total = 0;
-  
-  if (max < 0)
-    max = 0;
-
-  uvalue = value;
-
-  if(!(flags & DP_F_UNSIGNED))
-  {
-    if( value < 0 ) {
-      signvalue = '-';
-      uvalue = -value;
-    }
-    else
-      if (flags & DP_F_PLUS)  /* Do a sign (+/i) */
-	signvalue = '+';
-    else
-      if (flags & DP_F_SPACE)
-	signvalue = ' ';
-  }
-  
-  if (flags & DP_F_UP) caps = 1; /* Should characters be upper case? */
-
-  do {
-    convert[place++] =
-      (caps? "0123456789ABCDEF":"0123456789abcdef")
-      [uvalue % (unsigned)base  ];
-    uvalue = (uvalue / (unsigned)base );
-  } while(uvalue && (place < 20));
-  if (place == 20) place--;
-  convert[place] = 0;
-
-  zpadlen = max - place;
-  spadlen = min - MAX (max, place) - (signvalue ? 1 : 0);
-  if (zpadlen < 0) zpadlen = 0;
-  if (spadlen < 0) spadlen = 0;
-  if (flags & DP_F_ZERO)
-  {
-    zpadlen = MAX(zpadlen, spadlen);
-    spadlen = 0;
-  }
-  if (flags & DP_F_MINUS) 
-    spadlen = -spadlen; /* Left Justifty */
-
-#ifdef DEBUG_SNPRINTF
-  dprint (1, (debugfile, "zpad: %d, spad: %d, min: %d, max: %d, place: %d\n",
-      zpadlen, spadlen, min, max, place));
-#endif
-
-  /* Spaces */
-  while (spadlen > 0) 
-  {
-    total += dopr_outch (buffer, currlen, maxlen, ' ');
-    --spadlen;
-  }
-
-  /* Sign */
-  if (signvalue) 
-    total += dopr_outch (buffer, currlen, maxlen, signvalue);
-
-  /* Zeros */
-  if (zpadlen > 0) 
-  {
-    while (zpadlen > 0)
-    {
-      total += dopr_outch (buffer, currlen, maxlen, '0');
-      --zpadlen;
-    }
-  }
-
-  /* Digits */
-  while (place > 0) 
-    total += dopr_outch (buffer, currlen, maxlen, convert[--place]);
-  
-  /* Left Justified spaces */
-  while (spadlen < 0) {
-    total += dopr_outch (buffer, currlen, maxlen, ' ');
-    ++spadlen;
-  }
-
-  return total;
-}
-
-static LDOUBLE abs_val (LDOUBLE value)
-{
-  LDOUBLE result = value;
-
-  if (value < 0)
-    result = -value;
-
-  return result;
-}
-
-static LDOUBLE pow10_argp (int exp)
-{
-  LDOUBLE result = 1;
-
-  while (exp)
-  {
-    result *= 10;
-    exp--;
-  }
-  
-  return result;
-}
-
-static long round_argp (LDOUBLE value)
-{
-  long intpart;
-
-  intpart = value;
-  value = value - intpart;
-  if (value >= 0.5)
-    intpart++;
-
-  return intpart;
-}
-
-static int fmtfp (char *buffer, size_t *currlen, size_t maxlen,
-		  LDOUBLE fvalue, int min, int max, int flags)
-{
-  int signvalue = 0;
-  LDOUBLE ufvalue;
-  char iconvert[20];
-  char fconvert[20];
-  int iplace = 0;
-  int fplace = 0;
-  int padlen = 0; /* amount to pad */
-  int zpadlen = 0; 
-  int caps = 0;
-  int total = 0;
-  long intpart;
-  long fracpart;
-  
-  /* 
-   * AIX manpage says the default is 0, but Solaris says the default
-   * is 6, and sprintf on AIX defaults to 6
-   */
-  if (max < 0)
-    max = 6;
-
-  ufvalue = abs_val (fvalue);
-
-  if (fvalue < 0)
-    signvalue = '-';
-  else
-    if (flags & DP_F_PLUS)  /* Do a sign (+/i) */
-      signvalue = '+';
-    else
-      if (flags & DP_F_SPACE)
-	signvalue = ' ';
-
-#if 0
-  if (flags & DP_F_UP) caps = 1; /* Should characters be upper case? */
-#endif
-
-  intpart = ufvalue;
-
-  /* 
-   * Sorry, we only support 9 digits past the decimal because of our 
-   * conversion method
-   */
-  if (max > 9)
-    max = 9;
-
-  /* We "cheat" by converting the fractional part to integer by
-   * multiplying by a factor of 10
-   */
-  fracpart = round_argp ((pow10_argp (max)) * (ufvalue - intpart));
-
-  if (fracpart >= pow10_argp (max))
-  {
-    intpart++;
-    fracpart -= pow10_argp (max);
-  }
-
-#ifdef DEBUG_SNPRINTF
-  dprint (1, (debugfile, "fmtfp: %f =? %d.%d\n", fvalue, intpart, fracpart));
-#endif
-
-  /* Convert integer part */
-  do {
-    iconvert[iplace++] =
-      (caps? "0123456789ABCDEF":"0123456789abcdef")[intpart % 10];
-    intpart = (intpart / 10);
-  } while(intpart && (iplace < 20));
-  if (iplace == 20) iplace--;
-  iconvert[iplace] = 0;
-
-  /* Convert fractional part */
-  do {
-    fconvert[fplace++] =
-      (caps? "0123456789ABCDEF":"0123456789abcdef")[fracpart % 10];
-    fracpart = (fracpart / 10);
-  } while(fracpart && (fplace < 20));
-  if (fplace == 20) fplace--;
-  fconvert[fplace] = 0;
-
-  /* -1 for decimal point, another -1 if we are printing a sign */
-  padlen = min - iplace - max - 1 - ((signvalue) ? 1 : 0); 
-  zpadlen = max - fplace;
-  if (zpadlen < 0)
-    zpadlen = 0;
-  if (padlen < 0) 
-    padlen = 0;
-  if (flags & DP_F_MINUS) 
-    padlen = -padlen; /* Left Justifty */
-
-  if ((flags & DP_F_ZERO) && (padlen > 0)) 
-  {
-    if (signvalue) 
-    {
-      total += dopr_outch (buffer, currlen, maxlen, signvalue);
-      --padlen;
-      signvalue = 0;
-    }
-    while (padlen > 0)
-    {
-      total += dopr_outch (buffer, currlen, maxlen, '0');
-      --padlen;
-    }
-  }
-  while (padlen > 0)
-  {
-    total += dopr_outch (buffer, currlen, maxlen, ' ');
-    --padlen;
-  }
-  if (signvalue) 
-    total += dopr_outch (buffer, currlen, maxlen, signvalue);
-
-  while (iplace > 0) 
-    total += dopr_outch (buffer, currlen, maxlen, iconvert[--iplace]);
-
-  /*
-   * Decimal point.  This should probably use locale to find the correct
-   * char to print out.
-   */
-  if (max > 0)
-  {
-    total += dopr_outch (buffer, currlen, maxlen, '.');
-
-    while (fplace > 0) 
-      total += dopr_outch (buffer, currlen, maxlen, fconvert[--fplace]);
-  }
-
-  while (zpadlen > 0)
-  {
-    total += dopr_outch (buffer, currlen, maxlen, '0');
-    --zpadlen;
-  }
-
-  while (padlen < 0) 
-  {
-    total += dopr_outch (buffer, currlen, maxlen, ' ');
-    ++padlen;
-  }
-
-  return total;
-}
-
-static int dopr_outch (char *buffer, size_t *currlen, size_t maxlen, char c)
-{
-  if (*currlen + 1 < maxlen)
-    buffer[(*currlen)++] = c;
-  return 1;
-}
-
-#ifndef HAVE_VSNPRINTF
-int vsnprintf (char *str, size_t count, const char *fmt, va_list args)
-{
-  if (str != NULL)
-    str[0] = 0;
-  return dopr(str, count, fmt, args);
-}
-#endif /* !HAVE_VSNPRINTF */
-
-#ifndef HAVE_SNPRINTF
-/* VARARGS3 */
-#ifdef HAVE_STDARGS
-int snprintf (char *str,size_t count,const char *fmt,...)
-#else
-int snprintf (va_alist) va_dcl
-#endif
-{
-#ifndef HAVE_STDARGS
-  char *str;
-  size_t count;
-  char *fmt;
-#endif
-  VA_LOCAL_DECL;
-  int total;
-    
-  VA_START (fmt);
-  VA_SHIFT (str, char *);
-  VA_SHIFT (count, size_t );
-  VA_SHIFT (fmt, char *);
-  total = vsnprintf(str, count, fmt, ap);
-  VA_END;
-  return total;
-}
-#endif /* !HAVE_SNPRINTF */
-
-#ifdef TEST_SNPRINTF
-#ifndef LONG_STRING
-#define LONG_STRING 1024
-#endif
-int main (void)
-{
-  char buf1[LONG_STRING];
-  char buf2[LONG_STRING];
-  char *fp_fmt[] = {
-    "%-1.5f",
-    "%1.5f",
-    "%123.9f",
-    "%10.5f",
-    "% 10.5f",
-    "%+22.9f",
-    "%+4.9f",
-    "%01.3f",
-    "%4f",
-    "%3.1f",
-    "%3.2f",
-    "%.0f",
-    "%.1f",
-    NULL
-  };
-  double fp_nums[] = { -1.5, 134.21, 91340.2, 341.1234, 0203.9, 0.96, 0.996, 
-    0.9996, 1.996, 4.136, 0};
-  char *int_fmt[] = {
-    "%-1.5d",
-    "%1.5d",
-    "%123.9d",
-    "%5.5d",
-    "%10.5d",
-    "% 10.5d",
-    "%+22.33d",
-    "%01.3d",
-    "%4d",
-    NULL
-  };
-  long int_nums[] = { -1, 134, 91340, 341, 0203, 0};
-  int x, y;
-  int fail = 0;
-  int num = 0;
-
-  printf ("Testing snprintf format codes against system sprintf...\n");
-
-  for (x = 0; fp_fmt[x] != NULL ; x++)
-    for (y = 0; fp_nums[y] != 0 ; y++)
-    {
-      snprintf (buf1, sizeof (buf1), fp_fmt[x], fp_nums[y]);
-      sprintf (buf2, fp_fmt[x], fp_nums[y]);
-      if (strcmp (buf1, buf2))
-      {
-	printf("snprintf doesn't match Format: %s\n\tsnprintf = %s\n\tsprintf  = %s\n", 
-	    fp_fmt[x], buf1, buf2);
-	fail++;
-      }
-      num++;
-    }
-
-  for (x = 0; int_fmt[x] != NULL ; x++)
-    for (y = 0; int_nums[y] != 0 ; y++)
-    {
-      snprintf (buf1, sizeof (buf1), int_fmt[x], int_nums[y]);
-      sprintf (buf2, int_fmt[x], int_nums[y]);
-      if (strcmp (buf1, buf2))
-      {
-	printf("snprintf doesn't match Format: %s\n\tsnprintf = %s\n\tsprintf  = %s\n", 
-	    int_fmt[x], buf1, buf2);
-	fail++;
-      }
-      num++;
-    }
-  printf ("%d tests failed out of %d.\n", fail, num);
-}
-#endif /* SNPRINTF_TEST */
-
-#endif /* !HAVE_SNPRINTF */
diff --git a/autogen.sh b/autogen.sh
index e870855169e..c8cdc3f89fa 100755
--- a/autogen.sh
+++ b/autogen.sh
@@ -85,10 +85,7 @@ $TOOL --automake --copy --force
 echo Running ${AUTOCONF}...
 $AUTOCONF
 echo Running ${AUTOMAKE}...
-$AUTOMAKE --add-missing --copy --foreign
-
-# Run autogen in the argp-standalone sub-directory
-cd argp-standalone;./autogen.sh
+$AUTOMAKE --add-missing --force-missing --copy
 
 # Instruct user on next steps
 echo
diff --git a/build-aux/checkpatch.pl b/build-aux/checkpatch.pl
new file mode 100755
index 00000000000..17ae4e4d579
--- /dev/null
+++ b/build-aux/checkpatch.pl
@@ -0,0 +1,4326 @@
+#!/usr/bin/perl -w
+# (c) 2001, Dave Jones. (the file handling bit)
+# (c) 2005, Joel Schopp <jschopp@austin.ibm.com> (the ugly bit)
+# (c) 2007,2008, Andy Whitcroft <apw@uk.ibm.com> (new conditions, test suite)
+# (c) 2008-2010 Andy Whitcroft <apw@canonical.com>
+# (c) 2014 Gluster Community <gluster-devel@gluster.org>
+# Licensed under the terms of the GNU GPL License version 2
+
+use strict;
+use POSIX;
+
+my $P = $0;
+$P =~ s@.*/@@g;
+
+my $V = '0.32.1';
+
+use Getopt::Long qw(:config no_auto_abbrev);
+
+my $quiet = 0;
+my $tree = 1;
+my $chk_signoff = 1;
+my $chk_patch = 1;
+my $tst_only;
+my $emacs = 0;
+my $terse = 0;
+my $file = 0;
+my $check = 0;
+my $check_orig = 0;
+my $summary = 1;
+my $mailback = 0;
+my $summary_file = 0;
+my $show_types = 0;
+my $fix = 0;
+my $fix_inplace = 0;
+my $root;
+my %debug;
+my %camelcase = ();
+my %use_type = ();
+my @use = ();
+my %ignore_type = ();
+my @ignore = ();
+my $help = 0;
+my $configuration_file = ".checkpatch.conf";
+my $max_line_length = 80;
+my $ignore_perl_version = 0;
+my $minimum_perl_version = 5.10.0;
+my $gerrit_url = $ENV{GERRIT_URL};
+
+sub help {
+    my ($exitcode) = @_;
+
+    print << "EOM";
+Usage: $P [OPTION]... [FILE]...
+Version: $V
+
+Options:
+  -q, --quiet                quiet
+  --patch                    treat FILE as patchfile (default)
+  --emacs                    emacs compile window format
+  --gerrit-url=STRING        URL the patch was reviewed at
+  --terse                    one line per report
+  -f, --file                 treat FILE as regular source file
+  --subjective, --strict     enable more subjective tests
+  --types TYPE(,TYPE2...)    show only these comma separated message types
+  --ignore TYPE(,TYPE2...)   ignore various comma separated message types
+  --max-line-length=n        set the maximum line length, if exceeded, warn
+  --show-types               show the message "types" in the output
+  --root=PATH                PATH to the glusterfs tree root
+  --no-summary               suppress the per-file summary
+  --mailback                 only produce a report in case of warnings/errors
+  --summary-file             include the filename in summary
+  --debug KEY=[0|1]          turn on/off debugging of KEY, where KEY is one of
+                             'values', 'possible', 'type', and 'attr' (default
+                             is all off)
+  --test-only=WORD           report only warnings/errors containing WORD literally
+  --fix                      EXPERIMENTAL - may create horrible results
+                             If correctable single-line errors exist, create
+                             "<inputfile>.EXPERIMENTAL-checkpatch-fixes"
+                             with potential errors corrected to the preferred
+                             checkpatch style
+  --fix-inplace              EXPERIMENTAL - may create horrible results
+                             Is the same as --fix, but overwrites the input
+                             file.  It's your fault if there's no backup or git
+  --ignore-perl-version      override checking of perl version.  expect
+                             runtime errors.
+  -h, --help, --version      display this help and exit
+
+When FILE is - read standard input.
+EOM
+
+exit($exitcode);
+}
+
+my $conf = which_conf($configuration_file);
+if (-f $conf) {
+    my @conf_args;
+    open(my $conffile, '<', "$conf")
+        or warn "$P: Can't find a readable $configuration_file file $!\n";
+
+    while (<$conffile>) {
+        my $line = $_;
+
+        $line =~ s/\s*\n?$//g;
+        $line =~ s/^\s*//g;
+        $line =~ s/\s+/ /g;
+
+        next if ($line =~ m/^\s*#/);
+        next if ($line =~ m/^\s*$/);
+
+        my @words = split(" ", $line);
+        foreach my $word (@words) {
+            last if ($word =~ m/^#/);
+            push (@conf_args, $word);
+        }
+    }
+    close($conffile);
+    unshift(@ARGV, @conf_args) if @conf_args;
+}
+
+GetOptions(
+    'q|quiet+'  => \$quiet,
+    'patch!'    => \$chk_patch,
+    'emacs!'    => \$emacs,
+    'gerrit-url=s' => \$gerrit_url,
+    'terse!'    => \$terse,
+    'f|file!'   => \$file,
+    'subjective!'       => \$check,
+    'strict!'   => \$check,
+    'ignore=s'  => \@ignore,
+    'types=s'   => \@use,
+    'show-types!'       => \$show_types,
+    'max-line-length=i' => \$max_line_length,
+    'root=s'    => \$root,
+    'summary!'  => \$summary,
+    'mailback!' => \$mailback,
+    'summary-file!'     => \$summary_file,
+    'fix!'              => \$fix,
+    'fix-inplace!'      => \$fix_inplace,
+    'ignore-perl-version!' => \$ignore_perl_version,
+    'debug=s'   => \%debug,
+    'test-only=s'       => \$tst_only,
+    'h|help'    => \$help,
+    'version'   => \$help
+) or help(1);
+
+help(0) if ($help);
+
+$fix = 1 if ($fix_inplace);
+$check_orig = $check;
+
+my $exit = 0;
+
+if ($^V && $^V lt $minimum_perl_version) {
+    printf "$P: requires at least perl version %vd\n", $minimum_perl_version;
+    if (!$ignore_perl_version) {
+        exit(1);
+    }
+}
+
+if ($#ARGV < 0) {
+    print "$P: no input files\n";
+    exit(1);
+}
+
+sub hash_save_array_words {
+    my ($hashRef, $arrayRef) = @_;
+
+    my @array = split(/,/, join(',', @$arrayRef));
+    foreach my $word (@array) {
+        $word =~ s/\s*\n?$//g;
+        $word =~ s/^\s*//g;
+        $word =~ s/\s+/ /g;
+        $word =~ tr/[a-z]/[A-Z]/;
+
+        next if ($word =~ m/^\s*#/);
+        next if ($word =~ m/^\s*$/);
+
+        $hashRef->{$word}++;
+    }
+}
+
+sub hash_show_words {
+    my ($hashRef, $prefix) = @_;
+
+    if ($quiet == 0 && keys %$hashRef) {
+        print "NOTE: $prefix message types:";
+        foreach my $word (sort keys %$hashRef) {
+            print " $word";
+        }
+        print "\n\n";
+    }
+}
+
+hash_save_array_words(\%ignore_type, \@ignore);
+hash_save_array_words(\%use_type, \@use);
+
+my $dbg_values = 0;
+my $dbg_possible = 0;
+my $dbg_type = 0;
+my $dbg_attr = 0;
+for my $key (keys %debug) {
+    ## no critic
+    eval "\${dbg_$key} = '$debug{$key}';";
+    die "$@" if ($@);
+}
+
+my $rpt_cleaners = 0;
+
+if ($terse) {
+    $emacs = 1;
+    $quiet++;
+}
+
+if ($tree) {
+    if (defined $root) {
+        if (!top_of_glusterfs_tree($root)) {
+            die "$P: $root: --root does not point at a valid tree\n";
+        }
+    } else {
+        if (top_of_glusterfs_tree('.')) {
+            $root = '.';
+        } elsif ($0 =~ m@(.*)/extras/[^/]*$@ &&
+                 top_of_glusterfs_tree($1)) {
+            $root = $1;
+        }
+    }
+
+    if (!defined $root) {
+        print "Must be run from the top-level dir. of a GlusterFS tree\n";
+        exit(2);
+    }
+}
+
+my $emitted_corrupt = 0;
+
+our $Ident      = qr{
+                        [A-Za-z_][A-Za-z\d_]*
+                        (?:\s*\#\#\s*[A-Za-z_][A-Za-z\d_]*)*
+                }x;
+our $Storage    = qr{extern|static|asmlinkage};
+our $Sparse     = qr{
+                        __user|
+                        __kernel|
+                        __force|
+                        __iomem|
+                        __must_check|
+                        __init_refok|
+                        __kprobes|
+                        __ref|
+                        __rcu
+                }x;
+our $InitAttributePrefix = qr{__(?:mem|cpu|dev|net_|)};
+our $InitAttributeData = qr{$InitAttributePrefix(?:initdata\b)};
+our $InitAttributeConst = qr{$InitAttributePrefix(?:initconst\b)};
+our $InitAttributeInit = qr{$InitAttributePrefix(?:init\b)};
+our $InitAttribute = qr{$InitAttributeData|$InitAttributeConst|$InitAttributeInit};
+
+# Notes to $Attribute:
+# We need \b after 'init' otherwise 'initconst' will cause a false positive in a check
+our $Attribute  = qr{
+                        const|
+                        __percpu|
+                        __nocast|
+                        __safe|
+                        __bitwise__|
+                        __packed__|
+                        __packed2__|
+                        __naked|
+                        __maybe_unused|
+                        __always_unused|
+                        __noreturn|
+                        __used|
+                        __cold|
+                        __noclone|
+                        __deprecated|
+                        __read_mostly|
+                        __kprobes|
+                        $InitAttribute|
+                        ____cacheline_aligned|
+                        ____cacheline_aligned_in_smp|
+                        ____cacheline_internodealigned_in_smp|
+                        __weak
+                  }x;
+our $Modifier;
+our $Inline     = qr{inline|__always_inline|noinline|__inline|__inline__};
+our $Member     = qr{->$Ident|\.$Ident|\[[^]]*\]};
+our $Lval       = qr{$Ident(?:$Member)*};
+
+our $Int_type   = qr{(?i)llu|ull|ll|lu|ul|l|u};
+our $Binary     = qr{(?i)0b[01]+$Int_type?};
+our $Hex        = qr{(?i)0x[0-9a-f]+$Int_type?};
+our $Int        = qr{[0-9]+$Int_type?};
+our $Octal      = qr{0[0-7]+$Int_type?};
+our $Float_hex  = qr{(?i)0x[0-9a-f]+p-?[0-9]+[fl]?};
+our $Float_dec  = qr{(?i)(?:[0-9]+\.[0-9]*|[0-9]*\.[0-9]+)(?:e-?[0-9]+)?[fl]?};
+our $Float_int  = qr{(?i)[0-9]+e-?[0-9]+[fl]?};
+our $Float      = qr{$Float_hex|$Float_dec|$Float_int};
+our $Constant   = qr{$Float|$Binary|$Octal|$Hex|$Int};
+our $Assignment = qr{\*\=|/=|%=|\+=|-=|<<=|>>=|&=|\^=|\|=|=};
+our $Compare    = qr{<=|>=|==|!=|<|(?<!-)>};
+our $Arithmetic = qr{\+|-|\*|\/|%};
+our $Operators  = qr{
+                        <=|>=|==|!=|
+                        =>|->|<<|>>|<|>|!|~|
+                        &&|\|\||,|\^|\+\+|--|&|\||$Arithmetic
+                  }x;
+
+our $c90_Keywords = qr{do|for|while|if|else|return|goto|continue|switch|default|case|break}x;
+
+our $NonptrType;
+our $NonptrTypeWithAttr;
+our $Type;
+our $Declare;
+
+our $NON_ASCII_UTF8     = qr{
+        [\xC2-\xDF][\x80-\xBF]               # non-overlong 2-byte
+        |  \xE0[\xA0-\xBF][\x80-\xBF]        # excluding overlongs
+        | [\xE1-\xEC\xEE\xEF][\x80-\xBF]{2}  # straight 3-byte
+        |  \xED[\x80-\x9F][\x80-\xBF]        # excluding surrogates
+        |  \xF0[\x90-\xBF][\x80-\xBF]{2}     # planes 1-3
+        | [\xF1-\xF3][\x80-\xBF]{3}          # planes 4-15
+        |  \xF4[\x80-\x8F][\x80-\xBF]{2}     # plane 16
+}x;
+
+our $UTF8       = qr{
+        [\x09\x0A\x0D\x20-\x7E]              # ASCII
+        | $NON_ASCII_UTF8
+}x;
+
+our $typeTypedefs = qr{(?x:
+        (?:__)?(?:u|s|be|le)(?:8|16|32|64)|
+        atomic_t
+)};
+
+our $logFunctions = qr{(?x:
+        printk(?:_ratelimited|_once|)|
+        (?:[a-z0-9]+_){1,2}(?:printk|emerg|alert|crit|err|warning|warn|notice|info|debug|dbg|vdbg|devel|cont|WARN)(?:_ratelimited|_once|)|
+        WARN(?:_RATELIMIT|_ONCE|)|
+        panic|
+        MODULE_[A-Z_]+|
+        seq_vprintf|seq_printf|seq_puts
+)};
+
+our $signature_tags = qr{(?xi:
+        Signed-off-by:|
+        Acked-by:|
+        Tested-by:|
+        Reviewed-by:|
+        Reviewed-on:|
+        Reported-by:|
+        Original-author:|
+        Original-Author:|
+        Original-Authors:|
+        Suggested-by:|
+        To:|
+        Cc:
+)};
+
+our $url_tags = qr{http:|https:};
+
+our @typeList = (
+        qr{void},
+        qr{(?:unsigned\s+)?char},
+        qr{(?:unsigned\s+)?short},
+        qr{(?:unsigned\s+)?int},
+        qr{(?:unsigned\s+)?long},
+        qr{(?:unsigned\s+)?long\s+int},
+        qr{(?:unsigned\s+)?long\s+long},
+        qr{(?:unsigned\s+)?long\s+long\s+int},
+        qr{unsigned},
+        qr{float},
+        qr{double},
+        qr{bool},
+        qr{struct\s+$Ident},
+        qr{union\s+$Ident},
+        qr{enum\s+$Ident},
+        qr{${Ident}_t},
+        qr{${Ident}_handler},
+        qr{${Ident}_handler_fn},
+);
+our @typeListWithAttr = (
+        @typeList,
+        qr{struct\s+$InitAttribute\s+$Ident},
+        qr{union\s+$InitAttribute\s+$Ident},
+);
+
+our @modifierList = (
+        qr{fastcall},
+);
+
+our @mode_permission_funcs = (
+        ["module_param", 3],
+        ["module_param_(?:array|named|string)", 4],
+        ["module_param_array_named", 5],
+        ["debugfs_create_(?:file|u8|u16|u32|u64|x8|x16|x32|x64|size_t|atomic_t|bool|blob|regset32|u32_array)", 2],
+        ["proc_create(?:_data|)", 2],
+        ["(?:CLASS|DEVICE|SENSOR)_ATTR", 2],
+);
+
+#Create a search pattern for all these functions to speed up a loop below
+our $mode_perms_search = "";
+foreach my $entry (@mode_permission_funcs) {
+        $mode_perms_search .= '|' if ($mode_perms_search ne "");
+        $mode_perms_search .= $entry->[0];
+}
+
+our $declaration_macros = qr{(?x:
+        (?:$Storage\s+)?(?:DECLARE|DEFINE)_[A-Z]+\s*\(|
+        (?:$Storage\s+)?LIST_HEAD\s*\(
+)};
+
+our $allowed_asm_includes = qr{(?x:
+        irq|
+        memory
+)};
+# memory.h: ARM has a custom one
+
+sub build_types {
+    my $mods = "(?x:\n" . join("|\n  ", @modifierList) . "\n)";
+    my $all = "(?x:\n" . join("|\n  ", @typeList) . "\n)";
+    my $allWithAttr = "(?x:\n" . join("|\n  ", @typeListWithAttr) . "\n)";
+    $Modifier   = qr{(?:$Attribute|$Sparse|$mods)};
+    $NonptrType = qr{
+                        (?:$Modifier\s+|const\s+)*
+                        (?:
+                                (?:typeof|__typeof__)\s*\([^\)]*\)|
+                                (?:$typeTypedefs\b)|
+                                (?:${all}\b)
+                        )
+                        (?:\s+$Modifier|\s+const)*
+                  }x;
+        $NonptrTypeWithAttr     = qr{
+                        (?:$Modifier\s+|const\s+)*
+                        (?:
+                                (?:typeof|__typeof__)\s*\([^\)]*\)|
+                                (?:$typeTypedefs\b)|
+                                (?:${allWithAttr}\b)
+                        )
+                        (?:\s+$Modifier|\s+const)*
+                  }x;
+        $Type   = qr{
+                        $NonptrType
+                        (?:(?:\s|\*|\[\])+\s*const|(?:\s|\*|\[\])+|(?:\s*\[\s*\])+)?
+                        (?:\s+$Inline|\s+$Modifier)*
+                  }x;
+        $Declare        = qr{(?:$Storage\s+(?:$Inline\s+)?)?$Type};
+}
+build_types();
+
+our $Typecast   = qr{\s*(\(\s*$NonptrType\s*\)){0,1}\s*};
+
+# Using $balanced_parens, $LvalOrFunc, or $FuncArg
+# requires at least perl version v5.10.0
+# Any use must be runtime checked with $^V
+
+our $balanced_parens = qr/(\((?:[^\(\)]++|(?-1))*\))/;
+our $LvalOrFunc = qr{((?:[\&\*]\s*)?$Lval)\s*($balanced_parens{0,1})\s*};
+our $FuncArg = qr{$Typecast{0,1}($LvalOrFunc|$Constant)};
+
+sub deparenthesize {
+    my ($string) = @_;
+    return "" if (!defined($string));
+
+    while ($string =~ /^\s*\(.*\)\s*$/) {
+        $string =~ s@^\s*\(\s*@@;
+        $string =~ s@\s*\)\s*$@@;
+    }
+
+    $string =~ s@\s+@ @g;
+
+    return $string;
+}
+
+sub seed_camelcase_file {
+    my ($file) = @_;
+
+    return if (!(-f $file));
+
+    local $/;
+
+    open(my $include_file, '<', "$file")
+        or warn "$P: Can't read '$file' $!\n";
+    my $text = <$include_file>;
+    close($include_file);
+
+    my @lines = split('\n', $text);
+
+    foreach my $line (@lines) {
+        next if ($line !~ /(?:[A-Z][a-z]|[a-z][A-Z])/);
+        if ($line =~ /^[ \t]*(?:#[ \t]*define|typedef\s+$Type)\s+(\w*(?:[A-Z][a-z]|[a-z][A-Z])\w*)/) {
+            $camelcase{$1} = 1;
+        } elsif ($line =~ /^\s*$Declare\s+(\w*(?:[A-Z][a-z]|[a-z][A-Z])\w*)\s*[\(\[,;]/) {
+            $camelcase{$1} = 1;
+        } elsif ($line =~ /^\s*(?:union|struct|enum)\s+(\w*(?:[A-Z][a-z]|[a-z][A-Z])\w*)\s*[;\{]/) {
+            $camelcase{$1} = 1;
+        }
+    }
+}
+
+my $camelcase_seeded = 0;
+sub seed_camelcase_includes {
+    return if ($camelcase_seeded);
+
+    my $files;
+    my $camelcase_cache = "";
+    my @include_files = ();
+
+    $camelcase_seeded = 1;
+
+    if (-e ".git") {
+        my $git_last_include_commit = `git log --no-merges --pretty=format:"%h%n" -1 -- include`;
+        chomp $git_last_include_commit;
+        $camelcase_cache = ".checkpatch-camelcase.git.$git_last_include_commit";
+    } else {
+        my $last_mod_date = 0;
+        $files = `find $root/include -name "*.h"`;
+        @include_files = split('\n', $files);
+        foreach my $file (@include_files) {
+            my $date = POSIX::strftime("%Y%m%d%H%M",
+                                       localtime((stat $file)[9]));
+            $last_mod_date = $date if ($last_mod_date < $date);
+        }
+        $camelcase_cache = ".checkpatch-camelcase.date.$last_mod_date";
+    }
+
+    if ($camelcase_cache ne "" && -f $camelcase_cache) {
+        open(my $camelcase_file, '<', "$camelcase_cache")
+            or warn "$P: Can't read '$camelcase_cache' $!\n";
+        while (<$camelcase_file>) {
+            chomp;
+            $camelcase{$_} = 1;
+        }
+        close($camelcase_file);
+        return;
+    }
+
+    if (-e ".git") {
+        $files = `git ls-files "include/*.h"`;
+        @include_files = split('\n', $files);
+    }
+
+    foreach my $file (@include_files) {
+        seed_camelcase_file($file);
+    }
+
+    if ($camelcase_cache ne "") {
+        unlink glob ".checkpatch-camelcase.*";
+        open(my $camelcase_file, '>', "$camelcase_cache")
+            or warn "$P: Can't write '$camelcase_cache' $!\n";
+        foreach (sort { lc($a) cmp lc($b) } keys(%camelcase)) {
+            print $camelcase_file ("$_\n");
+        }
+        close($camelcase_file);
+    }
+}
+
+$chk_signoff = 0 if ($file);
+
+my @rawlines = ();
+my @lines = ();
+my @fixed = ();
+my $vname;
+for my $filename (@ARGV) {
+    my $FILE;
+    if ($file) {
+        open($FILE, '-|', "diff -u /dev/null $filename") ||
+            die "$P: $filename: diff failed - $!\n";
+    } elsif ($filename eq '-') {
+        open($FILE, '<&STDIN');
+    } else {
+        open($FILE, '<', "$filename") ||
+            die "$P: $filename: open failed - $!\n";
+    }
+    if ($filename eq '-') {
+        $vname = 'Your patch';
+    } else {
+        $vname = $filename;
+    }
+    while (<$FILE>) {
+        chomp;
+        push(@rawlines, $_);
+    }
+    close($FILE);
+    if (!process($filename)) {
+        $exit = 1;
+    }
+    @rawlines = ();
+    @lines = ();
+    @fixed = ();
+}
+
+exit($exit);
+
+sub top_of_glusterfs_tree {
+    my ($root) = @_;
+
+    # Add here if the tree changes
+    my @tree_check = (
+        "api",
+        "AUTHORS",
+        "autogen.sh",
+        "build-aux",
+        "ChangeLog",
+        "cli",
+        "configure.ac",
+        "contrib",
+        "CONTRIBUTING",
+        "COPYING-GPLV2",
+        "COPYING-LGPLV3",
+        "doc",
+        "extras",
+        "geo-replication",
+        "glusterfs-api.pc.in",
+        "glusterfsd",
+        "glusterfs.spec.in",
+        "heal",
+        "INSTALL",
+        "libgfchangelog.pc.in",
+        "libglusterfs",
+        "MAINTAINERS",
+        "Makefile.am",
+        "NEWS",
+        "README.md",
+        "rfc.sh",
+        "rpc",
+        "run-tests.sh",
+        "tests",
+        "THANKS",
+        "xlators",
+        );
+
+    foreach my $check (@tree_check) {
+        if (! -e $root . '/' . $check) {
+            return 0;
+        }
+    }
+    return 1;
+}
+
+sub parse_email {
+    my ($formatted_email) = @_;
+
+    my $name = "";
+    my $address = "";
+    my $comment = "";
+
+    if ($formatted_email =~ /^(.*)<(\S+\@\S+)>(.*)$/) {
+        $name = $1;
+        $address = $2;
+        $comment = $3 if defined $3;
+    } elsif ($formatted_email =~ /^\s*<(\S+\@\S+)>(.*)$/) {
+        $address = $1;
+        $comment = $2 if defined $2;
+    } elsif ($formatted_email =~ /(\S+\@\S+)(.*)$/) {
+        $address = $1;
+        $comment = $2 if defined $2;
+        $formatted_email =~ s/$address.*$//;
+        $name = $formatted_email;
+        $name = trim($name);
+        $name =~ s/^\"|\"$//g;
+        # If there's a name left after stripping spaces and
+        # leading quotes, and the address doesn't have both
+        # leading and trailing angle brackets, the address
+        # is invalid. ie:
+        #   "joe smith joe@smith.com" bad
+        #   "joe smith <joe@smith.com" bad
+        if ($name ne "" && $address !~ /^<[^>]+>$/) {
+            $name = "";
+            $address = "";
+            $comment = "";
+        }
+    }
+
+    $name = trim($name);
+    $name =~ s/^\"|\"$//g;
+    $address = trim($address);
+    $address =~ s/^\<|\>$//g;
+
+    if ($name =~ /[^\w \-]/i) { ##has "must quote" chars
+        $name =~ s/(?<!\\)"/\\"/g; ##escape quotes
+        $name = "\"$name\"";
+    }
+
+    return ($name, $address, $comment);
+}
+
+sub format_email {
+    my ($name, $address) = @_;
+
+    my $formatted_email;
+
+    $name = trim($name);
+    $name =~ s/^\"|\"$//g;
+    $address = trim($address);
+
+    if ($name =~ /[^\w \-]/i) { ##has "must quote" chars
+        $name =~ s/(?<!\\)"/\\"/g; ##escape quotes
+        $name = "\"$name\"";
+    }
+
+    if ("$name" eq "") {
+        $formatted_email = "$address";
+    } else {
+        $formatted_email = "$name <$address>";
+    }
+
+    return $formatted_email;
+}
+
+sub which_conf {
+    my ($conf) = @_;
+
+    foreach my $path (split(/:/, ".:$ENV{HOME}:.scripts")) {
+        if (-e "$path/$conf") {
+            return "$path/$conf";
+        }
+    }
+
+    return "";
+}
+
+sub expand_tabs {
+    my ($str) = @_;
+
+    my $res = '';
+    my $n = 0;
+    for my $c (split(//, $str)) {
+        if ($c eq "\t") {
+            $res .= ' ';
+            $n++;
+            for (; ($n % 8) != 0; $n++) {
+                $res .= ' ';
+            }
+            next;
+        }
+        $res .= $c;
+        $n++;
+    }
+    return $res;
+}
+sub copy_spacing {
+    (my $res = shift) =~ tr/\t/ /c;
+    return $res;
+}
+
+sub line_stats {
+    my ($line) = @_;
+
+    # Drop the diff line leader and expand tabs
+    $line =~ s/^.//;
+    $line = expand_tabs($line);
+
+    # Pick the indent from the front of the line.
+    my ($white) = ($line =~ /^(\s*)/);
+
+    return (length($line), length($white));
+}
+
+my $sanitise_quote = '';
+
+sub sanitise_line_reset {
+    my ($in_comment) = @_;
+
+    if ($in_comment) {
+        $sanitise_quote = '*/';
+    } else {
+        $sanitise_quote = '';
+    }
+}
+sub sanitise_line {
+    my ($line) = @_;
+
+    my $res = '';
+    my $l = '';
+
+    my $qlen = 0;
+    my $off = 0;
+    my $c;
+
+    # Always copy over the diff marker.
+    $res = substr($line, 0, 1);
+
+    for ($off = 1; $off < length($line); $off++) {
+        $c = substr($line, $off, 1);
+
+        # Comments we are wacking completly including the begin
+        # and end, all to $;.
+        if ($sanitise_quote eq '' && substr($line, $off, 2) eq '/*') {
+            $sanitise_quote = '*/';
+
+            substr($res, $off, 2, "$;$;");
+            $off++;
+            next;
+        }
+        if ($sanitise_quote eq '*/' && substr($line, $off, 2) eq '*/') {
+            $sanitise_quote = '';
+            substr($res, $off, 2, "$;$;");
+            $off++;
+            next;
+        }
+        if ($sanitise_quote eq '' && substr($line, $off, 2) eq '//') {
+            $sanitise_quote = '//';
+
+            substr($res, $off, 2, $sanitise_quote);
+            $off++;
+            next;
+        }
+
+        # A \ in a string means ignore the next character.
+        if (($sanitise_quote eq "'" || $sanitise_quote eq '"') &&
+            $c eq "\\") {
+            substr($res, $off, 2, 'XX');
+            $off++;
+            next;
+        }
+        # Regular quotes.
+        if ($c eq "'" || $c eq '"') {
+            if ($sanitise_quote eq '') {
+                $sanitise_quote = $c;
+
+                substr($res, $off, 1, $c);
+                next;
+            } elsif ($sanitise_quote eq $c) {
+                $sanitise_quote = '';
+            }
+        }
+
+        #print "c<$c> SQ<$sanitise_quote>\n";
+        if ($off != 0 && $sanitise_quote eq '*/' && $c ne "\t") {
+            substr($res, $off, 1, $;);
+        } elsif ($off != 0 && $sanitise_quote eq '//' && $c ne "\t") {
+            substr($res, $off, 1, $;);
+        } elsif ($off != 0 && $sanitise_quote && $c ne "\t") {
+            substr($res, $off, 1, 'X');
+        } else {
+            substr($res, $off, 1, $c);
+        }
+    }
+
+    if ($sanitise_quote eq '//') {
+        $sanitise_quote = '';
+    }
+
+    # The pathname on a #include may be surrounded by '<' and '>'.
+    if ($res =~ /^.\s*\#\s*include\s+\<(.*)\>/) {
+        my $clean = 'X' x length($1);
+        $res =~ s@\<.*\>@<$clean>@;
+
+    # The whole of a #error is a string.
+    } elsif ($res =~ /^.\s*\#\s*(?:error|warning)\s+(.*)\b/) {
+        my $clean = 'X' x length($1);
+        $res =~ s@(\#\s*(?:error|warning)\s+).*@$1$clean@;
+    }
+
+    return $res;
+}
+
+sub get_quoted_string {
+    my ($line, $rawline) = @_;
+
+    return "" if ($line !~ m/(\"[X]+\")/g);
+    return substr($rawline, $-[0], $+[0] - $-[0]);
+}
+
+sub ctx_statement_block {
+    my ($linenr, $remain, $off) = @_;
+    my $line = $linenr - 1;
+    my $blk = '';
+    my $soff = $off;
+    my $coff = $off - 1;
+    my $coff_set = 0;
+
+    my $loff = 0;
+
+    my $type = '';
+    my $level = 0;
+    my @stack = ();
+    my $p;
+    my $c;
+    my $len = 0;
+
+    my $remainder;
+    while (1) {
+        @stack = (['', 0]) if ($#stack == -1);
+
+        #warn "CSB: blk<$blk> remain<$remain>\n";
+        # If we are about to drop off the end, pull in more
+        # context.
+        if ($off >= $len) {
+            for (; $remain > 0; $line++) {
+                last if (!defined $lines[$line]);
+                next if ($lines[$line] =~ /^-/);
+                $remain--;
+                $loff = $len;
+                $blk .= $lines[$line] . "\n";
+                $len = length($blk);
+                $line++;
+                last;
+            }
+            # Bail if there is no further context.
+            #warn "CSB: blk<$blk> off<$off> len<$len>\n";
+            if ($off >= $len) {
+                last;
+            }
+            if ($level == 0 && substr($blk, $off) =~ /^.\s*#\s*define/) {
+                $level++;
+                $type = '#';
+            }
+        }
+        $p = $c;
+        $c = substr($blk, $off, 1);
+        $remainder = substr($blk, $off);
+
+        #warn "CSB: c<$c> type<$type> level<$level> remainder<$remainder> coff_set<$coff_set>\n";
+
+        # Handle nested #if/#else.
+        if ($remainder =~ /^#\s*(?:ifndef|ifdef|if)\s/) {
+            push(@stack, [ $type, $level ]);
+        } elsif ($remainder =~ /^#\s*(?:else|elif)\b/) {
+            ($type, $level) = @{$stack[$#stack - 1]};
+        } elsif ($remainder =~ /^#\s*endif\b/) {
+            ($type, $level) = @{pop(@stack)};
+        }
+
+        # Statement ends at the ';' or a close '}' at the
+        # outermost level.
+        if ($level == 0 && $c eq ';') {
+            last;
+        }
+
+        # An else is really a conditional as long as its not else if
+        if ($level == 0 && $coff_set == 0 &&
+            (!defined($p) || $p =~ /(?:\s|\}|\+)/) &&
+            $remainder =~ /^(else)(?:\s|{)/ &&
+            $remainder !~ /^else\s+if\b/) {
+            $coff = $off + length($1) - 1;
+            $coff_set = 1;
+            #warn "CSB: mark coff<$coff> soff<$soff> 1<$1>\n";
+            #warn "[" . substr($blk, $soff, $coff - $soff + 1) . "]\n";
+        }
+
+        if (($type eq '' || $type eq '(') && $c eq '(') {
+            $level++;
+            $type = '(';
+        }
+        if ($type eq '(' && $c eq ')') {
+            $level--;
+            $type = ($level != 0)? '(' : '';
+
+            if ($level == 0 && $coff < $soff) {
+                $coff = $off;
+                $coff_set = 1;
+                #warn "CSB: mark coff<$coff>\n";
+            }
+        }
+        if (($type eq '' || $type eq '{') && $c eq '{') {
+            $level++;
+            $type = '{';
+        }
+        if ($type eq '{' && $c eq '}') {
+            $level--;
+            $type = ($level != 0)? '{' : '';
+
+            if ($level == 0) {
+                if (substr($blk, $off + 1, 1) eq ';') {
+                    $off++;
+                }
+                last;
+            }
+        }
+        # Preprocessor commands end at the newline unless escaped.
+        if ($type eq '#' && $c eq "\n" && $p ne "\\") {
+            $level--;
+            $type = '';
+            $off++;
+            last;
+        }
+        $off++;
+    }
+    # We are truly at the end, so shuffle to the next line.
+    if ($off == $len) {
+        $loff = $len + 1;
+        $line++;
+        $remain--;
+    }
+
+    my $statement = substr($blk, $soff, $off - $soff + 1);
+    my $condition = substr($blk, $soff, $coff - $soff + 1);
+
+    #warn "STATEMENT<$statement>\n";
+    #warn "CONDITION<$condition>\n";
+
+    #print "coff<$coff> soff<$off> loff<$loff>\n";
+
+    return ($statement, $condition,
+            $line, $remain + 1, $off - $loff + 1, $level);
+}
+
+sub statement_lines {
+    my ($stmt) = @_;
+
+    # Strip the diff line prefixes and rip blank lines at start and end.
+    $stmt =~ s/(^|\n)./$1/g;
+    $stmt =~ s/^\s*//;
+    $stmt =~ s/\s*$//;
+
+    my @stmt_lines = ($stmt =~ /\n/g);
+
+    return $#stmt_lines + 2;
+}
+
+sub statement_rawlines {
+    my ($stmt) = @_;
+
+    my @stmt_lines = ($stmt =~ /\n/g);
+
+    return $#stmt_lines + 2;
+}
+
+sub statement_block_size {
+    my ($stmt) = @_;
+
+    $stmt =~ s/(^|\n)./$1/g;
+    $stmt =~ s/^\s*{//;
+    $stmt =~ s/}\s*$//;
+    $stmt =~ s/^\s*//;
+    $stmt =~ s/\s*$//;
+
+    my @stmt_lines = ($stmt =~ /\n/g);
+    my @stmt_statements = ($stmt =~ /;/g);
+
+    my $stmt_lines = $#stmt_lines + 2;
+    my $stmt_statements = $#stmt_statements + 1;
+
+    if ($stmt_lines > $stmt_statements) {
+        return $stmt_lines;
+    } else {
+        return $stmt_statements;
+    }
+}
+
+sub ctx_statement_full {
+    my ($linenr, $remain, $off) = @_;
+    my ($statement, $condition, $level);
+
+    my (@chunks);
+
+    # Grab the first conditional/block pair.
+    ($statement, $condition, $linenr, $remain, $off, $level) =
+        ctx_statement_block($linenr, $remain, $off);
+    #print "F: c<$condition> s<$statement> remain<$remain>\n";
+    push(@chunks, [ $condition, $statement ]);
+    if (!($remain > 0 && $condition =~ /^\s*(?:\n[+-])?\s*(?:if|else|do)\b/s)) {
+        return ($level, $linenr, @chunks);
+    }
+
+    # Pull in the following conditional/block pairs and see if they
+    # could continue the statement.
+    for (;;) {
+        ($statement, $condition, $linenr, $remain, $off, $level) =
+            ctx_statement_block($linenr, $remain, $off);
+        #print "C: c<$condition> s<$statement> remain<$remain>\n";
+        last if (!($remain > 0 && $condition =~ /^(?:\s*\n[+-])*\s*(?:else|do)\b/s));
+        #print "C: push\n";
+        push(@chunks, [ $condition, $statement ]);
+    }
+
+    return ($level, $linenr, @chunks);
+}
+
+sub ctx_block_get {
+    my ($linenr, $remain, $outer, $open, $close, $off) = @_;
+    my $line;
+    my $start = $linenr - 1;
+    my $blk = '';
+    my @o;
+    my @c;
+    my @res = ();
+
+    my $level = 0;
+    my @stack = ($level);
+    for ($line = $start; $remain > 0; $line++) {
+        next if ($rawlines[$line] =~ /^-/);
+        $remain--;
+
+        $blk .= $rawlines[$line];
+
+        # Handle nested #if/#else.
+        if ($lines[$line] =~ /^.\s*#\s*(?:ifndef|ifdef|if)\s/) {
+            push(@stack, $level);
+        } elsif ($lines[$line] =~ /^.\s*#\s*(?:else|elif)\b/) {
+            $level = $stack[$#stack - 1];
+        } elsif ($lines[$line] =~ /^.\s*#\s*endif\b/) {
+            $level = pop(@stack);
+        }
+
+        foreach my $c (split(//, $lines[$line])) {
+            ##print "C<$c>L<$level><$open$close>O<$off>\n";
+            if ($off > 0) {
+                $off--;
+                next;
+            }
+
+            if ($c eq $close && $level > 0) {
+                $level--;
+                last if ($level == 0);
+            } elsif ($c eq $open) {
+                $level++;
+            }
+        }
+
+        if (!$outer || $level <= 1) {
+            push(@res, $rawlines[$line]);
+        }
+
+        last if ($level == 0);
+    }
+
+    return ($level, @res);
+}
+sub ctx_block_outer {
+    my ($linenr, $remain) = @_;
+
+    my ($level, @r) = ctx_block_get($linenr, $remain, 1, '{', '}', 0);
+    return @r;
+}
+sub ctx_block {
+    my ($linenr, $remain) = @_;
+
+    my ($level, @r) = ctx_block_get($linenr, $remain, 0, '{', '}', 0);
+    return @r;
+}
+sub ctx_statement {
+    my ($linenr, $remain, $off) = @_;
+
+    my ($level, @r) = ctx_block_get($linenr, $remain, 0, '(', ')', $off);
+    return @r;
+}
+sub ctx_block_level {
+    my ($linenr, $remain) = @_;
+
+    return ctx_block_get($linenr, $remain, 0, '{', '}', 0);
+}
+sub ctx_statement_level {
+    my ($linenr, $remain, $off) = @_;
+
+    return ctx_block_get($linenr, $remain, 0, '(', ')', $off);
+}
+
+sub ctx_locate_comment {
+    my ($first_line, $end_line) = @_;
+
+    # Catch a comment on the end of the line itself.
+    my ($current_comment) = ($rawlines[$end_line - 1] =~ m@.*(/\*.*\*/)\s*(?:\\\s*)?$@);
+    return $current_comment if (defined $current_comment);
+
+    # Look through the context and try and figure out if there is a
+    # comment.
+    my $in_comment = 0;
+    $current_comment = '';
+    for (my $linenr = $first_line; $linenr < $end_line; $linenr++) {
+        my $line = $rawlines[$linenr - 1];
+        #warn "           $line\n";
+        if ($linenr == $first_line and $line =~ m@^.\s*\*@) {
+            $in_comment = 1;
+        }
+        if ($line =~ m@/\*@) {
+            $in_comment = 1;
+        }
+        if (!$in_comment && $current_comment ne '') {
+            $current_comment = '';
+        }
+        $current_comment .= $line . "\n" if ($in_comment);
+        if ($line =~ m@\*/@) {
+            $in_comment = 0;
+        }
+    }
+
+    chomp($current_comment);
+    return($current_comment);
+}
+sub ctx_has_comment {
+    my ($first_line, $end_line) = @_;
+    my $cmt = ctx_locate_comment($first_line, $end_line);
+
+    ##print "LINE: $rawlines[$end_line - 1 ]\n";
+    ##print "CMMT: $cmt\n";
+
+    return ($cmt ne '');
+}
+
+sub raw_line {
+    my ($linenr, $cnt) = @_;
+
+    my $offset = $linenr - 1;
+    $cnt++;
+
+    my $line;
+    while ($cnt) {
+        $line = $rawlines[$offset++];
+        next if (defined($line) && $line =~ /^-/);
+        $cnt--;
+    }
+    return $line;
+}
+
+sub cat_vet {
+    my ($vet) = @_;
+    my ($res, $coded);
+
+    $res = '';
+    while ($vet =~ /([^[:cntrl:]]*)([[:cntrl:]]|$)/g) {
+        $res .= $1;
+        if ($2 ne '') {
+            $coded = sprintf("^%c", unpack('C', $2) + 64);
+            $res .= $coded;
+        }
+    }
+    $res =~ s/$/\$/;
+    return $res;
+}
+
+my $av_preprocessor = 0;
+my $av_pending;
+my @av_paren_type;
+my $av_pend_colon;
+
+sub annotate_reset {
+    $av_preprocessor = 0;
+    $av_pending = '_';
+    @av_paren_type = ('E');
+    $av_pend_colon = 'O';
+}
+
+sub annotate_values {
+    my ($stream, $type) = @_;
+
+    my $res;
+    my $var = '_' x length($stream);
+    my $cur = $stream;
+
+    print "$stream\n" if ($dbg_values > 1);
+
+    while (length($cur)) {
+        @av_paren_type = ('E') if ($#av_paren_type < 0);
+        print " <" . join('', @av_paren_type) .
+            "> <$type> <$av_pending>" if ($dbg_values > 1);
+        if ($cur =~ /^(\s+)/o) {
+            print "WS($1)\n" if ($dbg_values > 1);
+            if ($1 =~ /\n/ && $av_preprocessor) {
+                $type = pop(@av_paren_type);
+                $av_preprocessor = 0;
+            }
+
+        } elsif ($cur =~ /^(\(\s*$Type\s*)\)/ && $av_pending eq '_') {
+            print "CAST($1)\n" if ($dbg_values > 1);
+            push(@av_paren_type, $type);
+            $type = 'c';
+
+        } elsif ($cur =~ /^($Type)\s*(?:$Ident|,|\)|\(|\s*$)/) {
+            print "DECLARE($1)\n" if ($dbg_values > 1);
+            $type = 'T';
+
+        } elsif ($cur =~ /^($Modifier)\s*/) {
+            print "MODIFIER($1)\n" if ($dbg_values > 1);
+            $type = 'T';
+
+        } elsif ($cur =~ /^(\#\s*define\s*$Ident)(\(?)/o) {
+            print "DEFINE($1,$2)\n" if ($dbg_values > 1);
+            $av_preprocessor = 1;
+            push(@av_paren_type, $type);
+            if ($2 ne '') {
+                $av_pending = 'N';
+            }
+            $type = 'E';
+
+        } elsif ($cur =~ /^(\#\s*(?:undef\s*$Ident|include\b))/o) {
+            print "UNDEF($1)\n" if ($dbg_values > 1);
+            $av_preprocessor = 1;
+            push(@av_paren_type, $type);
+
+        } elsif ($cur =~ /^(\#\s*(?:ifdef|ifndef|if))/o) {
+            print "PRE_START($1)\n" if ($dbg_values > 1);
+            $av_preprocessor = 1;
+
+            push(@av_paren_type, $type);
+            push(@av_paren_type, $type);
+            $type = 'E';
+
+        } elsif ($cur =~ /^(\#\s*(?:else|elif))/o) {
+            print "PRE_RESTART($1)\n" if ($dbg_values > 1);
+            $av_preprocessor = 1;
+
+            push(@av_paren_type, $av_paren_type[$#av_paren_type]);
+
+            $type = 'E';
+
+        } elsif ($cur =~ /^(\#\s*(?:endif))/o) {
+            print "PRE_END($1)\n" if ($dbg_values > 1);
+
+            $av_preprocessor = 1;
+
+            # Assume all arms of the conditional end as this
+            # one does, and continue as if the #endif was not here.
+            pop(@av_paren_type);
+            push(@av_paren_type, $type);
+            $type = 'E';
+
+        } elsif ($cur =~ /^(\\\n)/o) {
+            print "PRECONT($1)\n" if ($dbg_values > 1);
+
+        } elsif ($cur =~ /^(__attribute__)\s*\(?/o) {
+            print "ATTR($1)\n" if ($dbg_values > 1);
+            $av_pending = $type;
+            $type = 'N';
+
+        } elsif ($cur =~ /^(sizeof)\s*(\()?/o) {
+            print "SIZEOF($1)\n" if ($dbg_values > 1);
+            if (defined $2) {
+                $av_pending = 'V';
+            }
+            $type = 'N';
+
+        } elsif ($cur =~ /^(if|while|for)\b/o) {
+            print "COND($1)\n" if ($dbg_values > 1);
+            $av_pending = 'E';
+            $type = 'N';
+
+        } elsif ($cur =~/^(case)/o) {
+            print "CASE($1)\n" if ($dbg_values > 1);
+            $av_pend_colon = 'C';
+            $type = 'N';
+
+        } elsif ($cur =~/^(return|else|goto|typeof|__typeof__)\b/o) {
+            print "KEYWORD($1)\n" if ($dbg_values > 1);
+            $type = 'N';
+
+        } elsif ($cur =~ /^(\()/o) {
+            print "PAREN('$1')\n" if ($dbg_values > 1);
+            push(@av_paren_type, $av_pending);
+            $av_pending = '_';
+            $type = 'N';
+
+        } elsif ($cur =~ /^(\))/o) {
+            my $new_type = pop(@av_paren_type);
+            if ($new_type ne '_') {
+                $type = $new_type;
+                print "PAREN('$1') -> $type\n"
+                    if ($dbg_values > 1);
+            } else {
+                print "PAREN('$1')\n" if ($dbg_values > 1);
+            }
+
+        } elsif ($cur =~ /^($Ident)\s*\(/o) {
+            print "FUNC($1)\n" if ($dbg_values > 1);
+            $type = 'V';
+            $av_pending = 'V';
+
+        } elsif ($cur =~ /^($Ident\s*):(?:\s*\d+\s*(,|=|;))?/) {
+            if (defined $2 && $type eq 'C' || $type eq 'T') {
+                $av_pend_colon = 'B';
+            } elsif ($type eq 'E') {
+                $av_pend_colon = 'L';
+            }
+            print "IDENT_COLON($1,$type>$av_pend_colon)\n" if ($dbg_values > 1);
+            $type = 'V';
+
+        } elsif ($cur =~ /^($Ident|$Constant)/o) {
+            print "IDENT($1)\n" if ($dbg_values > 1);
+            $type = 'V';
+
+        } elsif ($cur =~ /^($Assignment)/o) {
+            print "ASSIGN($1)\n" if ($dbg_values > 1);
+            $type = 'N';
+
+        } elsif ($cur =~/^(;|{|})/) {
+            print "END($1)\n" if ($dbg_values > 1);
+            $type = 'E';
+            $av_pend_colon = 'O';
+
+        } elsif ($cur =~/^(,)/) {
+            print "COMMA($1)\n" if ($dbg_values > 1);
+            $type = 'C';
+
+        } elsif ($cur =~ /^(\?)/o) {
+            print "QUESTION($1)\n" if ($dbg_values > 1);
+            $type = 'N';
+
+        } elsif ($cur =~ /^(:)/o) {
+            print "COLON($1,$av_pend_colon)\n" if ($dbg_values > 1);
+
+            substr($var, length($res), 1, $av_pend_colon);
+            if ($av_pend_colon eq 'C' || $av_pend_colon eq 'L') {
+                $type = 'E';
+            } else {
+                $type = 'N';
+            }
+            $av_pend_colon = 'O';
+
+        } elsif ($cur =~ /^(\[)/o) {
+            print "CLOSE($1)\n" if ($dbg_values > 1);
+            $type = 'N';
+
+        } elsif ($cur =~ /^(-(?![->])|\+(?!\+)|\*|\&\&|\&)/o) {
+            my $variant;
+
+            print "OPV($1)\n" if ($dbg_values > 1);
+            if ($type eq 'V') {
+                $variant = 'B';
+            } else {
+                $variant = 'U';
+            }
+
+            substr($var, length($res), 1, $variant);
+            $type = 'N';
+
+        } elsif ($cur =~ /^($Operators)/o) {
+            print "OP($1)\n" if ($dbg_values > 1);
+            if ($1 ne '++' && $1 ne '--') {
+                $type = 'N';
+            }
+
+        } elsif ($cur =~ /(^.)/o) {
+            print "C($1)\n" if ($dbg_values > 1);
+        }
+        if (defined $1) {
+            $cur = substr($cur, length($1));
+            $res .= $type x length($1);
+        }
+    }
+
+    return ($res, $var);
+}
+
+sub possible {
+    my ($possible, $line) = @_;
+    my $notPermitted = qr{(?:
+                ^(?:
+                        $Modifier|
+                        $Storage|
+                        $Type|
+                        DEFINE_\S+
+                )$|
+                ^(?:
+                        goto|
+                        return|
+                        case|
+                        else|
+                        asm|__asm__|
+                        do|
+                        \#|
+                        \#\#|
+                )(?:\s|$)|
+                ^(?:typedef|struct|enum)\b
+            )}x;
+    warn "CHECK<$possible> ($line)\n" if ($dbg_possible > 2);
+    if ($possible !~ $notPermitted) {
+        # Check for modifiers.
+        $possible =~ s/\s*$Storage\s*//g;
+        $possible =~ s/\s*$Sparse\s*//g;
+        if ($possible =~ /^\s*$/) {
+
+        } elsif ($possible =~ /\s/) {
+            $possible =~ s/\s*$Type\s*//g;
+            for my $modifier (split(' ', $possible)) {
+                if ($modifier !~ $notPermitted) {
+                    warn "MODIFIER: $modifier ($possible) ($line)\n" if ($dbg_possible);
+                    push(@modifierList, $modifier);
+                }
+            }
+
+        } else {
+            warn "POSSIBLE: $possible ($line)\n" if ($dbg_possible);
+            push(@typeList, $possible);
+        }
+        build_types();
+    } else {
+        warn "NOTPOSS: $possible ($line)\n" if ($dbg_possible > 1);
+    }
+}
+
+my $prefix = '';
+
+sub show_type {
+    my ($type) = @_;
+
+    return defined $use_type{$type} if (scalar keys %use_type > 0);
+
+    return !defined $ignore_type{$type};
+}
+
+sub report {
+    my ($level, $type, $msg) = @_;
+
+    if (!show_type($type) ||
+        (defined $tst_only && $msg !~ /\Q$tst_only\E/)) {
+        return 0;
+    }
+    my $line;
+    if ($show_types) {
+        $line = "$prefix$level:$type: $msg\n";
+    } else {
+        $line = "$prefix$level: $msg\n";
+    }
+    $line = (split('\n', $line))[0] . "\n" if ($terse);
+
+    if ($quiet == 0) {
+        push(our @report, $line);
+    }
+    return 1;
+}
+
+sub report_dump {
+    our @report;
+}
+
+sub ERROR {
+    my ($type, $msg) = @_;
+
+    if (report("ERROR", $type, $msg)) {
+        our $clean = 0;
+        our $cnt_error++;
+        return 1;
+    }
+    return 0;
+}
+sub WARN {
+    my ($type, $msg) = @_;
+
+    if (report("WARNING", $type, $msg)) {
+        ## Warning is okay to submit
+        our $clean = 0;
+        our $cnt_warn++;
+        return 1;
+    }
+    return 0;
+}
+sub CHK {
+    my ($type, $msg) = @_;
+
+    if ($check && report("CHECK", $type, $msg)) {
+        our $clean = 0;
+        our $cnt_chk++;
+        return 1;
+    }
+    return 0;
+}
+
+sub check_absolute_file {
+    my ($absolute, $herecurr) = @_;
+    my $file = $absolute;
+
+    ##print "absolute<$absolute>\n";
+
+    # See if any suffix of this path is a path within the tree.
+    while ($file =~ s@^[^/]*/@@) {
+        if (-f "$root/$file") {
+            ##print "file<$file>\n";
+            last;
+        }
+    }
+    if (! -f _)  {
+        return 0;
+    }
+
+    # It is, so see if the prefix is acceptable.
+    my $prefix = $absolute;
+    substr($prefix, -length($file)) = '';
+
+    ##print "prefix<$prefix>\n";
+    if ($prefix ne ".../") {
+        WARN("USE_RELATIVE_PATH",
+             "use relative pathname instead of absolute in changelog text\n" . $herecurr);
+    }
+}
+
+sub trim {
+    my ($string) = @_;
+
+    $string =~ s/^\s+|\s+$//g;
+
+    return $string;
+}
+
+sub ltrim {
+    my ($string) = @_;
+
+    $string =~ s/^\s+//;
+
+    return $string;
+}
+
+sub rtrim {
+    my ($string) = @_;
+
+    $string =~ s/\s+$//;
+
+    return $string;
+}
+
+sub string_find_replace {
+    my ($string, $find, $replace) = @_;
+
+    $string =~ s/$find/$replace/g;
+
+    return $string;
+}
+
+sub tabify {
+    my ($leading) = @_;
+
+    my $source_indent = 8;
+    my $max_spaces_before_tab = $source_indent - 1;
+    my $spaces_to_tab = " " x $source_indent;
+
+    #convert leading spaces to tabs
+    1 while $leading =~ s@^([\t]*)$spaces_to_tab@$1\t@g;
+    #Remove spaces before a tab
+    1 while $leading =~ s@^([\t]*)( {1,$max_spaces_before_tab})\t@$1\t@g;
+    return "$leading";
+}
+
+sub pos_last_openparen {
+    my ($line) = @_;
+
+    my $pos = 0;
+
+    my $opens = $line =~ tr/\(/\(/;
+    my $closes = $line =~ tr/\)/\)/;
+
+    my $last_openparen = 0;
+
+    if (($opens == 0) || ($closes >= $opens)) {
+        return -1;
+    }
+
+    my $len = length($line);
+
+    for ($pos = 0; $pos < $len; $pos++) {
+        my $string = substr($line, $pos);
+        if ($string =~ /^($FuncArg|$balanced_parens)/) {
+            $pos += length($1) - 1;
+        } elsif (substr($line, $pos, 1) eq '(') {
+            $last_openparen = $pos;
+        } elsif (index($string, '(') == -1) {
+            last;
+        }
+    }
+
+    return length(expand_tabs(substr($line, 0, $last_openparen))) + 1;
+}
+
+sub process {
+    my $filename = shift;
+
+    my $linenr=0;
+    my $prevline="";
+    my $prevrawline="";
+    my $stashline="";
+    my $stashrawline="";
+
+    my $length;
+    my $indent;
+    my $previndent=0;
+    my $stashindent=0;
+
+    our $clean = 1;
+    my $signoff = 0;
+    my $subject_trailing_dot = 0;
+    my $is_patch = 0;
+
+    my $in_header_lines = 1;
+    my $in_commit_log = 0; #Scanning lines before patch
+
+    my $non_utf8_charset = 0;
+
+    our @report = ();
+    our $cnt_lines = 0;
+    our $cnt_error = 0;
+    our $cnt_warn = 0;
+    our $cnt_chk = 0;
+
+    # Trace the real file/line as we go.
+    my $realfile = '';
+    my $realline = 0;
+    my $realcnt = 0;
+    my $here = '';
+    my $in_comment = 0;
+    my $comment_edge = 0;
+    my $first_line = 0;
+    my $p1_prefix = '';
+
+    my $prev_values = 'E';
+
+    # suppression flags
+    my %suppress_ifbraces;
+    my %suppress_whiletrailers;
+    my %suppress_export;
+    my $suppress_statement = 0;
+
+    my %signatures = ();
+
+    # Pre-scan the patch sanitizing the lines.
+    # Pre-scan the patch looking for any __setup documentation.
+    #
+    my @setup_docs = ();
+    my $setup_docs = 0;
+
+    my $camelcase_file_seeded = 0;
+
+    sanitise_line_reset();
+    my $line;
+    foreach my $rawline (@rawlines) {
+        $linenr++;
+        $line = $rawline;
+
+        push(@fixed, $rawline) if ($fix);
+
+        if ($rawline=~/^\+\+\+\s+(\S+)/) {
+            $setup_docs = 0;
+            if ($1 =~ m@Documentation/kernel-parameters.txt$@) {
+                $setup_docs = 1;
+            }
+            #next;
+        }
+        if ($rawline=~/^\@\@ -\d+(?:,\d+)? \+(\d+)(,(\d+))? \@\@/) {
+            $realline=$1-1;
+            if (defined $2) {
+                $realcnt=$3+1;
+            } else {
+                $realcnt=1+1;
+            }
+            $in_comment = 0;
+
+            # Guestimate if this is a continuing comment.  Run
+            # the context looking for a comment "edge".  If this
+            # edge is a close comment then we must be in a comment
+            # at context start.
+            my $edge;
+            my $cnt = $realcnt;
+            for (my $ln = $linenr + 1; $cnt > 0; $ln++) {
+                next if (defined $rawlines[$ln - 1] &&
+                         $rawlines[$ln - 1] =~ /^-/);
+                $cnt--;
+                #print "RAW<$rawlines[$ln - 1]>\n";
+                last if (!defined $rawlines[$ln - 1]);
+                if ($rawlines[$ln - 1] =~ m@(/\*|\*/)@ &&
+                    $rawlines[$ln - 1] !~ m@"[^"]*(?:/\*|\*/)[^"]*"@) {
+                    ($edge) = $1;
+                    last;
+                }
+            }
+            if (defined $edge && $edge eq '*/') {
+                $in_comment = 1;
+            }
+
+            # Guestimate if this is a continuing comment.  If this
+            # is the start of a diff block and this line starts
+            # ' *' then it is very likely a comment.
+            if (!defined $edge &&
+                $rawlines[$linenr] =~ m@^.\s*(?:\*\*+| \*)(?:\s|$)@)
+            {
+                $in_comment = 1;
+            }
+
+            ##print "COMMENT:$in_comment edge<$edge> $rawline\n";
+            sanitise_line_reset($in_comment);
+
+        } elsif ($realcnt && $rawline =~ /^(?:\+| |$)/) {
+            # Standardise the strings and chars within the input to
+            # simplify matching -- only bother with positive lines.
+            $line = sanitise_line($rawline);
+        }
+        push(@lines, $line);
+
+        if ($realcnt > 1) {
+            $realcnt-- if ($line =~ /^(?:\+| |$)/);
+        } else {
+            $realcnt = 0;
+        }
+
+        #print "==>$rawline\n";
+        #print "-->$line\n";
+
+        if ($setup_docs && $line =~ /^\+/) {
+            push(@setup_docs, $line);
+        }
+    }
+
+    $prefix = '';
+
+    $realcnt = 0;
+    $linenr = 0;
+    foreach my $line (@lines) {
+        $linenr++;
+        my $sline = $line;      #copy of $line
+        $sline =~ s/$;/ /g;     #with comments as spaces
+
+        my $rawline = $rawlines[$linenr - 1];
+
+#extract the line range in the file after the patch is applied
+        if ($line=~/^\@\@ -\d+(?:,\d+)? \+(\d+)(,(\d+))? \@\@/) {
+            $is_patch = 1;
+            $first_line = $linenr + 1;
+            $realline=$1-1;
+            if (defined $2) {
+                $realcnt=$3+1;
+            } else {
+                $realcnt=1+1;
+            }
+            annotate_reset();
+            $prev_values = 'E';
+
+            %suppress_ifbraces = ();
+            %suppress_whiletrailers = ();
+            %suppress_export = ();
+            $suppress_statement = 0;
+            next;
+
+# track the line number as we move through the hunk, note that
+# new versions of GNU diff omit the leading space on completely
+# blank context lines so we need to count that too.
+        } elsif ($line =~ /^( |\+|$)/) {
+            $realline++;
+            $realcnt-- if ($realcnt != 0);
+
+            # Measure the line length and indent.
+            ($length, $indent) = line_stats($rawline);
+
+            # Track the previous line.
+            ($prevline, $stashline) = ($stashline, $line);
+            ($previndent, $stashindent) = ($stashindent, $indent);
+            ($prevrawline, $stashrawline) = ($stashrawline, $rawline);
+
+            #warn "line<$line>\n";
+
+        } elsif ($realcnt == 1) {
+            $realcnt--;
+        }
+
+        my $hunk_line = ($realcnt != 0);
+
+#make up the handle for any error we report on this line
+        $prefix = "$filename:$realline: " if ($emacs && $file);
+        $prefix = "$filename:$linenr: " if ($emacs && !$file);
+
+        $here = "#$linenr: " if (!$file);
+        $here = "#$realline: " if ($file);
+
+        my $found_file = 0;
+        # extract the filename as it passes
+        if ($line =~ /^diff --git.*?(\S+)$/) {
+            $realfile = $1;
+            $realfile =~ s@^([^/]*)/@@ if (!$file);
+            $in_commit_log = 0;
+            $found_file = 1;
+        } elsif ($line =~ /^\+\+\+\s+(\S+)/) {
+            $realfile = $1;
+            $realfile =~ s@^([^/]*)/@@ if (!$file);
+            $in_commit_log = 0;
+
+            $p1_prefix = $1;
+            if (!$file && $tree && $p1_prefix ne '' &&
+                -e "$root/$p1_prefix") {
+                WARN("PATCH_PREFIX",
+                     "patch prefix '$p1_prefix' exists, appears to be a -p0 patch\n");
+            }
+
+            $found_file = 1;
+        }
+
+        if ($found_file) {
+            if ($realfile =~ m@^(drivers/net/|net/)@) {
+                $check = 1;
+            } else {
+                $check = $check_orig;
+            }
+            next;
+        }
+
+        $here .= "FILE: $realfile:$realline:" if ($realcnt != 0);
+
+        my $hereline = "$here\n$rawline\n";
+        my $herecurr = "$here\n$rawline\n";
+        my $hereprev = "$here\n$prevrawline\n$rawline\n";
+
+        $cnt_lines++ if ($realcnt != 0);
+
+# Check for incorrect file permissions
+        if ($line =~ /^new (file )?mode.*[7531]\d{0,2}$/) {
+            my $permhere = $here . "FILE: $realfile\n";
+            if ($realfile !~ m@scripts/@ &&
+                $realfile !~ /\.(py|pl|awk|sh|t)$/) {
+                ERROR("EXECUTE_PERMISSIONS",
+                      "do not set execute permissions for source files\n" . $permhere);
+            }
+        }
+
+        next if ($realfile =~ /(checkpatch.pl)/);
+        next if ($realfile =~ /\.(md|txt|doc|8|pdf|tex)$/);
+
+# Check that the subject does not have a trailing dot
+        if ($in_header_lines &&
+            $line =~ /^Subject: \[PATCH\] (.+)\.(\s*)$/) {
+                $subject_trailing_dot++;
+        }
+
+# Check the patch for a signoff:
+        if ($line =~ /^\s*signed-off-by:/i) {
+            $signoff++;
+            $in_commit_log = 0;
+        }
+
+# Check signature styles
+        if (!$in_header_lines &&
+            $line =~ /^(\s*)([a-z0-9_-]+by:|$signature_tags)(\s*)(.*)/i) {
+            my $space_before = $1;
+            my $sign_off = $2;
+            my $space_after = $3;
+            my $email = $4;
+            my $ucfirst_sign_off = ucfirst(lc($sign_off));
+
+            if ($sign_off !~ /$signature_tags/) {
+                WARN("BAD_SIGN_OFF",
+                     "Non-standard signature: $sign_off\n" . $herecurr);
+            }
+            if (defined $space_before && $space_before ne "") {
+                if (WARN("BAD_SIGN_OFF",
+                         "Do not use whitespace before $ucfirst_sign_off\n" . $herecurr) &&
+                    $fix) {
+                    $fixed[$linenr - 1] =
+                        "$ucfirst_sign_off $email";
+                }
+            }
+            if ($sign_off =~ /-by:$/i && $sign_off ne $ucfirst_sign_off) {
+                if (WARN("BAD_SIGN_OFF",
+                         "'$ucfirst_sign_off' is the preferred signature form\n" . $herecurr) &&
+                    $fix) {
+                    $fixed[$linenr - 1] =
+                        "$ucfirst_sign_off $email";
+                }
+
+            }
+            if (!defined $space_after || $space_after ne " ") {
+                if (WARN("BAD_SIGN_OFF",
+                         "Use a single space after $ucfirst_sign_off\n" . $herecurr) &&
+                    $fix) {
+                    $fixed[$linenr - 1] =
+                        "$ucfirst_sign_off $email";
+                }
+            }
+
+            # Check if email is really Gerrit URL
+            if ($email =~ /^($url_tags)(.*)/) {
+                my $uri = $1;
+                my $url = $2;
+                if ($uri && $url !~ /$gerrit_url/) {
+                    ERROR("BAD_URL",
+                          "Unrecognized url address: '$email'\n" . $herecurr);
+                }
+            } else {
+                my ($email_name, $email_address, $comment) = parse_email($email);
+                my $suggested_email = format_email(($email_name, $email_address));
+                if ($suggested_email eq "") {
+                    ERROR("BAD_SIGN_OFF",
+                          "Unrecognized email address: '$email'\n" . $herecurr);
+                } else {
+                    my $dequoted = $suggested_email;
+                    $dequoted =~ s/^"//;
+                    $dequoted =~ s/" </ </;
+                    # Don't force email to have quotes
+                    # Allow just an angle bracketed address
+                    if ("$dequoted$comment" ne $email &&
+                        "<$email_address>$comment" ne $email &&
+                        "$suggested_email$comment" ne $email) {
+                        WARN("BAD_SIGN_OFF",
+                             "email address '$email' might be better as '$suggested_email$comment'\n" . $herecurr);
+                    }
+                }
+            }
+
+# Check for duplicate signatures
+            my $sig_nospace = $line;
+            $sig_nospace =~ s/\s//g;
+            $sig_nospace = lc($sig_nospace);
+            if (defined $signatures{$sig_nospace}) {
+                WARN("BAD_SIGN_OFF",
+                     "Duplicate signature\n" . $herecurr);
+            } else {
+                $signatures{$sig_nospace} = 1;
+            }
+        }
+
+# Check for wrappage within a valid hunk of the file
+        if ($realcnt != 0 && $line !~ m{^(?:\+|-| |\\ No newline|$)}) {
+            ERROR("CORRUPTED_PATCH",
+                  "patch seems to be corrupt (line wrapped?)\n" .
+                  $herecurr) if (!$emitted_corrupt++);
+        }
+
+# Check for absolute kernel paths.
+        if ($tree) {
+            while ($line =~ m{(?:^|\s)(/\S*)}g) {
+                my $file = $1;
+
+                if ($file =~ m{^(.*?)(?::\d+)+:?$} &&
+                    check_absolute_file($1, $herecurr)) {
+                    #
+                } else {
+                    check_absolute_file($file, $herecurr);
+                }
+            }
+        }
+
+# UTF-8 regex found at http://www.w3.org/International/questions/qa-forms-utf-8.en.php
+        if (($realfile =~ /^$/ || $line =~ /^\+/) &&
+            $rawline !~ m/^$UTF8*$/) {
+            my ($utf8_prefix) = ($rawline =~ /^($UTF8*)/);
+
+            my $blank = copy_spacing($rawline);
+            my $ptr = substr($blank, 0, length($utf8_prefix)) . "^";
+            my $hereptr = "$hereline$ptr\n";
+
+            CHK("INVALID_UTF8",
+                "Invalid UTF-8, patch and commit message should be encoded in UTF-8\n" . $hereptr);
+        }
+
+# Check if it's the start of a commit log
+# (not a header line and we haven't seen the patch filename)
+        if ($in_header_lines && $realfile =~ /^$/ &&
+            $rawline !~ /^(commit\b|from\b|[\w-]+:).+$/i) {
+            $in_header_lines = 0;
+            $in_commit_log = 1;
+        }
+
+# Check if there is UTF-8 in a commit log when a mail header has explicitly
+# declined it, i.e defined some charset where it is missing.
+        if ($in_header_lines &&
+            $rawline =~ /^Content-Type:.+charset="(.+)".*$/ &&
+            $1 !~ /utf-8/i) {
+            $non_utf8_charset = 1;
+        }
+
+        if ($in_commit_log && $non_utf8_charset && $realfile =~ /^$/ &&
+            $rawline =~ /$NON_ASCII_UTF8/) {
+            WARN("UTF8_BEFORE_PATCH",
+                 "8-bit UTF-8 used in possible commit log\n" . $herecurr);
+        }
+
+# ignore non-hunk lines and lines being removed
+        next if (!$hunk_line || $line =~ /^-/);
+
+#trailing whitespace
+        if ($line =~ /^\+.*\015/) {
+            my $herevet = "$here\n" . cat_vet($rawline) . "\n";
+            if (ERROR("DOS_LINE_ENDINGS",
+                      "DOS line endings\n" . $herevet) &&
+                $fix) {
+                $fixed[$linenr - 1] =~ s/[\s\015]+$//;
+            }
+        } elsif ($rawline =~ /^\+.*\S\s+$/ || $rawline =~ /^\+\s+$/) {
+            my $herevet = "$here\n" . cat_vet($rawline) . "\n";
+            if (ERROR("TRAILING_WHITESPACE",
+                      "trailing whitespace\n" . $herevet) &&
+                $fix) {
+                $fixed[$linenr - 1] =~ s/\s+$//;
+            }
+
+            $rpt_cleaners = 1;
+        }
+
+        if (($realfile =~ /Makefile.*/) &&
+            ($line =~ /\+(EXTRA_[A-Z]+FLAGS).*/)) {
+            my $flag = $1;
+            my $replacement = {
+                            'EXTRA_AFLAGS' =>   'asflags-y',
+                            'EXTRA_CFLAGS' =>   'ccflags-y',
+                            'EXTRA_CPPFLAGS' => 'cppflags-y',
+                            'EXTRA_LDFLAGS' =>  'ldflags-y',
+            };
+
+            WARN("DEPRECATED_VARIABLE",
+                 "Use of $flag is deprecated, please use \`$replacement->{$flag} instead.\n" . $herecurr) if ($replacement->{$flag});
+        }
+
+# check we are in .spec file, then ignore this hunk
+        next if ($realfile eq "glusterfs.spec.in");
+
+# check we are in a valid source file if not then ignore this hunk
+        next if ($realfile !~ /\.(h|c|pl|py|l|y|sh|in)$/);
+
+#line length limit
+        if ($line =~ /^\+/ && $prevrawline !~ /\/\*\*/ &&
+            $rawline !~ /^.\s*\*\s*\@$Ident\s/ &&
+            !($line =~ /^\+\s*$logFunctions\s*\(\s*(?:(KERN_\S+\s*|[^"]*))?"[X\t]*"\s*(?:|,|\)\s*;)\s*$/ ||
+              $line =~ /^\+\s*"[^"]*"\s*(?:\s*|,|\)\s*;)\s*$/) &&
+            $length > $max_line_length)
+        {
+            WARN("LONG_LINE",
+                 "line over $max_line_length characters\n" . $herecurr);
+        }
+
+# check for spaces before a quoted newline
+        if ($rawline =~ /^.*\".*\s\\n/) {
+            if (WARN("QUOTED_WHITESPACE_BEFORE_NEWLINE",
+                     "unnecessary whitespace before a quoted newline\n" . $herecurr) &&
+                $fix) {
+                $fixed[$linenr - 1] =~ s/^(\+.*\".*)\s+\\n/$1\\n/;
+            }
+
+        }
+
+# check for adding lines without a newline.
+        if ($line =~ /^\+/ && defined $lines[$linenr] && $lines[$linenr] =~ /^\\ No newline at end of file/) {
+            WARN("MISSING_EOF_NEWLINE",
+                 "adding a line without newline at end of file\n" . $herecurr);
+        }
+
+# check we are in a valid source file C or perl if not then ignore this hunk
+        next if ($realfile !~ /\.(h|c|pl)$/);
+
+# check for space before tabs.
+        if ($rawline =~ /^\+/ && $rawline =~ / \t/) {
+            my $herevet = "$here\n" . cat_vet($rawline) . "\n";
+            if (WARN("SPACE_BEFORE_TAB",
+                     "please, no space before tabs\n" . $herevet) &&
+                $fix) {
+                while ($fixed[$linenr - 1] =~
+                       s/(^\+.*) {8,8}\t/$1\t\t/) {}
+                while ($fixed[$linenr - 1] =~
+                       s/(^\+.*) +\t/$1\t/) {}
+            }
+        }
+
+# check for && or || at the start of a line
+        if ($rawline =~ /^\+\s*(&&|\|\|)/) {
+            CHK("LOGICAL_CONTINUATIONS",
+                "Logical continuations should be on the previous line\n" . $hereprev);
+        }
+
+# check multi-line statement indentation matches previous line
+        if ($^V && $^V ge 5.10.0 &&
+            $prevline =~ /^\+([ \t]*)((?:$c90_Keywords(?:\s+if)\s*)|(?:$Declare\s*)?(?:$Ident|\(\s*\*\s*$Ident\s*\))\s*|$Ident\s*=\s*$Ident\s*)\(.*(\&\&|\|\||,)\s*$/) {
+            $prevline =~ /^\+(\t*)(.*)$/;
+            my $oldindent = $1;
+            my $rest = $2;
+
+            my $pos = pos_last_openparen($rest);
+            if ($pos >= 0) {
+                $line =~ /^(\+| )([ \t]*)/;
+                my $newindent = $2;
+
+                my $goodtabindent = $oldindent .
+                    "\t" x ($pos / 8) .
+                    " "  x ($pos % 8);
+                my $goodspaceindent = $oldindent . " "  x $pos;
+
+                if ($newindent ne $goodtabindent &&
+                    $newindent ne $goodspaceindent) {
+
+                    if (CHK("PARENTHESIS_ALIGNMENT",
+                            "Alignment should match open parenthesis\n" . $hereprev) &&
+                        $fix && $line =~ /^\+/) {
+                        $fixed[$linenr - 1] =~
+                            s/^\+[ \t]*/\+$goodtabindent/;
+                    }
+                }
+            }
+        }
+
+        if ($line =~ /^\+.*\*[ \t]*\)[ \t]+(?!$Assignment|$Arithmetic)/) {
+            if (CHK("SPACING",
+                    "No space is necessary after a cast\n" . $hereprev) &&
+                $fix) {
+                $fixed[$linenr - 1] =~
+                    s/^(\+.*\*[ \t]*\))[ \t]+/$1/;
+            }
+        }
+
+
+# check for missing blank lines after declarations
+        if ($sline =~ /^\+\s+\S/ &&  #Not at char 1
+            # actual declarations
+            ($prevline =~ /^\+\s+$Declare\s*$Ident\s*[=,;:\[]/ ||
+             # foo bar; where foo is some local typedef or #define
+             $prevline =~ /^\+\s+$Ident(?:\s+|\s*\*\s*)$Ident\s*[=,;\[]/ ||
+             # known declaration macros
+             $prevline =~ /^\+\s+$declaration_macros/) &&
+            # for "else if" which can look like "$Ident $Ident"
+            !($prevline =~ /^\+\s+$c90_Keywords\b/ ||
+              # other possible extensions of declaration lines
+              $prevline =~ /(?:$Compare|$Assignment|$Operators)\s*$/ ||
+              # not starting a section or a macro "\" extended line
+              $prevline =~ /(?:\{\s*|\\)$/) &&
+            # looks like a declaration
+            !($sline =~ /^\+\s+$Declare\s*$Ident\s*[=,;:\[]/ ||
+              # foo bar; where foo is some local typedef or #define
+              $sline =~ /^\+\s+$Ident(?:\s+|\s*\*\s*)$Ident\s*[=,;\[]/ ||
+              # known declaration macros
+              $sline =~ /^\+\s+$declaration_macros/ ||
+              # start of struct or union or enum
+              $sline =~ /^\+\s+(?:union|struct|enum|typedef)\b/ ||
+              # start or end of block or continuation of declaration
+              $sline =~ /^\+\s+(?:$|[\{\}\.\#\"\?\:\(\[])/ ||
+              # bitfield continuation
+              $sline =~ /^\+\s+$Ident\s*:\s*\d+\s*[,;]/ ||
+              # other possible extensions of declaration lines
+              $sline =~ /^\+\s+\(?\s*(?:$Compare|$Assignment|$Operators)/) &&
+            # indentation of previous and current line are the same
+            (($prevline =~ /\+(\s+)\S/) && $sline =~ /^\+$1\S/)) {
+            WARN("SPACING",
+                 "Missing a blank line after declarations\n" . $hereprev);
+        }
+
+# check we are in a valid C source file if not then ignore this hunk
+        next if ($realfile !~ /\.(h|c)$/);
+
+# check for RCS/CVS revision markers
+        if ($rawline =~ /^\+.*\$(Revision|Log|Id)(?:\$|)/) {
+            WARN("CVS_KEYWORD",
+                 "CVS style keyword markers, these will _not_ be updated\n". $herecurr);
+        }
+
+# Check for potential 'bare' types
+        my ($stat, $cond, $line_nr_next, $remain_next, $off_next,
+            $realline_next);
+#print "LINE<$line>\n";
+        if ($linenr >= $suppress_statement &&
+            $realcnt && $sline =~ /.\s*\S/) {
+            ($stat, $cond, $line_nr_next, $remain_next, $off_next) =
+                ctx_statement_block($linenr, $realcnt, 0);
+            $stat =~ s/\n./\n /g;
+            $cond =~ s/\n./\n /g;
+
+#print "linenr<$linenr> <$stat>\n";
+            # If this statement has no statement boundaries within
+            # it there is no point in retrying a statement scan
+            # until we hit end of it.
+            my $frag = $stat; $frag =~ s/;+\s*$//;
+            if ($frag !~ /(?:{|;)/) {
+#print "skip<$line_nr_next>\n";
+                $suppress_statement = $line_nr_next;
+            }
+
+            # Find the real next line.
+            $realline_next = $line_nr_next;
+            if (defined $realline_next &&
+                (!defined $lines[$realline_next - 1] ||
+                 substr($lines[$realline_next - 1], $off_next) =~ /^\s*$/)) {
+                $realline_next++;
+            }
+
+            my $s = $stat;
+            $s =~ s/{.*$//s;
+
+            # Ignore goto labels.
+            if ($s =~ /$Ident:\*$/s) {
+
+                # Ignore functions being called
+            } elsif ($s =~ /^.\s*$Ident\s*\(/s) {
+
+            } elsif ($s =~ /^.\s*else\b/s) {
+
+                # declarations always start with types
+            } elsif ($prev_values eq 'E' && $s =~ /^.\s*(?:$Storage\s+)?(?:$Inline\s+)?(?:const\s+)?((?:\s*$Ident)+?)\b(?:\s+$Sparse)?\s*\**\s*(?:$Ident|\(\*[^\)]*\))(?:\s*$Modifier)?\s*(?:;|=|,|\()/s) {
+                my $type = $1;
+                $type =~ s/\s+/ /g;
+                possible($type, "A:" . $s);
+
+                # definitions in global scope can only start with types
+            } elsif ($s =~ /^.(?:$Storage\s+)?(?:$Inline\s+)?(?:const\s+)?($Ident)\b\s*(?!:)/s) {
+                possible($1, "B:" . $s);
+            }
+
+            # any (foo ... *) is a pointer cast, and foo is a type
+            while ($s =~ /\(($Ident)(?:\s+$Sparse)*[\s\*]+\s*\)/sg) {
+                possible($1, "C:" . $s);
+            }
+
+            # Check for any sort of function declaration.
+            # int foo(something bar, other baz);
+            # void (*store_gdt)(x86_descr_ptr *);
+            if ($prev_values eq 'E' && $s =~ /^(.(?:typedef\s*)?(?:(?:$Storage|$Inline)\s*)*\s*$Type\s*(?:\b$Ident|\(\*\s*$Ident\))\s*)\(/s) {
+                my ($name_len) = length($1);
+
+                my $ctx = $s;
+                substr($ctx, 0, $name_len + 1, '');
+                $ctx =~ s/\)[^\)]*$//;
+
+                for my $arg (split(/\s*,\s*/, $ctx)) {
+                    if ($arg =~ /^(?:const\s+)?($Ident)(?:\s+$Sparse)*\s*\**\s*(:?\b$Ident)?$/s || $arg =~ /^($Ident)$/s) {
+
+                        possible($1, "D:" . $s);
+                    }
+                }
+            }
+
+        }
+
+#
+# Checks which may be anchored in the context.
+#
+
+# Check for switch () and associated case and default
+# statements should be at the same indent.
+        if ($line=~/\bswitch\s*\(.*\)/) {
+            my $err = '';
+            my $sep = '';
+            my @ctx = ctx_block_outer($linenr, $realcnt);
+            shift(@ctx);
+            for my $ctx (@ctx) {
+                my ($clen, $cindent) = line_stats($ctx);
+                if ($ctx =~ /^\+\s*(case\s+|default:)/ &&
+                    $indent != $cindent) {
+                    $err .= "$sep$ctx\n";
+                    $sep = '';
+                } else {
+                    $sep = "[...]\n";
+                }
+            }
+            if ($err ne '') {
+                ERROR("SWITCH_CASE_INDENT_LEVEL",
+                      "switch and case should be at the same indent\n$hereline$err");
+            }
+        }
+
+# if/while/etc brace do not go on next line, unless defining a do while loop,
+# or if that brace on the next line is for something else
+        if ($line =~ /(.*)\b((?:if|while|for|switch)\s*\(|do\b|else\b)/ && $line !~ /^.\s*\#/) {
+            my $pre_ctx = "$1$2";
+
+            my ($level, @ctx) = ctx_statement_level($linenr, $realcnt, 0);
+
+            if ($line =~ /^\+\t{6,}/) {
+                WARN("DEEP_INDENTATION",
+                     "Too many leading tabs - consider code refactoring\n" . $herecurr);
+            }
+
+            my $ctx_cnt = $realcnt - $#ctx - 1;
+            my $ctx = join("\n", @ctx);
+
+            my $ctx_ln = $linenr;
+            my $ctx_skip = $realcnt;
+
+            while ($ctx_skip > $ctx_cnt || ($ctx_skip == $ctx_cnt &&
+                                            defined $lines[$ctx_ln - 1] &&
+                                            $lines[$ctx_ln - 1] =~ /^-/)) {
+                ##print "SKIP<$ctx_skip> CNT<$ctx_cnt>\n";
+                $ctx_skip-- if (!defined $lines[$ctx_ln - 1] || $lines[$ctx_ln - 1] !~ /^-/);
+                $ctx_ln++;
+            }
+
+            #print "realcnt<$realcnt> ctx_cnt<$ctx_cnt>\n";
+            #print "pre<$pre_ctx>\nline<$line>\nctx<$ctx>\nnext<$lines[$ctx_ln - 1]>\n";
+
+            if ($ctx !~ /{\s*/ && defined($lines[$ctx_ln -1]) && $lines[$ctx_ln - 1] =~ /^\+\s*{/) {
+                ERROR("OPEN_BRACE",
+                      "that open brace { should be on the previous line\n" .
+                      "$here\n$ctx\n$rawlines[$ctx_ln - 1]\n");
+            }
+            if ($level == 0 && $pre_ctx !~ /}\s*while\s*\($/ &&
+                $ctx =~ /\)\s*\;\s*$/ &&
+                defined $lines[$ctx_ln - 1])
+            {
+                my ($nlength, $nindent) = line_stats($lines[$ctx_ln - 1]);
+                if ($nindent > $indent) {
+                    WARN("TRAILING_SEMICOLON",
+                         "trailing semicolon indicates no statements, indent implies otherwise\n" .
+                         "$here\n$ctx\n$rawlines[$ctx_ln - 1]\n");
+                }
+            }
+        }
+
+# Check relative indent for conditionals and blocks.
+        if ($line =~ /\b(?:(?:if|while|for)\s*\(|do\b)/ && $line !~ /^.\s*#/ && $line !~ /\}\s*while\s*/) {
+            ($stat, $cond, $line_nr_next, $remain_next, $off_next) =
+                ctx_statement_block($linenr, $realcnt, 0)
+                if (!defined $stat);
+            my ($s, $c) = ($stat, $cond);
+
+            substr($s, 0, length($c), '');
+
+            # Make sure we remove the line prefixes as we have
+            # none on the first line, and are going to readd them
+            # where necessary.
+            $s =~ s/\n./\n/gs;
+
+            # Find out how long the conditional actually is.
+            my @newlines = ($c =~ /\n/gs);
+            my $cond_lines = 1 + $#newlines;
+
+            # We want to check the first line inside the block
+            # starting at the end of the conditional, so remove:
+            #  1) any blank line termination
+            #  2) any opening brace { on end of the line
+            #  3) any do (...) {
+            my $continuation = 0;
+            my $check = 0;
+            $s =~ s/^.*\bdo\b//;
+            $s =~ s/^\s*{//;
+            if ($s =~ s/^\s*\\//) {
+                $continuation = 1;
+            }
+            if ($s =~ s/^\s*?\n//) {
+                $check = 1;
+                $cond_lines++;
+            }
+
+            # Also ignore a loop construct at the end of a
+            # preprocessor statement.
+            if (($prevline =~ /^.\s*#\s*define\s/ ||
+                 $prevline =~ /\\\s*$/) && $continuation == 0) {
+                $check = 0;
+            }
+
+            my $cond_ptr = -1;
+            $continuation = 0;
+            while ($cond_ptr != $cond_lines) {
+                $cond_ptr = $cond_lines;
+
+                # If we see an #else/#elif then the code
+                # is not linear.
+                if ($s =~ /^\s*\#\s*(?:else|elif)/) {
+                    $check = 0;
+                }
+
+                # Ignore:
+                #  1) blank lines, they should be at 0,
+                #  2) preprocessor lines, and
+                #  3) labels.
+                if ($continuation ||
+                    $s =~ /^\s*?\n/ ||
+                    $s =~ /^\s*#\s*?/ ||
+                    $s =~ /^\s*$Ident\s*:/) {
+                    $continuation = ($s =~ /^.*?\\\n/) ? 1 : 0;
+                    if ($s =~ s/^.*?\n//) {
+                        $cond_lines++;
+                    }
+                }
+            }
+
+            my (undef, $sindent) = line_stats("+" . $s);
+            my $stat_real = raw_line($linenr, $cond_lines);
+
+            # Check if either of these lines are modified, else
+            # this is not this patch's fault.
+            if (!defined($stat_real) ||
+                $stat !~ /^\+/ && $stat_real !~ /^\+/) {
+                $check = 0;
+            }
+            if (defined($stat_real) && $cond_lines > 1) {
+                $stat_real = "[...]\n$stat_real";
+            }
+
+            #print "line<$line> prevline<$prevline> indent<$indent> sindent<$sindent> check<$check> continuation<$continuation> s<$s> cond_lines<$cond_lines> stat_real<$stat_real> stat<$stat>\n";
+
+            if ($check && (($sindent % 8) != 0 ||
+                           ($sindent <= $indent && $s ne ''))) {
+                WARN("SUSPECT_CODE_INDENT",
+                     "suspect code indent for conditional statements ($indent, $sindent)\n" . $herecurr . "$stat_real\n");
+            }
+        }
+
+        # Track the 'values' across context and added lines.
+        my $opline = $line; $opline =~ s/^./ /;
+        my ($curr_values, $curr_vars) =
+            annotate_values($opline . "\n", $prev_values);
+        $curr_values = $prev_values . $curr_values;
+        if ($dbg_values) {
+            my $outline = $opline; $outline =~ s/\t/ /g;
+            print "$linenr > .$outline\n";
+            print "$linenr > $curr_values\n";
+            print "$linenr >  $curr_vars\n";
+        }
+        $prev_values = substr($curr_values, -1);
+
+#ignore lines not being added
+        next if ($line =~ /^[^\+]/);
+
+# TEST: allow direct testing of the type matcher.
+        if ($dbg_type) {
+            if ($line =~ /^.\s*$Declare\s*$/) {
+                ERROR("TEST_TYPE",
+                      "TEST: is type\n" . $herecurr);
+            } elsif ($dbg_type > 1 && $line =~ /^.+($Declare)/) {
+                ERROR("TEST_NOT_TYPE",
+                      "TEST: is not type ($1 is)\n". $herecurr);
+            }
+            next;
+        }
+# TEST: allow direct testing of the attribute matcher.
+        if ($dbg_attr) {
+            if ($line =~ /^.\s*$Modifier\s*$/) {
+                ERROR("TEST_ATTR",
+                      "TEST: is attr\n" . $herecurr);
+            } elsif ($dbg_attr > 1 && $line =~ /^.+($Modifier)/) {
+                ERROR("TEST_NOT_ATTR",
+                      "TEST: is not attr ($1 is)\n". $herecurr);
+            }
+            next;
+        }
+
+# check for initialisation to aggregates open brace on the next line
+        if ($line =~ /^.\s*{/ &&
+            $prevline =~ /(?:^|[^=])=\s*$/) {
+            ERROR("OPEN_BRACE",
+                  "that open brace { should be on the previous line\n" . $hereprev);
+        }
+
+#
+# Checks which are anchored on the added line.
+#
+
+# check for malformed paths in #include statements (uses RAW line)
+        if ($rawline =~ m{^.\s*\#\s*include\s+[<"](.*)[">]}) {
+            my $path = $1;
+            if ($path =~ m{//}) {
+                ERROR("MALFORMED_INCLUDE",
+                      "malformed #include filename\n" . $herecurr);
+            }
+            if ($path =~ "^uapi/" && $realfile =~ m@\binclude/uapi/@) {
+                ERROR("UAPI_INCLUDE",
+                      "No #include in ...include/uapi/... should use a uapi/ path prefix\n" . $herecurr);
+            }
+        }
+
+# no C99 // comments
+        if ($line =~ m{//}) {
+            if (ERROR("C99_COMMENTS",
+                      "do not use C99 // comments\n" . $herecurr) &&
+                $fix) {
+                my $line = $fixed[$linenr - 1];
+                if ($line =~ /\/\/(.*)$/) {
+                    my $comment = trim($1);
+                    $fixed[$linenr - 1] =~ s@\/\/(.*)$@/\* $comment \*/@;
+                }
+            }
+        }
+        # Remove C99 comments.
+        $line =~ s@//.*@@;
+        $opline =~ s@//.*@@;
+
+# EXPORT_SYMBOL should immediately follow the thing it is exporting, consider
+# the whole statement.
+#print "APW <$lines[$realline_next - 1]>\n";
+        if (defined $realline_next &&
+            exists $lines[$realline_next - 1] &&
+            !defined $suppress_export{$realline_next} &&
+            ($lines[$realline_next - 1] =~ /EXPORT_SYMBOL.*\((.*)\)/ ||
+             $lines[$realline_next - 1] =~ /EXPORT_UNUSED_SYMBOL.*\((.*)\)/)) {
+            # Handle definitions which produce identifiers with
+            # a prefix:
+            #   XXX(foo);
+            #   EXPORT_SYMBOL(something_foo);
+            my $name = $1;
+            if ($stat =~ /^(?:.\s*}\s*\n)?.([A-Z_]+)\s*\(\s*($Ident)/ &&
+                $name =~ /^${Ident}_$2/) {
+#print "FOO C name<$name>\n";
+                $suppress_export{$realline_next} = 1;
+
+            } elsif ($stat !~ /(?:
+                                \n.}\s*$|
+                                ^.DEFINE_$Ident\(\Q$name\E\)|
+                                ^.DECLARE_$Ident\(\Q$name\E\)|
+                                ^.LIST_HEAD\(\Q$name\E\)|
+                                ^.(?:$Storage\s+)?$Type\s*\(\s*\*\s*\Q$name\E\s*\)\s*\(|
+                                \b\Q$name\E(?:\s+$Attribute)*\s*(?:;|=|\[|\()
+                            )/x) {
+#print "FOO A<$lines[$realline_next - 1]> stat<$stat> name<$name>\n";
+                $suppress_export{$realline_next} = 2;
+            } else {
+                $suppress_export{$realline_next} = 1;
+            }
+        }
+        if (!defined $suppress_export{$linenr} &&
+            $prevline =~ /^.\s*$/ &&
+            ($line =~ /EXPORT_SYMBOL.*\((.*)\)/ ||
+             $line =~ /EXPORT_UNUSED_SYMBOL.*\((.*)\)/)) {
+#print "FOO B <$lines[$linenr - 1]>\n";
+            $suppress_export{$linenr} = 2;
+        }
+        if (defined $suppress_export{$linenr} &&
+            $suppress_export{$linenr} == 2) {
+            WARN("EXPORT_SYMBOL",
+                 "EXPORT_SYMBOL(foo); should immediately follow its function/variable\n" . $herecurr);
+        }
+
+# check for global initialisers.
+        if ($line =~ /^\+(\s*$Type\s*$Ident\s*(?:\s+$Modifier))*\s*=\s*(0|NULL|false)\s*;/) {
+            if (ERROR("GLOBAL_INITIALISERS",
+                      "do not initialise globals to 0 or NULL\n" .
+                      $herecurr) &&
+                $fix) {
+                $fixed[$linenr - 1] =~ s/($Type\s*$Ident\s*(?:\s+$Modifier))*\s*=\s*(0|NULL|false)\s*;/$1;/;
+            }
+        }
+# check for static initialisers.
+        if ($line =~ /^\+.*\bstatic\s.*=\s*(0|NULL|false)\s*;/) {
+            if (ERROR("INITIALISED_STATIC",
+                      "do not initialise statics to 0 or NULL\n" .
+                      $herecurr) &&
+                $fix) {
+                $fixed[$linenr - 1] =~ s/(\bstatic\s.*?)\s*=\s*(0|NULL|false)\s*;/$1;/;
+            }
+        }
+
+# check for static const char * arrays.
+        if ($line =~ /\bstatic\s+const\s+char\s*\*\s*(\w+)\s*\[\s*\]\s*=\s*/) {
+            WARN("STATIC_CONST_CHAR_ARRAY",
+                 "static const char * array should probably be static const char * const\n" .
+                 $herecurr);
+        }
+
+# check for static char foo[] = "bar" declarations.
+        if ($line =~ /\bstatic\s+char\s+(\w+)\s*\[\s*\]\s*=\s*"/) {
+            WARN("STATIC_CONST_CHAR_ARRAY",
+                 "static char array declaration should probably be static const char\n" .
+                 $herecurr);
+        }
+
+# check for non-global char *foo[] = {"bar", ...} declarations.
+        if ($line =~ /^.\s+(?:static\s+|const\s+)?char\s+\*\s*\w+\s*\[\s*\]\s*=\s*\{/) {
+            WARN("STATIC_CONST_CHAR_ARRAY",
+                 "char * array declaration might be better as static const\n" .
+                 $herecurr);
+        }
+
+# check for function declarations without arguments like "int foo()"
+        if ($line =~ /(\b$Type\s+$Ident)\s*\(\s*\)/) {
+            if (ERROR("FUNCTION_WITHOUT_ARGS",
+                      "Bad function definition - $1() should probably be $1(void)\n" . $herecurr) &&
+                $fix) {
+                $fixed[$linenr - 1] =~ s/(\b($Type)\s+($Ident))\s*\(\s*\)/$2 $3(void)/;
+            }
+        }
+
+# * goes on variable not on type
+        # (char*[ const])
+        while ($line =~ m{(\($NonptrType(\s*(?:$Modifier\b\s*|\*\s*)+)\))}g) {
+            #print "AA<$1>\n";
+            my ($ident, $from, $to) = ($1, $2, $2);
+
+            # Should start with a space.
+            $to =~ s/^(\S)/ $1/;
+            # Should not end with a space.
+            $to =~ s/\s+$//;
+            # '*'s should not have spaces between.
+            while ($to =~ s/\*\s+\*/\*\*/) {
+            }
+
+##   print "1: from<$from> to<$to> ident<$ident>\n";
+            if ($from ne $to) {
+                if (ERROR("POINTER_LOCATION",
+                          "\"(foo$from)\" should be \"(foo$to)\"\n" .  $herecurr) &&
+                    $fix) {
+                    my $sub_from = $ident;
+                    my $sub_to = $ident;
+                    $sub_to =~ s/\Q$from\E/$to/;
+                    $fixed[$linenr - 1] =~
+                        s@\Q$sub_from\E@$sub_to@;
+                }
+            }
+        }
+        while ($line =~ m{(\b$NonptrType(\s*(?:$Modifier\b\s*|\*\s*)+)($Ident))}g) {
+            #print "BB<$1>\n";
+            my ($match, $from, $to, $ident) = ($1, $2, $2, $3);
+
+            # Should start with a space.
+            $to =~ s/^(\S)/ $1/;
+            # Should not end with a space.
+            $to =~ s/\s+$//;
+            # '*'s should not have spaces between.
+            while ($to =~ s/\*\s+\*/\*\*/) {
+            }
+            # Modifiers should have spaces.
+            $to =~ s/(\b$Modifier$)/$1 /;
+
+##  print "2: from<$from> to<$to> ident<$ident>\n";
+            if ($from ne $to && $ident !~ /^$Modifier$/) {
+                if (ERROR("POINTER_LOCATION",
+                          "\"foo${from}bar\" should be \"foo${to}bar\"\n" .  $herecurr) &&
+                    $fix) {
+
+                    my $sub_from = $match;
+                    my $sub_to = $match;
+                    $sub_to =~ s/\Q$from\E/$to/;
+                    $fixed[$linenr - 1] =~
+                        s@\Q$sub_from\E@$sub_to@;
+                }
+            }
+        }
+
+# function brace can't be on same line, except for #defines of do while,
+# or if closed on same line
+        if (($line=~/$Type\s*$Ident\(.*\).*\s\{/) and
+            !($line=~/\#\s*define.*do\s\{/) and !($line=~/}/)) {
+            ERROR("OPEN_BRACE",
+                  "open brace '{' following function declarations go on the next line\n" . $herecurr);
+        }
+
+# open braces for enum, union and struct go on the same line.
+        if ($line =~ /^.\s*{/ &&
+            $prevline =~ /^.\s*(?:typedef\s+)?(enum|union|struct)(?:\s+$Ident)?\s*$/) {
+            ERROR("OPEN_BRACE",
+                  "open brace '{' following $1 go on the same line\n" . $hereprev);
+        }
+
+# missing space after union, struct or enum definition
+        if ($line =~ /^.\s*(?:typedef\s+)?(enum|union|struct)(?:\s+$Ident){1,2}[=\{]/) {
+            if (WARN("SPACING",
+                     "missing space after $1 definition\n" . $herecurr) &&
+                $fix) {
+                $fixed[$linenr - 1] =~
+                    s/^(.\s*(?:typedef\s+)?(?:enum|union|struct)(?:\s+$Ident){1,2})([=\{])/$1 $2/;
+            }
+        }
+
+# Function pointer declarations
+# check spacing between type, funcptr, and args
+# canonical declaration is "type (*funcptr)(args...)"
+        if ($line =~ /^.\s*($Declare)\((\s*)\*(\s*)($Ident)(\s*)\)(\s*)\(/) {
+            my $declare = $1;
+            my $pre_pointer_space = $2;
+            my $post_pointer_space = $3;
+            my $funcname = $4;
+            my $post_funcname_space = $5;
+            my $pre_args_space = $6;
+
+# the $Declare variable will capture all spaces after the type
+# so check it for a missing trailing missing space but pointer return types
+# don't need a space so don't warn for those.
+            my $post_declare_space = "";
+            if ($declare =~ /(\s+)$/) {
+                $post_declare_space = $1;
+                $declare = rtrim($declare);
+            }
+            if ($declare !~ /\*$/ && $post_declare_space =~ /^$/) {
+                WARN("SPACING",
+                     "missing space after return type\n" . $herecurr);
+                $post_declare_space = " ";
+            }
+
+# unnecessary space "type ( *funcptr)(args...)"
+            if (defined $pre_pointer_space &&
+                $pre_pointer_space =~ /^\s/) {
+                WARN("SPACING",
+                     "Unnecessary space after function pointer open parenthesis\n" . $herecurr);
+            }
+
+# unnecessary space "type (* funcptr)(args...)"
+            if (defined $post_pointer_space &&
+                $post_pointer_space =~ /^\s/) {
+                WARN("SPACING",
+                     "Unnecessary space before function pointer name\n" . $herecurr);
+            }
+
+# unnecessary space "type (*funcptr )(args...)"
+            if (defined $post_funcname_space &&
+                $post_funcname_space =~ /^\s/) {
+                WARN("SPACING",
+                     "Unnecessary space after function pointer name\n" . $herecurr);
+            }
+
+# unnecessary space "type (*funcptr) (args...)"
+            if (defined $pre_args_space &&
+                $pre_args_space =~ /^\s/) {
+                WARN("SPACING",
+                     "Unnecessary space before function pointer arguments\n" . $herecurr);
+            }
+
+            if (show_type("SPACING") && $fix) {
+                $fixed[$linenr - 1] =~
+                    s/^(.\s*)$Declare\s*\(\s*\*\s*$Ident\s*\)\s*\(/$1 . $declare . $post_declare_space . '(*' . $funcname . ')('/ex;
+            }
+        }
+
+# check for spacing round square brackets; allowed:
+#  1. with a type on the left -- int [] a;
+#  2. at the beginning of a line for slice initialisers -- [0...10] = 5,
+#  3. inside a curly brace -- = { [0...10] = 5 }
+        while ($line =~ /(.*?\s)\[/g) {
+            my ($where, $prefix) = ($-[1], $1);
+            if ($prefix !~ /$Type\s+$/ &&
+                ($where != 0 || $prefix !~ /^.\s+$/) &&
+                $prefix !~ /[{,]\s+$/) {
+                if (ERROR("BRACKET_SPACE",
+                          "space prohibited before open square bracket '['\n" . $herecurr) &&
+                    $fix) {
+                    $fixed[$linenr - 1] =~
+                        s/^(\+.*?)\s+\[/$1\[/;
+                }
+            }
+        }
+
+# Check operator spacing.
+        if (!($line=~/\#\s*include/)) {
+            my $fixed_line = "";
+            my $line_fixed = 0;
+
+            my $ops = qr{
+                                <<=|>>=|<=|>=|==|!=|
+                                \+=|-=|\*=|\/=|%=|\^=|\|=|&=|
+                                =>|->|<<|>>|<|>|=|!|~|
+                                &&|\|\||,|\^|\+\+|--|&|\||\+|-|\*|\/|%|
+                                \?:|\?|:
+                        }x;
+            my @elements = split(/($ops|;)/, $opline);
+
+##                      print("element count: <" . $#elements . ">\n");
+##                      foreach my $el (@elements) {
+##                              print("el: <$el>\n");
+##                      }
+
+            my @fix_elements = ();
+            my $off = 0;
+
+            foreach my $el (@elements) {
+                push(@fix_elements, substr($rawline, $off, length($el)));
+                $off += length($el);
+            }
+
+            $off = 0;
+
+            my $blank = copy_spacing($opline);
+            my $last_after = -1;
+
+            for (my $n = 0; $n < $#elements; $n += 2) {
+
+                my $good = $fix_elements[$n] . $fix_elements[$n + 1];
+
+##                              print("n: <$n> good: <$good>\n");
+
+                $off += length($elements[$n]);
+
+                # Pick up the preceding and succeeding characters.
+                my $ca = substr($opline, 0, $off);
+                my $cc = '';
+                if (length($opline) >= ($off + length($elements[$n + 1]))) {
+                    $cc = substr($opline, $off + length($elements[$n + 1]));
+                }
+                my $cb = "$ca$;$cc";
+
+                my $a = '';
+                $a = 'V' if ($elements[$n] ne '');
+                $a = 'W' if ($elements[$n] =~ /\s$/);
+                $a = 'C' if ($elements[$n] =~ /$;$/);
+                $a = 'B' if ($elements[$n] =~ /(\[|\()$/);
+                $a = 'O' if ($elements[$n] eq '');
+                $a = 'E' if ($ca =~ /^\s*$/);
+
+                my $op = $elements[$n + 1];
+
+                my $c = '';
+                if (defined $elements[$n + 2]) {
+                    $c = 'V' if ($elements[$n + 2] ne '');
+                    $c = 'W' if ($elements[$n + 2] =~ /^\s/);
+                    $c = 'C' if ($elements[$n + 2] =~ /^$;/);
+                    $c = 'B' if ($elements[$n + 2] =~ /^(\)|\]|;)/);
+                    $c = 'O' if ($elements[$n + 2] eq '');
+                    $c = 'E' if ($elements[$n + 2] =~ /^\s*\\$/);
+                } else {
+                    $c = 'E';
+                }
+
+                my $ctx = "${a}x${c}";
+
+                my $at = "(ctx:$ctx)";
+
+                my $ptr = substr($blank, 0, $off) . "^";
+                my $hereptr = "$hereline$ptr\n";
+
+                # Pull out the value of this operator.
+                my $op_type = substr($curr_values, $off + 1, 1);
+
+                # Get the full operator variant.
+                my $opv = $op . substr($curr_vars, $off, 1);
+
+                # Ignore operators passed as parameters.
+                if ($op_type ne 'V' &&
+                    $ca =~ /\s$/ && $cc =~ /^\s*,/) {
+
+#                               # Ignore comments
+#                               } elsif ($op =~ /^$;+$/) {
+
+                    # ; should have either the end of line or a space or \ after it
+                } elsif ($op eq ';') {
+                    if ($ctx !~ /.x[WEBC]/ &&
+                        $cc !~ /^\\/ && $cc !~ /^;/) {
+                        if (ERROR("SPACING",
+                                  "space required after that '$op' $at\n" . $hereptr)) {
+                            $good = $fix_elements[$n] . trim($fix_elements[$n + 1]) . " ";
+                            $line_fixed = 1;
+                        }
+                    }
+
+                    # // is a comment
+                } elsif ($op eq '//') {
+
+                    #   :   when part of a bitfield
+                } elsif ($opv eq ':B') {
+                    # skip the bitfield test for now
+
+                    # No spaces for:
+                    #   ->
+                } elsif ($op eq '->') {
+                    if ($ctx =~ /Wx.|.xW/) {
+                        if (ERROR("SPACING",
+                                  "spaces prohibited around that '$op' $at\n" . $hereptr)) {
+                            $good = rtrim($fix_elements[$n]) . trim($fix_elements[$n + 1]);
+                            if (defined $fix_elements[$n + 2]) {
+                                $fix_elements[$n + 2] =~ s/^\s+//;
+                            }
+                            $line_fixed = 1;
+                        }
+                    }
+
+                    # , must have a space on the right.
+                } elsif ($op eq ',') {
+                    if ($ctx !~ /.x[WEC]/ && $cc !~ /^}/) {
+                        if (ERROR("SPACING",
+                                  "space required after that '$op' $at\n" . $hereptr)) {
+                            $good = $fix_elements[$n] . trim($fix_elements[$n + 1]) . " ";
+                            $line_fixed = 1;
+                            $last_after = $n;
+                        }
+                    }
+
+                    # '*' as part of a type definition -- reported already.
+                } elsif ($opv eq '*_') {
+                    #warn "'*' is part of type\n";
+
+                    # unary operators should have a space before and
+                    # none after.  May be left adjacent to another
+                    # unary operator, or a cast
+                } elsif ($op eq '!' || $op eq '~' ||
+                         $opv eq '*U' || $opv eq '-U' ||
+                         $opv eq '&U' || $opv eq '&&U') {
+                    if ($ctx !~ /[WEBC]x./ && $ca !~ /(?:\)|!|~|\*|-|\&|\||\+\+|\-\-|\{)$/) {
+                        if (ERROR("SPACING",
+                                  "space required before that '$op' $at\n" . $hereptr)) {
+                            if ($n != $last_after + 2) {
+                                $good = $fix_elements[$n] . " " . ltrim($fix_elements[$n + 1]);
+                                $line_fixed = 1;
+                            }
+                        }
+                    }
+                    if ($op eq '*' && $cc =~/\s*$Modifier\b/) {
+                        # A unary '*' may be const
+
+                    } elsif ($ctx =~ /.xW/) {
+                        if (ERROR("SPACING",
+                                  "space prohibited after that '$op' $at\n" . $hereptr)) {
+                            $good = $fix_elements[$n] . rtrim($fix_elements[$n + 1]);
+                            if (defined $fix_elements[$n + 2]) {
+                                $fix_elements[$n + 2] =~ s/^\s+//;
+                            }
+                            $line_fixed = 1;
+                        }
+                    }
+
+                    # unary ++ and unary -- are allowed no space on one side.
+                } elsif ($op eq '++' or $op eq '--') {
+                    if ($ctx !~ /[WEOBC]x[^W]/ && $ctx !~ /[^W]x[WOBEC]/) {
+                        if (ERROR("SPACING",
+                                  "space required one side of that '$op' $at\n" . $hereptr)) {
+                            $good = $fix_elements[$n] . trim($fix_elements[$n + 1]) . " ";
+                            $line_fixed = 1;
+                        }
+                    }
+                    if ($ctx =~ /Wx[BE]/ ||
+                        ($ctx =~ /Wx./ && $cc =~ /^;/)) {
+                        if (ERROR("SPACING",
+                                  "space prohibited before that '$op' $at\n" . $hereptr)) {
+                            $good = rtrim($fix_elements[$n]) . trim($fix_elements[$n + 1]);
+                            $line_fixed = 1;
+                        }
+                    }
+                    if ($ctx =~ /ExW/) {
+                        if (ERROR("SPACING",
+                                  "space prohibited after that '$op' $at\n" . $hereptr)) {
+                            $good = $fix_elements[$n] . trim($fix_elements[$n + 1]);
+                            if (defined $fix_elements[$n + 2]) {
+                                $fix_elements[$n + 2] =~ s/^\s+//;
+                            }
+                            $line_fixed = 1;
+                        }
+                    }
+
+                    # << and >> may either have or not have spaces both sides
+                } elsif ($op eq '<<' or $op eq '>>' or
+                         $op eq '&' or $op eq '^' or $op eq '|' or
+                         $op eq '+' or $op eq '-' or
+                         $op eq '*' or $op eq '/' or
+                         $op eq '%')
+                {
+                    if ($ctx =~ /Wx[^WCE]|[^WCE]xW/) {
+                        if (ERROR("SPACING",
+                                  "need consistent spacing around '$op' $at\n" . $hereptr)) {
+                            $good = rtrim($fix_elements[$n]) . " " . trim($fix_elements[$n + 1]) . " ";
+                            if (defined $fix_elements[$n + 2]) {
+                                $fix_elements[$n + 2] =~ s/^\s+//;
+                            }
+                            $line_fixed = 1;
+                        }
+                    }
+
+                    # A colon needs no spaces before when it is
+                    # terminating a case value or a label.
+                } elsif ($opv eq ':C' || $opv eq ':L') {
+                    if ($ctx =~ /Wx./) {
+                        if (ERROR("SPACING",
+                                  "space prohibited before that '$op' $at\n" . $hereptr)) {
+                            $good = rtrim($fix_elements[$n]) . trim($fix_elements[$n + 1]);
+                            $line_fixed = 1;
+                        }
+                    }
+
+                    # All the others need spaces both sides.
+                } elsif ($ctx !~ /[EWC]x[CWE]/) {
+                    my $ok = 0;
+
+                    # Ignore email addresses <foo@bar>
+                    if (($op eq '<' &&
+                         $cc =~ /^\S+\@\S+>/) ||
+                        ($op eq '>' &&
+                         $ca =~ /<\S+\@\S+$/))
+                    {
+                        $ok = 1;
+                    }
+
+                    # messages are ERROR, but ?: are CHK
+                    if ($ok == 0) {
+                        my $msg_type = \&ERROR;
+                        $msg_type = \&CHK if (($op eq '?:' || $op eq '?' || $op eq ':') && $ctx =~ /VxV/);
+
+                        if (&{$msg_type}("SPACING",
+                                         "spaces required around that '$op' $at\n" . $hereptr)) {
+                            $good = rtrim($fix_elements[$n]) . " " . trim($fix_elements[$n + 1]) . " ";
+                            if (defined $fix_elements[$n + 2]) {
+                                $fix_elements[$n + 2] =~ s/^\s+//;
+                            }
+                            $line_fixed = 1;
+                        }
+                    }
+                }
+                $off += length($elements[$n + 1]);
+
+##                              print("n: <$n> GOOD: <$good>\n");
+
+                $fixed_line = $fixed_line . $good;
+            }
+
+            if (($#elements % 2) == 0) {
+                $fixed_line = $fixed_line . $fix_elements[$#elements];
+            }
+
+            if ($fix && $line_fixed && $fixed_line ne $fixed[$linenr - 1]) {
+                $fixed[$linenr - 1] = $fixed_line;
+            }
+
+
+        }
+
+# check for whitespace before a non-naked semicolon
+        if ($line =~ /^\+.*\S\s+;\s*$/) {
+            if (WARN("SPACING",
+                     "space prohibited before semicolon\n" . $herecurr) &&
+                $fix) {
+                1 while $fixed[$linenr - 1] =~
+                    s/^(\+.*\S)\s+;/$1;/;
+            }
+        }
+
+# check for multiple assignments
+        if ($line =~ /^.\s*$Lval\s*=\s*$Lval\s*=(?!=)/) {
+            CHK("MULTIPLE_ASSIGNMENTS",
+                "multiple assignments should be avoided\n" . $herecurr);
+        }
+
+## # check for multiple declarations, allowing for a function declaration
+## # continuation.
+##              if ($line =~ /^.\s*$Type\s+$Ident(?:\s*=[^,{]*)?\s*,\s*$Ident.*/ &&
+##                  $line !~ /^.\s*$Type\s+$Ident(?:\s*=[^,{]*)?\s*,\s*$Type\s*$Ident.*/) {
+##
+##                      # Remove any bracketed sections to ensure we do not
+##                      # falsly report the parameters of functions.
+##                      my $ln = $line;
+##                      while ($ln =~ s/\([^\(\)]*\)//g) {
+##                      }
+##                      if ($ln =~ /,/) {
+##                              WARN("MULTIPLE_DECLARATION",
+##                                   "declaring multiple variables together should be avoided\n" . $herecurr);
+##                      }
+##              }
+
+#need space before brace following if, while, etc
+        if (($line =~ /\(.*\)\{/ && $line !~ /\($Type\)\{/) ||
+            $line =~ /do\{/) {
+            if (ERROR("SPACING",
+                      "space required before the open brace '{'\n" . $herecurr) &&
+                $fix) {
+                $fixed[$linenr - 1] =~ s/^(\+.*(?:do|\)))\{/$1 {/;
+            }
+        }
+
+## # check for blank lines before declarations
+##              if ($line =~ /^.\t+$Type\s+$Ident(?:\s*=.*)?;/ &&
+##                  $prevrawline =~ /^.\s*$/) {
+##                      WARN("SPACING",
+##                           "No blank lines before declarations\n" . $hereprev);
+##              }
+##
+
+# closing brace should have a space following it when it has anything
+# on the line
+        if ($line =~ /}(?!(?:,|;|\)))\S/) {
+            if (ERROR("SPACING",
+                      "space required after that close brace '}'\n" . $herecurr) &&
+                $fix) {
+                $fixed[$linenr - 1] =~
+                    s/}((?!(?:,|;|\)))\S)/} $1/;
+            }
+        }
+
+# check spacing on square brackets
+        if ($line =~ /\[\s/ && $line !~ /\[\s*$/) {
+            if (ERROR("SPACING",
+                      "space prohibited after that open square bracket '['\n" . $herecurr) &&
+                $fix) {
+                $fixed[$linenr - 1] =~
+                    s/\[\s+/\[/;
+            }
+        }
+        if ($line =~ /\s\]/) {
+            if (ERROR("SPACING",
+                      "space prohibited before that close square bracket ']'\n" . $herecurr) &&
+                $fix) {
+                $fixed[$linenr - 1] =~
+                    s/\s+\]/\]/;
+            }
+        }
+
+# check spacing on parentheses
+        if ($line =~ /\(\s/ && $line !~ /\(\s*(?:\\)?$/ &&
+            $line !~ /for\s*\(\s+;/) {
+            if (ERROR("SPACING",
+                      "space prohibited after that open parenthesis '('\n" . $herecurr) &&
+                $fix) {
+                $fixed[$linenr - 1] =~
+                    s/\(\s+/\(/;
+            }
+        }
+        if ($line =~ /(\s+)\)/ && $line !~ /^.\s*\)/ &&
+            $line !~ /for\s*\(.*;\s+\)/ &&
+            $line !~ /:\s+\)/) {
+            if (ERROR("SPACING",
+                      "space prohibited before that close parenthesis ')'\n" . $herecurr) &&
+                $fix) {
+                $fixed[$linenr - 1] =~
+                    s/\s+\)/\)/;
+            }
+        }
+
+#goto labels aren't indented, allow a single space however
+        if ($line=~/^.\s+[A-Za-z\d_]+:(?![0-9]+)/ and
+            !($line=~/^. [A-Za-z\d_]+:/) and !($line=~/^.\s+default:/)) {
+            if (WARN("INDENTED_LABEL",
+                     "labels should not be indented\n" . $herecurr) &&
+                $fix) {
+                $fixed[$linenr - 1] =~
+                    s/^(.)\s+/$1/;
+            }
+        }
+
+# return is not a function
+        if (defined($stat) && $stat =~ /^.\s*return(\s*)\(/s) {
+            my $spacing = $1;
+            if ($^V && $^V ge 5.10.0 &&
+                $stat =~ /^.\s*return\s*($balanced_parens)\s*;\s*$/) {
+                my $value = $1;
+                $value = deparenthesize($value);
+                if ($value =~ m/^\s*$FuncArg\s*(?:\?|$)/) {
+                    ERROR("RETURN_PARENTHESES",
+                          "return is not a function, parentheses are not required\n" . $herecurr);
+                }
+            } elsif ($spacing !~ /\s+/) {
+                ERROR("SPACING",
+                      "space required before the open parenthesis '('\n" . $herecurr);
+            }
+        }
+
+# unnecessary return in a void function
+# at end-of-function, with the previous line a single leading tab, then return;
+# and the line before that not a goto label target like "out:"
+        if ($sline =~ /^[ \+]}\s*$/ &&
+            $prevline =~ /^\+\treturn\s*;\s*$/ &&
+            $linenr >= 3 &&
+            $lines[$linenr - 3] =~ /^[ +]/ &&
+            $lines[$linenr - 3] !~ /^[ +]\s*$Ident\s*:/) {
+            WARN("RETURN_VOID",
+                 "void function return statements are not generally useful\n" . $hereprev);
+        }
+
+# if statements using unnecessary parentheses - ie: if ((foo == bar))
+        if ($^V && $^V ge 5.10.0 &&
+            $line =~ /\bif\s*((?:\(\s*){2,})/) {
+            my $openparens = $1;
+            my $count = $openparens =~ tr@\(@\(@;
+            my $msg = "";
+            if ($line =~ /\bif\s*(?:\(\s*){$count,$count}$LvalOrFunc\s*($Compare)\s*$LvalOrFunc(?:\s*\)){$count,$count}/) {
+                my $comp = $4;  #Not $1 because of $LvalOrFunc
+                $msg = " - maybe == should be = ?" if ($comp eq "==");
+                WARN("UNNECESSARY_PARENTHESES",
+                     "Unnecessary parentheses$msg\n" . $herecurr);
+            }
+        }
+
+# Return of what appears to be an errno should normally be -'ve
+        if ($line =~ /^.\s*return\s*(E[A-Z]*)\s*;/) {
+            my $name = $1;
+            if ($name ne 'EOF' && $name ne 'ERROR') {
+                WARN("USE_NEGATIVE_ERRNO",
+                     "return of an errno should typically be -ve (return -$1)\n" . $herecurr);
+            }
+        }
+
+# Need a space before open parenthesis after if, while etc
+        if ($line =~ /\b(if|while|for|switch)\(/) {
+            if (ERROR("SPACING",
+                      "space required before the open parenthesis '('\n" . $herecurr) &&
+                $fix) {
+                $fixed[$linenr - 1] =~
+                    s/\b(if|while|for|switch)\(/$1 \(/;
+            }
+        }
+
+# Check for illegal assignment in if conditional -- and check for trailing
+# statements after the conditional.
+        if ($line =~ /do\s*(?!{)/) {
+            ($stat, $cond, $line_nr_next, $remain_next, $off_next) =
+                ctx_statement_block($linenr, $realcnt, 0)
+                if (!defined $stat);
+            my ($stat_next) = ctx_statement_block($line_nr_next,
+                                                  $remain_next, $off_next);
+            $stat_next =~ s/\n./\n /g;
+            ##print "stat<$stat> stat_next<$stat_next>\n";
+
+            if ($stat_next =~ /^\s*while\b/) {
+                # If the statement carries leading newlines,
+                # then count those as offsets.
+                my ($whitespace) =
+                    ($stat_next =~ /^((?:\s*\n[+-])*\s*)/s);
+                my $offset =
+                    statement_rawlines($whitespace) - 1;
+
+                $suppress_whiletrailers{$line_nr_next +
+                                            $offset} = 1;
+            }
+        }
+        if (!defined $suppress_whiletrailers{$linenr} &&
+            defined($stat) && defined($cond) &&
+            $line =~ /\b(?:if|while|for)\s*\(/ && $line !~ /^.\s*#/) {
+            my ($s, $c) = ($stat, $cond);
+
+            if ($c =~ /\bif\s*\(.*[^<>!=]=[^=].*/s) {
+                ERROR("ASSIGN_IN_IF",
+                      "do not use assignment in if condition\n" . $herecurr);
+            }
+
+            # Find out what is on the end of the line after the
+            # conditional.
+            substr($s, 0, length($c), '');
+            $s =~ s/\n.*//g;
+            $s =~ s/$;//g;      # Remove any comments
+            if (length($c) && $s !~ /^\s*{?\s*\\*\s*$/ &&
+                $c !~ /}\s*while\s*/)
+            {
+                # Find out how long the conditional actually is.
+                my @newlines = ($c =~ /\n/gs);
+                my $cond_lines = 1 + $#newlines;
+                my $stat_real = '';
+
+                $stat_real = raw_line($linenr, $cond_lines)
+                    . "\n" if ($cond_lines);
+                if (defined($stat_real) && $cond_lines > 1) {
+                    $stat_real = "[...]\n$stat_real";
+                }
+
+                ERROR("TRAILING_STATEMENTS",
+                      "trailing statements should be on next line\n" . $herecurr . $stat_real);
+            }
+        }
+
+# Check for bitwise tests written as boolean
+        if ($line =~ /
+                        (?:
+                                (?:\[|\(|\&\&|\|\|)
+                                \s*0[xX][0-9]+\s*
+                                (?:\&\&|\|\|)
+                        |
+                                (?:\&\&|\|\|)
+                                \s*0[xX][0-9]+\s*
+                                (?:\&\&|\|\||\)|\])
+                        )/x)
+        {
+            WARN("HEXADECIMAL_BOOLEAN_TEST",
+                 "boolean test with hexadecimal, perhaps just 1 \& or \|?\n" . $herecurr);
+        }
+
+# if and else should not have general statements after it
+        if ($line =~ /^.\s*(?:}\s*)?else\b(.*)/) {
+            my $s = $1;
+            $s =~ s/$;//g;      # Remove any comments
+            if ($s !~ /^\s*(?:\sif|(?:{|)\s*\\?\s*$)/) {
+                ERROR("TRAILING_STATEMENTS",
+                      "trailing statements should be on next line\n" . $herecurr);
+            }
+        }
+# if should not continue a brace
+        if ($line =~ /}\s*if\b/) {
+            ERROR("TRAILING_STATEMENTS",
+                  "trailing statements should be on next line\n" .
+                  $herecurr);
+        }
+# case and default should not have general statements after them
+        if ($line =~ /^.\s*(?:case\s*.*|default\s*):/g &&
+            $line !~ /\G(?:
+                        (?:\s*$;*)(?:\s*{)?(?:\s*$;*)(?:\s*\\)?\s*$|
+                        \s*return\s+
+                    )/xg)
+        {
+            ERROR("TRAILING_STATEMENTS",
+                  "trailing statements should be on next line\n" . $herecurr);
+        }
+
+        # Check for }<nl>else {, these must be at the same
+        # indent level to be relevant to each other.
+        if ($prevline=~/}\s*$/ and $line=~/^.\s*else\s*/ and
+            $previndent == $indent) {
+            ERROR("ELSE_AFTER_BRACE",
+                  "else should follow close brace '}'\n" . $hereprev);
+        }
+
+        if ($prevline=~/}\s*$/ and $line=~/^.\s*while\s*/ and
+            $previndent == $indent) {
+            my ($s, $c) = ctx_statement_block($linenr, $realcnt, 0);
+
+            # Find out what is on the end of the line after the
+            # conditional.
+            substr($s, 0, length($c), '');
+            $s =~ s/\n.*//g;
+
+            if ($s =~ /^\s*;/) {
+                ERROR("WHILE_AFTER_BRACE",
+                      "while should follow close brace '}'\n" . $hereprev);
+            }
+        }
+
+#Specific variable tests
+        while ($line =~ m{($Constant|$Lval)}g) {
+            my $var = $1;
+
+#gcc binary extension
+            if ($var =~ /^$Binary$/) {
+                if (WARN("GCC_BINARY_CONSTANT",
+                         "Avoid gcc v4.3+ binary constant extension: <$var>\n" . $herecurr) &&
+                    $fix) {
+                    my $hexval = sprintf("0x%x", oct($var));
+                    $fixed[$linenr - 1] =~
+                        s/\b$var\b/$hexval/;
+                }
+            }
+
+#CamelCase
+            if ($var !~ /^$Constant$/ &&
+                $var =~ /[A-Z][a-z]|[a-z][A-Z]/ &&
+#Ignore Page<foo> variants
+                $var !~ /^(?:Clear|Set|TestClear|TestSet|)Page[A-Z]/ &&
+#Ignore SI style variants like nS, mV and dB (ie: max_uV, regulator_min_uA_show)
+                $var !~ /^(?:[a-z_]*?)_?[a-z][A-Z](?:_[a-z_]+)?$/) {
+                while ($var =~ m{($Ident)}g) {
+                    my $word = $1;
+                    next if ($word !~ /[A-Z][a-z]|[a-z][A-Z]/);
+                    if ($check) {
+                        seed_camelcase_includes();
+                        if (!$file && !$camelcase_file_seeded) {
+                            seed_camelcase_file($realfile);
+                            $camelcase_file_seeded = 1;
+                        }
+                    }
+                    if (!defined $camelcase{$word}) {
+                        $camelcase{$word} = 1;
+                        CHK("CAMELCASE",
+                            "Avoid CamelCase: <$word>\n" . $herecurr);
+                    }
+                }
+            }
+        }
+
+#no spaces allowed after \ in define
+        if ($line =~ /\#\s*define.*\\\s+$/) {
+            if (WARN("WHITESPACE_AFTER_LINE_CONTINUATION",
+                     "Whitespace after \\ makes next lines useless\n" . $herecurr) &&
+                $fix) {
+                $fixed[$linenr - 1] =~ s/\s+$//;
+            }
+        }
+
+#warn if <asm/foo.h> is #included and <linux/foo.h> is available (uses RAW line)
+        if ($tree && $rawline =~ m{^.\s*\#\s*include\s*\<asm\/(.*)\.h\>}) {
+            my $file = "$1.h";
+            my $checkfile = "include/linux/$file";
+            if (-f "$root/$checkfile" &&
+                $realfile ne $checkfile &&
+                $1 !~ /$allowed_asm_includes/)
+            {
+                if ($realfile =~ m{^arch/}) {
+                    CHK("ARCH_INCLUDE_LINUX",
+                        "Consider using #include <linux/$file> instead of <asm/$file>\n" . $herecurr);
+                } else {
+                    WARN("INCLUDE_LINUX",
+                         "Use #include <linux/$file> instead of <asm/$file>\n" . $herecurr);
+                }
+            }
+        }
+
+# multi-statement macros should be enclosed in a do while loop, grab the
+# first statement and ensure its the whole macro if its not enclosed
+# in a known good container
+        if ($realfile !~ m@/vmlinux.lds.h$@ &&
+            $line =~ /^.\s*\#\s*define\s*$Ident(\()?/) {
+            my $ln = $linenr;
+            my $cnt = $realcnt;
+            my ($off, $dstat, $dcond, $rest);
+            my $ctx = '';
+            ($dstat, $dcond, $ln, $cnt, $off) =
+                ctx_statement_block($linenr, $realcnt, 0);
+            $ctx = $dstat;
+            #print "dstat<$dstat> dcond<$dcond> cnt<$cnt> off<$off>\n";
+            #print "LINE<$lines[$ln-1]> len<" . length($lines[$ln-1]) . "\n";
+
+            $dstat =~ s/^.\s*\#\s*define\s+$Ident(?:\([^\)]*\))?\s*//;
+            $dstat =~ s/$;//g;
+            $dstat =~ s/\\\n.//g;
+            $dstat =~ s/^\s*//s;
+            $dstat =~ s/\s*$//s;
+
+            # Flatten any parentheses and braces
+            while ($dstat =~ s/\([^\(\)]*\)/1/ ||
+                   $dstat =~ s/\{[^\{\}]*\}/1/ ||
+                   $dstat =~ s/\[[^\[\]]*\]/1/)
+            {
+            }
+
+            # Flatten any obvious string concatentation.
+            while ($dstat =~ s/("X*")\s*$Ident/$1/ ||
+                   $dstat =~ s/$Ident\s*("X*")/$1/)
+            {
+            }
+
+            my $exceptions = qr{
+                                $Declare|
+                                module_param_named|
+                                MODULE_PARM_DESC|
+                                DECLARE_PER_CPU|
+                                DEFINE_PER_CPU|
+                                __typeof__\(|
+                                union|
+                                struct|
+                                \.$Ident\s*=\s*|
+                                ^\"|\"$
+                        }x;
+            #print "REST<$rest> dstat<$dstat> ctx<$ctx>\n";
+            if ($dstat ne '' &&
+                $dstat !~ /^(?:$Ident|-?$Constant),$/ &&                        # 10, // foo(),
+                $dstat !~ /^(?:$Ident|-?$Constant);$/ &&                        # foo();
+                $dstat !~ /^[!~-]?(?:$Lval|$Constant)$/ &&              # 10 // foo() // !foo // ~foo // -foo // foo->bar // foo.bar->baz
+                $dstat !~ /^'X'$/ &&                                    # character constants
+                $dstat !~ /$exceptions/ &&
+                $dstat !~ /^\.$Ident\s*=/ &&                            # .foo =
+                $dstat !~ /^(?:\#\s*$Ident|\#\s*$Constant)\s*$/ &&              # stringification #foo
+                $dstat !~ /^do\s*$Constant\s*while\s*$Constant;?$/ &&   # do {...} while (...); // do {...} while (...)
+                $dstat !~ /^for\s*$Constant$/ &&                                # for (...)
+                $dstat !~ /^for\s*$Constant\s+(?:$Ident|-?$Constant)$/ &&       # for (...) bar()
+                $dstat !~ /^do\s*{/ &&                                  # do {...
+                $dstat !~ /^\(\{/ &&                                             # ({...
+                $ctx !~ /^.\s*#\s*define\s+TRACE_(?:SYSTEM|INCLUDE_FILE|INCLUDE_PATH)\b/)
+            {
+                $ctx =~ s/\n*$//;
+                my $herectx = $here . "\n";
+                my $cnt = statement_rawlines($ctx);
+
+                for (my $n = 0; $n < $cnt; $n++) {
+                    $herectx .= raw_line($linenr, $n) . "\n";
+                }
+
+                if ($dstat =~ /;/) {
+                    ERROR("MULTISTATEMENT_MACRO_USE_DO_WHILE",
+                          "Macros with multiple statements should be enclosed in a do - while loop\n" . "$herectx");
+                } else {
+                    ERROR("COMPLEX_MACRO",
+                          "Macros with complex values should be enclosed in parenthesis\n" . "$herectx");
+                }
+            }
+
+# check for line continuations outside of #defines, preprocessor #, and asm
+
+        } else {
+            if ($prevline !~ /^..*\\$/ &&
+                $line !~ /^\+\s*\#.*\\$/ &&             # preprocessor
+                $line !~ /^\+.*\b(__asm__|asm)\b.*\\$/ &&       # asm
+                $line =~ /^\+.*\\$/) {
+                WARN("LINE_CONTINUATIONS",
+                     "Avoid unnecessary line continuations\n" . $herecurr);
+            }
+        }
+
+# do {} while (0) macro tests:
+# single-statement macros do not need to be enclosed in do while (0) loop,
+# macro should not end with a semicolon
+        if ($^V && $^V ge 5.10.0 &&
+            $realfile !~ m@/vmlinux.lds.h$@ &&
+            $line =~ /^.\s*\#\s*define\s+$Ident(\()?/) {
+            my $ln = $linenr;
+            my $cnt = $realcnt;
+            my ($off, $dstat, $dcond, $rest);
+            my $ctx = '';
+            ($dstat, $dcond, $ln, $cnt, $off) =
+                ctx_statement_block($linenr, $realcnt, 0);
+            $ctx = $dstat;
+
+            $dstat =~ s/\\\n.//g;
+
+            if ($dstat =~ /^\+\s*#\s*define\s+$Ident\s*${balanced_parens}\s*do\s*{(.*)\s*}\s*while\s*\(\s*0\s*\)\s*([;\s]*)\s*$/) {
+                my $stmts = $2;
+                my $semis = $3;
+
+                $ctx =~ s/\n*$//;
+                my $cnt = statement_rawlines($ctx);
+                my $herectx = $here . "\n";
+
+                for (my $n = 0; $n < $cnt; $n++) {
+                    $herectx .= raw_line($linenr, $n) . "\n";
+                }
+
+                if (($stmts =~ tr/;/;/) == 1 &&
+                    $stmts !~ /^\s*(if|while|for|switch)\b/) {
+                    WARN("SINGLE_STATEMENT_DO_WHILE_MACRO",
+                         "Single statement macros should not use a do {} while (0) loop\n" . "$herectx");
+                }
+                if (defined $semis && $semis ne "") {
+                    WARN("DO_WHILE_MACRO_WITH_TRAILING_SEMICOLON",
+                         "do {} while (0) macros should not be semicolon terminated\n" . "$herectx");
+                }
+            } elsif ($dstat =~ /^\+\s*#\s*define\s+$Ident.*;\s*$/) {
+                $ctx =~ s/\n*$//;
+                my $cnt = statement_rawlines($ctx);
+                my $herectx = $here . "\n";
+
+                for (my $n = 0; $n < $cnt; $n++) {
+                    $herectx .= raw_line($linenr, $n) . "\n";
+                }
+
+                WARN("TRAILING_SEMICOLON",
+                     "macros should not use a trailing semicolon\n" . "$herectx");
+            }
+        }
+
+# make sure symbols are always wrapped with VMLINUX_SYMBOL() ...
+# all assignments may have only one of the following with an assignment:
+#       .
+#       ALIGN(...)
+#       VMLINUX_SYMBOL(...)
+        if ($realfile eq 'vmlinux.lds.h' && $line =~ /(?:(?:^|\s)$Ident\s*=|=\s*$Ident(?:\s|$))/) {
+            WARN("MISSING_VMLINUX_SYMBOL",
+                 "vmlinux.lds.h needs VMLINUX_SYMBOL() around C-visible symbols\n" . $herecurr);
+        }
+
+# check for redundant bracing round if etc
+        if ($line =~ /(^.*)\bif\b/ && $1 !~ /else\s*$/) {
+            my ($level, $endln, @chunks) =
+                ctx_statement_full($linenr, $realcnt, 1);
+            #print "chunks<$#chunks> linenr<$linenr> endln<$endln> level<$level>\n";
+            #print "APW: <<$chunks[1][0]>><<$chunks[1][1]>>\n";
+            if ($#chunks > 0 && $level == 0) {
+                my @allowed = ();
+                my $allow = 0;
+                my $seen = 0;
+                my $herectx = $here . "\n";
+                my $ln = $linenr - 1;
+                for my $chunk (@chunks) {
+                    my ($cond, $block) = @{$chunk};
+
+                    # If the condition carries leading newlines, then count those as offsets.
+                    my ($whitespace) = ($cond =~ /^((?:\s*\n[+-])*\s*)/s);
+                    my $offset = statement_rawlines($whitespace) - 1;
+
+                    $allowed[$allow] = 0;
+                    #print "COND<$cond> whitespace<$whitespace> offset<$offset>\n";
+
+                    # We have looked at and allowed this specific line.
+                    $suppress_ifbraces{$ln + $offset} = 1;
+
+                    $herectx .= "$rawlines[$ln + $offset]\n[...]\n";
+                    $ln += statement_rawlines($block) - 1;
+
+                    substr($block, 0, length($cond), '');
+
+                    $seen++ if ($block =~ /^\s*{/);
+
+                    #print "cond<$cond> block<$block> allowed<$allowed[$allow]>\n";
+                    if (statement_lines($cond) > 1) {
+                        #print "APW: ALLOWED: cond<$cond>\n";
+                        $allowed[$allow] = 1;
+                    }
+                    if ($block =~/\b(?:if|for|while)\b/) {
+                        #print "APW: ALLOWED: block<$block>\n";
+                        $allowed[$allow] = 1;
+                    }
+                    if (statement_block_size($block) > 1) {
+                        #print "APW: ALLOWED: lines block<$block>\n";
+                        $allowed[$allow] = 1;
+                    }
+                    $allow++;
+                }
+                if ($seen) {
+                    my $sum_allowed = 0;
+                    foreach (@allowed) {
+                        $sum_allowed += $_;
+                    }
+                    if ($sum_allowed != 0 && $sum_allowed != $allow
+		    			  && $seen != $allow) {
+                        CHK("BRACES",
+                            "braces {} should be used on all arms of this statement\n" . $herectx);
+                    }
+                }
+            }
+        }
+        if (!defined $suppress_ifbraces{$linenr - 1} &&
+            $line =~ /\b(if|while|for|else)\b/) {
+            my $allowed = 0;
+
+            # Check the pre-context.
+            if (substr($line, 0, $-[0]) =~ /(\}\s*)$/) {
+                #print "APW: ALLOWED: pre<$1>\n";
+                $allowed = 1;
+            }
+
+            my ($level, $endln, @chunks) =
+                ctx_statement_full($linenr, $realcnt, $-[0]);
+
+            # Check the condition.
+            my ($cond, $block) = @{$chunks[0]};
+            #print "CHECKING<$linenr> cond<$cond> block<$block>\n";
+            if (defined $cond) {
+                substr($block, 0, length($cond), '');
+            }
+            if (statement_lines($cond) > 1) {
+                #print "APW: ALLOWED: cond<$cond>\n";
+                $allowed = 1;
+            }
+            if ($block =~/\b(?:if|for|while)\b/) {
+                #print "APW: ALLOWED: block<$block>\n";
+                $allowed = 1;
+            }
+            if (statement_block_size($block) > 1) {
+                #print "APW: ALLOWED: lines block<$block>\n";
+                $allowed = 1;
+            }
+            # Check the post-context.
+            if (defined $chunks[1]) {
+                my ($cond, $block) = @{$chunks[1]};
+                if (defined $cond) {
+                    substr($block, 0, length($cond), '');
+                }
+                if ($block =~ /^\s*\{/) {
+                    #print "APW: ALLOWED: chunk-1 block<$block>\n";
+                    $allowed = 1;
+                }
+            }
+        }
+
+# check for unnecessary blank lines around braces
+        if (($line =~ /^.\s*}\s*$/ && $prevrawline =~ /^.\s*$/)) {
+            CHK("BRACES",
+                "Blank lines aren't necessary before a close brace '}'\n" . $hereprev);
+        }
+        if (($rawline =~ /^.\s*$/ && $prevline =~ /^..*{\s*$/)) {
+            CHK("BRACES",
+                "Blank lines aren't necessary after an open brace '{'\n" . $hereprev);
+        }
+
+# no volatiles please
+        my $asm_volatile = qr{\b(__asm__|asm)\s+(__volatile__|volatile)\b};
+        if ($line =~ /\bvolatile\b/ && $line !~ /$asm_volatile/) {
+            WARN("VOLATILE",
+                 "Use of volatile is usually wrong: see Documentation/volatile-considered-harmful.txt\n" . $herecurr);
+        }
+
+# warn about #if 0
+        if ($line =~ /^.\s*\#\s*if\s+0\b/) {
+            CHK("REDUNDANT_CODE",
+                "if this code is redundant consider removing it\n" .
+                $herecurr);
+        }
+
+# check for needless "if (<foo>) fn(<foo>)" uses
+        if ($prevline =~ /\bif\s*\(\s*($Lval)\s*\)/) {
+            my $expr = '\s*\(\s*' . quotemeta($1) . '\s*\)\s*;';
+            if ($line =~ /\b(kfree|usb_free_urb|debugfs_remove(?:_recursive)?)$expr/) {
+                WARN('NEEDLESS_IF',
+                     "$1(NULL) is safe this check is probably not required\n" . $hereprev);
+            }
+        }
+
+# check for bad placement of section $InitAttribute (e.g.: __initdata)
+        if ($line =~ /(\b$InitAttribute\b)/) {
+            my $attr = $1;
+            if ($line =~ /^\+\s*static\s+(?:const\s+)?(?:$attr\s+)?($NonptrTypeWithAttr)\s+(?:$attr\s+)?($Ident(?:\[[^]]*\])?)\s*[=;]/) {
+                my $ptr = $1;
+                my $var = $2;
+                if ((($ptr =~ /\b(union|struct)\s+$attr\b/ &&
+                      ERROR("MISPLACED_INIT",
+                            "$attr should be placed after $var\n" . $herecurr)) ||
+                     ($ptr !~ /\b(union|struct)\s+$attr\b/ &&
+                      WARN("MISPLACED_INIT",
+                           "$attr should be placed after $var\n" . $herecurr))) &&
+                    $fix) {
+                    $fixed[$linenr - 1] =~ s/(\bstatic\s+(?:const\s+)?)(?:$attr\s+)?($NonptrTypeWithAttr)\s+(?:$attr\s+)?($Ident(?:\[[^]]*\])?)\s*([=;])\s*/"$1" . trim(string_find_replace($2, "\\s*$attr\\s*", " ")) . " " . trim(string_find_replace($3, "\\s*$attr\\s*", "")) . " $attr" . ("$4" eq ";" ? ";" : " = ")/e;
+                }
+            }
+        }
+
+# check for $InitAttributeData (ie: __initdata) with const
+        if ($line =~ /\bconst\b/ && $line =~ /($InitAttributeData)/) {
+            my $attr = $1;
+            $attr =~ /($InitAttributePrefix)(.*)/;
+            my $attr_prefix = $1;
+            my $attr_type = $2;
+            if (ERROR("INIT_ATTRIBUTE",
+                      "Use of const init definition must use ${attr_prefix}initconst\n" . $herecurr) &&
+                $fix) {
+                $fixed[$linenr - 1] =~
+                    s/$InitAttributeData/${attr_prefix}initconst/;
+            }
+        }
+
+# check for $InitAttributeConst (ie: __initconst) without const
+        if ($line !~ /\bconst\b/ && $line =~ /($InitAttributeConst)/) {
+            my $attr = $1;
+            if (ERROR("INIT_ATTRIBUTE",
+                      "Use of $attr requires a separate use of const\n" . $herecurr) &&
+                $fix) {
+                my $lead = $fixed[$linenr - 1] =~
+                    /(^\+\s*(?:static\s+))/;
+                $lead = rtrim($1);
+                $lead = "$lead " if ($lead !~ /^\+$/);
+                $lead = "${lead}const ";
+                $fixed[$linenr - 1] =~ s/(^\+\s*(?:static\s+))/$lead/;
+            }
+        }
+
+# don't use __constant_<foo> functions outside of include/uapi/
+        if ($realfile !~ m@^include/uapi/@ &&
+            $line =~ /(__constant_(?:htons|ntohs|[bl]e(?:16|32|64)_to_cpu|cpu_to_[bl]e(?:16|32|64)))\s*\(/) {
+            my $constant_func = $1;
+            my $func = $constant_func;
+            $func =~ s/^__constant_//;
+            if (WARN("CONSTANT_CONVERSION",
+                     "$constant_func should be $func\n" . $herecurr) &&
+                $fix) {
+                $fixed[$linenr - 1] =~ s/\b$constant_func\b/$func/g;
+            }
+        }
+
+# prefer usleep_range over udelay
+        if ($line =~ /\budelay\s*\(\s*(\d+)\s*\)/) {
+            my $delay = $1;
+            # ignore udelay's < 10, however
+            if (! ($delay < 10) ) {
+                CHK("USLEEP_RANGE",
+                    "usleep_range is preferred over udelay; see Documentation/timers/timers-howto.txt\n" . $herecurr);
+            }
+            if ($delay > 2000) {
+                WARN("LONG_UDELAY",
+                     "long udelay - prefer mdelay; see arch/arm/include/asm/delay.h\n" . $herecurr);
+            }
+        }
+
+# warn about unexpectedly long msleep's
+        if ($line =~ /\bmsleep\s*\((\d+)\);/) {
+            if ($1 < 20) {
+                WARN("MSLEEP",
+                     "msleep < 20ms can sleep for up to 20ms; see Documentation/timers/timers-howto.txt\n" . $herecurr);
+            }
+        }
+
+# check for comparisons of jiffies
+        if ($line =~ /\bjiffies\s*$Compare|$Compare\s*jiffies\b/) {
+            WARN("JIFFIES_COMPARISON",
+                 "Comparing jiffies is almost always wrong; prefer time_after, time_before and friends\n" . $herecurr);
+        }
+
+# check for comparisons of get_jiffies_64()
+        if ($line =~ /\bget_jiffies_64\s*\(\s*\)\s*$Compare|$Compare\s*get_jiffies_64\s*\(\s*\)/) {
+            WARN("JIFFIES_COMPARISON",
+                 "Comparing get_jiffies_64() is almost always wrong; prefer time_after64, time_before64 and friends\n" . $herecurr);
+        }
+
+# warn about spacing in #ifdefs
+        if ($line =~ /^.\s*\#\s*(ifdef|ifndef|elif)\s\s+/) {
+            if (ERROR("SPACING",
+                      "exactly one space required after that #$1\n" . $herecurr) &&
+                $fix) {
+                $fixed[$linenr - 1] =~
+                    s/^(.\s*\#\s*(ifdef|ifndef|elif))\s{2,}/$1 /;
+            }
+
+        }
+
+# check for spinlock_t definitions without a comment.
+        if ($line =~ /^.\s*(struct\s+mutex|spinlock_t)\s+\S+;/ ||
+            $line =~ /^.\s*(DEFINE_MUTEX)\s*\(/) {
+            my $which = $1;
+            if (!ctx_has_comment($first_line, $linenr)) {
+                CHK("UNCOMMENTED_DEFINITION",
+                    "$1 definition without comment\n" . $herecurr);
+            }
+        }
+# check for memory barriers without a comment.
+        if ($line =~ /\b(mb|rmb|wmb|read_barrier_depends|smp_mb|smp_rmb|smp_wmb|smp_read_barrier_depends)\(/) {
+            if (!ctx_has_comment($first_line, $linenr)) {
+                WARN("MEMORY_BARRIER",
+                     "memory barrier without comment\n" . $herecurr);
+            }
+        }
+# check of hardware specific defines
+        if ($line =~ m@^.\s*\#\s*if.*\b(__i386__|__powerpc64__|__sun__|__s390x__)\b@ && $realfile !~ m@include/asm-@) {
+            CHK("ARCH_DEFINES",
+                "architecture specific defines should be avoided\n" .  $herecurr);
+        }
+
+# Check that the storage class is at the beginning of a declaration
+        if ($line =~ /\b$Storage\b/ && $line !~ /^.\s*$Storage\b/) {
+            WARN("STORAGE_CLASS",
+                 "storage class should be at the beginning of the declaration\n" . $herecurr)
+        }
+
+# check the location of the inline attribute, that it is between
+# storage class and type.
+        if ($line =~ /\b$Type\s+$Inline\b/ ||
+            $line =~ /\b$Inline\s+$Storage\b/) {
+            ERROR("INLINE_LOCATION",
+                  "inline keyword should sit between storage class and type\n" . $herecurr);
+        }
+
+# Check for __inline__ and __inline, prefer inline
+        if ($realfile !~ m@\binclude/uapi/@ &&
+            $line =~ /\b(__inline__|__inline)\b/) {
+            if (WARN("INLINE",
+                     "plain inline is preferred over $1\n" . $herecurr) &&
+                $fix) {
+                $fixed[$linenr - 1] =~ s/\b(__inline__|__inline)\b/inline/;
+
+            }
+        }
+
+# Check for __attribute__ packed, prefer __packed
+        if ($realfile !~ m@\binclude/uapi/@ &&
+            $line =~ /\b__attribute__\s*\(\s*\(.*\bpacked\b/) {
+            WARN("PREFER_PACKED",
+                 "__packed is preferred over __attribute__((packed))\n" . $herecurr);
+        }
+
+# Check for __attribute__ aligned, prefer __aligned
+        if ($realfile !~ m@\binclude/uapi/@ &&
+            $line =~ /\b__attribute__\s*\(\s*\(.*aligned/) {
+            WARN("PREFER_ALIGNED",
+                 "__aligned(size) is preferred over __attribute__((aligned(size)))\n" . $herecurr);
+        }
+
+# Check for __attribute__ format(printf, prefer __printf
+        if ($realfile !~ m@\binclude/uapi/@ &&
+            $line =~ /\b__attribute__\s*\(\s*\(\s*format\s*\(\s*printf/) {
+            if (WARN("PREFER_PRINTF",
+                     "__printf(string-index, first-to-check) is preferred over __attribute__((format(printf, string-index, first-to-check)))\n" . $herecurr) &&
+                $fix) {
+                $fixed[$linenr - 1] =~ s/\b__attribute__\s*\(\s*\(\s*format\s*\(\s*printf\s*,\s*(.*)\)\s*\)\s*\)/"__printf(" . trim($1) . ")"/ex;
+
+            }
+        }
+
+# Check for __attribute__ format(scanf, prefer __scanf
+        if ($realfile !~ m@\binclude/uapi/@ &&
+            $line =~ /\b__attribute__\s*\(\s*\(\s*format\s*\(\s*scanf\b/) {
+            if (WARN("PREFER_SCANF",
+                     "__scanf(string-index, first-to-check) is preferred over __attribute__((format(scanf, string-index, first-to-check)))\n" . $herecurr) &&
+                $fix) {
+                $fixed[$linenr - 1] =~ s/\b__attribute__\s*\(\s*\(\s*format\s*\(\s*scanf\s*,\s*(.*)\)\s*\)\s*\)/"__scanf(" . trim($1) . ")"/ex;
+            }
+        }
+
+# check for sizeof(&)
+        if ($line =~ /\bsizeof\s*\(\s*\&/) {
+            WARN("SIZEOF_ADDRESS",
+                 "sizeof(& should be avoided\n" . $herecurr);
+        }
+
+# check for sizeof without parenthesis
+        if ($line =~ /\bsizeof\s+((?:\*\s*|)$Lval|$Type(?:\s+$Lval|))/) {
+            if (WARN("SIZEOF_PARENTHESIS",
+                     "sizeof $1 should be sizeof($1)\n" . $herecurr) &&
+                $fix) {
+                $fixed[$linenr - 1] =~ s/\bsizeof\s+((?:\*\s*|)$Lval|$Type(?:\s+$Lval|))/"sizeof(" . trim($1) . ")"/ex;
+            }
+        }
+
+# check for line continuations in quoted strings with odd counts of "
+        if ($rawline =~ /\\$/ && $rawline =~ tr/"/"/ % 2) {
+            WARN("LINE_CONTINUATIONS",
+                 "Avoid line continuations in quoted strings\n" . $herecurr);
+        }
+
+# check for struct spinlock declarations
+        if ($line =~ /^.\s*\bstruct\s+spinlock\s+\w+\s*;/) {
+            WARN("USE_SPINLOCK_T",
+                 "struct spinlock should be spinlock_t\n" . $herecurr);
+        }
+
+# check for seq_printf uses that could be seq_puts
+        if ($sline =~ /\bseq_printf\s*\(.*"\s*\)\s*;\s*$/) {
+            my $fmt = get_quoted_string($line, $rawline);
+            if ($fmt ne "" && $fmt !~ /[^\\]\%/) {
+                if (WARN("PREFER_SEQ_PUTS",
+                         "Prefer seq_puts to seq_printf\n" . $herecurr) &&
+                    $fix) {
+                    $fixed[$linenr - 1] =~ s/\bseq_printf\b/seq_puts/;
+                }
+            }
+        }
+
+# Check for misused memsets
+        if ($^V && $^V ge 5.10.0 &&
+            defined $stat &&
+            $stat =~ /^\+(?:.*?)\bmemset\s*\(\s*$FuncArg\s*,\s*$FuncArg\s*\,\s*$FuncArg\s*\)/s) {
+
+            my $ms_addr = $2;
+            my $ms_val = $7;
+            my $ms_size = $12;
+
+            if ($ms_size =~ /^(0x|)0$/i) {
+                ERROR("MEMSET",
+                      "memset to 0's uses 0 as the 2nd argument, not the 3rd\n" . "$here\n$stat\n");
+            } elsif ($ms_size =~ /^(0x|)1$/i) {
+                WARN("MEMSET",
+                     "single byte memset is suspicious. Swapped 2nd/3rd argument?\n" . "$here\n$stat\n");
+            }
+        }
+
+# typecasts on min/max could be min_t/max_t
+        if ($^V && $^V ge 5.10.0 &&
+            defined $stat &&
+            $stat =~ /^\+(?:.*?)\b(min|max)\s*\(\s*$FuncArg\s*,\s*$FuncArg\s*\)/) {
+            if (defined $2 || defined $7) {
+                my $call = $1;
+                my $cast1 = deparenthesize($2);
+                my $arg1 = $3;
+                my $cast2 = deparenthesize($7);
+                my $arg2 = $8;
+                my $cast;
+
+                if ($cast1 ne "" && $cast2 ne "" && $cast1 ne $cast2) {
+                    $cast = "$cast1 or $cast2";
+                } elsif ($cast1 ne "") {
+                    $cast = $cast1;
+                } else {
+                    $cast = $cast2;
+                }
+                WARN("MINMAX",
+                     "$call() should probably be ${call}_t($cast, $arg1, $arg2)\n" . "$here\n$stat\n");
+            }
+        }
+
+# check usleep_range arguments
+        if ($^V && $^V ge 5.10.0 &&
+            defined $stat &&
+            $stat =~ /^\+(?:.*?)\busleep_range\s*\(\s*($FuncArg)\s*,\s*($FuncArg)\s*\)/) {
+            my $min = $1;
+            my $max = $7;
+            if ($min eq $max) {
+                WARN("USLEEP_RANGE",
+                     "usleep_range should not use min == max args; see Documentation/timers/timers-howto.txt\n" . "$here\n$stat\n");
+            } elsif ($min =~ /^\d+$/ && $max =~ /^\d+$/ &&
+                     $min > $max) {
+                WARN("USLEEP_RANGE",
+                     "usleep_range args reversed, use min then max; see Documentation/timers/timers-howto.txt\n" . "$here\n$stat\n");
+            }
+        }
+
+# check for naked sscanf
+        if ($^V && $^V ge 5.10.0 &&
+            defined $stat &&
+            $line =~ /\bsscanf\b/ &&
+            ($stat !~ /$Ident\s*=\s*sscanf\s*$balanced_parens/ &&
+             $stat !~ /\bsscanf\s*$balanced_parens\s*(?:$Compare)/ &&
+             $stat !~ /(?:$Compare)\s*\bsscanf\s*$balanced_parens/)) {
+            my $lc = $stat =~ tr@\n@@;
+            $lc = $lc + $linenr;
+            my $stat_real = raw_line($linenr, 0);
+            for (my $count = $linenr + 1; $count <= $lc; $count++) {
+                $stat_real = $stat_real . "\n" . raw_line($count, 0);
+            }
+            WARN("NAKED_SSCANF",
+                 "unchecked sscanf return value\n" . "$here\n$stat_real\n");
+        }
+
+# check for simple sscanf that should be kstrto<foo>
+        if ($^V && $^V ge 5.10.0 &&
+            defined $stat &&
+            $line =~ /\bsscanf\b/) {
+            my $lc = $stat =~ tr@\n@@;
+            $lc = $lc + $linenr;
+            my $stat_real = raw_line($linenr, 0);
+            for (my $count = $linenr + 1; $count <= $lc; $count++) {
+                $stat_real = $stat_real . "\n" . raw_line($count, 0);
+            }
+            if ($stat_real =~ /\bsscanf\b\s*\(\s*$FuncArg\s*,\s*("[^"]+")/) {
+                my $format = $6;
+                my $count = $format =~ tr@%@%@;
+                if ($count == 1 &&
+                    $format =~ /^"\%(?i:ll[udxi]|[udxi]ll|ll|[hl]h?[udxi]|[udxi][hl]h?|[hl]h?|[udxi])"$/) {
+                    WARN("SSCANF_TO_KSTRTO",
+                         "Prefer kstrto<type> to single variable sscanf\n" . "$here\n$stat_real\n");
+                }
+            }
+        }
+
+# check for new externs in .h files.
+        if ($realfile =~ /\.h$/ &&
+            $line =~ /^\+\s*(extern\s+)$Type\s*$Ident\s*\(/s) {
+            if (CHK("AVOID_EXTERNS",
+                    "extern prototypes should be avoided in .h files\n" . $herecurr) &&
+                $fix) {
+                $fixed[$linenr - 1] =~ s/(.*)\bextern\b\s*(.*)/$1$2/;
+            }
+        }
+
+# check for new externs in .c files.
+        if ($realfile =~ /\.c$/ && defined $stat &&
+            $stat =~ /^.\s*(?:extern\s+)?$Type\s+($Ident)(\s*)\(/s)
+        {
+            my $function_name = $1;
+            my $paren_space = $2;
+
+            my $s = $stat;
+            if (defined $cond) {
+                substr($s, 0, length($cond), '');
+            }
+            if ($s =~ /^\s*;/ &&
+                $function_name ne 'uninitialized_var')
+            {
+                WARN("AVOID_EXTERNS",
+                     "externs should be avoided in .c files\n" .  $herecurr);
+            }
+
+            if ($paren_space =~ /\n/) {
+                WARN("FUNCTION_ARGUMENTS",
+                     "arguments for function declarations should follow identifier\n" . $herecurr);
+            }
+
+        } elsif ($realfile =~ /\.c$/ && defined $stat &&
+                 $stat =~ /^.\s*extern\s+/)
+        {
+            WARN("AVOID_EXTERNS",
+                 "externs should be avoided in .c files\n" .  $herecurr);
+        }
+
+# check for pointless casting of kmalloc return
+        if ($line =~ /\*\s*\)\s*[kv][czm]alloc(_node){0,1}\b/) {
+            WARN("UNNECESSARY_CASTS",
+                 "unnecessary cast may hide bugs, see http://c-faq.com/malloc/mallocnocast.html\n" . $herecurr);
+        }
+
+# alloc style
+# p = alloc(sizeof(struct foo), ...) should be p = alloc(sizeof(*p), ...)
+        if ($^V && $^V ge 5.10.0 &&
+            $line =~ /\b($Lval)\s*\=\s*(?:$balanced_parens)?\s*([kv][mz]alloc(?:_node)?)\s*\(\s*(sizeof\s*\(\s*struct\s+$Lval\s*\))/) {
+            CHK("ALLOC_SIZEOF_STRUCT",
+                "Prefer $3(sizeof(*$1)...) over $3($4...)\n" . $herecurr);
+        }
+
+# check for multiple semicolons
+        if ($line =~ /;\s*;\s*$/) {
+            if (WARN("ONE_SEMICOLON",
+                     "Statements terminations use 1 semicolon\n" . $herecurr) &&
+                $fix) {
+                $fixed[$linenr - 1] =~ s/(\s*;\s*){2,}$/;/g;
+            }
+        }
+
+# check for case / default statements not preceeded by break/fallthrough/switch
+        if ($line =~ /^.\s*(?:case\s+(?:$Ident|$Constant)\s*|default):/) {
+            my $has_break = 0;
+            my $has_statement = 0;
+            my $count = 0;
+            my $prevline = $linenr;
+            while ($prevline > 1 && $count < 3 && !$has_break) {
+                $prevline--;
+                my $rline = $rawlines[$prevline - 1];
+                my $fline = $lines[$prevline - 1];
+                last if ($fline =~ /^\@\@/);
+                next if ($fline =~ /^\-/);
+                next if ($fline =~ /^.(?:\s*(?:case\s+(?:$Ident|$Constant)[\s$;]*|default):[\s$;]*)*$/);
+                $has_break = 1 if ($rline =~ /fall[\s_-]*(through|thru)/i);
+                next if ($fline =~ /^.[\s$;]*$/);
+                $has_statement = 1;
+                $count++;
+                $has_break = 1 if ($fline =~ /\bswitch\b|\b(?:break\s*;[\s$;]*$|return\b|goto\b|continue\b)/);
+            }
+            if (!$has_break && $has_statement) {
+                WARN("MISSING_BREAK",
+                     "Possible switch case/default not preceeded by break or fallthrough comment\n" . $herecurr);
+            }
+        }
+
+# check for switch/default statements without a break;
+        if ($^V && $^V ge 5.10.0 &&
+            defined $stat &&
+            $stat =~ /^\+[$;\s]*(?:case[$;\s]+\w+[$;\s]*:[$;\s]*|)*[$;\s]*\bdefault[$;\s]*:[$;\s]*;/g) {
+            my $ctx = '';
+            my $herectx = $here . "\n";
+            my $cnt = statement_rawlines($stat);
+            for (my $n = 0; $n < $cnt; $n++) {
+                $herectx .= raw_line($linenr, $n) . "\n";
+            }
+            WARN("DEFAULT_NO_BREAK",
+                 "switch default: should use break\n" . $herectx);
+        }
+
+# check for comparisons against true and false
+        if ($line =~ /\+\s*(.*?)\b(true|false|$Lval)\s*(==|\!=)\s*(true|false|$Lval)\b(.*)$/i) {
+            my $lead = $1;
+            my $arg = $2;
+            my $test = $3;
+            my $otype = $4;
+            my $trail = $5;
+            my $op = "!";
+
+            ($arg, $otype) = ($otype, $arg) if ($arg =~ /^(?:true|false)$/i);
+
+            my $type = lc($otype);
+            if ($type =~ /^(?:true|false)$/) {
+                if (("$test" eq "==" && "$type" eq "true") ||
+                    ("$test" eq "!=" && "$type" eq "false")) {
+                    $op = "";
+                }
+
+                CHK("BOOL_COMPARISON",
+                    "Using comparison to $otype is error prone\n" . $herecurr);
+            }
+        }
+
+# check for semaphores initialized locked
+        if ($line =~ /^.\s*sema_init.+,\W?0\W?\)/) {
+            WARN("CONSIDER_COMPLETION",
+                 "consider using a completion\n" . $herecurr);
+        }
+
+# check for %L{u,d,i} in strings
+        my $string;
+        while ($line =~ /(?:^|")([X\t]*)(?:"|$)/g) {
+            $string = substr($rawline, $-[1], $+[1] - $-[1]);
+            $string =~ s/%%/__/g;
+            if ($string =~ /(?<!%)%L[udi]/) {
+                WARN("PRINTF_L",
+                     "\%Ld/%Lu are not-standard C, use %lld/%llu\n" . $herecurr);
+                last;
+            }
+        }
+
+
+# Mode permission misuses where it seems decimal should be octal
+# This uses a shortcut match to avoid unnecessary uses of a slow foreach loop
+        if ($^V && $^V ge 5.10.0 &&
+            $line =~ /$mode_perms_search/) {
+            foreach my $entry (@mode_permission_funcs) {
+                my $func = $entry->[0];
+                my $arg_pos = $entry->[1];
+
+                my $skip_args = "";
+                if ($arg_pos > 1) {
+                    $arg_pos--;
+                    $skip_args = "(?:\\s*$FuncArg\\s*,\\s*){$arg_pos,$arg_pos}";
+                }
+                my $test = "\\b$func\\s*\\(${skip_args}([\\d]+)\\s*[,\\)]";
+                if ($line =~ /$test/) {
+                    my $val = $1;
+                    $val = $6 if ($skip_args ne "");
+
+                    if ($val !~ /^0$/ &&
+                        (($val =~ /^$Int$/ && $val !~ /^$Octal$/) ||
+                         length($val) ne 4)) {
+                        ERROR("NON_OCTAL_PERMISSIONS",
+                              "Use 4 digit octal (0777) not decimal permissions\n" . $herecurr);
+                    }
+                }
+            }
+        }
+    }
+
+    # If we have no input at all, then there is nothing to report on
+    # so just keep quiet.
+    if ($#rawlines == -1) {
+        exit(0);
+    }
+
+    # In mailback mode only produce a report in the negative, for
+    # things that appear to be patches.
+    if ($mailback && ($clean == 1 || !$is_patch)) {
+        exit(0);
+    }
+
+    # This is not a patch, and we are are in 'no-patch' mode so
+    # just keep quiet.
+    if (!$chk_patch && !$is_patch) {
+        exit(0);
+    }
+
+    if (!$is_patch) {
+        ERROR("NOT_UNIFIED_DIFF",
+              "Does not appear to be a unified-diff format patch\n");
+    }
+    if ($is_patch && $subject_trailing_dot != 0) {
+        ERROR("SUBJECT_TRAILING_DOT",
+              "The subject of the patch should not end with a dot.\n");
+    }
+    if ($is_patch && $chk_signoff && $signoff == 0) {
+        ERROR("MISSING_SIGN_OFF",
+              "Missing Signed-off-by: line(s)\n");
+    }
+
+    print report_dump();
+    if ($summary && !($clean == 1 && $quiet == 1)) {
+        print "$filename " if ($summary_file);
+        if ($cnt_error > 0) {
+            print "Patch not according to coding guidelines! please fix.\n";
+            print "total: $cnt_error errors, $cnt_warn warnings, " .
+                (($check)? "$cnt_chk checks, " : "") .
+                "$cnt_lines lines checked\n"; exit 1;
+        } else {
+            print "total: $cnt_warn warnings, " .
+                (($check)? "$cnt_chk checks, " : "") .
+                "$cnt_lines lines checked\n";
+            print "Patch found to have warnings, please fix if necessary.\n" if ($cnt_warn > 0);
+            exit 2;
+        }
+        print "\n" if ($quiet == 0);
+    }
+
+    if ($quiet == 0) {
+
+        if ($^V lt 5.10.0) {
+            print("NOTE: perl $^V is not modern enough to detect all possible issues.\n");
+            print("An upgrade to at least perl v5.10.0 is suggested.\n\n");
+        }
+
+        # If there were whitespace errors which cleanpatch can fix
+        # then suggest that.
+        if ($rpt_cleaners) {
+            print "NOTE: whitespace errors detected, you may wish to use scripts/cleanpatch or\n";
+            print "      scripts/cleanfile\n\n";
+            $rpt_cleaners = 0;
+        }
+    }
+
+    hash_show_words(\%use_type, "Used");
+    hash_show_words(\%ignore_type, "Ignored");
+
+    if ($clean == 0 && $fix && "@rawlines" ne "@fixed") {
+        my $newfile = $filename;
+        $newfile .= ".EXPERIMENTAL-checkpatch-fixes" if (!$fix_inplace);
+        my $linecount = 0;
+        my $f;
+
+        open($f, '>', $newfile)
+            or die "$P: Can't open $newfile for write\n";
+        foreach my $fixed_line (@fixed) {
+            $linecount++;
+            if ($file) {
+                if ($linecount > 3) {
+                    $fixed_line =~ s/^\+//;
+                    print $f $fixed_line. "\n";
+                }
+            } else {
+                print $f $fixed_line . "\n";
+            }
+        }
+        close($f);
+
+        if (!$quiet) {
+            print << "EOM";
+Wrote EXPERIMENTAL --fix correction(s) to '$newfile'
+
+Do _NOT_ trust the results written to this file.
+Do _NOT_ submit these changes without inspecting them for correctness.
+
+This EXPERIMENTAL file is simply a convenience to help rewrite patches.
+No warranties, expressed or implied...
+EOM
+        }
+    }
+
+    if ($clean == 1 && $quiet == 0) {
+        print "$vname has no obvious style problems and is ready for submission.\n"
+    }
+    if ($clean == 0 && $quiet == 0) {
+        print << "EOM";
+$vname has style problems, please review.
+
+If any of these errors are false positives, please report
+them to the maintainer, see MAINTAINERS
+EOM
+    }
+
+    return $clean;
+}
diff --git a/build-aux/config.guess.dist b/build-aux/config.guess.dist
new file mode 100755
index 00000000000..881ba7a0438
--- /dev/null
+++ b/build-aux/config.guess.dist
@@ -0,0 +1,14 @@
+#!/bin/sh
+#
+# This script is intentionally left empty. Distributions that package GlusterFS
+# may want to to replace it with an updated copy from the automake project.
+#
+
+cat << EOM
+It is not expected to execute this script. When you are building from a
+released tarball (generated with 'make dist'), you are expected to pass
+--build=... and --host=... to ./configure or replace this config.guess script
+in the sources with an updated version.
+EOM
+
+exit 0
diff --git a/build-aux/config.sub.dist b/build-aux/config.sub.dist
new file mode 100755
index 00000000000..c5a0dbad282
--- /dev/null
+++ b/build-aux/config.sub.dist
@@ -0,0 +1,14 @@
+#!/bin/sh
+#
+# This script is intentionally left empty. Distributions that package GlusterFS
+# may want to to replace it with an updated copy from the automake project.
+#
+
+cat << EOM
+It is not expected to execute this script. When you are building from a
+released tarball (generated with 'make dist'), you are expected to pass
+--build=... and --host=... to ./configure or replace this config.sub script in
+the sources with an updated version.
+EOM
+
+exit 0
diff --git a/build-aux/pkg-version b/build-aux/pkg-version
new file mode 100755
index 00000000000..17ceab70c03
--- /dev/null
+++ b/build-aux/pkg-version
@@ -0,0 +1,64 @@
+#!/bin/bash
+
+# To override version/release from git,
+# create VERSION file containing text with version/release
+# eg. v3.4.0-1
+
+# One thing to note, If one does 'git clone --depth N glusterfs.git',
+# the git describe command doesn't work. Hence you notice below that
+# we have added timestamp as version (YYYY.MM.DD) and release (HH.mmss)
+PKG_VERSION=`cat VERSION 2> /dev/null || git describe --tags --match "v[0-9]*" 2>/dev/null`
+
+get_version()
+{
+    # tags and output versions:
+    #   - v3.4.0   => 3.4.0 (upstream clean)
+    #   - v3.4.0-1 => 3.4.0 (downstream clean)
+    #   - v3.4.0-2-g34e62f   => 3.4.0 (upstream dirty)
+    #   - v3.4.0-1-2-g34e62f => 3.4.0 (downstream dirty)
+    AWK_VERSION='
+    BEGIN { FS="-" }
+    /^v[0-9]/ {
+      sub(/^v/,"") ; print $1
+    }'
+
+    version=$(echo $PKG_VERSION | awk "$AWK_VERSION" | tr -cd '[:alnum:].')
+    if [ "x${version}" == "x" ] ; then
+       version=$(date +%Y.%m.%d | tr -d '\n')
+    fi
+    echo $version | tr -d '\n'
+}
+
+get_release()
+{
+    # tags and output releases:
+    #   - v3.4.0   => 0 (upstream clean)
+    #   - v3.4.0-1 => 1 (downstream clean)
+    #   - v3.4.0-2-g34e62f1   => 2.git34e62f1 (upstream dirty)
+    #   - v3.4.0-1-2-g34e62f1 => 1.2.git34e62f1 (downstream dirty)
+    AWK_RELEASE='
+    BEGIN { FS="-"; OFS="." }
+    /^v[0-9]/ {
+      if (NF == 1) print 0
+      else if (NF == 2) print $2
+      else if (NF == 3) print $2, "git" substr($3, 2)
+      else if (NF == 4) print $2, $3, "git" substr($4, 2)
+    }'
+
+    release=$(echo $PKG_VERSION | awk "$AWK_RELEASE" | tr -cd '[:alnum:].')
+    if [ "x${release}" == "x" ] ; then
+       release=$(date +%H.%M%S | tr -d '\n')
+    fi
+    echo $release | tr -d '\n'
+}
+
+if test "x$1" = "x--full"; then
+    echo -n "v$(get_version)-$(get_release)"
+elif test "x$1" = "x--version"; then
+    get_version
+elif test "x$1" = "x--release"; then
+    get_release
+else
+    echo "usage: $0 [--full|--version|--release]"
+    exit 1
+fi
diff --git a/cli/src/Makefile.am b/cli/src/Makefile.am
index 300c00ee922..16063f27c7f 100644
--- a/cli/src/Makefile.am
+++ b/cli/src/Makefile.am
@@ -1,32 +1,37 @@
 sbin_PROGRAMS = gluster
 
-gluster_SOURCES = cli.c registry.c input.c cli-cmd.c cli-rl.c \
+gluster_SOURCES = cli.c registry.c input.c cli-cmd.c cli-rl.c cli-cmd-global.c \
 	 cli-cmd-volume.c cli-cmd-peer.c cli-rpc-ops.c cli-cmd-parser.c\
-	 cli-cmd-system.c cli-cmd-misc.c cli-xml-output.c cli-quotad-client.c
+	 cli-cmd-system.c cli-cmd-misc.c cli-xml-output.c cli-quotad-client.c cli-cmd-snapshot.c
 
-gluster_LDADD = $(top_builddir)/libglusterfs/src/libglusterfs.la $(GF_LDADD)\
+gluster_LDADD = $(top_builddir)/libglusterfs/src/libglusterfs.la $(GF_LDADD) \
+		$(top_builddir)/libglusterd/src/libglusterd.la \
 		$(RLLIBS) $(top_builddir)/rpc/xdr/src/libgfxdr.la \
 		$(top_builddir)/rpc/rpc-lib/src/libgfrpc.la \
-		$(GF_GLUSTERFS_LIBS) $(XML_LIBS)
+		$(XML_LIBS)
 
 gluster_LDFLAGS = $(GF_LDFLAGS)
 noinst_HEADERS = cli.h cli-mem-types.h cli-cmd.h cli-quotad-client.h
 
 AM_CPPFLAGS = $(GF_CPPFLAGS) \
-	-I$(top_srcdir)/libglusterfs/src -I$(top_srcdir)/rpc/rpc-lib/src\
-	-I$(top_srcdir)/rpc/xdr/src\
+	-I$(top_srcdir)/libglusterfs/src -I$(top_srcdir)/rpc/rpc-lib/src \
+	-I$(top_srcdir)/rpc/xdr/src -I$(top_srcdir)/libglusterd/src \
+	-I$(top_builddir)/rpc/xdr/src \
 	-DDATADIR=\"$(localstatedir)\" \
 	-DCONFDIR=\"$(sysconfdir)/glusterfs\" \
-	-DGSYNCD_PREFIX=\"$(libexecdir)/glusterfs\"\
-	-DSYNCDAEMON_COMPILE=$(SYNCDAEMON_COMPILE) -DSBIN_DIR=\"$(sbindir)\"\
-	$(XML_CPPFLAGS)
+	-DGSYNCD_PREFIX=\"$(GLUSTERFS_LIBEXECDIR)\"\
+	-DGLFSHEAL_PREFIX=\"$(GLUSTERFS_LIBEXECDIR)\"\
+	-DSYNCDAEMON_COMPILE=$(SYNCDAEMON_COMPILE)
 
-AM_CFLAGS = -Wall $(GF_GLUSTERFS_CFLAGS)
+AM_CFLAGS = -Wall $(GF_CFLAGS) $(XML_CFLAGS)
 
 CLEANFILES =
 
 $(top_builddir)/libglusterfs/src/libglusterfs.la:
 	$(MAKE) -C $(top_builddir)/libglusterfs/src/ all
 
+$(top_builddir)/libglusterd/src/libglusterd.la:
+	$(MAKE) -C $(top_builddir)/libglusterd/src/ all
+
 install-data-hook:
 	$(mkdir_p) $(DESTDIR)$(localstatedir)/run/gluster
diff --git a/cli/src/cli-cmd-global.c b/cli/src/cli-cmd-global.c
new file mode 100644
index 00000000000..2c9a5f01bb1
--- /dev/null
+++ b/cli/src/cli-cmd-global.c
@@ -0,0 +1,195 @@
+/*
+   Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#include <stdio.h>
+#include <string.h>
+#include <stdlib.h>
+#include <stdint.h>
+#include <pthread.h>
+
+#include <sys/socket.h>
+#include <netdb.h>
+#include <sys/types.h>
+#include <sys/wait.h>
+#include <netinet/in.h>
+
+#include "cli.h"
+#include "cli-cmd.h"
+#include "cli-mem-types.h"
+#include "cli1-xdr.h"
+#include <glusterfs/run.h>
+#include <glusterfs/syscall.h>
+#include <glusterfs/common-utils.h>
+
+int
+cli_cmd_global_help_cbk(struct cli_state *state, struct cli_cmd_word *in_word,
+                        const char **words, int wordcount);
+int
+cli_cmd_get_state_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                      const char **words, int wordcount);
+
+int
+cli_cmd_ganesha_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                    const char **words, int wordcount);
+
+struct cli_cmd global_cmds[] = {
+    {
+        "global help",
+        cli_cmd_global_help_cbk,
+        "list global commands",
+    },
+    {
+        "get-state [<daemon>] [[odir </path/to/output/dir/>] "
+        "[file <filename>]] [detail|volumeoptions]",
+        cli_cmd_get_state_cbk,
+        "Get local state representation of mentioned daemon",
+    },
+    {
+        "nfs-ganesha {enable| disable} ",
+        cli_cmd_ganesha_cbk,
+        "Enable/disable NFS-Ganesha support",
+    },
+    {NULL, NULL, NULL}};
+
+int
+cli_cmd_global_help_cbk(struct cli_state *state, struct cli_cmd_word *in_word,
+                        const char **words, int wordcount)
+{
+    struct cli_cmd *cmd = NULL;
+    struct cli_cmd *global_cmd = NULL;
+    int count = 0;
+
+    cmd = GF_MALLOC(sizeof(global_cmds), cli_mt_cli_cmd);
+    memcpy(cmd, global_cmds, sizeof(global_cmds));
+    count = (sizeof(global_cmds) / sizeof(struct cli_cmd));
+    cli_cmd_sort(cmd, count);
+
+    cli_out("\ngluster global commands");
+    cli_out("========================\n");
+    for (global_cmd = cmd; global_cmd->pattern; global_cmd++)
+        if (_gf_false == global_cmd->disable)
+            cli_out("%s - %s", global_cmd->pattern, global_cmd->desc);
+
+    cli_out("\n");
+    GF_FREE(cmd);
+    return 0;
+}
+
+int
+cli_cmd_global_register(struct cli_state *state)
+{
+    int ret = 0;
+    struct cli_cmd *cmd = NULL;
+    for (cmd = global_cmds; cmd->pattern; cmd++) {
+        ret = cli_cmd_register(&state->tree, cmd);
+        if (ret)
+            goto out;
+    }
+out:
+    return ret;
+}
+
+int
+cli_cmd_ganesha_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                    const char **words, int wordcount)
+
+{
+    int sent = 0;
+    int parse_error = 0;
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *options = NULL;
+    cli_local_t *local = NULL;
+    char *op_errstr = NULL;
+
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_GANESHA];
+
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame)
+        goto out;
+
+    ret = cli_cmd_ganesha_parse(state, words, wordcount, &options, &op_errstr);
+    if (ret) {
+        if (op_errstr) {
+            cli_err("%s", op_errstr);
+            GF_FREE(op_errstr);
+        } else
+            cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
+
+    CLI_LOCAL_INIT(local, words, frame, options);
+
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, options);
+    }
+
+out:
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("Setting global option failed");
+    }
+
+    CLI_STACK_DESTROY(frame);
+    return ret;
+}
+
+int
+cli_cmd_get_state_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                      const char **words, int wordcount)
+{
+    int sent = 0;
+    int parse_error = 0;
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *options = NULL;
+    cli_local_t *local = NULL;
+    char *op_errstr = NULL;
+
+    ret = cli_cmd_get_state_parse(state, words, wordcount, &options,
+                                  &op_errstr);
+
+    if (ret) {
+        if (op_errstr) {
+            cli_err("%s", op_errstr);
+            cli_usage_out(word->pattern);
+            GF_FREE(op_errstr);
+        } else
+            cli_usage_out(word->pattern);
+
+        parse_error = 1;
+        goto out;
+    }
+
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
+
+    CLI_LOCAL_INIT(local, words, frame, options);
+
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_GET_STATE];
+    if (proc->fn)
+        ret = proc->fn(frame, THIS, options);
+out:
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("Getting daemon state failed");
+    }
+
+    CLI_STACK_DESTROY(frame);
+
+    return ret;
+}
diff --git a/cli/src/cli-cmd-misc.c b/cli/src/cli-cmd-misc.c
index 4aec8dd35b4..e961d88da86 100644
--- a/cli/src/cli-cmd-misc.c
+++ b/cli/src/cli-cmd-misc.c
@@ -13,85 +13,105 @@
 #include <stdint.h>
 #include <pthread.h>
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 #include "cli.h"
 #include "cli-cmd.h"
 #include "cli-mem-types.h"
 #include "protocol-common.h"
 
-extern struct rpc_clnt *global_rpc;
-
-extern rpc_clnt_prog_t *cli_rpc_prog;
-
 extern struct cli_cmd volume_cmds[];
+extern struct cli_cmd bitrot_cmds[];
+extern struct cli_cmd quota_cmds[];
 extern struct cli_cmd cli_probe_cmds[];
 extern struct cli_cmd cli_log_cmds[];
 extern struct cli_cmd cli_system_cmds[];
 extern struct cli_cmd cli_bd_cmds[];
+extern struct cli_cmd snapshot_cmds[];
+extern struct cli_cmd global_cmds[];
 struct cli_cmd cli_misc_cmds[];
 
 int
-cli_cmd_quit_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                   const char **words, int wordcount)
+cli_cmd_quit_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                 const char **words, int wordcount)
 {
-        exit (0);
+    exit(0);
 }
 
-int
-cli_cmd_display_help (struct cli_state *state, struct cli_cmd_word *in_word,
-                      const char **words, int wordcount)
+static gf_boolean_t
+cli_is_help_command(const char *pattern)
 {
-        struct cli_cmd        *cmd[] = {volume_cmds, cli_probe_cmds,
-                                       cli_misc_cmds,
-                                       NULL};
-        struct cli_cmd        *cmd_ind = NULL;
-        int                   i = 0;
-
-         /* cli_system_cmds commands for internal usage
-           they are not exposed
-         */
-        for (i=0; cmd[i]!=NULL; i++)
-                for (cmd_ind = cmd[i]; cmd_ind->pattern; cmd_ind++)
-                        if (_gf_false == cmd_ind->disable)
-                                cli_out ("%s - %s", cmd_ind->pattern,
-                                         cmd_ind->desc);
-
-        return 0;
+    /* FixFixFix
+     * This is not the best way to determine whether
+     * this is a help command
+     */
+    if (strstr(pattern, "help"))
+        return _gf_true;
+
+    return _gf_false;
 }
 
-struct cli_cmd cli_misc_cmds[] = {
-        { "quit",
-          cli_cmd_quit_cbk,
-          "quit"},
+int
+cli_cmd_display_help(struct cli_state *state, struct cli_cmd_word *in_word,
+                     const char **words, int wordcount)
+{
+    static struct cli_cmd *cmd[] = {
+        cli_misc_cmds, cli_probe_cmds, volume_cmds, bitrot_cmds,
+        quota_cmds,    snapshot_cmds,  global_cmds, NULL};
+    struct cli_cmd *cmd_ind = NULL;
+    int i = 0;
+    gf_boolean_t list_all = _gf_false;
+
+    /* cli_system_cmds commands for internal usage
+      they are not exposed
+    */
+
+    /* If "help all" */
+    if (wordcount == 2)
+        list_all = _gf_true;
+
+    for (i = 0; cmd[i] != NULL; i++) {
+        for (cmd_ind = cmd[i]; cmd_ind->pattern; cmd_ind++) {
+            if ((_gf_false == cmd_ind->disable) &&
+                cli_is_help_command(cmd_ind->pattern)) {
+                if (list_all && (cmd_ind->cbk)) {
+                    cmd_ind->cbk(state, in_word, words, wordcount);
+                } else {
+                    cli_out(" %-25s- %s", cmd_ind->pattern, cmd_ind->desc);
+                }
+            }
+        }
+    }
+
+    cli_out("\n");
+    return 0;
+}
 
-        { "help",
-           cli_cmd_display_help,
-           "display command options"},
+struct cli_cmd cli_help_cmds[] = {
+    {"help [all]", cli_cmd_display_help, "display help for command classes"},
 
-        { "exit",
-          cli_cmd_quit_cbk,
-          "exit"},
+    {NULL, NULL, NULL}};
 
-        { NULL, NULL, NULL }
-};
+struct cli_cmd cli_misc_cmds[] = {{"quit", cli_cmd_quit_cbk, "quit"},
+                                  {"exit", cli_cmd_quit_cbk, "exit"},
 
+                                  {NULL, NULL, NULL}};
 
 int
-cli_cmd_misc_register (struct cli_state *state)
+cli_cmd_misc_register(struct cli_state *state)
 {
-        int  ret = 0;
-        struct cli_cmd *cmd = NULL;
-
-        for (cmd = cli_misc_cmds; cmd->pattern; cmd++) {
-
-                ret = cli_cmd_register (&state->tree, cmd);
-                if (ret)
-                        goto out;
-        }
+    int ret = 0;
+    struct cli_cmd *cmd = NULL;
+
+    for (cmd = cli_misc_cmds; cmd->pattern; cmd++) {
+        ret = cli_cmd_register(&state->tree, cmd);
+        if (ret)
+            goto out;
+    }
+
+    for (cmd = cli_help_cmds; cmd->pattern; cmd++) {
+        ret = cli_cmd_register(&state->tree, cmd);
+        if (ret)
+            goto out;
+    }
 out:
-        return ret;
+    return ret;
 }
diff --git a/cli/src/cli-cmd-parser.c b/cli/src/cli-cmd-parser.c
index a77c57c82ed..34620b4a31b 100644
--- a/cli/src/cli-cmd-parser.c
+++ b/cli/src/cli-cmd-parser.c
@@ -1,5 +1,5 @@
 /*
-   Copyright (c) 2010-2012 Red Hat, Inc. <http://www.redhat.com>
+   Copyright (c) 2010-2013 Red Hat, Inc. <http://www.redhat.com>
    This file is part of GlusterFS.
 
    This file is licensed to you under your choice of the GNU Lesser
@@ -12,2834 +12,5935 @@
 #include <stdlib.h>
 #include <stdint.h>
 #include <pthread.h>
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
+#include <fnmatch.h>
+#include <time.h>
 
 #include "cli.h"
 #include "cli-cmd.h"
 #include "cli-mem-types.h"
-#include "dict.h"
+#include <glusterfs/dict.h>
+#include <glusterfs/list.h>
 
 #include "protocol-common.h"
 #include "cli1-xdr.h"
 
+#define MAX_SNAP_DESCRIPTION_LEN 1024
+
+static struct snap_config_opt_vals_ snap_confopt_vals[] = {
+    {.op_name = "snap-max-hard-limit",
+     .question = "Changing snapshot-max-hard-limit "
+                 "will limit the creation of new snapshots "
+                 "if they exceed the new limit.\n"
+                 "Do you want to continue?"},
+    {.op_name = "snap-max-soft-limit",
+     .question = "If Auto-delete is enabled, snap-max-soft-limit will"
+                 " trigger deletion of oldest snapshot, on the "
+                 "creation of new snapshot, when the "
+                 "snap-max-soft-limit is reached.\n"
+                 "Do you want to change the snap-max-soft-limit?"},
+    {.op_name = "both",
+     .question = "Changing snapshot-max-hard-limit "
+                 "will limit the creation of new snapshots "
+                 "if they exceed the new snapshot-max-hard-limit.\n"
+                 "If Auto-delete is enabled, snap-max-soft-limit will"
+                 " trigger deletion of oldest snapshot, on the "
+                 "creation of new snapshot, when the "
+                 "snap-max-soft-limit is reached.\n"
+                 "Do you want to continue?"},
+    {
+        .op_name = NULL,
+    }};
+
+enum cli_snap_config_set_types {
+    GF_SNAP_CONFIG_SET_HARD = 0,
+    GF_SNAP_CONFIG_SET_SOFT = 1,
+    GF_SNAP_CONFIG_SET_BOTH = 2,
+};
+typedef enum cli_snap_config_set_types cli_snap_config_set_types;
+
+typedef struct _cli_brick {
+    struct list_head list;
+    const char *name;
+    int32_t len;
+} cli_brick_t;
+
+int
+cli_cmd_validate_volume(char *volname);
+
 static const char *
-id_sel (void *wcon)
+id_sel(void *wcon)
 {
-        return (const char *)wcon;
+    return (const char *)wcon;
 }
 
 static char *
-str_getunamb (const char *tok, char **opwords)
+str_getunamb(const char *tok, char **opwords)
 {
-        return (char *)cli_getunamb (tok, (void **)opwords, id_sel);
+    return (char *)cli_getunamb(tok, (void **)opwords, id_sel);
 }
 
 int32_t
-cli_cmd_bricks_parse (const char **words, int wordcount, int brick_index,
-                      char **bricks, int *brick_count)
+cli_cmd_ta_brick_parse(const char **words, int wordcount, char **ta_brick)
 {
-        int                    ret = 0;
-        char             *tmp_list = NULL;
-        char    brick_list[120000] = {0,};
-        char                *space = " ";
-        char            *delimiter = NULL;
-        char            *host_name = NULL;
-        char        *free_list_ptr = NULL;
-        char               *tmpptr = NULL;
-        int                      j = 0;
-        int         brick_list_len = 0;
-        char             *tmp_host = NULL;
-
-        GF_ASSERT (words);
-        GF_ASSERT (wordcount);
-        GF_ASSERT (bricks);
-        GF_ASSERT (brick_index > 0);
-        GF_ASSERT (brick_index < wordcount);
-
-        strncpy (brick_list, space, strlen (space));
-        brick_list_len++;
-        while (brick_index < wordcount) {
-                if (validate_brick_name ((char *)words[brick_index])) {
-                        cli_err ("Wrong brick type: %s, use <HOSTNAME>:"
-                                 "<export-dir-abs-path>", words[brick_index]);
-                        ret = -1;
-                        goto out;
-                } else {
-                        delimiter = strrchr (words[brick_index], ':');
-                        ret = gf_canonicalize_path (delimiter + 1);
-                        if (ret)
-                                goto out;
-                }
-
-                if ((brick_list_len + strlen (words[brick_index]) + 1) > sizeof (brick_list)) {
-                        cli_err ("Total brick list is larger than a request. "
-                                 "Can take (brick_count %d)", *brick_count);
-                        ret = -1;
-                        goto out;
-                }
+    char *host_name = NULL;
+    char *tmp_host = NULL;
+    char *delimiter = NULL;
+    cli_brick_t *brick = NULL;
+    int ret = 0;
+
+    GF_ASSERT(words);
+    GF_ASSERT(wordcount);
+
+    if (validate_brick_name((char *)words[wordcount - 1])) {
+        cli_err(
+            "Wrong brick type: %s, use <HOSTNAME>:"
+            "<export-dir-abs-path>",
+            words[wordcount - 1]);
+        ret = -1;
+        goto out;
+    } else {
+        delimiter = strrchr(words[wordcount - 1], ':');
+        ret = gf_canonicalize_path(delimiter + 1);
+        if (ret)
+            goto out;
+    }
 
-                tmp_host = gf_strdup ((char *)words[brick_index]);
-                if (!tmp_host) {
-                        gf_log ("cli", GF_LOG_ERROR, "Out of memory");
-                        ret = -1;
-                        goto out;
-                }
-                get_host_name (tmp_host, &host_name);
-                if (!host_name) {
-                        ret = -1;
-                        gf_log("cli",GF_LOG_ERROR, "Unable to allocate "
-                               "memory");
-                        goto out;
-                }
+    tmp_host = gf_strdup((char *)words[wordcount - 1]);
+    if (!tmp_host) {
+        gf_log("cli", GF_LOG_ERROR, "Out of memory");
+        ret = -1;
+        goto out;
+    }
+    get_host_name(tmp_host, &host_name);
+    if (!host_name) {
+        ret = -1;
+        gf_log("cli", GF_LOG_ERROR,
+               "Unable to retrieve "
+               "hostname");
+        goto out;
+    }
+
+    if (!(strcmp(host_name, "localhost") && strcmp(host_name, "127.0.0.1") &&
+          strncmp(host_name, "0.", 2))) {
+        cli_err(
+            "Please provide a valid hostname/ip other "
+            "than localhost, 127.0.0.1 or loopback "
+            "address (0.0.0.0 to 0.255.255.255).");
+        ret = -1;
+        goto out;
+    }
+    if (!valid_internet_address(host_name, _gf_false, _gf_false)) {
+        cli_err(
+            "internet address '%s' does not conform to "
+            "standards",
+            host_name);
+    }
+
+    brick = GF_MALLOC(sizeof(cli_brick_t), gf_common_list_node);
+    if (brick == NULL) {
+        ret = -1;
+        gf_log("cli", GF_LOG_ERROR, "Out of memory");
+        goto out;
+    }
 
-                if (!(strcmp (host_name, "localhost") &&
-                      strcmp (host_name, "127.0.0.1") &&
-                      strncmp (host_name, "0.", 2))) {
-                        cli_err ("Please provide a valid hostname/ip other "
-                                 "than localhost, 127.0.0.1 or loopback "
-                                 "address (0.0.0.0 to 0.255.255.255).");
-                        ret = -1;
-                        GF_FREE (tmp_host);
-                        goto out;
-                }
-                if (!valid_internet_address (host_name, _gf_false)) {
-                        cli_err ("internet address '%s' does not conform to "
-                                 "standards", host_name);
-                }
-                GF_FREE (tmp_host);
-                tmp_list = gf_strdup (brick_list + 1);
-                if (free_list_ptr) {
-                        GF_FREE (free_list_ptr);
-                        free_list_ptr = NULL;
-                }
-                free_list_ptr = tmp_list;
-                j = 0;
-                while(j < *brick_count) {
-                        strtok_r (tmp_list, " ", &tmpptr);
-                        if (!(strcmp (tmp_list, words[brick_index]))) {
-                                ret = -1;
-                                cli_err ("Found duplicate"
-                                         " exports %s",words[brick_index]);
-                                goto out;
-                       }
-                       tmp_list = tmpptr;
-                       j++;
-                }
-                strcat (brick_list, words[brick_index]);
-                strcat (brick_list, " ");
-                brick_list_len += (strlen (words[brick_index]) + 1);
-                ++(*brick_count);
-                ++brick_index;
-        }
+    brick->name = words[wordcount - 1];
+    brick->len = strlen(words[wordcount - 1]);
+    *ta_brick = GF_MALLOC(brick->len + 3, gf_common_mt_char);
+    if (*ta_brick == NULL) {
+        ret = -1;
+        gf_log("cli", GF_LOG_ERROR, "Out of memory");
+        goto out;
+    }
 
-        *bricks = gf_strdup (brick_list);
-        if (!*bricks)
-                ret = -1;
+    strcat(*ta_brick, " ");
+    strcat(*ta_brick, brick->name);
+    strcat(*ta_brick, " ");
 out:
-        GF_FREE (free_list_ptr);
-        return ret;
+    if (tmp_host) {
+        GF_FREE(tmp_host);
+        tmp_host = NULL;
+    }
+    if (brick) {
+        GF_FREE(brick);
+        brick = NULL;
+    }
+
+    return ret;
 }
 
 int32_t
-cli_cmd_volume_create_parse (const char **words, int wordcount, dict_t **options)
+cli_cmd_bricks_parse(const char **words, int wordcount, int brick_index,
+                     char **bricks, int *brick_count)
 {
-        dict_t  *dict = NULL;
-        char    *volname = NULL;
-        int     ret = -1;
-        gf1_cluster_type type = GF_CLUSTER_TYPE_NONE;
-        int     count = 1;
-        int     sub_count = 1;
-        int     brick_index = 0;
-        int     i = 0;
-        char    *trans_type = NULL;
-        int32_t index = 0;
-        char    *bricks = NULL;
-        int32_t brick_count = 0;
-        char    *opwords[] = { "replica", "stripe", "transport", NULL };
-
-        char    *invalid_volnames[] = {"volume", "type", "subvolumes", "option",
-                                       "end-volume", "all", "volume_not_in_ring",
-                                       NULL};
-        char    *w = NULL;
-        int      op_count = 0;
-        int32_t  replica_count = 1;
-        int32_t  stripe_count = 1;
-        gf_boolean_t is_force = _gf_false;
-        int wc = wordcount;
-
-        GF_ASSERT (words);
-        GF_ASSERT (options);
+    int ret = 0;
+    char *delimiter = NULL;
+    char *host_name = NULL;
+    char *tmp_host = NULL;
+    char *bricks_str = NULL;
+    int len = 0;
+    int brick_list_len = 1; /* For initial space */
+    struct list_head brick_list = {
+        0,
+    };
+    cli_brick_t *brick = NULL;
+
+    GF_ASSERT(words);
+    GF_ASSERT(wordcount);
+    GF_ASSERT(bricks);
+    GF_ASSERT(brick_index > 0);
+    GF_ASSERT(brick_index < wordcount);
+
+    INIT_LIST_HEAD(&brick_list);
+
+    while (brick_index < wordcount) {
+        if (validate_brick_name((char *)words[brick_index])) {
+            cli_err(
+                "Wrong brick type: %s, use <HOSTNAME>:"
+                "<export-dir-abs-path>",
+                words[brick_index]);
+            ret = -1;
+            goto out;
+        } else {
+            delimiter = strrchr(words[brick_index], ':');
+            ret = gf_canonicalize_path(delimiter + 1);
+            if (ret)
+                goto out;
+        }
 
-        dict = dict_new ();
+        tmp_host = gf_strdup((char *)words[brick_index]);
+        if (!tmp_host) {
+            gf_log("cli", GF_LOG_ERROR, "Out of memory");
+            ret = -1;
+            goto out;
+        }
+        get_host_name(tmp_host, &host_name);
+        if (!host_name) {
+            ret = -1;
+            gf_log("cli", GF_LOG_ERROR,
+                   "Unable to allocate "
+                   "memory");
+            GF_FREE(tmp_host);
+            goto out;
+        }
 
-        if (!dict)
+        if (!(strcmp(host_name, "localhost") &&
+              strcmp(host_name, "127.0.0.1") && strncmp(host_name, "0.", 2))) {
+            cli_err(
+                "Please provide a valid hostname/ip other "
+                "than localhost, 127.0.0.1 or loopback "
+                "address (0.0.0.0 to 0.255.255.255).");
+            ret = -1;
+            GF_FREE(tmp_host);
+            goto out;
+        }
+        if (!valid_internet_address(host_name, _gf_false, _gf_false)) {
+            cli_err(
+                "internet address '%s' does not conform to "
+                "standards",
+                host_name);
+        }
+        GF_FREE(tmp_host);
+        list_for_each_entry(brick, &brick_list, list)
+        {
+            if (strcmp(brick->name, words[brick_index]) == 0) {
+                ret = -1;
+                cli_err("Found duplicate exports %s", words[brick_index]);
                 goto out;
+            }
+        }
 
-        if (wordcount < 3)
-                goto out;
+        brick = GF_MALLOC(sizeof(cli_brick_t), gf_common_list_node);
+        if (brick == NULL) {
+            ret = -1;
+            gf_log("cli", GF_LOG_ERROR, "Out of memory");
+            goto out;
+        }
+        len = strlen(words[brick_index]);
+        brick->name = words[brick_index];
+        brick->len = len;
+        list_add_tail(&brick->list, &brick_list);
+
+        brick_list_len += len + 1; /* Brick name + space */
+        ++(*brick_count);
+        ++brick_index;
+    }
+
+    /* If brick count is not valid exit here */
+    if (!*brick_count) {
+        cli_err("No bricks specified");
+        ret = -1;
+        goto out;
+    }
 
-        volname = (char *)words[2];
+    brick_list_len++; /* For terminating null char */
 
-        GF_ASSERT (volname);
+    bricks_str = GF_MALLOC(brick_list_len, gf_common_mt_char);
+    if (bricks_str == NULL) {
+        ret = -1;
+        gf_log("cli", GF_LOG_ERROR, "Out of memory");
+        goto out;
+    }
+    *bricks = bricks_str;
+    *bricks_str = ' ';
+    bricks_str++;
+    while (!list_empty(&brick_list)) {
+        brick = list_first_entry(&brick_list, cli_brick_t, list);
+        list_del_init(&brick->list);
+        memcpy(bricks_str, brick->name, brick->len);
+        bricks_str[brick->len] = ' ';
+        bricks_str += brick->len + 1;
+        GF_FREE(brick);
+    }
+    *bricks_str = 0;
 
-        /* Validate the volume name here itself */
-        {
-                if (volname[0] == '-')
-                        goto out;
+out:
+    while (!list_empty(&brick_list)) {
+        brick = list_first_entry(&brick_list, cli_brick_t, list);
+        list_del_init(&brick->list);
+        GF_FREE(brick);
+    }
 
-                for (i = 0; invalid_volnames[i]; i++) {
-                        if (!strcmp (volname, invalid_volnames[i])) {
-                                cli_err ("\"%s\" cannot be the name of a volume.",
-                                         volname);
-                                goto out;
-                        }
-                }
+    return ret;
+}
 
-                if (strchr (volname, '/'))
-                        goto out;
+int32_t
+cli_cmd_create_disperse_check(struct cli_state *state, int *disperse,
+                              int *redundancy, int *data, int count)
+{
+    int i = 0;
+    int tmp = 0;
+    gf_answer_t answer = GF_ANSWER_NO;
+    char question[128];
+
+    const char *question1 =
+        "There isn't an optimal redundancy value "
+        "for this configuration. Do you want to "
+        "create the volume with redundancy 1 ?";
+
+    const char *question2 =
+        "The optimal redundancy for this "
+        "configuration is %d. Do you want to create "
+        "the volume with this value ?";
+
+    const char *question3 =
+        "This configuration is not optimal on most "
+        "workloads. Do you want to use it ?";
+
+    const char *question4 =
+        "Redundancy for this configuration is %d. "
+        "Do you want to create "
+        "the volume with this value ?";
+
+    if (*data > 0) {
+        if (*disperse > 0 && *redundancy > 0) {
+            if (*disperse != (*data + *redundancy)) {
+                cli_err(
+                    "Disperse count(%d) should be equal "
+                    "to sum of disperse-data count(%d) and "
+                    "redundancy count(%d)",
+                    *disperse, *data, *redundancy);
+                return -1;
+            }
+        } else if (*redundancy > 0) {
+            *disperse = *data + *redundancy;
+        } else if (*disperse > 0) {
+            *redundancy = *disperse - *data;
+        } else {
+            if ((count - *data) >= *data) {
+                cli_err(
+                    "Please provide redundancy count "
+                    "along with disperse-data count");
+                return -1;
+            } else {
+                sprintf(question, question4, count - *data);
+                answer = cli_cmd_get_confirmation(state, question);
+                if (answer == GF_ANSWER_NO)
+                    return -1;
+                *redundancy = count - *data;
+                *disperse = count;
+            }
+        }
+    }
 
-                if (strlen (volname) > 512)
-                        goto out;
+    if (*disperse <= 0) {
+        if (count < 3) {
+            cli_err(
+                "number of bricks must be greater "
+                "than 2");
 
-                for (i = 0; i < strlen (volname); i++)
-                        if (!isalnum (volname[i]) && (volname[i] != '_') && (volname[i] != '-'))
-                                goto out;
+            return -1;
         }
+        *disperse = count;
+    }
 
-        if (wordcount < 4) {
-                ret = -1;
-                goto out;
+    if (*redundancy == -1) {
+        tmp = *disperse - 1;
+        for (i = tmp / 2; (i > 0) && ((tmp & -tmp) != tmp); i--, tmp--)
+            ;
+
+        if (i == 0) {
+            answer = cli_cmd_get_confirmation(state, question1);
+            if (answer == GF_ANSWER_NO)
+                return -1;
+
+            *redundancy = 1;
+        } else {
+            *redundancy = *disperse - tmp;
+            if (*redundancy > 1) {
+                sprintf(question, question2, *redundancy);
+                answer = cli_cmd_get_confirmation(state, question);
+                if (answer == GF_ANSWER_NO)
+                    return -1;
+            }
         }
 
-        type = GF_CLUSTER_TYPE_NONE;
-        index = 3;
+        tmp = 0;
+    } else {
+        tmp = *disperse - *redundancy;
+    }
 
-        while (op_count < 3) {
-                ret = -1;
-                w = str_getunamb (words[index], opwords);
-                if (!w) {
-                        break;
-                } else if ((strcmp (w, "replica")) == 0) {
-                        switch (type) {
-                        case GF_CLUSTER_TYPE_STRIPE_REPLICATE:
-                        case GF_CLUSTER_TYPE_REPLICATE:
-                                cli_err ("replica option given twice");
-                                goto out;
-                        case GF_CLUSTER_TYPE_NONE:
-                                type = GF_CLUSTER_TYPE_REPLICATE;
-                                break;
-                        case GF_CLUSTER_TYPE_STRIPE:
-                                type = GF_CLUSTER_TYPE_STRIPE_REPLICATE;
-                                break;
-                        }
+    if ((*redundancy < 1) || (*redundancy > (*disperse - 1) / 2)) {
+        cli_err(
+            "redundancy must be greater than or equal to 1 and "
+            "less than %d for a disperse %d volume",
+            (*disperse + 1) / 2, *disperse);
 
-                        if (wordcount < (index+2)) {
-                                ret = -1;
-                                goto out;
-                        }
-                        replica_count = strtol (words[index+1], NULL, 0);
-                        if (replica_count < 2) {
-                                cli_err ("replica count should be greater"
-                                         " than 1");
-                                ret = -1;
-                                goto out;
-                        }
-                        ret = dict_set_int32 (dict, "replica-count", replica_count);
-                        if (ret)
-                                goto out;
-
-                        index += 2;
-
-                } else if ((strcmp (w, "stripe")) == 0) {
-                        switch (type) {
-                        case GF_CLUSTER_TYPE_STRIPE_REPLICATE:
-                        case GF_CLUSTER_TYPE_STRIPE:
-                                cli_err ("stripe option given twice");
-                                goto out;
-                        case GF_CLUSTER_TYPE_NONE:
-                                type = GF_CLUSTER_TYPE_STRIPE;
-                                break;
-                        case GF_CLUSTER_TYPE_REPLICATE:
-                                type = GF_CLUSTER_TYPE_STRIPE_REPLICATE;
-                                break;
-                        }
-                        if (wordcount < (index + 2)) {
-                                ret = -1;
-                                goto out;
-                        }
-                        stripe_count = strtol (words[index+1], NULL, 0);
-                        if (stripe_count < 2) {
-                                cli_err ("stripe count should be greater"
-                                         " than 1");
-                                ret = -1;
-                                goto out;
-                        }
-                        ret = dict_set_int32 (dict, "stripe-count", stripe_count);
-                        if (ret)
-                                goto out;
+        return -1;
+    }
 
-                        index += 2;
+    if ((tmp & -tmp) != tmp) {
+        answer = cli_cmd_get_confirmation(state, question3);
+        if (answer == GF_ANSWER_NO)
+            return -1;
+    }
 
-                } else if ((strcmp (w, "transport")) == 0) {
-                        if (trans_type) {
-                                cli_err ("'transport' option given more"
-                                         " than one time");
-                                goto out;
-                        }
-                        if ((strcasecmp (words[index+1], "tcp") == 0)) {
-                                trans_type = gf_strdup ("tcp");
-                        } else if ((strcasecmp (words[index+1], "rdma") == 0)) {
-                                trans_type = gf_strdup ("rdma");
-                        } else if ((strcasecmp (words[index+1], "tcp,rdma") == 0) ||
-                                   (strcasecmp (words[index+1], "rdma,tcp") == 0)) {
-                                trans_type = gf_strdup ("tcp,rdma");
-                        } else {
-                                gf_log ("", GF_LOG_ERROR, "incorrect transport"
-                                        " protocol specified");
-                                ret = -1;
-                                goto out;
-                        }
-                        index += 2;
-                }              else {
-                        GF_ASSERT (!"opword mismatch");
-                        ret = -1;
+    return 0;
+}
+
+static int32_t
+cli_validate_disperse_volume(char *word, gf1_cluster_type type,
+                             const char **words, int32_t wordcount,
+                             int32_t index, int32_t *disperse_count,
+                             int32_t *redundancy_count, int32_t *data_count)
+{
+    int ret = -1;
+
+    switch (type) {
+        case GF_CLUSTER_TYPE_NONE:
+        case GF_CLUSTER_TYPE_DISPERSE:
+            if (strcmp(word, "disperse") == 0) {
+                if (*disperse_count >= 0) {
+                    cli_err("disperse option given twice");
+                    goto out;
+                }
+                if (wordcount < (index + 2)) {
+                    goto out;
+                }
+                ret = gf_string2int(words[index + 1], disperse_count);
+                if (ret == -1 && errno == EINVAL) {
+                    *disperse_count = 0;
+                    ret = 1;
+                } else if (ret == -1) {
+                    goto out;
+                } else {
+                    if (*disperse_count < 3) {
+                        cli_err(
+                            "disperse count must "
+                            "be greater than 2");
                         goto out;
+                    }
+                    ret = 2;
                 }
-                op_count++;
-        }
+            } else if (strcmp(word, "disperse-data") == 0) {
+                if (*data_count >= 0) {
+                    cli_err("disperse-data option given twice");
+                    goto out;
+                }
+                if (wordcount < (index + 2)) {
+                    goto out;
+                }
+                ret = gf_string2int(words[index + 1], data_count);
+                if (ret == -1 || *data_count < 2) {
+                    cli_err("disperse-data must be greater than 1");
+                    goto out;
+                }
+                ret = 2;
+            } else if (strcmp(word, "redundancy") == 0) {
+                if (*redundancy_count >= 0) {
+                    cli_err("redundancy option given twice");
+                    goto out;
+                }
+                if (wordcount < (index + 2)) {
+                    goto out;
+                }
+                ret = gf_string2int(words[index + 1], redundancy_count);
+                if (ret == -1 || *redundancy_count < 1) {
+                    cli_err("redundancy must be greater than 0");
+                    goto out;
+                }
+                ret = 2;
+            }
+            break;
+        case GF_CLUSTER_TYPE_REPLICATE:
+            cli_err(
+                "replicated-dispersed volume is not "
+                "supported");
+            goto out;
+        default:
+            cli_err("Invalid type given");
+            break;
+    }
+out:
+    return ret;
+}
 
-        if (!trans_type)
-                trans_type = gf_strdup ("tcp");
+int32_t
+cli_validate_volname(const char *volname)
+{
+    int32_t ret = -1;
+    int32_t i = -1;
+    int volname_len;
+    static const char *const invalid_volnames[] = {"volume",
+                                                   "type",
+                                                   "subvolumes",
+                                                   "option",
+                                                   "end-volume",
+                                                   "all",
+                                                   "volume_not_in_ring",
+                                                   "description",
+                                                   "force",
+                                                   "snap-max-hard-limit",
+                                                   "snap-max-soft-limit",
+                                                   "auto-delete",
+                                                   "activate-on-create",
+                                                   NULL};
+
+    if (volname[0] == '-')
+        goto out;
 
-        sub_count = stripe_count * replica_count;
+    for (i = 0; invalid_volnames[i]; i++) {
+        if (!strcmp(volname, invalid_volnames[i])) {
+            cli_err("\"%s\" cannot be the name of a volume.", volname);
+            goto out;
+        }
+    }
 
-        /* reset the count value now */
-        count = 1;
+    if (strchr(volname, '/'))
+        goto out;
 
-        if (index >= wordcount) {
-                ret = -1;
-                goto out;
+    volname_len = strlen(volname);
+    if (volname_len > GD_VOLUME_NAME_MAX) {
+        cli_err("Volume name exceeds %d characters.", GD_VOLUME_NAME_MAX);
+        goto out;
+    }
+
+    for (i = 0; i < volname_len; i++) {
+        if (!isalnum(volname[i]) && (volname[i] != '_') &&
+            (volname[i] != '-')) {
+            cli_err(
+                "Volume name should not contain \"%c\""
+                " character.\nVolume names can only"
+                "contain alphanumeric, '-' and '_' "
+                "characters.",
+                volname[i]);
+            goto out;
         }
+    }
+
+    ret = 0;
+out:
+    return ret;
+}
 
-        brick_index = index;
+int32_t
+cli_cmd_volume_create_parse(struct cli_state *state, const char **words,
+                            int wordcount, dict_t **options, char **brick_list)
+{
+    dict_t *dict = NULL;
+    char *volname = NULL;
+    int ret = -1;
+    gf1_cluster_type type = GF_CLUSTER_TYPE_NONE;
+    int sub_count = 1;
+    int brick_index = 0;
+    char *trans_type = NULL;
+    int32_t index = 0;
+    char *bricks = NULL;
+    char *ta_brick = NULL;
+    int32_t brick_count = 0;
+    static char *opwords[] = {"replica",  "stripe",       "transport",
+                              "disperse", "redundancy",   "disperse-data",
+                              "arbiter",  "thin-arbiter", NULL};
+
+    char *w = NULL;
+    int op_count = 0;
+    int32_t replica_count = 1;
+    int32_t arbiter_count = 0;
+    int32_t thin_arbiter_count = 0;
+    int32_t stripe_count = 1;
+    int32_t disperse_count = -1;
+    int32_t redundancy_count = -1;
+    int32_t disperse_data_count = -1;
+    gf_boolean_t is_force = _gf_false;
+    int wc = wordcount;
+    gf_answer_t answer = GF_ANSWER_NO;
+    const char *question = NULL;
+
+    GF_ASSERT(words);
+    GF_ASSERT(options);
+
+    dict = dict_new();
+
+    if (!dict)
+        goto out;
 
-        if (strcmp (words[wordcount - 1], "force") == 0) {
-                is_force = _gf_true;
-                wc = wordcount - 1;
-        }
+    if (wordcount < 3)
+        goto out;
 
-        ret = cli_cmd_bricks_parse (words, wc, brick_index, &bricks,
-                                    &brick_count);
-        if (ret)
-                goto out;
+    volname = (char *)words[2];
+
+    GF_ASSERT(volname);
+
+    /* Validate the volume name here itself */
+    if (cli_validate_volname(volname) < 0)
+        goto out;
+
+    if (wordcount < 4) {
+        ret = -1;
+        goto out;
+    }
+
+    type = GF_CLUSTER_TYPE_NONE;
+    index = 3;
 
-        /* If brick-count is not valid when replica or stripe is
-           given, exit here */
-        if (!brick_count) {
-                cli_err ("No bricks specified");
+    while (op_count < 3) {
+        ret = -1;
+        w = str_getunamb(words[index], opwords);
+        if (!w) {
+            break;
+        } else if ((strcmp(w, "replica")) == 0) {
+            switch (type) {
+                case GF_CLUSTER_TYPE_STRIPE_REPLICATE:
+                case GF_CLUSTER_TYPE_REPLICATE:
+                    cli_err("replica option given twice");
+                    goto out;
+                case GF_CLUSTER_TYPE_NONE:
+                    type = GF_CLUSTER_TYPE_REPLICATE;
+                    break;
+                case GF_CLUSTER_TYPE_STRIPE:
+                    cli_err("stripe option not supported");
+                    goto out;
+                case GF_CLUSTER_TYPE_DISPERSE:
+                    cli_err(
+                        "replicated-dispersed volume is not "
+                        "supported");
+                    goto out;
+                default:
+                    cli_err("Invalid type given");
+                    goto out;
+            }
+
+            if (wordcount < (index + 2)) {
                 ret = -1;
                 goto out;
-        }
+            }
 
-        if (brick_count % sub_count) {
-                if (type == GF_CLUSTER_TYPE_STRIPE)
-                        cli_err ("number of bricks is not a multiple of "
-                                 "stripe count");
-                else if (type == GF_CLUSTER_TYPE_REPLICATE)
-                        cli_err ("number of bricks is not a multiple of "
-                                 "replica count");
-                else
-                        cli_err ("number of bricks given doesn't match "
-                                 "required count");
+            replica_count = strtol(words[index + 1], NULL, 0);
+            if (replica_count < 2) {
+                cli_err(
+                    "replica count should be greater"
+                    " than 1");
+                ret = -1;
+                goto out;
+            }
+
+            index += 2;
+            if (words[index]) {
+                if (!strcmp(words[index], "arbiter")) {
+                    ret = gf_string2int(words[index + 1], &arbiter_count);
+                    if ((ret == -1) || (arbiter_count != 1)) {
+                        cli_err(
+                            "For arbiter "
+                            "configuration, "
+                            "replica count must be"
+                            " 2 and arbiter count "
+                            "must be 1. The 3rd "
+                            "brick of the replica "
+                            "will be the arbiter");
+                        ret = -1;
+                        goto out;
+                    }
+                    ret = dict_set_int32(dict, "arbiter-count", arbiter_count);
+                    if (ret)
+                        goto out;
+                    index += 2;
+                } else if (!strcmp(words[index], "thin-arbiter")) {
+                    ret = gf_string2int(words[index + 1], &thin_arbiter_count);
+                    if ((ret == -1) || (thin_arbiter_count != 1) ||
+                        (replica_count != 2)) {
+                        cli_err(
+                            "For thin-arbiter "
+                            "configuration, "
+                            "replica count must be"
+                            " 2 and thin-arbiter count "
+                            "must be 1. The 3rd "
+                            "brick of the replica "
+                            "will be the thin-arbiter brick");
+                        ret = -1;
+                        goto out;
+                    }
+                    ret = dict_set_int32(dict, "thin-arbiter-count",
+                                         thin_arbiter_count);
+                    if (ret)
+                        goto out;
+                    index += 2;
+                }
+            }
+
+            /* Do this to keep glusterd happy with sending
+               "replica 3 arbiter 1" options to server */
+            if ((arbiter_count == 1) && (replica_count == 2))
+                replica_count += arbiter_count;
+
+            if (replica_count == 2 && thin_arbiter_count == 0) {
+                if (strcmp(words[wordcount - 1], "force")) {
+                    question =
+                        "Replica 2 volumes are prone"
+                        " to split-brain. Use "
+                        "Arbiter or Replica 3 to "
+                        "avoid this. See: "
+                        "http://docs.gluster.org/en/latest/"
+                        "Administrator%20Guide/"
+                        "Split%20brain%20and%20ways%20to%20deal%20with%20it/."
+                        "\nDo you still want to "
+                        "continue?\n";
+                    answer = cli_cmd_get_confirmation(state, question);
+                    if (GF_ANSWER_NO == answer) {
+                        gf_log("cli", GF_LOG_ERROR,
+                               "Volume create "
+                               "cancelled, exiting");
+                        ret = -1;
+                        goto out;
+                    }
+                }
+            }
+            ret = dict_set_int32(dict, "replica-count", replica_count);
+            if (ret)
+                goto out;
 
+        } else if ((strcmp(w, "stripe")) == 0) {
+            cli_err("stripe option not supported");
+            goto out;
+        } else if ((strcmp(w, "transport")) == 0) {
+            if (trans_type) {
+                cli_err(
+                    "'transport' option given more"
+                    " than one time");
+                goto out;
+            }
+            if ((strcasecmp(words[index + 1], "tcp") == 0)) {
+                trans_type = gf_strdup("tcp");
+            } else if ((strcasecmp(words[index + 1], "rdma") == 0)) {
+                trans_type = gf_strdup("rdma");
+            } else if ((strcasecmp(words[index + 1], "tcp,rdma") == 0) ||
+                       (strcasecmp(words[index + 1], "rdma,tcp") == 0)) {
+                trans_type = gf_strdup("tcp,rdma");
+            } else {
+                gf_log("", GF_LOG_ERROR,
+                       "incorrect transport"
+                       " protocol specified");
                 ret = -1;
                 goto out;
+            }
+            index += 2;
+
+        } else if ((strcmp(w, "disperse") == 0) ||
+                   (strcmp(w, "redundancy") == 0) ||
+                   (strcmp(w, "disperse-data") == 0)) {
+            ret = cli_validate_disperse_volume(
+                w, type, words, wordcount, index, &disperse_count,
+                &redundancy_count, &disperse_data_count);
+            if (ret < 0)
+                goto out;
+            index += ret;
+            type = GF_CLUSTER_TYPE_DISPERSE;
+        } else if ((strcmp(w, "arbiter") == 0)) {
+            cli_err(
+                "arbiter option must be preceded by replica "
+                "option.");
+            ret = -1;
+            goto out;
+        } else if ((strcmp(w, "thin-arbiter") == 0)) {
+            cli_err(
+                "thin-arbiter option must be preceded by replica "
+                "option.");
+            ret = -1;
+            goto out;
+        } else {
+            GF_ASSERT(!"opword mismatch");
+            ret = -1;
+            goto out;
         }
+        op_count++;
+    }
 
-        /* Everything if parsed fine. start setting info in dict */
-        ret = dict_set_str (dict, "volname", volname);
-        if (ret)
-                goto out;
+    if (!trans_type)
+        trans_type = gf_strdup("tcp");
 
-        ret = dict_set_int32 (dict, "type", type);
-        if (ret)
-                goto out;
+    if (index >= wordcount) {
+        ret = -1;
+        goto out;
+    }
 
-        ret = dict_set_dynstr (dict, "transport", trans_type);
-        if (ret)
-                goto out;
-        trans_type = NULL;
+    brick_index = index;
 
-        ret = dict_set_dynstr (dict, "bricks", bricks);
+    if (strcmp(words[wordcount - 1], "force") == 0) {
+        is_force = _gf_true;
+        wc = wordcount - 1;
+    }
+
+    // Exclude the thin-arbiter-brick i.e. last brick in the bricks list
+    if (thin_arbiter_count == 1) {
+        ret = cli_cmd_bricks_parse(words, wc - 1, brick_index, &bricks,
+                                   &brick_count);
         if (ret)
-                goto out;
+            goto out;
+
+        ret = cli_cmd_ta_brick_parse(words, wc, &ta_brick);
 
-        ret = dict_set_int32 (dict, "count", brick_count);
+    } else {
+        ret = cli_cmd_bricks_parse(words, wc, brick_index, &bricks,
+                                   &brick_count);
+    }
+
+    if (ret)
+        goto out;
+
+    if (type == GF_CLUSTER_TYPE_DISPERSE) {
+        ret = cli_cmd_create_disperse_check(state, &disperse_count,
+                                            &redundancy_count,
+                                            &disperse_data_count, brick_count);
+        if (!ret)
+            ret = dict_set_int32(dict, "disperse-count", disperse_count);
+        if (!ret)
+            ret = dict_set_int32(dict, "redundancy-count", redundancy_count);
         if (ret)
-                goto out;
+            goto out;
+
+        sub_count = disperse_count;
+    } else
+        sub_count = stripe_count * replica_count;
+
+    if (brick_count % sub_count) {
+        if (type == GF_CLUSTER_TYPE_STRIPE)
+            cli_err(
+                "number of bricks is not a multiple of "
+                "stripe count");
+        else if (type == GF_CLUSTER_TYPE_REPLICATE)
+            cli_err(
+                "number of bricks is not a multiple of "
+                "replica count");
+        else if (type == GF_CLUSTER_TYPE_DISPERSE)
+            cli_err(
+                "number of bricks is not a multiple of "
+                "disperse count");
+        else
+            cli_err(
+                "number of bricks given doesn't match "
+                "required count");
+
+        ret = -1;
+        goto out;
+    }
+
+    /* Everything is parsed fine. start setting info in dict */
+    ret = dict_set_str(dict, "volname", volname);
+    if (ret)
+        goto out;
+
+    ret = dict_set_int32(dict, "type", type);
+    if (ret)
+        goto out;
+
+    ret = dict_set_dynstr(dict, "transport", trans_type);
+    if (ret)
+        goto out;
+    trans_type = NULL;
+
+    ret = dict_set_dynstr(dict, "bricks", bricks);
+    if (ret)
+        goto out;
 
-        ret = dict_set_int32 (dict, "force", is_force);
+    if (thin_arbiter_count == 1) {
+        ret = dict_set_dynstr(dict, "ta-brick", ta_brick);
         if (ret)
-                goto out;
+            goto out;
+    }
 
-        *options = dict;
+    ret = dict_set_int32(dict, "count", brick_count);
+    if (ret)
+        goto out;
 
+    ret = dict_set_int32(dict, "force", is_force);
+    if (ret)
+        goto out;
+
+    *options = dict;
+    *brick_list = bricks;
 out:
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Unable to parse create volume CLI");
-                if (dict)
-                        dict_destroy (dict);
-        }
+    if (ret) {
+        GF_FREE(bricks);
+        GF_FREE(ta_brick);
+        gf_log("cli", GF_LOG_ERROR, "Unable to parse create volume CLI");
+        if (dict)
+            dict_unref(dict);
+    }
 
-        GF_FREE (trans_type);
+    GF_FREE(trans_type);
 
-        return ret;
+    return ret;
 }
 
 int32_t
-cli_cmd_volume_reset_parse (const char **words, int wordcount, dict_t **options)
+cli_cmd_volume_reset_parse(const char **words, int wordcount, dict_t **options)
 {
-        dict_t  *dict = NULL;
-        char    *volname = NULL;
-        int     ret = -1;
+    dict_t *dict = NULL;
+    char *volname = NULL;
+    int ret = -1;
 
-        GF_ASSERT (words);
-        GF_ASSERT (options);
+    GF_ASSERT(words);
+    GF_ASSERT(options);
 
-        dict = dict_new ();
+    dict = dict_new();
 
-        if (!dict)
-                goto out;
+    if (!dict)
+        goto out;
 
-        if (wordcount < 3)
-                goto out;
+    if (wordcount < 3)
+        goto out;
 
-        if (wordcount > 5)
-                goto out;
+    if (wordcount > 5)
+        goto out;
 
-        volname = (char *)words[2];
+    volname = (char *)words[2];
 
-        if (!volname) {
-                ret = -1;
-                goto out;
-        }
+    if (!volname) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_set_str(dict, "volname", volname);
+    if (ret)
+        goto out;
 
-        ret = dict_set_str (dict, "volname", volname);
+    if (wordcount == 3) {
+        ret = dict_set_str(dict, "key", "all");
         if (ret)
-                goto out;
+            goto out;
+    }
 
-        if (wordcount == 3) {
-                ret = dict_set_str (dict, "key", "all");
-                if (ret)
-                        goto out;
+    if (wordcount >= 4) {
+        if (!strcmp("force", (char *)words[3])) {
+            ret = dict_set_int32(dict, "force", 1);
+            if (ret)
+                goto out;
+            ret = dict_set_str(dict, "key", "all");
+            if (ret)
+                goto out;
+        } else {
+            ret = dict_set_str(dict, "key", (char *)words[3]);
+            if (ret)
+                goto out;
         }
+    }
 
-        if (wordcount >= 4) {
-                if (!strcmp ("force", (char*)words[3])) {
-                        ret = dict_set_int32 (dict, "force", 1);
-                        if (ret)
-                                goto out;
-                        ret = dict_set_str (dict, "key", "all");
-                        if (ret)
-                                goto out;
-                } else {
-                        ret = dict_set_str (dict, "key", (char *)words[3]);
-                        if (ret)
-                                goto out;
-                }
+    if (wordcount == 5) {
+        if (strcmp("force", (char *)words[4])) {
+            ret = -1;
+            goto out;
+        } else {
+            ret = dict_set_int32(dict, "force", 1);
+            if (ret)
+                goto out;
         }
+    }
+
+    *options = dict;
+
+out:
+    if (ret && dict) {
+        dict_unref(dict);
+    }
 
-        if (wordcount == 5) {
-                if (strcmp ("force", (char*)words[4])) {
+    return ret;
+}
+
+int32_t
+cli_cmd_get_state_parse(struct cli_state *state, const char **words,
+                        int wordcount, dict_t **options, char **op_errstr)
+{
+    dict_t *dict = NULL;
+    int ret = -1;
+    char *odir = NULL;
+    char *filename = NULL;
+    char *daemon_name = NULL;
+    int count = 0;
+    uint32_t cmd = 0;
+
+    GF_VALIDATE_OR_GOTO("cli", options, out);
+    GF_VALIDATE_OR_GOTO("cli", words, out);
+
+    dict = dict_new();
+    if (!dict)
+        goto out;
+
+    if (wordcount < 1 || wordcount > 7) {
+        *op_errstr = gf_strdup(
+            "Problem parsing arguments."
+            " Check usage.");
+        goto out;
+    }
+
+    if (wordcount >= 1) {
+        gf_asprintf(&daemon_name, "%s", "glusterd");
+
+        for (count = 1; count < wordcount; count++) {
+            if (strcmp(words[count], "odir") == 0 ||
+                strcmp(words[count], "file") == 0) {
+                if (strcmp(words[count], "odir") == 0) {
+                    if (++count < wordcount) {
+                        odir = (char *)words[count];
+                        continue;
+                    } else {
                         ret = -1;
                         goto out;
+                    }
+                } else if (strcmp(words[count], "file") == 0) {
+                    if (++count < wordcount) {
+                        filename = (char *)words[count];
+                        continue;
+                    } else {
+                        ret = -1;
+                        goto out;
+                    }
+                }
+            } else {
+                if (count > 1) {
+                    if (count == wordcount - 1) {
+                        if (strcmp(words[count], "detail") == 0) {
+                            cmd = GF_CLI_GET_STATE_DETAIL;
+                            continue;
+                        } else if (strcmp(words[count], "volumeoptions") == 0) {
+                            cmd = GF_CLI_GET_STATE_VOLOPTS;
+                            continue;
+                        }
+                    } else {
+                        *op_errstr = gf_strdup(
+                            "Problem"
+                            " parsing arguments. "
+                            "Check usage.");
+                        ret = -1;
+                        goto out;
+                    }
+                }
+                if (strcmp(words[count], "glusterd") == 0) {
+                    continue;
                 } else {
-                        ret = dict_set_int32 (dict, "force", 1);
-                        if (ret)
-                                goto out;
+                    if (count == wordcount - 1) {
+                        if (strcmp(words[count], "detail") == 0) {
+                            cmd = GF_CLI_GET_STATE_DETAIL;
+                            continue;
+                        } else if (strcmp(words[count], "volumeoptions") == 0) {
+                            cmd = GF_CLI_GET_STATE_VOLOPTS;
+                            continue;
+                        }
+                    }
+
+                    *op_errstr = gf_strdup(
+                        "glusterd is "
+                        "the only supported daemon.");
+                    ret = -1;
+                    goto out;
                 }
+            }
         }
 
-        *options = dict;
+        ret = dict_set_dynstr(dict, "daemon", daemon_name);
+        if (ret) {
+            *op_errstr = gf_strdup(
+                "Command failed. Please check "
+                " log file for more details.");
+            gf_log(THIS->name, GF_LOG_ERROR,
+                   "Setting daemon name to dictionary failed");
+            goto out;
+        }
+        daemon_name = NULL;
+
+        if (odir) {
+            ret = dict_set_str(dict, "odir", odir);
+            if (ret) {
+                *op_errstr = gf_strdup(
+                    "Command failed. Please"
+                    " check log file for"
+                    " more details.");
+                gf_log(THIS->name, GF_LOG_ERROR,
+                       "Setting output directory to"
+                       "dictionary failed");
+                goto out;
+            }
+        }
 
-out:
-        if (ret && dict) {
-                dict_destroy (dict);
+        if (filename) {
+            ret = dict_set_str(dict, "filename", filename);
+            if (ret) {
+                *op_errstr = gf_strdup(
+                    "Command failed. Please"
+                    " check log file for"
+                    " more  details.");
+                gf_log(THIS->name, GF_LOG_ERROR,
+                       "Setting filename to dictionary failed");
+                goto out;
+            }
         }
 
-                return ret;
+        if (cmd) {
+            ret = dict_set_uint32(dict, "getstate-cmd", cmd);
+            if (ret) {
+                *op_errstr = gf_strdup(
+                    "Command failed. Please"
+                    " check log file for"
+                    " more details.");
+                gf_log(THIS->name, GF_LOG_ERROR,
+                       "Setting "
+                       "get-state command type to dictionary "
+                       "failed");
+                goto out;
+            }
+        }
+    }
+
+out:
+    if (dict)
+        *options = dict;
+
+    if (ret && dict)
+        dict_unref(dict);
+
+    GF_FREE(daemon_name);
+
+    return ret;
 }
 
 int32_t
-cli_cmd_quota_parse (const char **words, int wordcount, dict_t **options)
+cli_cmd_inode_quota_parse(const char **words, int wordcount, dict_t **options)
 {
-        dict_t          *dict    = NULL;
-        char            *volname = NULL;
-        int              ret     = -1;
-        int              i       = 0;
-        char             key[20] = {0, };
-        uint64_t         value   = 0;
-        gf_quota_type    type    = GF_QUOTA_OPTION_TYPE_NONE;
-        char           *opwords[] = { "enable", "disable", "limit-usage",
-                                      "remove", "list", "alert-time",
-                                      "soft-timeout", "hard-timeout",
-                                      "default-soft-limit", NULL};
-        char            *w       = NULL;
-        uint32_t         time    = 0;
-        double           percent = 0;
+    dict_t *dict = NULL;
+    char *volname = NULL;
+    int ret = -1;
 
-        GF_ASSERT (words);
-        GF_ASSERT (options);
+    GF_ASSERT(words);
+    GF_ASSERT(options);
 
-        dict = dict_new ();
-        if (!dict)
-                goto out;
+    dict = dict_new();
+    if (!dict) {
+        gf_log("cli", GF_LOG_ERROR, "dict_new failed");
+        goto out;
+    }
 
-        if (wordcount < 4)
-                goto out;
+    if (wordcount != 4)
+        goto out;
 
-        volname = (char *)words[2];
-        if (!volname) {
-                ret = -1;
-                goto out;
-        }
+    volname = (char *)words[2];
+    if (!volname) {
+        ret = -1;
+        goto out;
+    }
 
-        /* Validate the volume name here itself */
-        {
-                if (volname[0] == '-')
-                        goto out;
+    /* Validate the volume name here itself */
+    if (cli_validate_volname(volname) < 0)
+        goto out;
 
-                if (!strcmp (volname, "all")) {
-                        cli_err ("\"all\" cannot be the name of a volume.");
-                        goto out;
-                }
+    ret = dict_set_str(dict, "volname", volname);
+    if (ret < 0)
+        goto out;
 
-                if (strchr (volname, '/'))
-                        goto out;
+    if (strcmp(words[3], "enable") != 0) {
+        cli_out("Invalid quota option : %s", words[3]);
+        ret = -1;
+        goto out;
+    }
 
-                if (strlen (volname) > 512)
-                        goto out;
+    ret = dict_set_int32(dict, "type", GF_QUOTA_OPTION_TYPE_ENABLE_OBJECTS);
+    if (ret < 0)
+        goto out;
+
+    *options = dict;
+out:
+    if (ret < 0) {
+        if (dict)
+            dict_unref(dict);
+    }
+
+    return ret;
+}
 
-                for (i = 0; i < strlen (volname); i++)
-                        if (!isalnum (volname[i]) && (volname[i] != '_') && (volname[i] != '-'))
-                                goto out;
+int32_t
+cli_cmd_quota_parse(const char **words, int wordcount, dict_t **options)
+{
+    dict_t *dict = NULL;
+    char *volname = NULL;
+    int ret = -1;
+    int i = -1;
+    char key[20] = {
+        0,
+    };
+    int64_t value = 0;
+    gf_quota_type type = GF_QUOTA_OPTION_TYPE_NONE;
+    static char *opwords[] = {"enable",
+                              "disable",
+                              "limit-usage",
+                              "remove",
+                              "list",
+                              "alert-time",
+                              "soft-timeout",
+                              "hard-timeout",
+                              "default-soft-limit",
+                              "limit-objects",
+                              "list-objects",
+                              "remove-objects",
+                              NULL};
+    char *w = NULL;
+    uint32_t time = 0;
+    double percent = 0;
+    char *end_ptr = NULL;
+    int64_t limit = 0;
+
+    GF_ASSERT(words);
+    GF_ASSERT(options);
+
+    dict = dict_new();
+    if (!dict) {
+        gf_log("cli", GF_LOG_ERROR, "dict_new failed");
+        goto out;
+    }
+
+    if (wordcount < 4) {
+        if ((wordcount == 3) && !(strcmp(words[2], "help"))) {
+            ret = 1;
         }
+        goto out;
+    }
 
-        ret = dict_set_str (dict, "volname", volname);
-        if (ret < 0)
-                goto out;
+    volname = (char *)words[2];
+    if (!volname) {
+        ret = -1;
+        goto out;
+    }
 
-        w = str_getunamb (words[3], opwords);
-        if (!w) {
-                cli_out ("Invalid quota option : %s", words[3]);
-                ret = - 1;
-                goto out;
+    /* Validate the volume name here itself */
+    if (cli_validate_volname(volname) < 0)
+        goto out;
+
+    ret = dict_set_str(dict, "volname", volname);
+    if (ret < 0)
+        goto out;
+
+    w = str_getunamb(words[3], opwords);
+    if (!w) {
+        cli_out("Invalid quota option : %s", words[3]);
+        ret = -1;
+        goto out;
+    }
+
+    if (strcmp(w, "enable") == 0) {
+        if (wordcount == 4) {
+            type = GF_QUOTA_OPTION_TYPE_ENABLE;
+            ret = 0;
+            goto set_type;
+        } else {
+            ret = -1;
+            goto out;
         }
+    }
 
-        if (strcmp (w, "enable") == 0) {
-                if (wordcount == 4) {
-                        type = GF_QUOTA_OPTION_TYPE_ENABLE;
-                        ret = 0;
-                        goto set_type;
-                } else {
-                        ret = -1;
-                        goto out;
-                }
+    if (strcmp(w, "disable") == 0) {
+        if (wordcount == 4) {
+            type = GF_QUOTA_OPTION_TYPE_DISABLE;
+            ret = 0;
+            goto set_type;
+        } else {
+            ret = -1;
+            goto out;
+        }
+    }
+
+    if (strcmp(w, "limit-usage") == 0) {
+        type = GF_QUOTA_OPTION_TYPE_LIMIT_USAGE;
+    } else if (strcmp(w, "limit-objects") == 0) {
+        type = GF_QUOTA_OPTION_TYPE_LIMIT_OBJECTS;
+    }
+
+    if (type == GF_QUOTA_OPTION_TYPE_LIMIT_USAGE ||
+        type == GF_QUOTA_OPTION_TYPE_LIMIT_OBJECTS) {
+        if (wordcount < 6 || wordcount > 7) {
+            ret = -1;
+            goto out;
         }
 
-        if (strcmp (w, "disable") == 0) {
-                if (wordcount == 4) {
-                        type = GF_QUOTA_OPTION_TYPE_DISABLE;
-                        ret = 0;
-                        goto set_type;
-                } else {
-                        ret = -1;
-                        goto out;
-                }
+        if (words[4][0] != '/') {
+            cli_err("Please enter absolute path");
+            ret = -1;
+            goto out;
         }
+        ret = dict_set_str(dict, "path", (char *)words[4]);
+        if (ret)
+            goto out;
 
-        if (strcmp (w, "limit-usage") == 0) {
+        if (!words[5]) {
+            cli_err("Please enter the limit value to be set");
+            ret = -1;
+            goto out;
+        }
 
-                if (wordcount < 6 || wordcount > 7) {
-                        ret = -1;
-                        goto out;
-                }
+        if (type == GF_QUOTA_OPTION_TYPE_LIMIT_USAGE) {
+            ret = gf_string2bytesize_int64(words[5], &value);
+            if (ret != 0 || value <= 0) {
+                if (errno == ERANGE || value <= 0) {
+                    ret = -1;
+                    cli_err(
+                        "Please enter an integer "
+                        "value in the range of "
+                        "(1 - %" PRId64 ")",
+                        INT64_MAX);
+                } else
+                    cli_err(
+                        "Please enter a correct "
+                        "value");
+                goto out;
+            }
+        } else {
+            errno = 0;
+            limit = strtol(words[5], &end_ptr, 10);
+            if (errno == ERANGE || errno == EINVAL || limit <= 0 ||
+                strcmp(end_ptr, "") != 0) {
+                ret = -1;
+                cli_err(
+                    "Please enter an integer value in "
+                    "the range 1 - %" PRId64,
+                    INT64_MAX);
+                goto out;
+            }
+        }
 
-                type = GF_QUOTA_OPTION_TYPE_LIMIT_USAGE;
+        ret = dict_set_str(dict, "hard-limit", (char *)words[5]);
+        if (ret < 0)
+            goto out;
 
-                if (words[4][0] != '/') {
-                        cli_err ("Please enter absolute path");
-                        ret = -1;
-                        goto out;
-                }
-                ret = dict_set_str (dict, "path", (char *) words[4]);
-                if (ret)
-                        goto out;
+        if (wordcount == 7) {
+            ret = gf_string2percent(words[6], &percent);
+            if (ret != 0 || percent > 100) {
+                ret = -1;
+                cli_err(
+                    "Please enter a correct value "
+                    "in the range of 0 to 100");
+                goto out;
+            }
 
-                if (!words[5]) {
-                        cli_err ("Please enter the limit value to be set");
-                        ret = -1;
-                        goto out;
-                }
+            ret = dict_set_str(dict, "soft-limit", (char *)words[6]);
+            if (ret < 0)
+                goto out;
+        }
 
-                ret = gf_string2bytesize (words[5], &value);
-                if (ret != 0) {
-                        if (errno == ERANGE)
-                                cli_err ("Value too large: %s", words[5]);
-                        else
-                                cli_err ("Please enter a correct value");
-                        goto out;
-                }
+        goto set_type;
+    }
 
-                ret  = dict_set_str (dict, "hard-limit", (char *) words[5]);
-                if (ret < 0)
-                        goto out;
+    if (strcmp(w, "remove") == 0) {
+        if (wordcount != 5) {
+            ret = -1;
+            goto out;
+        }
 
-                if (wordcount == 7) {
+        type = GF_QUOTA_OPTION_TYPE_REMOVE;
 
-                        ret = gf_string2percent (words[6], &percent);
-                        if (ret != 0) {
-                                cli_err ("Please enter a correct value");
-                                goto out;
-                        }
+        if (words[4][0] != '/') {
+            cli_err("Please enter absolute path");
+            ret = -1;
+            goto out;
+        }
 
-                        ret = dict_set_str (dict, "soft-limit",
-                                            (char *) words[6]);
-                        if (ret < 0)
-                                goto out;
-                }
+        ret = dict_set_str(dict, "path", (char *)words[4]);
+        if (ret < 0)
+            goto out;
+        goto set_type;
+    }
+
+    if (strcmp(w, "remove-objects") == 0) {
+        if (wordcount != 5) {
+            ret = -1;
+            goto out;
+        }
 
-                goto set_type;
+        type = GF_QUOTA_OPTION_TYPE_REMOVE_OBJECTS;
+
+        if (words[4][0] != '/') {
+            cli_err("Please enter absolute path");
+            ret = -1;
+            goto out;
         }
-        if (strcmp (w, "remove") == 0) {
-                if (wordcount != 5) {
-                        ret = -1;
-                        goto out;
-                }
 
-                type = GF_QUOTA_OPTION_TYPE_REMOVE;
+        ret = dict_set_str(dict, "path", (char *)words[4]);
+        if (ret < 0)
+            goto out;
+        goto set_type;
+    }
 
-                if (words[4][0] != '/') {
-                        cli_err ("Please enter absolute path");
-                        ret = -1;
-                        goto out;
-                }
+    if (strcmp(w, "list") == 0) {
+        type = GF_QUOTA_OPTION_TYPE_LIST;
 
-                ret = dict_set_str (dict, "path", (char *) words[4]);
-                if (ret < 0)
-                        goto out;
-                goto set_type;
+        if (words[4] && words[4][0] != '/') {
+            cli_err("Please enter absolute path");
+            ret = -1;
+            goto out;
         }
 
-        if (strcmp (w, "list") == 0) {
-                if (wordcount < 4) {
-                        ret = -1;
-                        goto out;
-                }
+        i = 4;
+        while (i < wordcount) {
+            snprintf(key, 20, "path%d", i - 4);
 
-                type = GF_QUOTA_OPTION_TYPE_LIST;
+            ret = dict_set_str(dict, key, (char *)words[i++]);
+            if (ret < 0)
+                goto out;
+        }
 
-                i = 4;
-                while (i < wordcount) {
-                        snprintf (key, 20, "path%d", i-4);
+        ret = dict_set_int32(dict, "count", i - 4);
+        if (ret < 0)
+            goto out;
 
-                        ret = dict_set_str (dict, key, (char *) words [i++]);
-                        if (ret < 0)
-                                goto out;
-                }
+        goto set_type;
+    }
 
-                ret = dict_set_int32 (dict, "count", i - 4);
-                if (ret < 0)
-                        goto out;
+    if (strcmp(w, "list-objects") == 0) {
+        type = GF_QUOTA_OPTION_TYPE_LIST_OBJECTS;
 
-                goto set_type;
+        i = 4;
+        while (i < wordcount) {
+            snprintf(key, 20, "path%d", i - 4);
+
+            ret = dict_set_str(dict, key, (char *)words[i++]);
+            if (ret < 0) {
+                gf_log("cli", GF_LOG_ERROR,
+                       "Failed to set "
+                       "quota patch in request dictionary");
+                goto out;
+            }
         }
 
+        ret = dict_set_int32(dict, "count", i - 4);
+        if (ret < 0) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Failed to set quota "
+                   "limit count in request dictionary");
+            goto out;
+        }
 
-        if (strcmp (w, "alert-time") == 0) {
-                if (wordcount != 5) {
-                        ret = -1;
-                        goto out;
-                }
-                type = GF_QUOTA_OPTION_TYPE_ALERT_TIME;
+        goto set_type;
+    }
 
-                ret = gf_string2time (words[4], &time);
-                if (ret) {
-                        cli_err ("Invalid argument %s. Please enter a valid "
-                                 "string", words[4]);
-                        goto out;
-                }
+    if (strcmp(w, "alert-time") == 0) {
+        if (wordcount != 5) {
+            ret = -1;
+            goto out;
+        }
+        type = GF_QUOTA_OPTION_TYPE_ALERT_TIME;
 
-                ret = dict_set_str (dict, "value", (char *)words[4]);
-                if (ret < 0)
-                        goto out;
-                goto set_type;
+        ret = gf_string2time(words[4], &time);
+        if (ret) {
+            cli_err(
+                "Invalid argument %s. Please enter a valid "
+                "string",
+                words[4]);
+            goto out;
         }
-        if (strcmp (w, "soft-timeout") == 0) {
-                if (wordcount != 5) {
-                        ret = -1;
-                        goto out;
-                }
-                type = GF_QUOTA_OPTION_TYPE_SOFT_TIMEOUT;
 
-                ret = gf_string2time (words[4], &time);
-                if (ret) {
-                        cli_err ("Invalid argument %s. Please enter a valid "
-                                 "string", words[4]);
-                        goto out;
-                }
+        ret = dict_set_str(dict, "value", (char *)words[4]);
+        if (ret < 0)
+            goto out;
+        goto set_type;
+    }
+
+    if (strcmp(w, "soft-timeout") == 0) {
+        if (wordcount != 5) {
+            ret = -1;
+            goto out;
+        }
+        type = GF_QUOTA_OPTION_TYPE_SOFT_TIMEOUT;
 
-                ret = dict_set_str (dict, "value", (char *)words[4]);
-                if (ret < 0)
-                        goto out;
-                goto set_type;
+        ret = gf_string2time(words[4], &time);
+        if (ret) {
+            cli_err(
+                "Invalid argument %s. Please enter a valid "
+                "string",
+                words[4]);
+            goto out;
         }
-        if (strcmp (w, "hard-timeout") == 0) {
-                if(wordcount != 5) {
-                        ret = -1;
-                        goto out;
-                }
-                type = GF_QUOTA_OPTION_TYPE_HARD_TIMEOUT;
 
-                ret = gf_string2time (words[4], &time);
-                if (ret) {
-                        cli_err ("Invalid argument %s. Please enter a valid "
-                                 "string", words[4]);
-                        goto out;
-                }
+        ret = dict_set_str(dict, "value", (char *)words[4]);
+        if (ret < 0)
+            goto out;
+        goto set_type;
+    }
+
+    if (strcmp(w, "hard-timeout") == 0) {
+        if (wordcount != 5) {
+            ret = -1;
+            goto out;
+        }
+        type = GF_QUOTA_OPTION_TYPE_HARD_TIMEOUT;
 
-                ret = dict_set_str (dict, "value", (char *)words[4]);
-                if (ret < 0)
-                        goto out;
-                goto set_type;
+        ret = gf_string2time(words[4], &time);
+        if (ret) {
+            cli_err(
+                "Invalid argument %s. Please enter a valid "
+                "string",
+                words[4]);
+            goto out;
         }
-        if (strcmp (w, "default-soft-limit") == 0) {
-                if(wordcount != 5) {
-                        ret = -1;
-                        goto out;
-                }
-                type = GF_QUOTA_OPTION_TYPE_DEFAULT_SOFT_LIMIT;
 
-                ret = dict_set_str (dict, "value", (char *)words[4]);
-                if (ret < 0)
-                        goto out;
-                goto set_type;
-        } else {
-                GF_ASSERT (!"opword mismatch");
+        ret = dict_set_str(dict, "value", (char *)words[4]);
+        if (ret < 0)
+            goto out;
+        goto set_type;
+    }
+    if (strcmp(w, "default-soft-limit") == 0) {
+        if (wordcount != 5) {
+            ret = -1;
+            goto out;
         }
+        type = GF_QUOTA_OPTION_TYPE_DEFAULT_SOFT_LIMIT;
 
-set_type:
-        ret = dict_set_int32 (dict, "type", type);
+        ret = dict_set_str(dict, "value", (char *)words[4]);
         if (ret < 0)
-                goto out;
+            goto out;
+        goto set_type;
+    } else {
+        GF_ASSERT(!"opword mismatch");
+    }
 
-        *options = dict;
+set_type:
+    ret = dict_set_int32(dict, "type", type);
+    if (ret < 0)
+        goto out;
+
+    *options = dict;
 out:
-        if (ret < 0) {
-                if (dict)
-                        dict_destroy (dict);
-        }
+    if (ret < 0) {
+        if (dict)
+            dict_unref(dict);
+    }
 
-        return ret;
+    return ret;
 }
 
-static inline gf_boolean_t
-cli_is_key_spl (char *key)
+static gf_boolean_t
+cli_is_key_spl(char *key)
 {
-        return (strcmp (key, "group") == 0);
+    return (strcmp(key, "group") == 0);
 }
 
-#define GLUSTERD_DEFAULT_WORKDIR "/var/lib/glusterd"
-static int
-cli_add_key_group (dict_t *dict, char *key, char *value, char **op_errstr)
+static int32_t
+cli_add_key_group_value(dict_t *dict, const char *name, const char *value,
+                        int32_t id, char **op_errstr)
 {
-        int             ret = -1;
-        int             opt_count = 0;
-        char            iter_key[1024] = {0,};
-        char            iter_val[1024] = {0,};
-        char            *saveptr = NULL;
-        char            *tok_key = NULL;
-        char            *tok_val = NULL;
-        char            *dkey = NULL;
-        char            *dval = NULL;
-        char            *tagpath = NULL;
-        char            *buf = NULL;
-        char            line[PATH_MAX + 256] = {0,};
-        char            errstr[2048] = "";
-        FILE            *fp = NULL;
-
-        ret = gf_asprintf (&tagpath, "%s/groups/%s",
-                           GLUSTERD_DEFAULT_WORKDIR, value);
-        if (ret == -1) {
-                tagpath = NULL;
-                goto out;
-        }
-
-        fp = fopen (tagpath, "r");
-        if (!fp) {
-                ret = -1;
-                snprintf(errstr, sizeof(errstr), "Unable to open file '%s'."
-                         " Error: %s", tagpath, strerror (errno));
-                if (op_errstr)
-                        *op_errstr = gf_strdup(errstr);
-                goto out;
-        }
+    char *key = NULL;
+    char *data = NULL;
+    int32_t ret = -1;
 
-        opt_count = 0;
-        buf = line;
-        while (fscanf (fp, "%s", buf) != EOF) {
+    ret = gf_asprintf(&key, "%s%d", name, id);
+    if (ret < 0) {
+        goto out;
+    }
+    data = gf_strdup(value);
+    if (data == NULL) {
+        gf_log(THIS->name, GF_LOG_ERROR, "Failed to allocate memory for data");
+        ret = -1;
+        goto out;
+    }
 
-                opt_count++;
-                tok_key = strtok_r (line, "=", &saveptr);
-                tok_val = strtok_r (NULL, "=", &saveptr);
-                if (!tok_key || !tok_val) {
-                        ret = -1;
-                        snprintf(errstr, sizeof(errstr), "'%s' file format "
-                                 "not valid.", tagpath);
-                        if (op_errstr)
-                                *op_errstr = gf_strdup(errstr);
-                        goto out;
-                }
+    ret = dict_set_dynstr(dict, key, data);
+    if (ret == 0) {
+        data = NULL;
+    }
 
-                snprintf (iter_key, sizeof (iter_key), "key%d", opt_count);
-                dkey = gf_strdup (tok_key);
-                ret = dict_set_dynstr (dict, iter_key, dkey);
-                if (ret)
-                        goto out;
-                dkey = NULL;
+out:
+    GF_FREE(key);
+    GF_FREE(data);
 
-                snprintf (iter_val, sizeof (iter_val), "value%d", opt_count);
-                dval = gf_strdup (tok_val);
-                ret = dict_set_dynstr (dict, iter_val, dval);
-                if (ret)
-                        goto out;
-                dval = NULL;
+    if ((ret != 0) && (op_errstr != NULL)) {
+        *op_errstr = gf_strdup("Failed to allocate memory");
+    }
+
+    return ret;
+}
 
+static int
+cli_add_key_group(dict_t *dict, char *key, char *value, char **op_errstr)
+{
+    int ret = -1;
+    int opt_count = 0;
+    char *saveptr = NULL;
+    char *tok_key = NULL;
+    char *tok_val = NULL;
+    char *tagpath = NULL;
+    char line[PATH_MAX + 256] = {
+        0,
+    };
+    FILE *fp = NULL;
+
+    ret = gf_asprintf(&tagpath, "%s/groups/%s", GLUSTERD_DEFAULT_WORKDIR,
+                      value);
+    if (ret == -1) {
+        tagpath = NULL;
+        goto out;
+    }
+
+    fp = fopen(tagpath, "r");
+    if (!fp) {
+        ret = -1;
+        if (op_errstr) {
+            gf_asprintf(op_errstr,
+                        "Unable to open file '%s'. "
+                        "Error: %s",
+                        tagpath, strerror(errno));
+        }
+        goto out;
+    }
+
+    opt_count = 0;
+    while (fgets(line, sizeof(line), fp) != NULL) {
+        if (strlen(line) >= sizeof(line) - 1) {
+            ret = -1;
+            if (op_errstr != NULL) {
+                *op_errstr = gf_strdup("Line too long");
+            }
+            goto out;
         }
 
-        if (!opt_count) {
-                ret = -1;
-                snprintf(errstr, sizeof(errstr), "'%s' file format "
-                         "not valid.", tagpath);
-                if (op_errstr)
-                        *op_errstr = gf_strdup(errstr);
-                goto out;
+        /* Treat line that start with "#" as comments */
+        if ('#' == line[0])
+            continue;
+
+        opt_count++;
+        tok_key = strtok_r(line, "=", &saveptr);
+        tok_val = strtok_r(NULL, "\r\n", &saveptr);
+        if (!tok_key || !tok_val) {
+            ret = -1;
+            if (op_errstr) {
+                gf_asprintf(op_errstr,
+                            "'%s' file format "
+                            "not valid.",
+                            tagpath);
+            }
+            goto out;
         }
-        ret = dict_set_int32 (dict, "count", opt_count);
-out:
 
-        GF_FREE (tagpath);
+        ret = cli_add_key_group_value(dict, "key", tok_key, opt_count,
+                                      op_errstr);
+        if (ret != 0) {
+            goto out;
+        }
+        ret = cli_add_key_group_value(dict, "value", tok_val, opt_count,
+                                      op_errstr);
+        if (ret != 0) {
+            goto out;
+        }
+    }
 
-        if (ret) {
-                GF_FREE (dkey);
-                GF_FREE (dval);
+    if (!opt_count) {
+        ret = -1;
+        if (op_errstr) {
+            gf_asprintf(op_errstr, "'%s' file format not valid.", tagpath);
         }
+        goto out;
+    }
+    ret = dict_set_int32(dict, "count", opt_count);
+out:
 
-        if (fp)
-                fclose (fp);
+    GF_FREE(tagpath);
 
-        return ret;
+    if (fp)
+        fclose(fp);
+
+    return ret;
 }
-#undef GLUSTERD_DEFAULT_WORKDIR
 
 int32_t
-cli_cmd_volume_set_parse (const char **words, int wordcount, dict_t **options,
-                          char **op_errstr)
+cli_cmd_volume_set_parse(struct cli_state *state, const char **words,
+                         int wordcount, dict_t **options, char **op_errstr)
 {
-        dict_t                  *dict = NULL;
-        char                    *volname = NULL;
-        int                     ret = -1;
-        int                     count = 0;
-        char                    *key = NULL;
-        char                    *value = NULL;
-        int                     i = 0;
-        char                    str[50] = {0,};
+    dict_t *dict = NULL;
+    char *volname = NULL;
+    int ret = -1;
+    int count = 0;
+    char *key = NULL;
+    char *value = NULL;
+    int i = 0;
+    char str[50] = {
+        0,
+    };
+    const char *question = NULL;
+    gf_answer_t answer = GF_ANSWER_NO;
+
+    GF_ASSERT(words);
+    GF_ASSERT(options);
+
+    dict = dict_new();
+
+    if (!dict)
+        goto out;
 
-        GF_ASSERT (words);
-        GF_ASSERT (options);
+    if (wordcount < 3)
+        goto out;
 
-        dict = dict_new ();
+    volname = (char *)words[2];
 
-        if (!dict)
-                goto out;
+    GF_ASSERT(volname);
 
-        if (wordcount < 3)
-                goto out;
+    ret = dict_set_str(dict, "volname", volname);
 
-        volname = (char *)words[2];
+    if (ret)
+        goto out;
 
-        GF_ASSERT (volname);
+    if (!strcmp(volname, "all")) {
+        ret = dict_set_str(dict, "globalname", "All");
+        if (ret) {
+            gf_log(THIS->name, GF_LOG_ERROR, "dict set on global key failed.");
+            goto out;
+        }
 
-        ret = dict_set_str (dict, "volname", volname);
+        ret = dict_set_int32(dict, "hold_global_locks", _gf_true);
+        if (ret) {
+            gf_log(THIS->name, GF_LOG_ERROR, "dict set on global key failed.");
+            goto out;
+        }
+    }
 
+    if ((!strcmp(volname, "help") || !strcmp(volname, "help-xml")) &&
+        wordcount == 3) {
+        ret = dict_set_str(dict, volname, volname);
         if (ret)
-                goto out;
-
-        if ((!strcmp (volname, "help") || !strcmp (volname, "help-xml"))
-            && wordcount == 3 ) {
-                ret = dict_set_str (dict, volname, volname);
-                if (ret)
-                        goto out;
+            goto out;
 
-        } else if (wordcount < 5) {
-                ret = -1;
-                goto out;
-
-        } else if (wordcount == 5  && cli_is_key_spl ((char *)words[3])) {
-                key = (char *) words[3];
-                value = (char *) words[4];
-                if ( !key || !value) {
-                        ret = -1;
-                        goto out;
-                }
+    } else if (wordcount < 5) {
+        ret = -1;
+        goto out;
 
-                ret = gf_strip_whitespace (value, strlen (value));
-                if (ret == -1)
-                        goto out;
+    } else if (wordcount == 5 && cli_is_key_spl((char *)words[3])) {
+        key = (char *)words[3];
+        value = (char *)words[4];
+        if (!key || !value) {
+            ret = -1;
+            goto out;
+        }
 
-                if (strlen (value) == 0) {
-                        ret = -1;
-                        goto out;
-                }
+        ret = gf_strip_whitespace(value, strlen(value));
+        if (ret == -1)
+            goto out;
 
-                ret = cli_add_key_group (dict, key, value, op_errstr);
-                if (ret == 0)
-                        *options = dict;
-                goto out;
+        if (strlen(value) == 0) {
+            ret = -1;
+            goto out;
         }
 
-        for (i = 3; i < wordcount; i+=2) {
+        ret = cli_add_key_group(dict, key, value, op_errstr);
+        if (ret == 0)
+            *options = dict;
+        goto out;
+    }
 
-                key = (char *) words[i];
-                value = (char *) words[i+1];
+    for (i = 3; i < wordcount; i += 2) {
+        key = (char *)words[i];
+        value = (char *)words[i + 1];
 
-                if ( !key || !value) {
-                        ret = -1;
-                        goto out;
-                }
+        if (!key || !value) {
+            ret = -1;
+            goto out;
+        }
 
-                count++;
+        count++;
 
-                ret = gf_strip_whitespace (value, strlen (value));
-                if (ret == -1)
-                        goto out;
+        if (fnmatch("user.*", key, FNM_NOESCAPE) != 0) {
+            ret = gf_strip_whitespace(value, strlen(value));
+            if (ret == -1)
+                goto out;
+        }
 
-                if (strlen (value) == 0) {
-                        ret = -1;
-                        goto out;
-                }
+        if (strlen(value) == 0) {
+            ret = -1;
+            goto out;
+        }
 
-                if (cli_is_key_spl (key)) {
-                        ret = -1;
-                        goto out;
-                }
+        if (cli_is_key_spl(key)) {
+            ret = -1;
+            goto out;
+        }
 
-                sprintf (str, "key%d", count);
-                ret = dict_set_str (dict, str, key);
-                if (ret)
-                        goto out;
+        sprintf(str, "key%d", count);
+        ret = dict_set_str(dict, str, key);
+        if (ret)
+            goto out;
 
-                sprintf (str, "value%d", count);
-                ret = dict_set_str (dict, str, value);
+        sprintf(str, "value%d", count);
+        ret = dict_set_str(dict, str, value);
 
-                if (ret)
-                        goto out;
+        if (ret)
+            goto out;
+
+        if ((!strcmp(key, "cluster.enable-shared-storage")) &&
+            (!strcmp(value, "disable"))) {
+            question =
+                "Disabling cluster.enable-shared-storage "
+                "will delete the shared storage volume"
+                "(gluster_shared_storage), which is used "
+                "by snapshot scheduler, geo-replication "
+                "and NFS-Ganesha. Do you still want to "
+                "continue?";
+            answer = cli_cmd_get_confirmation(state, question);
+            if (GF_ANSWER_NO == answer) {
+                gf_log("cli", GF_LOG_ERROR,
+                       "Operation "
+                       "cancelled, exiting");
+                *op_errstr = gf_strdup("Aborted by user.");
+                ret = -1;
+                goto out;
+            }
+        }
+        if ((!strcmp(key, "nfs.disable")) && (!strcmp(value, "off"))) {
+            question =
+                "Gluster NFS is being deprecated in favor "
+                "of NFS-Ganesha Enter \"yes\" to continue "
+                "using Gluster NFS";
+            answer = cli_cmd_get_confirmation(state, question);
+            if (GF_ANSWER_NO == answer) {
+                gf_log("cli", GF_LOG_ERROR,
+                       "Operation "
+                       "cancelled, exiting");
+                *op_errstr = gf_strdup("Aborted by user.");
+                ret = -1;
+                goto out;
+            }
         }
+    }
 
-        ret = dict_set_int32 (dict, "count", wordcount-3);
+    ret = dict_set_int32(dict, "count", wordcount - 3);
 
-        if (ret)
-                goto out;
+    if (ret)
+        goto out;
 
-        *options = dict;
+    *options = dict;
 
 out:
-        if (ret)
-                dict_destroy (dict);
+    if (ret && dict)
+        dict_unref(dict);
 
-        return ret;
+    return ret;
 }
 
 int32_t
-cli_cmd_volume_add_brick_parse (const char **words, int wordcount,
-                                dict_t **options)
+cli_cmd_volume_add_brick_parse(struct cli_state *state, const char **words,
+                               int wordcount, dict_t **options, int *ret_type)
 {
-        dict_t  *dict = NULL;
-        char    *volname = NULL;
-        int     ret = -1;
-        int     brick_count = 0, brick_index = 0;
-        char    *bricks = NULL;
-        char    *opwords_cl[] = { "replica", "stripe", NULL };
-        gf1_cluster_type type = GF_CLUSTER_TYPE_NONE;
-        int     count = 1;
-        char    *w = NULL;
-        int     index;
-        gf_boolean_t is_force = _gf_false;
-        int wc = wordcount;
+    dict_t *dict = NULL;
+    char *volname = NULL;
+    int ret = -1;
+    int brick_count = 0, brick_index = 0;
+    char *bricks = NULL;
+    static char *opwords_cl[] = {"replica", "stripe", NULL};
+    gf1_cluster_type type = GF_CLUSTER_TYPE_NONE;
+    int count = 1;
+    int arbiter_count = 0;
+    char *w = NULL;
+    int index;
+    gf_boolean_t is_force = _gf_false;
+    int wc = wordcount;
+    gf_answer_t answer = GF_ANSWER_NO;
+    const char *question = NULL;
+
+    GF_ASSERT(words);
+    GF_ASSERT(options);
+
+    dict = dict_new();
+
+    if (!dict)
+        goto out;
 
-        GF_ASSERT (words);
-        GF_ASSERT (options);
+    if (wordcount < 3)
+        goto out;
 
-        dict = dict_new ();
+    volname = (char *)words[2];
 
-        if (!dict)
-                goto out;
+    GF_ASSERT(volname);
 
-        if (wordcount < 3)
-                goto out;
-
-        volname = (char *)words[2];
+    ret = dict_set_str(dict, "volname", volname);
 
-        GF_ASSERT (volname);
+    if (ret)
+        goto out;
 
-        ret = dict_set_str (dict, "volname", volname);
+    if (wordcount < 4) {
+        ret = -1;
+        goto out;
+    }
+    if (wordcount < 6) {
+        /* seems no options are given, go directly to the parse_brick */
+        brick_index = 3;
+        type = GF_CLUSTER_TYPE_NONE;
+        goto parse_bricks;
+    }
 
+    w = str_getunamb(words[3], opwords_cl);
+    if (!w) {
+        type = GF_CLUSTER_TYPE_NONE;
+        index = 3;
+    } else if ((strcmp(w, "replica")) == 0) {
+        type = GF_CLUSTER_TYPE_REPLICATE;
+        count = strtol(words[4], NULL, 0);
+        if (!count || (count < 2)) {
+            cli_err("replica count should be greater than 1");
+            ret = -1;
+            goto out;
+        }
+        ret = dict_set_int32(dict, "replica-count", count);
         if (ret)
-                goto out;
-
-        if (wordcount < 4) {
+            goto out;
+        index = 5;
+        if (words[index] && !strcmp(words[index], "arbiter")) {
+            arbiter_count = strtol(words[6], NULL, 0);
+            if (arbiter_count != 1 || count != 3) {
+                cli_err(
+                    "For arbiter configuration, replica "
+                    "count must be 3 and arbiter count "
+                    "must be 1. The 3rd brick of the "
+                    "replica will be the arbiter");
                 ret = -1;
                 goto out;
-        }
-        if (wordcount < 6) {
-                /* seems no options are given, go directly to the parse_brick */
-                brick_index = 3;
-                type = GF_CLUSTER_TYPE_NONE;
-                goto parse_bricks;
+            }
+            ret = dict_set_int32(dict, "arbiter-count", arbiter_count);
+            if (ret)
+                goto out;
+            index = 7;
         }
 
-        w = str_getunamb (words[3], opwords_cl);
-        if (!w) {
-                type = GF_CLUSTER_TYPE_NONE;
-                index = 3;
-        } else if ((strcmp (w, "replica")) == 0) {
-                type = GF_CLUSTER_TYPE_REPLICATE;
-                if (wordcount < 5) {
-                        ret = -1;
-                        goto out;
-                }
-                count = strtol (words[4], NULL, 0);
-                if (!count || (count < 2)) {
-                        cli_err ("replica count should be greater than 1");
-                        ret = -1;
-                        goto out;
-                }
-                ret = dict_set_int32 (dict, "replica-count", count);
-                if (ret)
-                        goto out;
-                index = 5;
-        } else if ((strcmp (w, "stripe")) == 0) {
-                type = GF_CLUSTER_TYPE_STRIPE;
-                if (wordcount < 5) {
-                        ret = -1;
-                        goto out;
-                }
-                count = strtol (words[4], NULL, 0);
-                if (!count || (count < 2)) {
-                        cli_err ("stripe count should be greater than 1");
-                        ret = -1;
-                        goto out;
+        if (count == 2) {
+            if (strcmp(words[wordcount - 1], "force")) {
+                question =
+                    "Replica 2 volumes are prone to "
+                    "split-brain. Use Arbiter or "
+                    "Replica 3 to avoid this. See: "
+                    "http://docs.gluster.org/en/latest/Administrator%20Guide/"
+                    "Split%20brain%20and%20ways%20to%20deal%20with%20it/."
+                    "\nDo you still want to continue?\n";
+                answer = cli_cmd_get_confirmation(state, question);
+                if (GF_ANSWER_NO == answer) {
+                    gf_log("cli", GF_LOG_ERROR,
+                           "Add brick"
+                           " cancelled, exiting");
+                    ret = -1;
+                    goto out;
                 }
-                ret = dict_set_int32 (dict, "stripe-count", count);
-                if (ret)
-                        goto out;
-                index = 5;
-        } else {
-                GF_ASSERT (!"opword mismatch");
-                ret = -1;
-                goto out;
+            }
         }
+    } else if ((strcmp(w, "stripe")) == 0) {
+        cli_err("stripe option not supported");
+        goto out;
+    } else {
+        GF_ASSERT(!"opword mismatch");
+        ret = -1;
+        goto out;
+    }
 
-        brick_index = index;
+    brick_index = index;
 
 parse_bricks:
 
-        if (strcmp (words[wordcount - 1], "force") == 0) {
-                is_force = _gf_true;
-                wc = wordcount - 1;
-        }
+    if (strcmp(words[wordcount - 1], "force") == 0) {
+        is_force = _gf_true;
+        wc = wordcount - 1;
+    }
 
-        ret = cli_cmd_bricks_parse (words, wc, brick_index, &bricks,
-                                    &brick_count);
-        if (ret)
-                goto out;
+    ret = cli_cmd_bricks_parse(words, wc, brick_index, &bricks, &brick_count);
+    if (ret)
+        goto out;
 
-        ret = dict_set_dynstr (dict, "bricks", bricks);
-        if (ret)
-                goto out;
+    ret = dict_set_dynstr(dict, "bricks", bricks);
+    if (ret)
+        goto out;
 
-        ret = dict_set_int32 (dict, "count", brick_count);
+    ret = dict_set_int32(dict, "count", brick_count);
 
-        if (ret)
-                goto out;
+    if (ret)
+        goto out;
 
-        ret = dict_set_int32 (dict, "force", is_force);
-        if (ret)
-                goto out;
+    ret = dict_set_int32(dict, "force", is_force);
+    if (ret)
+        goto out;
 
-        *options = dict;
+    *options = dict;
 
 out:
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Unable to parse add-brick CLI");
-                if (dict)
-                        dict_destroy (dict);
-        }
+    if (ret_type)
+        *ret_type = type;
 
-        return ret;
-}
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Unable to parse add-brick CLI");
+        if (dict)
+            dict_unref(dict);
+    }
 
+    return ret;
+}
 
 int32_t
-cli_cmd_volume_remove_brick_parse (const char **words, int wordcount,
-                                   dict_t **options, int *question)
+cli_cmd_volume_remove_brick_parse(struct cli_state *state, const char **words,
+                                  int wordcount, dict_t **options,
+                                  int *question, int *brick_count,
+                                  int32_t *comm)
 {
-        dict_t  *dict = NULL;
-        char    *volname = NULL;
-        char    *delimiter = NULL;
-        int     ret = -1;
-        char    key[50];
-        int     brick_count = 0, brick_index = 0;
-        int32_t tmp_index = 0;
-        int32_t j = 0;
-        char    *tmp_brick = NULL;
-        char    *tmp_brick1 = NULL;
-        char    *type_opword[] = { "replica", NULL };
-        char    *opwords[] = { "start", "commit", "stop", "status",
-                               "force", NULL };
-        char    *w = NULL;
-        int32_t  command = GF_OP_CMD_NONE;
-        long     count = 0;
-
-        GF_ASSERT (words);
-        GF_ASSERT (options);
+    dict_t *dict = NULL;
+    char *volname = NULL;
+    char *delimiter = NULL;
+    int ret = -1;
+    char key[50];
+    int brick_index = 0;
+    int32_t tmp_index = 0;
+    int32_t j = 0;
+    char *tmp_brick = NULL;
+    char *tmp_brick1 = NULL;
+    static char *type_opword[] = {"replica", NULL};
+    static char *opwords[] = {"start",  "commit", "stop",
+                              "status", "force",  NULL};
+    char *w = NULL;
+    int32_t command = GF_OP_CMD_NONE;
+    long count = 0;
+    gf_answer_t answer = GF_ANSWER_NO;
+    const char *ques = NULL;
+
+    GF_ASSERT(words);
+    GF_ASSERT(options);
+
+    if (wordcount < 5)
+        goto out;
 
-        if (wordcount < 4)
-                goto out;
+    dict = dict_new();
+    if (!dict)
+        goto out;
 
-        dict = dict_new ();
-        if (!dict)
-                goto out;
+    volname = (char *)words[2];
 
-        volname = (char *)words[2];
+    GF_ASSERT(volname);
 
-        GF_ASSERT (volname);
+    ret = dict_set_str(dict, "volname", volname);
+    if (ret)
+        goto out;
 
-        ret = dict_set_str (dict, "volname", volname);
-        if (ret)
-                goto out;
+    brick_index = 3;
+    w = str_getunamb(words[3], type_opword);
+    if (w && !strcmp("replica", w)) {
+        if (wordcount < 6) {
+            ret = -1;
+            goto out;
+        }
+        count = strtol(words[4], NULL, 0);
+        if (count < 1) {
+            cli_err(
+                "replica count should be greater than 0 in "
+                "case of remove-brick");
+            ret = -1;
+            goto out;
+        }
 
-        brick_index = 3;
-        w = str_getunamb (words[3], type_opword);
-        if (w && !strcmp ("replica", w)) {
-                if (wordcount < 5) {
-                        ret = -1;
-                        goto out;
+        if (count == 2) {
+            if (strcmp(words[wordcount - 1], "force")) {
+                ques =
+                    "Replica 2 volumes are prone to "
+                    "split-brain. Use Arbiter or Replica 3 "
+                    "to avoid this. See: "
+                    "http://docs.gluster.org/en/latest/Administrator%20Guide/"
+                    "Split%20brain%20and%20ways%20to%20deal%20with%20it/."
+                    "\nDo you still want to continue?\n";
+                answer = cli_cmd_get_confirmation(state, ques);
+                if (GF_ANSWER_NO == answer) {
+                    gf_log("cli", GF_LOG_ERROR,
+                           "Remove "
+                           "brick cancelled, exiting");
+                    ret = -1;
+                    goto out;
                 }
-                count = strtol (words[4], NULL, 0);
-                if (count < 1) {
-                        cli_err ("replica count should be greater than 0 in "
-                                 "case of remove-brick");
-                        ret = -1;
-                        goto out;
-                }
-
-                ret = dict_set_int32 (dict, "replica-count", count);
-                if (ret)
-                        goto out;
-                brick_index = 5;
-        } else if (w) {
-                GF_ASSERT (!"opword mismatch");
+            }
         }
 
-        w = str_getunamb (words[wordcount - 1], opwords);
-        if (!w) {
-                /* Should be default 'force' */
-                command = GF_OP_CMD_COMMIT_FORCE;
-                cli_out("WARNING: running remove-brick commands without an "
-                        "explicit option is deprecated, and will be removed "
-                        "in the next version of GlusterFS.\n"
-                        "To forcibly remove a brick in the next version of "
-                        "GlusterFS, you will need to use \"remove-brick "
-                        "force\".");
-                if (question)
-                        *question = 1;
+        ret = dict_set_int32(dict, "replica-count", count);
+        if (ret)
+            goto out;
+        brick_index = 5;
+    } else if (w) {
+        GF_ASSERT(!"opword mismatch");
+    }
+
+    w = str_getunamb(words[wordcount - 1], opwords);
+    if (!w) {
+        ret = -1;
+        goto out;
+    } else {
+        /* handled this option */
+        wordcount--;
+        if (!strcmp("start", w)) {
+            command = GF_OP_CMD_START;
+            if (question)
+                *question = 1;
+        } else if (!strcmp("commit", w)) {
+            command = GF_OP_CMD_COMMIT;
+        } else if (!strcmp("stop", w)) {
+            command = GF_OP_CMD_STOP;
+        } else if (!strcmp("status", w)) {
+            command = GF_OP_CMD_STATUS;
+        } else if (!strcmp("force", w)) {
+            command = GF_OP_CMD_COMMIT_FORCE;
+            if (question)
+                *question = 1;
         } else {
-                /* handled this option */
-                wordcount--;
-                if (!strcmp ("start", w)) {
-                        command = GF_OP_CMD_START;
-                } else if (!strcmp ("commit", w)) {
-                        command = GF_OP_CMD_COMMIT;
-                        if (question)
-                                *question = 1;
-                } else if (!strcmp ("stop", w)) {
-                        command = GF_OP_CMD_STOP;
-                } else if (!strcmp ("status", w)) {
-                        command = GF_OP_CMD_STATUS;
-                } else if (!strcmp ("force", w)) {
-                        command = GF_OP_CMD_COMMIT_FORCE;
-                        if (question)
-                                *question = 1;
-                } else {
-                        GF_ASSERT (!"opword mismatch");
-                        ret = -1;
-                        goto out;
-                }
+            GF_ASSERT(!"opword mismatch");
+            ret = -1;
+            goto out;
         }
+    }
 
-        if (wordcount < 4) {
-                ret = -1;
-                goto out;
-        }
+    ret = dict_set_int32(dict, "command", command);
+    if (ret)
+        gf_log("cli", GF_LOG_INFO, "failed to set 'command' %d", command);
 
-        ret = dict_set_int32 (dict, "command", command);
-        if (ret)
-                gf_log ("cli", GF_LOG_INFO, "failed to set 'command' %d",
-                        command);
+    tmp_index = brick_index;
+    tmp_brick = GF_MALLOC(2048 * sizeof(*tmp_brick), gf_common_mt_char);
 
+    if (!tmp_brick) {
+        gf_log("", GF_LOG_ERROR,
+               "cli_cmd_volume_remove_brick_parse: "
+               "Unable to get memory");
+        ret = -1;
+        goto out;
+    }
 
-        tmp_index = brick_index;
-        tmp_brick = GF_MALLOC(2048 * sizeof(*tmp_brick), gf_common_mt_char);
+    tmp_brick1 = GF_MALLOC(2048 * sizeof(*tmp_brick1), gf_common_mt_char);
 
-        if (!tmp_brick) {
-                gf_log ("",GF_LOG_ERROR,"cli_cmd_volume_remove_brick_parse: "
-                        "Unable to get memory");
-                ret = -1;
+    if (!tmp_brick1) {
+        gf_log("", GF_LOG_ERROR,
+               "cli_cmd_volume_remove_brick_parse: "
+               "Unable to get memory");
+        ret = -1;
+        goto out;
+    }
+
+    while (brick_index < wordcount) {
+        if (validate_brick_name((char *)words[brick_index])) {
+            cli_err(
+                "wrong brick type: %s, use <HOSTNAME>:"
+                "<export-dir-abs-path>",
+                words[brick_index]);
+            ret = -1;
+            goto out;
+        } else {
+            delimiter = strrchr(words[brick_index], ':');
+            ret = gf_canonicalize_path(delimiter + 1);
+            if (ret)
                 goto out;
         }
 
-        tmp_brick1 = GF_MALLOC(2048 * sizeof(*tmp_brick1), gf_common_mt_char);
-
-        if (!tmp_brick1) {
-                gf_log ("",GF_LOG_ERROR,"cli_cmd_volume_remove_brick_parse: "
-                        "Unable to get memory");
+        j = tmp_index;
+        strcpy(tmp_brick, words[brick_index]);
+        while (j < brick_index) {
+            strcpy(tmp_brick1, words[j]);
+            if (!(strcmp(tmp_brick, tmp_brick1))) {
+                gf_log("", GF_LOG_ERROR,
+                       "Duplicate bricks"
+                       " found %s",
+                       words[brick_index]);
+                cli_err("Duplicate bricks found %s", words[brick_index]);
                 ret = -1;
                 goto out;
+            }
+            j++;
         }
+        snprintf(key, 50, "brick%d", ++(*brick_count));
+        ret = dict_set_str(dict, key, (char *)words[brick_index++]);
 
-        while (brick_index < wordcount) {
-                if (validate_brick_name ((char *)words[brick_index])) {
-                        cli_err ("wrong brick type: %s, use <HOSTNAME>:"
-                                 "<export-dir-abs-path>", words[brick_index]);
-                        ret = -1;
-                        goto out;
-                } else {
-                        delimiter = strrchr(words[brick_index], ':');
-                        ret = gf_canonicalize_path (delimiter + 1);
-                        if (ret)
-                                goto out;
-                }
-
-                j = tmp_index;
-                strcpy(tmp_brick, words[brick_index]);
-                while ( j < brick_index) {
-                        strcpy(tmp_brick1, words[j]);
-                        if (!(strcmp (tmp_brick, tmp_brick1))) {
-                                gf_log("",GF_LOG_ERROR, "Duplicate bricks"
-                                       " found %s", words[brick_index]);
-                                cli_err("Duplicate bricks found %s",
-                                        words[brick_index]);
-                                ret = -1;
-                                goto out;
-                        }
-                        j++;
-                }
-                snprintf (key, 50, "brick%d", ++brick_count);
-                ret = dict_set_str (dict, key, (char *)words[brick_index++]);
-
-                if (ret)
-                        goto out;
-        }
+        if (ret)
+            goto out;
+    }
 
-        ret = dict_set_int32 (dict, "count", brick_count);
+    if (command != GF_OP_CMD_STATUS && command != GF_OP_CMD_STOP) {
+        ret = dict_set_int32(dict, "count", *brick_count);
         if (ret)
-                goto out;
+            goto out;
+    }
 
-        *options = dict;
+    *options = dict;
 
 out:
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Unable to parse remove-brick CLI");
-                if (dict)
-                        dict_destroy (dict);
-        }
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Unable to parse remove-brick CLI");
+        if (dict)
+            dict_unref(dict);
+    }
 
-        GF_FREE (tmp_brick);
-        GF_FREE (tmp_brick1);
+    GF_FREE(tmp_brick);
+    GF_FREE(tmp_brick1);
 
-        return ret;
-}
+    *comm = command;
 
+    return ret;
+}
 
 int32_t
-cli_cmd_volume_replace_brick_parse (const char **words, int wordcount,
-                                   dict_t **options)
+cli_cmd_brick_op_validate_bricks(const char **words, dict_t *dict, int src,
+                                 int dst)
 {
-        dict_t  *dict = NULL;
-        char    *volname = NULL;
-        int     ret = -1;
-        int     op_index = 0;
-        char    *delimiter = NULL;
-        gf1_cli_replace_op replace_op = GF_REPLACE_OP_NONE;
-        char    *opwords[] = { "start", "commit", "pause", "abort", "status",
-                                NULL };
-        char    *w = NULL;
-        gf_boolean_t is_force = _gf_false;
+    int ret = -1;
+    char *delimiter = NULL;
+
+    if (validate_brick_name((char *)words[src])) {
+        cli_err(
+            "wrong brick type: %s, use "
+            "<HOSTNAME>:<export-dir-abs-path>",
+            words[3]);
+        ret = -1;
+        goto out;
+    } else {
+        delimiter = strrchr((char *)words[src], '/');
+        ret = gf_canonicalize_path(delimiter);
+        if (ret)
+            goto out;
+    }
 
-        GF_ASSERT (words);
-        GF_ASSERT (options);
+    ret = dict_set_str(dict, "src-brick", (char *)words[src]);
+    if (ret)
+        goto out;
 
-        dict = dict_new ();
+    if (dst == -1) {
+        ret = 0;
+        goto out;
+    }
 
-        if (!dict)
-                goto out;
+    if (validate_brick_name((char *)words[dst])) {
+        cli_err(
+            "wrong brick type: %s, use "
+            "<HOSTNAME>:<export-dir-abs-path>",
+            words[dst]);
+        ret = -1;
+        goto out;
+    } else {
+        delimiter = strrchr((char *)words[dst], '/');
+        ret = gf_canonicalize_path(delimiter);
+        if (ret)
+            goto out;
+    }
 
-        if (wordcount < 3)
-                goto out;
+    ret = dict_set_str(dict, "dst-brick", (char *)words[dst]);
+    if (ret)
+        goto out;
+    ret = 0;
+out:
+    return ret;
+}
 
-        volname = (char *)words[2];
+int32_t
+cli_cmd_volume_reset_brick_parse(const char **words, int wordcount,
+                                 dict_t **options)
+{
+    int ret = -1;
+    char *volname = NULL;
+    dict_t *dict = NULL;
 
-        GF_ASSERT (volname);
+    if (wordcount < 5 || wordcount > 7)
+        goto out;
 
-        ret = dict_set_str (dict, "volname", volname);
+    dict = dict_new();
 
-        if (ret)
-                goto out;
+    if (!dict)
+        goto out;
 
-        if (wordcount < 4) {
-                ret = -1;
-                goto out;
-        }
+    volname = (char *)words[2];
 
-        if (validate_brick_name ((char *)words[3])) {
-                cli_err ("wrong brick type: %s, use "
-                         "<HOSTNAME>:<export-dir-abs-path>", words[3]);
-                ret = -1;
-                goto out;
-        } else {
-                delimiter = strrchr ((char *)words[3], ':');
-                ret = gf_canonicalize_path (delimiter + 1);
-                if (ret)
-                        goto out;
+    ret = dict_set_str(dict, "volname", volname);
+    if (ret)
+        goto out;
+
+    if (wordcount == 5) {
+        if (strcmp(words[4], "start")) {
+            cli_err(
+                "Invalid option '%s' for reset-brick. Please "
+                "enter valid reset-brick command",
+                words[4]);
+            ret = -1;
+            goto out;
         }
-        ret = dict_set_str (dict, "src-brick", (char *)words[3]);
 
+        ret = cli_cmd_brick_op_validate_bricks(words, dict, 3, -1);
         if (ret)
-                goto out;
+            goto out;
 
-        if (wordcount < 5) {
-                ret = -1;
-                goto out;
+        ret = dict_set_str(dict, "operation", "GF_RESET_OP_START");
+        if (ret)
+            goto out;
+    } else if (wordcount == 6) {
+        if (strcmp(words[5], "commit")) {
+            cli_err(
+                "Invalid option '%s' for reset-brick. Please "
+                "enter valid reset-brick command",
+                words[5]);
+            ret = -1;
+            goto out;
         }
 
-        if (validate_brick_name ((char *)words[4])) {
-                cli_err ("wrong brick type: %s, use "
-                         "<HOSTNAME>:<export-dir-abs-path>", words[4]);
-                ret = -1;
-                goto out;
-        } else {
-                delimiter = strrchr ((char *)words[4], ':');
-                ret = gf_canonicalize_path (delimiter + 1);
-                if (ret)
-                        goto out;
-        }
+        ret = cli_cmd_brick_op_validate_bricks(words, dict, 3, 4);
+        if (ret)
+            goto out;
 
+        ret = dict_set_str(dict, "operation", "GF_RESET_OP_COMMIT");
+        if (ret)
+            goto out;
+    } else if (wordcount == 7) {
+        if (strcmp(words[5], "commit") || strcmp(words[6], "force")) {
+            cli_err(
+                "Invalid option '%s %s' for reset-brick. Please "
+                "enter valid reset-brick command",
+                words[5], words[6]);
+            ret = -1;
+            goto out;
+        }
 
-        ret = dict_set_str (dict, "dst-brick", (char *)words[4]);
+        ret = cli_cmd_brick_op_validate_bricks(words, dict, 3, 4);
+        if (ret)
+            goto out;
 
+        ret = dict_set_str(dict, "operation", "GF_RESET_OP_COMMIT_FORCE");
         if (ret)
-                goto out;
+            goto out;
+    }
 
-        op_index = 5;
-        if ((wordcount < (op_index + 1))) {
-                ret = -1;
-                goto out;
-        }
+    *options = dict;
 
-        w = str_getunamb (words[op_index], opwords);
+out:
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Unable to parse reset-brick CLI");
+        if (dict)
+            dict_unref(dict);
+    }
 
-        if (!w) {
-        } else if (!strcmp ("start", w)) {
-                replace_op = GF_REPLACE_OP_START;
-        } else if (!strcmp ("commit", w)) {
-                replace_op = GF_REPLACE_OP_COMMIT;
-        } else if (!strcmp ("pause", w)) {
-                replace_op = GF_REPLACE_OP_PAUSE;
-        } else if (!strcmp ("abort", w)) {
-                replace_op = GF_REPLACE_OP_ABORT;
-        } else if (!strcmp ("status", w)) {
-                replace_op = GF_REPLACE_OP_STATUS;
-        } else
-                GF_ASSERT (!"opword mismatch");
-
-        /* commit force option */
-
-        op_index = 6;
-
-        if (wordcount > (op_index + 1)) {
-                ret = -1;
-                goto out;
-        }
+    return ret;
+}
 
-        if (wordcount == (op_index + 1)) {
-                if ((replace_op != GF_REPLACE_OP_COMMIT) &&
-                    (replace_op != GF_REPLACE_OP_START)) {
-                        ret = -1;
-                        goto out;
-                }
-                if (!strcmp ("force", words[op_index])) {
-                        if (replace_op == GF_REPLACE_OP_COMMIT)
-                                replace_op = GF_REPLACE_OP_COMMIT_FORCE;
+int32_t
+cli_cmd_volume_replace_brick_parse(const char **words, int wordcount,
+                                   dict_t **options)
+{
+    int ret = -1;
+    char *volname = NULL;
+    dict_t *dict = NULL;
 
-                        else if (replace_op == GF_REPLACE_OP_START)
-                                is_force = _gf_true;
-                }
-        }
+    GF_ASSERT(words);
+    GF_ASSERT(options);
 
-        if (replace_op == GF_REPLACE_OP_NONE) {
-                ret = -1;
-                goto out;
-        }
+    if (wordcount != 7) {
+        ret = -1;
+        goto out;
+    }
 
-        ret = dict_set_int32 (dict, "operation", (int32_t) replace_op);
+    dict = dict_new();
 
-        if (ret)
-                goto out;
+    if (!dict) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to allocate dictionary");
+        goto out;
+    }
 
-        ret = dict_set_int32 (dict, "force", is_force);
-        if (ret)
-                goto out;
+    volname = (char *)words[2];
 
-        *options = dict;
+    GF_ASSERT(volname);
+
+    ret = dict_set_str(dict, "volname", volname);
+    if (ret)
+        goto out;
+
+    ret = cli_cmd_brick_op_validate_bricks(words, dict, 3, 4);
+    if (ret)
+        goto out;
+
+    /* commit force option */
+    if (strcmp("commit", words[5]) || strcmp("force", words[6])) {
+        cli_err(
+            "Invalid option '%s' '%s' for replace-brick. Please "
+            "enter valid replace-brick command",
+            words[5], words[6]);
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_set_str(dict, "operation", "GF_REPLACE_OP_COMMIT_FORCE");
+    if (ret)
+        goto out;
+
+    *options = dict;
 
 out:
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Unable to parse replace-brick CLI");
-                if (dict)
-                        dict_destroy (dict);
-        }
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Unable to parse reset-brick CLI");
+        if (dict)
+            dict_unref(dict);
+    }
 
-        return ret;
+    return ret;
 }
 
 int32_t
-cli_cmd_log_filename_parse (const char **words, int wordcount, dict_t **options)
+cli_cmd_log_filename_parse(const char **words, int wordcount, dict_t **options)
 {
-        dict_t  *dict = NULL;
-        char    *volname = NULL;
-        char    *str = NULL;
-        int     ret = -1;
-        char    *delimiter = NULL;
+    dict_t *dict = NULL;
+    char *volname = NULL;
+    char *str = NULL;
+    int ret = -1;
+    char *delimiter = NULL;
 
-        GF_ASSERT (words);
-        GF_ASSERT (options);
+    GF_ASSERT(words);
+    GF_ASSERT(options);
 
-        dict = dict_new ();
-        if (!dict)
-                goto out;
+    dict = dict_new();
+    if (!dict)
+        goto out;
 
-        volname = (char *)words[3];
-        GF_ASSERT (volname);
+    volname = (char *)words[3];
+    GF_ASSERT(volname);
 
-        ret = dict_set_str (dict, "volname", volname);
-        if (ret)
-                goto out;
+    ret = dict_set_str(dict, "volname", volname);
+    if (ret)
+        goto out;
 
-        str = (char *)words[4];
-        if (strchr (str, ':')) {
-                delimiter = strchr (words[4], ':');
-                if (!delimiter || delimiter == words[4]
-                    || *(delimiter+1) != '/') {
-                        cli_err ("wrong brick type: %s, use <HOSTNAME>:"
-                                 "<export-dir-abs-path>", words[4]);
-                        ret = -1;
-                        goto out;
-                } else {
-                        ret = gf_canonicalize_path (delimiter + 1);
-                        if (ret)
-                                goto out;
-                }
-                ret = dict_set_str (dict, "brick", str);
-                if (ret)
-                        goto out;
-                /* Path */
-                str = (char *)words[5];
-                ret = dict_set_str (dict, "path", str);
-                if (ret)
-                        goto out;
+    str = (char *)words[4];
+    if (strchr(str, ':')) {
+        delimiter = strchr(words[4], ':');
+        if (!delimiter || delimiter == words[4] || *(delimiter + 1) != '/') {
+            cli_err(
+                "wrong brick type: %s, use <HOSTNAME>:"
+                "<export-dir-abs-path>",
+                words[4]);
+            ret = -1;
+            goto out;
         } else {
-                ret = dict_set_str (dict, "path", str);
-                if (ret)
-                        goto out;
+            ret = gf_canonicalize_path(delimiter + 1);
+            if (ret)
+                goto out;
         }
+        ret = dict_set_str(dict, "brick", str);
+        if (ret)
+            goto out;
+        /* Path */
+        str = (char *)words[5];
+        ret = dict_set_str(dict, "path", str);
+        if (ret)
+            goto out;
+    } else {
+        ret = dict_set_str(dict, "path", str);
+        if (ret)
+            goto out;
+    }
 
-        *options = dict;
+    *options = dict;
 
 out:
-        if (ret && dict)
-                dict_destroy (dict);
+    if (ret && dict)
+        dict_unref(dict);
 
-        return ret;
+    return ret;
 }
 
 int32_t
-cli_cmd_log_level_parse (const char **words, int worcount, dict_t **options)
+cli_cmd_log_level_parse(const char **words, int worcount, dict_t **options)
 {
-        dict_t *dict            = NULL;
-        int     ret             = -1;
-
-        GF_ASSERT (words);
-        GF_ASSERT (options);
-
-        /*
-         * loglevel command format:
-         *  > volume log level <VOL> <XLATOR[*]> <LOGLEVEL>
-         *  > volume log level colon-o posix WARNING
-         *  > volume log level colon-o replicate* DEBUG
-         *  > volume log level coon-o * TRACE
-         */
-
-        GF_ASSERT ((strncmp(words[0], "volume", 6) == 0));
-        GF_ASSERT ((strncmp(words[1], "log", 3) == 0));
-        GF_ASSERT ((strncmp(words[2], "level", 5) == 0));
-
-        ret = glusterd_check_log_level(words[5]);
-        if (ret == -1) {
-                cli_err("Invalid log level [%s] specified", words[5]);
-                cli_err("Valid values for loglevel: (DEBUG|WARNING|ERROR"
-                        "|CRITICAL|NONE|TRACE)");
-                goto out;
-        }
+    dict_t *dict = NULL;
+    int ret = -1;
+
+    GF_ASSERT(words);
+    GF_ASSERT(options);
+
+    /*
+     * loglevel command format:
+     *  > volume log level <VOL> <XLATOR[*]> <LOGLEVEL>
+     *  > volume log level colon-o posix WARNING
+     *  > volume log level colon-o replicate* DEBUG
+     *  > volume log level coon-o * TRACE
+     */
+
+    GF_ASSERT((strncmp(words[0], "volume", 6) == 0));
+    GF_ASSERT((strncmp(words[1], "log", 3) == 0));
+    GF_ASSERT((strncmp(words[2], "level", 5) == 0));
+
+    ret = glusterd_check_log_level(words[5]);
+    if (ret == -1) {
+        cli_err("Invalid log level [%s] specified", words[5]);
+        cli_err(
+            "Valid values for loglevel: (DEBUG|WARNING|ERROR"
+            "|CRITICAL|NONE|TRACE)");
+        goto out;
+    }
 
-        dict = dict_new ();
-        if (!dict)
-                goto out;
+    dict = dict_new();
+    if (!dict)
+        goto out;
 
-        GF_ASSERT(words[3]);
-        GF_ASSERT(words[4]);
+    GF_ASSERT(words[3]);
+    GF_ASSERT(words[4]);
 
-        ret = dict_set_str (dict, "volname", (char *)words[3]);
-        if (ret)
-                goto out;
+    ret = dict_set_str(dict, "volname", (char *)words[3]);
+    if (ret)
+        goto out;
 
-        ret = dict_set_str (dict, "xlator", (char *)words[4]);
-        if (ret)
-                goto out;
+    ret = dict_set_str(dict, "xlator", (char *)words[4]);
+    if (ret)
+        goto out;
 
-        ret = dict_set_str (dict, "loglevel", (char *)words[5]);
-        if (ret)
-                goto out;
+    ret = dict_set_str(dict, "loglevel", (char *)words[5]);
+    if (ret)
+        goto out;
 
-        *options = dict;
+    *options = dict;
 
- out:
-        if (ret && dict)
-                dict_destroy (dict);
+out:
+    if (ret && dict)
+        dict_unref(dict);
 
-        return ret;
+    return ret;
 }
 
 int32_t
-cli_cmd_log_locate_parse (const char **words, int wordcount, dict_t **options)
+cli_cmd_log_locate_parse(const char **words, int wordcount, dict_t **options)
 {
-        dict_t  *dict = NULL;
-        char    *volname = NULL;
-        char    *str = NULL;
-        int     ret = -1;
-        char    *delimiter = NULL;
+    dict_t *dict = NULL;
+    char *volname = NULL;
+    char *str = NULL;
+    int ret = -1;
+    char *delimiter = NULL;
 
-        GF_ASSERT (words);
-        GF_ASSERT (options);
+    GF_ASSERT(words);
+    GF_ASSERT(options);
 
-        dict = dict_new ();
-        if (!dict)
-                goto out;
+    dict = dict_new();
+    if (!dict)
+        goto out;
 
-        volname = (char *)words[3];
-        GF_ASSERT (volname);
+    volname = (char *)words[3];
+    GF_ASSERT(volname);
 
-        ret = dict_set_str (dict, "volname", volname);
-        if (ret)
-                goto out;
+    ret = dict_set_str(dict, "volname", volname);
+    if (ret)
+        goto out;
 
-        if (words[4]) {
-                delimiter = strchr (words[4], ':');
-                if (!delimiter || delimiter == words[4]
-                    || *(delimiter+1) != '/') {
-                        cli_err ("wrong brick type: %s, use <HOSTNAME>:"
-                                 "<export-dir-abs-path>", words[4]);
-                        ret = -1;
-                        goto out;
-                } else {
-                        ret = gf_canonicalize_path (delimiter + 1);
-                        if (ret)
-                                goto out;
-                }
-                str = (char *)words[4];
-                ret = dict_set_str (dict, "brick", str);
-                if (ret)
-                        goto out;
+    if (words[4]) {
+        delimiter = strchr(words[4], ':');
+        if (!delimiter || delimiter == words[4] || *(delimiter + 1) != '/') {
+            cli_err(
+                "wrong brick type: %s, use <HOSTNAME>:"
+                "<export-dir-abs-path>",
+                words[4]);
+            ret = -1;
+            goto out;
+        } else {
+            ret = gf_canonicalize_path(delimiter + 1);
+            if (ret)
+                goto out;
         }
+        str = (char *)words[4];
+        ret = dict_set_str(dict, "brick", str);
+        if (ret)
+            goto out;
+    }
 
-        *options = dict;
+    *options = dict;
 
 out:
-        if (ret && dict)
-                dict_destroy (dict);
+    if (ret && dict)
+        dict_unref(dict);
 
-        return ret;
+    return ret;
 }
 
 int32_t
-cli_cmd_log_rotate_parse (const char **words, int wordcount, dict_t **options)
+cli_cmd_log_rotate_parse(const char **words, int wordcount, dict_t **options)
 {
-        dict_t  *dict = NULL;
-        char    *volname = NULL;
-        char    *str = NULL;
-        int     ret = -1;
-        char    *delimiter = NULL;
+    dict_t *dict = NULL;
+    char *volname = NULL;
+    char *str = NULL;
+    int ret = -1;
+    char *delimiter = NULL;
 
-        GF_ASSERT (words);
-        GF_ASSERT (options);
+    GF_ASSERT(words);
+    GF_ASSERT(options);
 
-        dict = dict_new ();
-        if (!dict)
-                goto out;
+    dict = dict_new();
+    if (!dict)
+        goto out;
 
-        volname = (char *)words[3];
-        GF_ASSERT (volname);
+    if (strcmp("rotate", words[3]) == 0)
+        volname = (char *)words[2];
+    GF_ASSERT(volname);
 
-        ret = dict_set_str (dict, "volname", volname);
-        if (ret)
-                goto out;
+    ret = dict_set_str(dict, "volname", volname);
+    if (ret)
+        goto out;
 
-        if (words[4]) {
-                delimiter = strchr (words[4], ':');
-                if (!delimiter || delimiter == words[4]
-                    || *(delimiter+1) != '/') {
-                        cli_err ("wrong brick type: %s, use <HOSTNAME>:"
-                                 "<export-dir-abs-path>", words[4]);
-                        ret = -1;
-                        goto out;
-                } else {
-                        ret = gf_canonicalize_path (delimiter + 1);
-                        if (ret)
-                                goto out;
-                }
-                str = (char *)words[4];
-                ret = dict_set_str (dict, "brick", str);
-                if (ret)
-                        goto out;
+    if (words[4]) {
+        delimiter = strchr(words[4], ':');
+        if (!delimiter || delimiter == words[4] || *(delimiter + 1) != '/') {
+            cli_err(
+                "wrong brick type: %s, use <HOSTNAME>:"
+                "<export-dir-abs-path>",
+                words[4]);
+            ret = -1;
+            goto out;
+        } else {
+            ret = gf_canonicalize_path(delimiter + 1);
+            if (ret)
+                goto out;
         }
+        str = (char *)words[4];
+        ret = dict_set_str(dict, "brick", str);
+        if (ret)
+            goto out;
+    }
 
-        *options = dict;
+    *options = dict;
 
 out:
-        if (ret && dict)
-                dict_destroy (dict);
+    if (ret && dict)
+        dict_unref(dict);
 
-        return ret;
+    return ret;
 }
 
 static gf_boolean_t
-gsyncd_url_check (const char *w)
+gsyncd_url_check(const char *w)
 {
-        return !!strpbrk (w, ":/");
+    return !!strpbrk(w, ":/");
 }
 
 static gf_boolean_t
-gsyncd_glob_check (const char *w)
+valid_slave_gsyncd_url(const char *w)
 {
-        return !!strpbrk (w, "*?[");
+    if (strstr(w, ":::"))
+        return _gf_false;
+    else if (strstr(w, "::"))
+        return _gf_true;
+    else
+        return _gf_false;
 }
 
-static int
-config_parse (const char **words, int wordcount, dict_t *dict,
-              unsigned cmdi, unsigned glob)
+static gf_boolean_t
+gsyncd_glob_check(const char *w)
 {
-        int32_t            ret     = -1;
-        int32_t            i       = -1;
-        char               *append_str = NULL;
-        size_t             append_len = 0;
-        char               *subop = NULL;
+    return !!strpbrk(w, "*?[");
+}
 
-        switch ((wordcount - 1) - cmdi) {
+static int
+config_parse(const char **words, int wordcount, dict_t *dict, unsigned cmdi,
+             unsigned glob)
+{
+    int32_t ret = -1;
+    int32_t i = -1;
+    char *append_str = NULL;
+    size_t append_len = 0;
+    char *subop = NULL;
+    char *ret_chkpt = NULL;
+    struct tm checkpoint_time;
+    char chkpt_buf[20] = "";
+
+    switch ((wordcount - 1) - cmdi) {
         case 0:
-                subop = gf_strdup ("get-all");
-                break;
+            subop = gf_strdup("get-all");
+            break;
         case 1:
-                if (words[cmdi + 1][0] == '!') {
-                        (words[cmdi + 1])++;
-                        if (gf_asprintf (&subop, "del%s",
-                                         glob ? "-glob" : "") == -1)
-                                subop = NULL;
-                } else
-                        subop = gf_strdup ("get");
-
-                ret = dict_set_str (dict, "op_name", ((char *)words[cmdi + 1]));
-                if (ret < 0)
-                        goto out;
-                break;
+            if (words[cmdi + 1][0] == '!') {
+                (words[cmdi + 1])++;
+                if (gf_asprintf(&subop, "del%s", glob ? "-glob" : "") == -1)
+                    subop = NULL;
+            } else
+                subop = gf_strdup("get");
+
+            ret = dict_set_str(dict, "op_name", ((char *)words[cmdi + 1]));
+            if (ret < 0)
+                goto out;
+            break;
         default:
-                if (gf_asprintf (&subop, "set%s", glob ? "-glob" : "") == -1)
-                        subop = NULL;
+            if (gf_asprintf(&subop, "set%s", glob ? "-glob" : "") == -1)
+                subop = NULL;
 
-                ret = dict_set_str (dict, "op_name", ((char *)words[cmdi + 1]));
-                if (ret < 0)
-                        goto out;
+            ret = dict_set_str(dict, "op_name", ((char *)words[cmdi + 1]));
+            if (ret < 0)
+                goto out;
+
+            /* join the varargs by spaces to get the op_value */
 
-                /* join the varargs by spaces to get the op_value */
+            for (i = cmdi + 2; i < wordcount; i++)
+                append_len += (strlen(words[i]) + 1);
+            /* trailing strcat will add two bytes, make space for that */
+            append_len++;
 
-                for (i = cmdi + 2; i < wordcount; i++)
-                        append_len += (strlen (words[i]) + 1);
-                /* trailing strcat will add two bytes, make space for that */
-                append_len++;
+            /* strcat is used on this allocation and hence expected to be
+             * initiatlized to 0. So GF_CALLOC is used.
+             */
+            append_str = GF_CALLOC(1, append_len, cli_mt_append_str);
+            if (!append_str) {
+                ret = -1;
+                goto out;
+            }
+
+            for (i = cmdi + 2; i < wordcount; i++) {
+                strcat(append_str, words[i]);
+                strcat(append_str, " ");
+            }
+            append_str[append_len - 2] = '\0';
+            /* "checkpoint now" is special: we resolve that "now" */
+            if ((strcmp(words[cmdi + 1], "checkpoint") == 0) &&
+                (strcmp(append_str, "now") == 0)) {
+                struct timeval tv = {
+                    0,
+                };
+
+                ret = gettimeofday(&tv, NULL);
+                if (ret == -1)
+                    goto out;
 
-                append_str = GF_CALLOC (1, append_len, cli_mt_append_str);
+                GF_FREE(append_str);
+                append_str = GF_MALLOC(300, cli_mt_append_str);
                 if (!append_str) {
-                        ret = -1;
-                        goto out;
+                    ret = -1;
+                    goto out;
                 }
-
-                for (i = cmdi + 2; i < wordcount; i++) {
-                        strcat (append_str, words[i]);
-                        strcat (append_str, " ");
+                snprintf(append_str, 300, "%" GF_PRI_SECOND, tv.tv_sec);
+            } else if ((strcmp(words[cmdi + 1], "checkpoint") == 0) &&
+                       (strcmp(append_str, "now") != 0)) {
+                memset(&checkpoint_time, 0, sizeof(struct tm));
+                ret_chkpt = strptime(append_str, "%Y-%m-%d %H:%M:%S",
+                                     &checkpoint_time);
+
+                if (ret_chkpt == NULL || *ret_chkpt != '\0') {
+                    ret = -1;
+                    cli_err(
+                        "Invalid Checkpoint label. Use format "
+                        "\"Y-m-d H:M:S\", Example: 2016-10-25 15:30:45");
+                    goto out;
                 }
-                append_str[append_len - 2] = '\0';
-                /* "checkpoint now" is special: we resolve that "now" */
-                if ((strcmp (words[cmdi + 1], "checkpoint") == 0) &&
-                    (strcmp (append_str, "now") == 0)) {
-                        struct timeval tv = {0,};
-
-                        ret = gettimeofday (&tv, NULL);
-                        if (ret == -1)
-                                goto out;
-
-                        GF_FREE (append_str);
-                        append_str = GF_CALLOC (1, 300, cli_mt_append_str);
-                        if (!append_str) {
-                                ret = -1;
-                                goto out;
-                        }
-                        snprintf (append_str, 300, "now:%ld.%06ld",
-                                  tv.tv_sec, tv.tv_usec);
+                GF_FREE(append_str);
+                append_str = GF_MALLOC(300, cli_mt_append_str);
+                if (!append_str) {
+                    ret = -1;
+                    goto out;
                 }
+                strftime(chkpt_buf, sizeof(chkpt_buf), "%s", &checkpoint_time);
+                snprintf(append_str, 300, "%s", chkpt_buf);
+            }
 
-                ret = dict_set_dynstr (dict, "op_value", append_str);
-        }
+            ret = dict_set_dynstr(dict, "op_value", append_str);
+            if (ret != 0) {
+                goto out;
+            }
+            append_str = NULL;
+    }
 
-        ret = -1;
-        if (subop) {
-                ret = dict_set_dynstr (dict, "subop", subop);
-                if (!ret)
-                      subop = NULL;
-        }
+    ret = -1;
+    if (subop) {
+        ret = dict_set_dynstr(dict, "subop", subop);
+        if (!ret)
+            subop = NULL;
+    }
 
 out:
-        if (ret && append_str)
-                GF_FREE (append_str);
-
-        GF_FREE (subop);
+    GF_FREE(append_str);
+    GF_FREE(subop);
 
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
 }
 
+/* ssh_port_parse: Parses and validates when ssh_port is given.
+ *                 ssh_index refers to index of ssh_port and
+ *                 type refers to either push-pem or no-verify
+ */
+
 static int32_t
-force_push_pem_parse (const char **words, int wordcount,
-                      dict_t *dict, unsigned *cmdi)
+parse_ssh_port(const char **words, int wordcount, dict_t *dict, unsigned *cmdi,
+               int ssh_index, char *type)
 {
-        int32_t            ret     = 0;
+    int ret = 0;
+    char *end_ptr = NULL;
+    int64_t limit = 0;
+
+    if (!strcmp((char *)words[ssh_index], "ssh-port")) {
+        if (strcmp((char *)words[ssh_index - 1], "create")) {
+            ret = -1;
+            goto out;
+        }
+        (*cmdi)++;
+        limit = strtol(words[ssh_index + 1], &end_ptr, 10);
+        if (errno == ERANGE || errno == EINVAL || limit <= 0 ||
+            strcmp(end_ptr, "") != 0) {
+            ret = -1;
+            cli_err("Please enter an integer value for ssh_port ");
+            goto out;
+        }
 
-        if (!strcmp ((char *)words[wordcount-1], "force")) {
-                if ((strcmp ((char *)words[wordcount-2], "start")) &&
-                    (strcmp ((char *)words[wordcount-2], "stop")) &&
-                    (strcmp ((char *)words[wordcount-2], "create")) &&
-                    (strcmp ((char *)words[wordcount-2], "push-pem"))) {
-                        ret = -1;
-                        goto out;
-                }
-                ret = dict_set_uint32 (dict, "force",
-                                       _gf_true);
-                if (ret)
-                        goto out;
-                (*cmdi)++;
+        ret = dict_set_int32(dict, "ssh_port", limit);
+        if (ret)
+            goto out;
+        (*cmdi)++;
+    } else if (strcmp((char *)words[ssh_index + 1], "create")) {
+        ret = -1;
+        goto out;
+    }
 
-                if (!strcmp ((char *)words[wordcount-2], "push-pem")) {
-                        if (strcmp ((char *)words[wordcount-3], "create")) {
-                                ret = -1;
-                                goto out;
-                        }
-                        ret = dict_set_int32 (dict, "push_pem", 1);
-                        if (ret)
-                                goto out;
-                        (*cmdi)++;
-                }
-        } else if (!strcmp ((char *)words[wordcount-1], "push-pem")) {
-                if (strcmp ((char *)words[wordcount-2], "create")) {
-                        ret = -1;
-                        goto out;
-                }
-                ret = dict_set_int32 (dict, "push_pem", 1);
-                if (ret)
-                        goto out;
-                (*cmdi)++;
-        }
+    ret = dict_set_int32(dict, type, 1);
+    if (ret)
+        goto out;
+    (*cmdi)++;
 
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    return ret;
 }
 
-
-int32_t
-cli_cmd_gsync_set_parse (const char **words, int wordcount, dict_t **options)
+static int32_t
+force_push_pem_no_verify_parse(const char **words, int wordcount, dict_t *dict,
+                               unsigned *cmdi)
 {
-        int32_t            ret     = -1;
-        dict_t             *dict   = NULL;
-        gf1_cli_gsync_set  type    = GF_GSYNC_OPTION_TYPE_NONE;
-        int                i       = 0;
-        unsigned           masteri = 0;
-        unsigned           slavei  = 0;
-        unsigned           glob    = 0;
-        unsigned           cmdi    = 0;
-        char               *opwords[] = { "create", "status", "start", "stop",
-                                          "config", "force", "delete",
-                                          "push-pem", "detail", NULL };
-        char               *w = NULL;
-
-        GF_ASSERT (words);
-        GF_ASSERT (options);
-
-        dict = dict_new ();
-        if (!dict)
-                goto out;
-
-        /* new syntax:
-         *
-         * volume geo-replication $m $s create [push-pem] [force]
-         * volume geo-replication [$m [$s]] status [detail]
-         * volume geo-replication [$m] $s config [[!]$opt [$val]]
-         * volume geo-replication $m $s start|stop [force]
-         * volume geo-replication $m $s delete
-         */
+    int32_t ret = 0;
+
+    if (!strcmp((char *)words[wordcount - 1], "force")) {
+        if ((strcmp((char *)words[wordcount - 2], "start")) &&
+            (strcmp((char *)words[wordcount - 2], "stop")) &&
+            (strcmp((char *)words[wordcount - 2], "create")) &&
+            (strcmp((char *)words[wordcount - 2], "no-verify")) &&
+            (strcmp((char *)words[wordcount - 2], "push-pem")) &&
+            (strcmp((char *)words[wordcount - 2], "pause")) &&
+            (strcmp((char *)words[wordcount - 2], "resume"))) {
+            ret = -1;
+            goto out;
+        }
+        ret = dict_set_int32n(dict, "force", SLEN("force"), 1);
+        if (ret)
+            goto out;
+        (*cmdi)++;
 
-        if (wordcount < 3)
+        if (!strcmp((char *)words[wordcount - 2], "push-pem")) {
+            ret = parse_ssh_port(words, wordcount, dict, cmdi, wordcount - 4,
+                                 "push_pem");
+            if (ret)
+                goto out;
+        } else if (!strcmp((char *)words[wordcount - 2], "no-verify")) {
+            ret = parse_ssh_port(words, wordcount, dict, cmdi, wordcount - 4,
+                                 "no_verify");
+            if (ret)
                 goto out;
-
-        for (i = 2; i <= 3 && i < wordcount - 1; i++) {
-                if (gsyncd_glob_check (words[i]))
-                        glob = i;
-                if (gsyncd_url_check (words[i])) {
-                        slavei = i;
-                        break;
-                }
         }
+    } else if (!strcmp((char *)words[wordcount - 1], "push-pem")) {
+        ret = parse_ssh_port(words, wordcount, dict, cmdi, wordcount - 3,
+                             "push_pem");
+        if (ret)
+            goto out;
+    } else if (!strcmp((char *)words[wordcount - 1], "no-verify")) {
+        ret = parse_ssh_port(words, wordcount, dict, cmdi, wordcount - 3,
+                             "no_verify");
+        if (ret)
+            goto out;
+    }
 
-        if (glob && !slavei)
-                /* glob is allowed only for config, thus it implies there is a
-                 * slave argument; but that might have not been recognized on
-                 * the first scan as it's url characteristics has been covered
-                 * by the glob syntax.
-                 *
-                 * In this case, the slave is perforce the last glob-word -- the
-                 * upcoming one is neither glob, nor url, so it's definitely not
-                 * the slave.
-                 */
-                slavei = glob;
-        if (slavei) {
-                cmdi = slavei + 1;
-                if (slavei == 3)
-                        masteri = 2;
-        } else if (i <= 3) {
-                if (!strcmp ((char *)words[wordcount-1], "detail")) {
-                        /* For status detail it is mandatory to provide
-                         * both master and slave */
-                        ret = -1;
-                        goto out;
-                }
+out:
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
+}
 
-                /* no $s, can only be status cmd
-                 * (with either a single $m before it or nothing)
-                 * -- these conditions imply that i <= 3 after
-                 * the iteration and that i is the successor of
-                 * the (0 or 1 length) sequence of $m-s.
-                 */
-                cmdi = i;
-                if (i == 3)
-                        masteri = 2;
-        } else
-                goto out;
-
-        /* now check if input really complies syntax
-         * (in a somewhat redundant way, in favor
-         * transparent soundness)
-         */
+int32_t
+cli_cmd_gsync_set_parse(struct cli_state *state, const char **words,
+                        int wordcount, dict_t **options, char **errstr)
+{
+    int32_t ret = -1;
+    dict_t *dict = NULL;
+    gf1_cli_gsync_set type = GF_GSYNC_OPTION_TYPE_NONE;
+    int i = 0;
+    unsigned masteri = 0;
+    unsigned slavei = 0;
+    unsigned glob = 0;
+    unsigned cmdi = 0;
+    static char *opwords[] = {"create",    "status",   "start",  "stop",
+                              "config",    "force",    "delete", "ssh-port",
+                              "no-verify", "push-pem", "detail", "pause",
+                              "resume",    NULL};
+    char *w = NULL;
+    char *save_ptr = NULL;
+    char *slave_temp = NULL;
+    char *token = NULL;
+    gf_answer_t answer = GF_ANSWER_NO;
+    const char *question = NULL;
+
+    GF_ASSERT(words);
+    GF_ASSERT(options);
+
+    dict = dict_new();
+    if (!dict)
+        goto out;
 
-        if (masteri && gsyncd_url_check (words[masteri]))
-                goto out;
-        if (slavei && !glob && !gsyncd_url_check (words[slavei]))
-                goto out;
+    /* new syntax:
+     *
+     * volume geo-replication $m $s create [[ssh-port n] [[no-verify] |
+     * [push-pem]]] [force] volume geo-replication [$m [$s]] status [detail]
+     * volume geo-replication [$m] $s config [[!]$opt [$val]]
+     * volume geo-replication $m $s start|stop [force]
+     * volume geo-replication $m $s delete [reset-sync-time]
+     * volume geo-replication $m $s pause [force]
+     * volume geo-replication $m $s resume [force]
+     */
+
+    if (wordcount < 3)
+        goto out;
 
-        w = str_getunamb (words[cmdi], opwords);
-        if (!w)
-                goto out;
+    for (i = 2; i <= 3 && i < wordcount - 1; i++) {
+        if (gsyncd_glob_check(words[i]))
+            glob = i;
+        if (gsyncd_url_check(words[i])) {
+            slavei = i;
+            break;
+        }
+    }
 
-        if (strcmp (w, "create") == 0) {
-                type = GF_GSYNC_OPTION_TYPE_CREATE;
+    if (glob && !slavei)
+        /* glob is allowed only for config, thus it implies there is a
+         * slave argument; but that might have not been recognized on
+         * the first scan as it's url characteristics has been covered
+         * by the glob syntax.
+         *
+         * In this case, the slave is perforce the last glob-word -- the
+         * upcoming one is neither glob, nor url, so it's definitely not
+         * the slave.
+         */
+        slavei = glob;
+    if (slavei) {
+        cmdi = slavei + 1;
+        if (slavei == 3)
+            masteri = 2;
+    } else if (i <= 4) {
+        if (strtail("detail", (char *)words[wordcount - 1])) {
+            cmdi = wordcount - 2;
+            if (i == 4)
+                masteri = 2;
+        } else {
+            /* no $s, can only be status cmd
+             * (with either a single $m before it or nothing)
+             * -- these conditions imply that i <= 3 after
+             * the iteration and that i is the successor of
+             * the (0 or 1 length) sequence of $m-s.
+             */
+            cmdi = i;
+            if (i == 3)
+                masteri = 2;
+        }
+    } else
+        goto out;
 
-                if (!masteri || !slavei)
-                        goto out;
-        } else if (strcmp (w, "status") == 0) {
-                type = GF_GSYNC_OPTION_TYPE_STATUS;
+    /* now check if input really complies syntax
+     * (in a somewhat redundant way, in favor
+     * transparent soundness)
+     */
 
-                if (slavei && !masteri)
-                        goto out;
-        } else if (strcmp (w, "config") == 0) {
-                type = GF_GSYNC_OPTION_TYPE_CONFIG;
+    if (masteri && gsyncd_url_check(words[masteri]))
+        goto out;
 
-                if (!slavei)
-                        goto out;
-        } else if (strcmp (w, "start") == 0) {
-                type = GF_GSYNC_OPTION_TYPE_START;
+    if (slavei && !glob && !valid_slave_gsyncd_url(words[slavei])) {
+        gf_asprintf(errstr, "Invalid slave url: %s", words[slavei]);
+        goto out;
+    }
 
-                if (!masteri || !slavei)
-                        goto out;
-        } else if (strcmp (w, "stop") == 0) {
-                type = GF_GSYNC_OPTION_TYPE_STOP;
+    w = str_getunamb(words[cmdi], opwords);
+    if (!w)
+        goto out;
 
-                if (!masteri || !slavei)
-                        goto out;
-        } else if (strcmp (w, "delete") == 0) {
-                type = GF_GSYNC_OPTION_TYPE_DELETE;
+    if (strcmp(w, "create") == 0) {
+        type = GF_GSYNC_OPTION_TYPE_CREATE;
+
+        if (!masteri || !slavei)
+            goto out;
+    } else if (strcmp(w, "status") == 0) {
+        type = GF_GSYNC_OPTION_TYPE_STATUS;
+
+        if (slavei && !masteri)
+            goto out;
+    } else if (strcmp(w, "config") == 0) {
+        type = GF_GSYNC_OPTION_TYPE_CONFIG;
+
+        if (!slavei)
+            goto out;
+    } else if (strcmp(w, "start") == 0) {
+        type = GF_GSYNC_OPTION_TYPE_START;
+
+        if (!masteri || !slavei)
+            goto out;
+    } else if (strcmp(w, "stop") == 0) {
+        type = GF_GSYNC_OPTION_TYPE_STOP;
+
+        if (!masteri || !slavei)
+            goto out;
+    } else if (strcmp(w, "delete") == 0) {
+        type = GF_GSYNC_OPTION_TYPE_DELETE;
+
+        if (!masteri || !slavei)
+            goto out;
+    } else if (strcmp(w, "pause") == 0) {
+        type = GF_GSYNC_OPTION_TYPE_PAUSE;
+
+        if (!masteri || !slavei)
+            goto out;
+    } else if (strcmp(w, "resume") == 0) {
+        type = GF_GSYNC_OPTION_TYPE_RESUME;
+
+        if (!masteri || !slavei)
+            goto out;
+    } else
+        GF_ASSERT(!"opword mismatch");
+
+    ret = force_push_pem_no_verify_parse(words, wordcount, dict, &cmdi);
+    if (ret)
+        goto out;
 
-                if (!masteri || !slavei)
-                        goto out;
-        } else
-                GF_ASSERT (!"opword mismatch");
+    if (strtail("detail", (char *)words[wordcount - 1])) {
+        if (!strtail("status", (char *)words[wordcount - 2])) {
+            ret = -1;
+            goto out;
+        }
 
-        ret = force_push_pem_parse (words, wordcount, dict, &cmdi);
+        ret = dict_set_uint32(dict, "status-detail", _gf_true);
         if (ret)
-                goto out;
-
-        if (!strcmp ((char *)words[wordcount-1], "detail")) {
-                if (strcmp ((char *)words[wordcount-2], "status")) {
-                        ret = -1;
-                        goto out;
-                }
-                if (!slavei || !masteri) {
-                        ret = -1;
-                        goto out;
-                }
-                ret = dict_set_uint32 (dict, "status-detail", _gf_true);
-                if (ret)
-                        goto out;
-                cmdi++;
+            goto out;
+        cmdi++;
+    }
+
+    if (type == GF_GSYNC_OPTION_TYPE_DELETE &&
+        !strcmp((char *)words[wordcount - 1], "reset-sync-time")) {
+        if (strcmp((char *)words[wordcount - 2], "delete")) {
+            ret = -1;
+            goto out;
         }
+        ret = dict_set_uint32(dict, "reset-sync-time", _gf_true);
+        if (ret)
+            goto out;
+        cmdi++;
+    }
 
-        if (type != GF_GSYNC_OPTION_TYPE_CONFIG &&
-            (cmdi < wordcount - 1 || glob))
-                goto out;
+    if (type != GF_GSYNC_OPTION_TYPE_CONFIG && (cmdi < wordcount - 1 || glob)) {
+        ret = -1;
+        goto out;
+    }
 
-        /* If got so far, input is valid, assemble the message */
+    /* If got so far, input is valid, assemble the message */
 
-        ret = 0;
+    ret = 0;
 
-        if (masteri) {
-                ret = dict_set_str (dict, "master", (char *)words[masteri]);
-                if (!ret)
-                        ret = dict_set_str (dict, "volname",
-                                            (char *)words[masteri]);
-        }
-        if (!ret && slavei)
-                ret = dict_set_str (dict, "slave", (char *)words[slavei]);
+    if (masteri) {
+        ret = dict_set_str(dict, "master", (char *)words[masteri]);
         if (!ret)
-                ret = dict_set_int32 (dict, "type", type);
-        if (!ret && type == GF_GSYNC_OPTION_TYPE_CONFIG)
-                ret = config_parse (words, wordcount, dict, cmdi, glob);
+            ret = dict_set_str(dict, "volname", (char *)words[masteri]);
+    }
+    if (!ret && slavei) {
+        /* If geo-rep is created with root user using the syntax
+         * gluster vol geo-rep <mastervol> root@<slavehost> ...
+         * pass down only <slavehost> else pass as it is.
+         */
+        slave_temp = gf_strdup(words[slavei]);
+        if (slave_temp == NULL) {
+            ret = -1;
+            goto out;
+        }
+        token = strtok_r(slave_temp, "@", &save_ptr);
+        if (token && !strcmp(token, "root")) {
+            ret = dict_set_str(dict, "slave", (char *)words[slavei] + 5);
+        } else {
+            ret = dict_set_str(dict, "slave", (char *)words[slavei]);
+        }
+    }
+    if (!ret)
+        ret = dict_set_int32(dict, "type", type);
+    if (!ret && type == GF_GSYNC_OPTION_TYPE_CONFIG) {
+        if (!strcmp((char *)words[wordcount - 2], "ignore-deletes") &&
+            !strcmp((char *)words[wordcount - 1], "true")) {
+            question =
+                "There exists ~15 seconds delay for the option to take"
+                " effect from stime of the corresponding brick. Please"
+                " check the log for the time, the option is effective."
+                " Proceed";
+
+            answer = cli_cmd_get_confirmation(state, question);
+
+            if (GF_ANSWER_NO == answer) {
+                gf_log("cli", GF_LOG_INFO,
+                       "Operation "
+                       "cancelled, exiting");
+                *errstr = gf_strdup("Aborted by user.");
+                ret = -1;
+                goto out;
+            }
+        }
 
-out:
-        if (ret) {
-                if (dict)
-                        dict_destroy (dict);
-        } else
-                *options = dict;
+        ret = config_parse(words, wordcount, dict, cmdi, glob);
+    }
 
+out:
+    if (slave_temp)
+        GF_FREE(slave_temp);
+    if (ret && dict)
+        dict_unref(dict);
+    else
+        *options = dict;
 
-        return ret;
+    return ret;
 }
 
 int32_t
-cli_cmd_volume_profile_parse (const char **words, int wordcount,
-                              dict_t **options)
+cli_cmd_volume_profile_parse(const char **words, int wordcount,
+                             dict_t **options)
 {
-        dict_t    *dict       = NULL;
-        char      *volname    = NULL;
-        int       ret         = -1;
-        gf1_cli_stats_op op = GF_CLI_STATS_NONE;
-        char      *opwords[] = { "start", "stop", "info", NULL };
-        char      *w = NULL;
+    dict_t *dict = NULL;
+    char *volname = NULL;
+    int ret = -1;
+    gf1_cli_stats_op op = GF_CLI_STATS_NONE;
+    gf1_cli_info_op info_op = GF_CLI_INFO_NONE;
+    gf_boolean_t is_peek = _gf_false;
 
-        GF_ASSERT (words);
-        GF_ASSERT (options);
+    static char *opwords[] = {"start", "stop", "info", NULL};
+    char *w = NULL;
 
-        dict = dict_new ();
-        if (!dict)
-                goto out;
+    GF_ASSERT(words);
+    GF_ASSERT(options);
 
-        if (wordcount < 4 || wordcount >5)
-                goto out;
+    dict = dict_new();
+    if (!dict)
+        goto out;
 
-        volname = (char *)words[2];
+    if (wordcount < 4)
+        goto out;
 
-        ret = dict_set_str (dict, "volname", volname);
-        if (ret)
-                goto out;
+    volname = (char *)words[2];
 
-        w = str_getunamb (words[3], opwords);
-        if (!w) {
-                ret = -1;
-                goto out;
-        }
-        if (strcmp (w, "start") == 0) {
-                op = GF_CLI_STATS_START;
-        } else if (strcmp (w, "stop") == 0) {
-                op = GF_CLI_STATS_STOP;
-        } else if (strcmp (w, "info") == 0) {
-                op = GF_CLI_STATS_INFO;
-        } else
-                GF_ASSERT (!"opword mismatch");
+    ret = dict_set_str(dict, "volname", volname);
+    if (ret)
+        goto out;
 
-        ret = dict_set_int32 (dict, "op", (int32_t)op);
-        if (ret)
-                goto out;
+    w = str_getunamb(words[3], opwords);
+    if (!w) {
+        ret = -1;
+        goto out;
+    }
+
+    if ((strcmp(w, "start") == 0 || strcmp(w, "stop") == 0) && wordcount > 5) {
+        ret = -1;
+        goto out;
+    }
 
-        if (wordcount == 5) {
-                if (!strcmp (words[4], "nfs")) {
-                        ret = dict_set_int32 (dict, "nfs", _gf_true);
-                        if (ret)
-                                goto out;
+    if (strcmp(w, "info") == 0 && wordcount > 7) {
+        ret = -1;
+        goto out;
+    }
+
+    if (strcmp(w, "start") == 0) {
+        op = GF_CLI_STATS_START;
+    } else if (strcmp(w, "stop") == 0) {
+        op = GF_CLI_STATS_STOP;
+    } else if (strcmp(w, "info") == 0) {
+        op = GF_CLI_STATS_INFO;
+        info_op = GF_CLI_INFO_ALL;
+        if (wordcount > 4) {
+            if (strcmp(words[4], "incremental") == 0) {
+                info_op = GF_CLI_INFO_INCREMENTAL;
+                if (wordcount > 5 && strcmp(words[5], "peek") == 0) {
+                    is_peek = _gf_true;
                 }
+            } else if (strcmp(words[4], "cumulative") == 0) {
+                info_op = GF_CLI_INFO_CUMULATIVE;
+            } else if (strcmp(words[4], "clear") == 0) {
+                info_op = GF_CLI_INFO_CLEAR;
+            } else if (strcmp(words[4], "peek") == 0) {
+                is_peek = _gf_true;
+            }
         }
+    } else
+        GF_ASSERT(!"opword mismatch");
 
-        *options = dict;
+    ret = dict_set_int32(dict, "op", (int32_t)op);
+    if (ret)
+        goto out;
+
+    ret = dict_set_int32(dict, "info-op", (int32_t)info_op);
+    if (ret)
+        goto out;
+
+    ret = dict_set_int32(dict, "peek", is_peek);
+    if (ret)
+        goto out;
+
+    if (!strcmp(words[wordcount - 1], "nfs")) {
+        ret = dict_set_int32(dict, "nfs", _gf_true);
+        if (ret)
+            goto out;
+    }
+
+    *options = dict;
 out:
-        if (ret && dict)
-                dict_destroy (dict);
-        return ret;
+    if (ret && dict)
+        dict_unref(dict);
+    return ret;
 }
 
 int32_t
-cli_cmd_volume_top_parse (const char **words, int wordcount,
-                              dict_t **options)
+cli_cmd_volume_top_parse(const char **words, int wordcount, dict_t **options)
 {
-        dict_t  *dict           = NULL;
-        char    *volname        = NULL;
-        char    *value          = NULL;
-        char    *key            = NULL;
-        int      ret            = -1;
-        gf1_cli_stats_op op = GF_CLI_STATS_NONE;
-        gf1_cli_top_op    top_op = GF_CLI_TOP_NONE;
-        int32_t  list_cnt       = -1;
-        int      index          = 0;
-        int      perf           = 0;
-        uint32_t  blk_size      = 0;
-        uint32_t  count         = 0;
-        gf_boolean_t nfs        = _gf_false;
-        char    *delimiter      = NULL;
-        char    *opwords[]      = { "open", "read", "write", "opendir",
-                                    "readdir", "read-perf", "write-perf",
-                                    "clear", NULL };
-        char    *w = NULL;
-
-        GF_ASSERT (words);
-        GF_ASSERT (options);
-
-        dict = dict_new ();
-        if (!dict)
-                goto out;
-
-        if (wordcount < 4)
-                goto out;
+    dict_t *dict = NULL;
+    char *volname = NULL;
+    char *value = NULL;
+    char *key = NULL;
+    int ret = -1;
+    gf1_cli_stats_op op = GF_CLI_STATS_NONE;
+    gf1_cli_top_op top_op = GF_CLI_TOP_NONE;
+    int32_t list_cnt = -1;
+    int index = 0;
+    int perf = 0;
+    int32_t blk_size = 0;
+    int count = 0;
+    gf_boolean_t nfs = _gf_false;
+    char *delimiter = NULL;
+    static char *opwords[] = {"open",       "read",    "write",
+                              "opendir",    "readdir", "read-perf",
+                              "write-perf", "clear",   NULL};
+    char *w = NULL;
+
+    GF_ASSERT(words);
+    GF_ASSERT(options);
+
+    dict = dict_new();
+    if (!dict)
+        goto out;
 
-        volname = (char *)words[2];
+    if (wordcount < 4)
+        goto out;
 
-        ret = dict_set_str (dict, "volname", volname);
-        if (ret)
-                goto out;
+    volname = (char *)words[2];
 
-        op = GF_CLI_STATS_TOP;
-        ret = dict_set_int32 (dict, "op", (int32_t)op);
-        if (ret)
-                goto out;
+    ret = dict_set_str(dict, "volname", volname);
+    if (ret)
+        goto out;
 
-        w = str_getunamb (words[3], opwords);
-        if (!w) {
-                ret = -1;
-                goto out;
-        }
-        if (strcmp (w, "open") == 0) {
-                top_op = GF_CLI_TOP_OPEN;
-        } else if (strcmp (w, "read") == 0) {
-                top_op = GF_CLI_TOP_READ;
-        } else if (strcmp (w, "write") == 0) {
-                top_op = GF_CLI_TOP_WRITE;
-        } else if (strcmp (w, "opendir") == 0) {
-                top_op = GF_CLI_TOP_OPENDIR;
-        } else if (strcmp (w, "readdir") == 0) {
-                top_op = GF_CLI_TOP_READDIR;
-        } else if (strcmp (w, "read-perf") == 0) {
-                top_op = GF_CLI_TOP_READ_PERF;
-                perf = 1;
-        } else if (strcmp (w, "write-perf") == 0) {
-                top_op = GF_CLI_TOP_WRITE_PERF;
-                perf = 1;
-        } else if (strcmp (w, "clear") == 0) {
-                ret = dict_set_int32 (dict, "clear-stats", 1);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Could not set clear-stats in dict");
-                        goto out;
-                }
-        } else
-                GF_ASSERT (!"opword mismatch");
-        ret = dict_set_int32 (dict, "top-op", (int32_t)top_op);
-        if (ret)
-                goto out;
+    op = GF_CLI_STATS_TOP;
+    ret = dict_set_int32(dict, "op", (int32_t)op);
+    if (ret)
+        goto out;
 
-        if ((wordcount > 4) && !strcmp (words[4], "nfs")) {
-                nfs = _gf_true;
-                ret = dict_set_int32 (dict, "nfs", nfs);
-                if (ret)
-                        goto out;
-                index = 5;
-        } else {
-                index = 4;
+    w = str_getunamb(words[3], opwords);
+    if (!w) {
+        ret = -1;
+        goto out;
+    }
+    if (strcmp(w, "open") == 0) {
+        top_op = GF_CLI_TOP_OPEN;
+    } else if (strcmp(w, "read") == 0) {
+        top_op = GF_CLI_TOP_READ;
+    } else if (strcmp(w, "write") == 0) {
+        top_op = GF_CLI_TOP_WRITE;
+    } else if (strcmp(w, "opendir") == 0) {
+        top_op = GF_CLI_TOP_OPENDIR;
+    } else if (strcmp(w, "readdir") == 0) {
+        top_op = GF_CLI_TOP_READDIR;
+    } else if (strcmp(w, "read-perf") == 0) {
+        top_op = GF_CLI_TOP_READ_PERF;
+        perf = 1;
+    } else if (strcmp(w, "write-perf") == 0) {
+        top_op = GF_CLI_TOP_WRITE_PERF;
+        perf = 1;
+    } else if (strcmp(w, "clear") == 0) {
+        ret = dict_set_int32(dict, "clear-stats", 1);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR, "Could not set clear-stats in dict");
+            goto out;
         }
+    } else
+        GF_ASSERT(!"opword mismatch");
+    ret = dict_set_int32(dict, "top-op", (int32_t)top_op);
+    if (ret)
+        goto out;
 
-        for (; index < wordcount; index+=2) {
-
-                key = (char *) words[index];
-                value = (char *) words[index+1];
-
-                if ( key && !value ) {
-                        ret = -1;
-                        goto out;
-                }
-                if (!strcmp (key, "brick")) {
-                        delimiter = strchr (value, ':');
-                        if (!delimiter || delimiter == value
-                            || *(delimiter+1) != '/') {
-                                cli_err ("wrong brick type: %s, use <HOSTNAME>:"
-                                         "<export-dir-abs-path>", value);
-                                ret = -1;
-                                goto out;
-                        } else {
-                                ret = gf_canonicalize_path (delimiter + 1);
-                                if (ret)
-                                        goto out;
-                        }
-                        ret = dict_set_str (dict, "brick", value);
-
-                } else if (!strcmp (key, "list-cnt")) {
-                        ret = gf_is_str_int (value);
-                        if (!ret)
-                                list_cnt = atoi (value);
-                        if (ret || (list_cnt < 0) || (list_cnt > 100)) {
-                                cli_err ("list-cnt should be between 0 to 100");
-                                ret = -1;
-                                goto out;
-                        }
-                } else if (perf && !nfs && !strcmp (key, "bs")) {
-                        ret = gf_is_str_int (value);
-                        if (!ret)
-                                blk_size = atoi (value);
-                        if (ret || (blk_size <= 0)) {
-                                if (blk_size < 0)
-                                        cli_err ("block size is an invalid"
-                                                 " number");
-                                else
-                                        cli_err ("block size should be an "
-                                                 "integer greater than zero");
-                                ret = -1;
-                                goto out;
-                        }
-                        ret = dict_set_uint32 (dict, "blk-size", blk_size);
-                } else if (perf && !nfs && !strcmp (key, "count")) {
-                        ret = gf_is_str_int (value);
-                        if (!ret)
-                                count = atoi(value);
-                        if (ret || (count <= 0)) {
-                                if (count < 0)
-                                        cli_err ("count is an invalid number");
-                                else
-                                        cli_err ("count should be an integer "
-                                                 "greater than zero");
-
-                                ret = -1;
-                                goto out;
-                        }
-                        ret = dict_set_uint32 (dict, "blk-cnt", count);
-                } else {
-                        ret = -1;
-                        goto out;
-                }
-                if (ret) {
-                        gf_log ("", GF_LOG_WARNING, "Dict set failed for "
-                                "key %s", key);
-                        goto out;
-                }
+    if ((wordcount > 4) && !strcmp(words[4], "nfs")) {
+        nfs = _gf_true;
+        ret = dict_set_int32(dict, "nfs", nfs);
+        if (ret)
+            goto out;
+        index = 5;
+    } else {
+        index = 4;
+    }
+
+    for (; index < wordcount; index += 2) {
+        key = (char *)words[index];
+        value = (char *)words[index + 1];
+
+        if (!key || !value) {
+            ret = -1;
+            goto out;
         }
-        if (list_cnt == -1)
-                list_cnt = 100;
-        ret = dict_set_int32 (dict, "list-cnt", list_cnt);
-        if (ret) {
-                gf_log ("", GF_LOG_WARNING, "Dict set failed for list_cnt");
+        if (!strcmp(key, "brick")) {
+            delimiter = strchr(value, ':');
+            if (!delimiter || delimiter == value || *(delimiter + 1) != '/') {
+                cli_err(
+                    "wrong brick type: %s, use <HOSTNAME>:"
+                    "<export-dir-abs-path>",
+                    value);
+                ret = -1;
                 goto out;
-        }
-
-        if ((blk_size > 0) ^ (count > 0)) {
-                cli_err ("Need to give both 'bs' and 'count'");
+            } else {
+                ret = gf_canonicalize_path(delimiter + 1);
+                if (ret)
+                    goto out;
+            }
+            ret = dict_set_str(dict, "brick", value);
+
+        } else if (!strcmp(key, "list-cnt")) {
+            ret = gf_is_str_int(value);
+            if (!ret)
+                list_cnt = atoi(value);
+            if (ret || (list_cnt < 0) || (list_cnt > 100)) {
+                cli_err("list-cnt should be between 0 to 100");
+                ret = -1;
+                goto out;
+            }
+        } else if (perf && !nfs && !strcmp(key, "bs")) {
+            ret = gf_is_str_int(value);
+            if (!ret)
+                blk_size = atoi(value);
+            if (ret || (blk_size <= 0)) {
+                if (blk_size < 0)
+                    cli_err(
+                        "block size is an invalid"
+                        " number");
+                else
+                    cli_err(
+                        "block size should be an "
+                        "integer greater than zero");
                 ret = -1;
                 goto out;
-        } else if (((uint64_t)blk_size * count) > (10 * GF_UNIT_GB)) {
-                cli_err ("'bs * count' value %"PRIu64" is greater than "
-                         "maximum allowed value of 10GB",
-                         ((uint64_t)blk_size * count));
+            }
+            ret = dict_set_uint32(dict, "blk-size", (uint32_t)blk_size);
+        } else if (perf && !nfs && !strcmp(key, "count")) {
+            ret = gf_is_str_int(value);
+            if (!ret)
+                count = atoi(value);
+            if (ret || (count <= 0)) {
+                if (count < 0)
+                    cli_err("count is an invalid number");
+                else
+                    cli_err(
+                        "count should be an integer "
+                        "greater than zero");
+
                 ret = -1;
                 goto out;
+            }
+            ret = dict_set_uint32(dict, "blk-cnt", count);
+        } else {
+            ret = -1;
+            goto out;
+        }
+        if (ret) {
+            gf_log("", GF_LOG_WARNING,
+                   "Dict set failed for "
+                   "key %s",
+                   key);
+            goto out;
         }
+    }
+    if (list_cnt == -1)
+        list_cnt = 100;
+    ret = dict_set_int32(dict, "list-cnt", list_cnt);
+    if (ret) {
+        gf_log("", GF_LOG_WARNING, "Dict set failed for list_cnt");
+        goto out;
+    }
 
-        *options = dict;
+    if ((blk_size > 0) ^ (count > 0)) {
+        cli_err("Need to give both 'bs' and 'count'");
+        ret = -1;
+        goto out;
+    } else if (((uint64_t)blk_size * count) > (10 * GF_UNIT_GB)) {
+        cli_err("'bs * count' value %" PRIu64
+                " is greater than "
+                "maximum allowed value of 10GB",
+                ((uint64_t)blk_size * count));
+        ret = -1;
+        goto out;
+    }
+
+    *options = dict;
 out:
-        if (ret && dict)
-                dict_destroy (dict);
-        return ret;
+    if (ret && dict)
+        dict_unref(dict);
+    return ret;
 }
 
 uint32_t
-cli_cmd_get_statusop (const char *arg)
+cli_cmd_get_statusop(const char *arg)
 {
-        int        i         = 0;
-        uint32_t   ret       = GF_CLI_STATUS_NONE;
-        char      *w         = NULL;
-        char      *opwords[] = {"detail", "mem", "clients", "fd",
-                                "inode", "callpool", "tasks", NULL};
-        struct {
-                char      *opname;
-                uint32_t   opcode;
-        } optable[] = {
-                { "detail",   GF_CLI_STATUS_DETAIL   },
-                { "mem",      GF_CLI_STATUS_MEM      },
-                { "clients",  GF_CLI_STATUS_CLIENTS  },
-                { "fd",       GF_CLI_STATUS_FD       },
-                { "inode",    GF_CLI_STATUS_INODE    },
-                { "callpool", GF_CLI_STATUS_CALLPOOL },
-                { "tasks",    GF_CLI_STATUS_TASKS    },
-                { NULL }
-        };
-
-        w = str_getunamb (arg, opwords);
-        if (!w) {
-                gf_log ("cli", GF_LOG_DEBUG,
-                        "Not a status op  %s", arg);
-                goto out;
-        }
+    int i = 0;
+    uint32_t ret = GF_CLI_STATUS_NONE;
+    char *w = NULL;
+    static char *opwords[] = {"detail",   "mem",   "clients",     "fd", "inode",
+                              "callpool", "tasks", "client-list", NULL};
+    static struct {
+        char *opname;
+        uint32_t opcode;
+    } optable[] = {{"detail", GF_CLI_STATUS_DETAIL},
+                   {"mem", GF_CLI_STATUS_MEM},
+                   {"clients", GF_CLI_STATUS_CLIENTS},
+                   {"fd", GF_CLI_STATUS_FD},
+                   {"inode", GF_CLI_STATUS_INODE},
+                   {"callpool", GF_CLI_STATUS_CALLPOOL},
+                   {"tasks", GF_CLI_STATUS_TASKS},
+                   {"client-list", GF_CLI_STATUS_CLIENT_LIST},
+                   {NULL}};
+
+    w = str_getunamb(arg, opwords);
+    if (!w) {
+        gf_log("cli", GF_LOG_DEBUG, "Not a status op  %s", arg);
+        goto out;
+    }
 
-        for (i = 0; optable[i].opname; i++) {
-                if (!strcmp (w, optable[i].opname)) {
-                        ret = optable[i].opcode;
-                        break;
-                }
+    for (i = 0; optable[i].opname; i++) {
+        if (!strcmp(w, optable[i].opname)) {
+            ret = optable[i].opcode;
+            break;
         }
+    }
 
- out:
-        return ret;
+out:
+    return ret;
 }
 
 int
-cli_cmd_volume_status_parse (const char **words, int wordcount,
-                             dict_t **options)
+cli_cmd_volume_status_parse(const char **words, int wordcount, dict_t **options)
 {
-        dict_t    *dict            = NULL;
-        int        ret             = -1;
-        uint32_t   cmd             = 0;
+    dict_t *dict = NULL;
+    int ret = -1;
+    uint32_t cmd = 0;
 
-        GF_ASSERT (options);
+    GF_ASSERT(options);
 
-        dict = dict_new ();
-        if (!dict)
-                goto out;
-
-        switch (wordcount) {
+    dict = dict_new();
+    if (!dict)
+        goto out;
 
+    switch (wordcount) {
         case 2:
+            cmd = GF_CLI_STATUS_ALL;
+            ret = 0;
+            break;
+
+        case 3:
+            if (!strcmp(words[2], "all")) {
                 cmd = GF_CLI_STATUS_ALL;
                 ret = 0;
-                break;
 
-        case 3:
-                if (!strcmp (words[2], "all")) {
-                        cmd = GF_CLI_STATUS_ALL;
-                        ret = 0;
+            } else {
+                cmd = GF_CLI_STATUS_VOL;
+                ret = dict_set_str(dict, "volname", (char *)words[2]);
+            }
 
-                } else {
-                        cmd = GF_CLI_STATUS_VOL;
-                        ret = dict_set_str (dict, "volname", (char *)words[2]);
-                }
-
-                break;
+            break;
 
         case 4:
-                cmd = cli_cmd_get_statusop (words[3]);
-
-                if (!strcmp (words[2], "all")) {
-                        if (cmd == GF_CLI_STATUS_NONE) {
-                                cli_err ("%s is not a valid status option",
-                                         words[3]);
-                                ret = -1;
-                                goto out;
-                        }
-                        cmd |= GF_CLI_STATUS_ALL;
-                        ret  = 0;
-
-                } else {
-                        ret = dict_set_str (dict, "volname",
-                                            (char *)words[2]);
-                        if (ret)
-                                goto out;
-
-                        if (cmd == GF_CLI_STATUS_NONE) {
-                                if (!strcmp (words[3], "nfs")) {
-                                        cmd |= GF_CLI_STATUS_NFS;
-                                } else if (!strcmp (words[3], "shd")) {
-                                        cmd |= GF_CLI_STATUS_SHD;
-                                } else if (!strcmp (words[3], "quotad")) {
-                                        cmd |= GF_CLI_STATUS_QUOTAD;
-                                } else {
-                                        cmd = GF_CLI_STATUS_BRICK;
-                                        ret = dict_set_str (dict, "brick",
-                                                            (char *)words[3]);
-                                }
-
-                        } else {
-                                cmd |= GF_CLI_STATUS_VOL;
-                                ret  = 0;
-                        }
-                }
-
-                break;
-
-        case 5:
-                if (!strcmp (words[2], "all")) {
-                        cli_err ("Cannot specify brick/nfs for \"all\"");
-                        ret = -1;
-                        goto out;
-                }
+            cmd = cli_cmd_get_statusop(words[3]);
 
-                cmd = cli_cmd_get_statusop (words[4]);
+            if (!strcmp(words[2], "all")) {
                 if (cmd == GF_CLI_STATUS_NONE) {
-                        cli_err ("%s is not a valid status option",
-                                 words[4]);
-                        ret = -1;
-                        goto out;
+                    cli_err("%s is not a valid status option", words[3]);
+                    ret = -1;
+                    goto out;
                 }
+                cmd |= GF_CLI_STATUS_ALL;
+                ret = 0;
 
-
-                ret = dict_set_str (dict, "volname", (char *)words[2]);
+            } else {
+                ret = dict_set_str(dict, "volname", (char *)words[2]);
                 if (ret)
-                        goto out;
+                    goto out;
 
-                if (!strcmp (words[3], "nfs")) {
-                        if (cmd == GF_CLI_STATUS_FD ||
-                            cmd == GF_CLI_STATUS_DETAIL ||
-                            cmd == GF_CLI_STATUS_TASKS) {
-                                cli_err ("Detail/FD/Tasks status not available"
-                                         " for NFS Servers");
-                                ret = -1;
-                                goto out;
-                        }
+                if (cmd == GF_CLI_STATUS_NONE) {
+                    if (!strcmp(words[3], "nfs")) {
                         cmd |= GF_CLI_STATUS_NFS;
-                } else if (!strcmp (words[3], "shd")){
-                        if (cmd == GF_CLI_STATUS_FD ||
-                            cmd == GF_CLI_STATUS_CLIENTS ||
-                            cmd == GF_CLI_STATUS_DETAIL ||
-                            cmd == GF_CLI_STATUS_TASKS) {
-                                cli_err ("Detail/FD/Clients/Tasks status not "
-                                         "available for Self-heal Daemons");
-                                ret = -1;
-                                goto out;
-                        }
+                    } else if (!strcmp(words[3], "shd")) {
                         cmd |= GF_CLI_STATUS_SHD;
-                } else if (!strcmp (words[3], "quotad")) {
-                        if (cmd == GF_CLI_STATUS_FD ||
-                            cmd == GF_CLI_STATUS_CLIENTS ||
-                            cmd == GF_CLI_STATUS_DETAIL ||
-                            cmd == GF_CLI_STATUS_INODE) {
-                                cli_err ("Detail/FD/Clients/Inode status not "
-                                         "available for Quota Daemon");
-                                ret = -1;
-                                goto out;
-                        }
+                    } else if (!strcmp(words[3], "quotad")) {
                         cmd |= GF_CLI_STATUS_QUOTAD;
+                    } else if (!strcmp(words[3], "snapd")) {
+                        cmd |= GF_CLI_STATUS_SNAPD;
+                    } else if (!strcmp(words[3], "bitd")) {
+                        cmd |= GF_CLI_STATUS_BITD;
+                    } else if (!strcmp(words[3], "scrub")) {
+                        cmd |= GF_CLI_STATUS_SCRUB;
+                    } else {
+                        cmd = GF_CLI_STATUS_BRICK;
+                        ret = dict_set_str(dict, "brick", (char *)words[3]);
+                    }
+
                 } else {
-                        if (cmd == GF_CLI_STATUS_TASKS) {
-                                cli_err ("Tasks status not available for "
-                                         "bricks");
-                                ret = -1;
-                                goto out;
-                        }
-                        cmd |= GF_CLI_STATUS_BRICK;
-                        ret = dict_set_str (dict, "brick", (char *)words[3]);
+                    cmd |= GF_CLI_STATUS_VOL;
+                    ret = 0;
                 }
-                break;
+            }
 
-        default:
+            break;
+
+        case 5:
+            if (!strcmp(words[2], "all")) {
+                cli_err("Cannot specify brick/nfs for \"all\"");
+                ret = -1;
                 goto out;
-        }
+            }
 
-        if (ret)
+            cmd = cli_cmd_get_statusop(words[4]);
+            if (cmd == GF_CLI_STATUS_NONE) {
+                cli_err("%s is not a valid status option", words[4]);
+                ret = -1;
                 goto out;
+            }
 
-        ret = dict_set_int32 (dict, "cmd", cmd);
-        if (ret)
+            ret = dict_set_str(dict, "volname", (char *)words[2]);
+            if (ret)
                 goto out;
 
-        *options = dict;
+            if (!strcmp(words[3], "nfs")) {
+                if (cmd == GF_CLI_STATUS_FD || cmd == GF_CLI_STATUS_DETAIL ||
+                    cmd == GF_CLI_STATUS_TASKS) {
+                    cli_err(
+                        "Detail/FD/Tasks status not available"
+                        " for NFS Servers");
+                    ret = -1;
+                    goto out;
+                }
+                cmd |= GF_CLI_STATUS_NFS;
+            } else if (!strcmp(words[3], "shd")) {
+                if (cmd == GF_CLI_STATUS_FD || cmd == GF_CLI_STATUS_CLIENTS ||
+                    cmd == GF_CLI_STATUS_DETAIL || cmd == GF_CLI_STATUS_TASKS) {
+                    cli_err(
+                        "Detail/FD/Clients/Tasks status not "
+                        "available for Self-heal Daemons");
+                    ret = -1;
+                    goto out;
+                }
+                cmd |= GF_CLI_STATUS_SHD;
+            } else if (!strcmp(words[3], "quotad")) {
+                if (cmd == GF_CLI_STATUS_FD || cmd == GF_CLI_STATUS_CLIENTS ||
+                    cmd == GF_CLI_STATUS_DETAIL || cmd == GF_CLI_STATUS_INODE) {
+                    cli_err(
+                        "Detail/FD/Clients/Inode status not "
+                        "available for Quota Daemon");
+                    ret = -1;
+                    goto out;
+                }
+                cmd |= GF_CLI_STATUS_QUOTAD;
+            } else if (!strcmp(words[3], "snapd")) {
+                if (cmd == GF_CLI_STATUS_FD || cmd == GF_CLI_STATUS_CLIENTS ||
+                    cmd == GF_CLI_STATUS_DETAIL || cmd == GF_CLI_STATUS_INODE) {
+                    cli_err(
+                        "Detail/FD/Clients/Inode status not "
+                        "available for snap daemon");
+                    ret = -1;
+                    goto out;
+                }
+                cmd |= GF_CLI_STATUS_SNAPD;
+            } else {
+                if (cmd == GF_CLI_STATUS_TASKS) {
+                    cli_err(
+                        "Tasks status not available for "
+                        "bricks");
+                    ret = -1;
+                    goto out;
+                }
+                cmd |= GF_CLI_STATUS_BRICK;
+                ret = dict_set_str(dict, "brick", (char *)words[3]);
+            }
+            break;
 
- out:
-        if (ret && dict)
-                dict_destroy (dict);
+        default:
+            goto out;
+    }
 
-        return ret;
+    if (ret)
+        goto out;
+
+    ret = dict_set_int32(dict, "cmd", cmd);
+    if (ret)
+        goto out;
+
+    *options = dict;
+
+out:
+    if (ret && dict)
+        dict_unref(dict);
+
+    return ret;
 }
 
 gf_boolean_t
-cli_cmd_validate_dumpoption (const char *arg, char **option)
+cli_cmd_validate_dumpoption(const char *arg, char **option)
 {
-        char    *opwords[] = {"all", "nfs", "mem", "iobuf", "callpool", "priv",
-                              "fd", "inode", "history", "inodectx", "fdctx",
-                              "quotad", NULL};
-        char    *w = NULL;
+    static char *opwords[] = {"all",   "nfs",    "mem",   "iobuf",   "callpool",
+                              "priv",  "fd",     "inode", "history", "inodectx",
+                              "fdctx", "quotad", NULL};
+    char *w = NULL;
+
+    w = str_getunamb(arg, opwords);
+    if (!w) {
+        gf_log("cli", GF_LOG_DEBUG, "Unknown statedump option  %s", arg);
+        return _gf_false;
+    }
+    *option = w;
+    return _gf_true;
+}
 
-        w = str_getunamb (arg, opwords);
-        if (!w) {
-                gf_log ("cli", GF_LOG_DEBUG, "Unknown statedump option  %s",
-                        arg);
-                return _gf_false;
+int
+cli_cmd_volume_statedump_options_parse(const char **words, int wordcount,
+                                       dict_t **options)
+{
+    int ret = 0;
+    int i = 0;
+    dict_t *dict = NULL;
+    int option_cnt = 0;
+    char *option = NULL;
+    char *option_str = NULL;
+    char *tmp_str = NULL;
+    char *tmp = NULL;
+    char *ip_addr = NULL;
+    char *pid = NULL;
+
+    if ((wordcount >= 5) && ((strcmp(words[3], "client")) == 0)) {
+        tmp = gf_strdup(words[4]);
+        if (!tmp) {
+            ret = -1;
+            goto out;
         }
-        *option = w;
-        return _gf_true;
+        ip_addr = strtok(tmp, ":");
+        pid = strtok(NULL, ":");
+        if (valid_internet_address(ip_addr, _gf_true, _gf_false) && pid &&
+            gf_valid_pid(pid, strlen(pid))) {
+            ret = gf_asprintf(&option_str, "%s %s %s", words[3], ip_addr, pid);
+            if (ret < 0) {
+                goto out;
+            }
+            option_cnt = 3;
+        } else {
+            ret = -1;
+            goto out;
+        }
+    } else {
+        for (i = 3; i < wordcount; i++, option_cnt++) {
+            if (!cli_cmd_validate_dumpoption(words[i], &option)) {
+                ret = -1;
+                goto out;
+            }
+            tmp_str = option_str;
+            option_str = NULL;
+            ret = gf_asprintf(&option_str, "%s%s ", tmp_str ? tmp_str : "",
+                              option);
+            GF_FREE(tmp_str);
+            if (ret < 0) {
+                goto out;
+            }
+        }
+        if (option_str && (strstr(option_str, "nfs")) &&
+            strstr(option_str, "quotad")) {
+            ret = -1;
+            goto out;
+        }
+    }
+
+    dict = dict_new();
+    if (!dict) {
+        ret = -1;
+        goto out;
+    }
+
+    /* dynamic string in dict is freed up when dict is freed up, and hence
+    if option_str is NULL pass in an duplicate empty string to the same */
+    ret = dict_set_dynstr(dict, "options",
+                          (option_str ? option_str : gf_strdup("")));
+    if (ret)
+        goto out;
+    option_str = NULL;
+
+    ret = dict_set_int32(dict, "option_cnt", option_cnt);
+    if (ret)
+        goto out;
+
+    *options = dict;
+out:
+    GF_FREE(tmp);
+    GF_FREE(option_str);
+    if (ret && dict)
+        dict_unref(dict);
+    if (ret)
+        gf_log("cli", GF_LOG_ERROR, "Error parsing dumpoptions");
+    return ret;
 }
 
 int
-cli_cmd_volume_statedump_options_parse (const char **words, int wordcount,
-                                        dict_t **options)
+cli_cmd_volume_clrlks_opts_parse(const char **words, int wordcount,
+                                 dict_t **options)
 {
-        int     ret = 0;
-        int     i = 0;
-        dict_t  *dict = NULL;
-        int     option_cnt = 0;
-        char    *option = NULL;
-        char    option_str[100] = {0,};
+    int ret = -1;
+    int i = 0;
+    dict_t *dict = NULL;
+    char *kind_opts[4] = {"blocked", "granted", "all", NULL};
+    char *types[4] = {"inode", "entry", "posix", NULL};
+    char *free_ptr = NULL;
+
+    dict = dict_new();
+    if (!dict)
+        goto out;
 
-        for (i = 3; i < wordcount; i++, option_cnt++) {
-                if (!cli_cmd_validate_dumpoption (words[i], &option)) {
-                        ret = -1;
-                        goto out;
-                }
-                strncat (option_str, option, strlen (option));
-                strncat (option_str, " ", 1);
-        }
-        if((strstr (option_str, "nfs")) && strstr (option_str, "quotad")) {
-                ret = -1;
+    if (strcmp(words[4], "kind"))
+        goto out;
+
+    for (i = 0; kind_opts[i]; i++) {
+        if (!strcmp(words[5], kind_opts[i])) {
+            free_ptr = gf_strdup(words[5]);
+            ret = dict_set_dynstr(dict, "kind", free_ptr);
+            if (ret)
                 goto out;
+            free_ptr = NULL;
+            break;
         }
+    }
+    if (i == 3)
+        goto out;
 
-        dict = dict_new ();
-        if (!dict)
+    ret = -1;
+    for (i = 0; types[i]; i++) {
+        if (!strcmp(words[6], types[i])) {
+            free_ptr = gf_strdup(words[6]);
+            ret = dict_set_dynstr(dict, "type", free_ptr);
+            if (ret)
                 goto out;
+            free_ptr = NULL;
+            break;
+        }
+    }
+    if (i == 3)
+        goto out;
 
-        ret = dict_set_dynstr (dict, "options", gf_strdup (option_str));
+    if (wordcount == 8) {
+        free_ptr = gf_strdup(words[7]);
+        ret = dict_set_dynstr(dict, "opts", free_ptr);
         if (ret)
+            goto out;
+        free_ptr = NULL;
+    }
+
+    ret = 0;
+    *options = dict;
+out:
+    if (ret) {
+        GF_FREE(free_ptr);
+        dict_unref(dict);
+    }
+
+    return ret;
+}
+
+static int
+extract_hostname_path_from_token(const char *tmp_words, char **hostname,
+                                 char **path)
+{
+    int ret = 0;
+    char *delimiter = NULL;
+    char *tmp_host = NULL;
+    char *host_name = NULL;
+    char *words = NULL;
+    int str_len = 0;
+    *hostname = NULL;
+    *path = NULL;
+
+    str_len = strlen(tmp_words) + 1;
+    words = GF_MALLOC(str_len, gf_common_mt_char);
+    if (!words) {
+        ret = -1;
+        goto out;
+    }
+
+    snprintf(words, str_len, "%s", tmp_words);
+
+    if (validate_brick_name(words)) {
+        cli_err(
+            "Wrong brick type: %s, use <HOSTNAME>:"
+            "<export-dir-abs-path>",
+            words);
+        ret = -1;
+        goto out;
+    } else {
+        delimiter = strrchr(words, ':');
+        ret = gf_canonicalize_path(delimiter + 1);
+        if (ret) {
+            goto out;
+        } else {
+            str_len = strlen(delimiter + 1) + 1;
+            *path = GF_MALLOC(str_len, gf_common_mt_char);
+            if (!*path) {
+                ret = -1;
                 goto out;
+            }
+            snprintf(*path, str_len, "%s", delimiter + 1);
+        }
+    }
 
-        ret = dict_set_int32 (dict, "option_cnt", option_cnt);
-        if (ret)
+    tmp_host = gf_strdup(words);
+    if (!tmp_host) {
+        gf_log("cli", GF_LOG_ERROR, "Out of memory");
+        ret = -1;
+        goto out;
+    }
+    get_host_name(tmp_host, &host_name);
+    if (!host_name) {
+        ret = -1;
+        gf_log("cli", GF_LOG_ERROR,
+               "Unable to allocate "
+               "memory");
+        goto out;
+    }
+    if (!(strcmp(host_name, "localhost") && strcmp(host_name, "127.0.0.1") &&
+          strncmp(host_name, "0.", 2))) {
+        cli_err(
+            "Please provide a valid hostname/ip other "
+            "than localhost, 127.0.0.1 or loopback "
+            "address (0.0.0.0 to 0.255.255.255).");
+        ret = -1;
+        goto out;
+    }
+    if (!valid_internet_address(host_name, _gf_false, _gf_false)) {
+        cli_err(
+            "internet address '%s' does not conform to "
+            "standards",
+            host_name);
+        ret = -1;
+        goto out;
+    }
+
+    str_len = strlen(host_name) + 1;
+    *hostname = GF_MALLOC(str_len, gf_common_mt_char);
+    if (!*hostname) {
+        ret = -1;
+        goto out;
+    }
+    snprintf(*hostname, str_len, "%s", host_name);
+    ret = 0;
+
+out:
+    GF_FREE(words);
+    GF_FREE(tmp_host);
+    return ret;
+}
+
+static int
+set_hostname_path_in_dict(const char *token, dict_t *dict, int heal_op)
+{
+    char *hostname = NULL;
+    char *path = NULL;
+    int ret = 0;
+
+    ret = extract_hostname_path_from_token(token, &hostname, &path);
+    if (ret)
+        goto out;
+
+    switch (heal_op) {
+        case GF_SHD_OP_SBRAIN_HEAL_FROM_BRICK:
+            ret = dict_set_dynstr(dict, "heal-source-hostname", hostname);
+            if (ret)
+                goto out;
+            hostname = NULL;
+            ret = dict_set_dynstr(dict, "heal-source-brickpath", path);
+            if (ret) {
                 goto out;
+            }
+            path = NULL;
+            break;
+        case GF_SHD_OP_STATISTICS_HEAL_COUNT_PER_REPLICA:
+            ret = dict_set_dynstr(dict, "per-replica-cmd-hostname", hostname);
+            if (ret)
+                goto out;
+            hostname = NULL;
+            ret = dict_set_dynstr(dict, "per-replica-cmd-path", path);
+            if (ret) {
+                goto out;
+            }
+            path = NULL;
+            break;
+        default:
+            ret = -1;
+            break;
+    }
 
-        *options = dict;
 out:
-        if (ret && dict)
-                dict_destroy (dict);
-        if (ret)
-                gf_log ("cli", GF_LOG_ERROR, "Error parsing dumpoptions");
-        return ret;
+    GF_FREE(hostname);
+    GF_FREE(path);
+    return ret;
+}
+
+static int
+heal_command_type_get(const char *command)
+{
+    int i = 0;
+    /* subcommands are set as NULL */
+    char *heal_cmds[GF_SHD_OP_HEAL_DISABLE + 1] = {
+        [GF_SHD_OP_INVALID] = NULL,
+        [GF_SHD_OP_HEAL_INDEX] = NULL,
+        [GF_SHD_OP_HEAL_FULL] = "full",
+        [GF_SHD_OP_INDEX_SUMMARY] = "info",
+        [GF_SHD_OP_SPLIT_BRAIN_FILES] = NULL,
+        [GF_SHD_OP_STATISTICS] = "statistics",
+        [GF_SHD_OP_STATISTICS_HEAL_COUNT] = NULL,
+        [GF_SHD_OP_STATISTICS_HEAL_COUNT_PER_REPLICA] = NULL,
+        [GF_SHD_OP_SBRAIN_HEAL_FROM_BIGGER_FILE] = NULL,
+        [GF_SHD_OP_SBRAIN_HEAL_FROM_BRICK] = NULL,
+        [GF_SHD_OP_HEAL_ENABLE] = "enable",
+        [GF_SHD_OP_HEAL_DISABLE] = "disable",
+    };
+
+    for (i = 0; i <= GF_SHD_OP_HEAL_DISABLE; i++) {
+        if (heal_cmds[i] && (strcmp(heal_cmds[i], command) == 0))
+            return i;
+    }
+
+    return GF_SHD_OP_INVALID;
 }
 
 int
-cli_cmd_volume_clrlks_opts_parse (const char **words, int wordcount,
+cli_cmd_volume_heal_options_parse(const char **words, int wordcount,
                                   dict_t **options)
 {
-        int     ret = -1;
-        int     i = 0;
-        dict_t  *dict = NULL;
-        char    *kind_opts[4] = {"blocked", "granted", "all", NULL};
-        char    *types[4] = {"inode", "entry", "posix", NULL};
-        char    *free_ptr = NULL;
+    int ret = 0;
+    dict_t *dict = NULL;
+    gf_xl_afr_op_t op = GF_SHD_OP_INVALID;
 
-        dict = dict_new ();
-        if (!dict)
-                goto out;
+    dict = dict_new();
+    if (!dict) {
+        gf_log(THIS->name, GF_LOG_ERROR, "Failed to create the dict");
+        ret = -1;
+        goto out;
+    }
 
-        if (strcmp (words[4], "kind"))
-                goto out;
+    ret = dict_set_str(dict, "volname", (char *)words[2]);
+    if (ret) {
+        gf_log(THIS->name, GF_LOG_ERROR, "failed to set volname");
+        goto out;
+    }
+
+    if (wordcount == 3) {
+        ret = dict_set_int32(dict, "heal-op", GF_SHD_OP_HEAL_INDEX);
+        goto done;
+    }
+
+    if (wordcount == 4) {
+        op = heal_command_type_get(words[3]);
+        if (op == GF_SHD_OP_INVALID) {
+            ret = -1;
+            goto out;
+        }
 
-        for (i = 0; kind_opts[i]; i++) {
-               if (!strcmp (words[5], kind_opts[i])) {
-                       free_ptr = gf_strdup (words[5]);
-                       ret = dict_set_dynstr (dict, "kind", free_ptr);
-                       if (ret)
-                               goto out;
-                       free_ptr = NULL;
-                       break;
-               }
+        ret = dict_set_int32(dict, "heal-op", op);
+        goto done;
+    }
+
+    if (wordcount == 5) {
+        if (strcmp(words[3], "info") && strcmp(words[3], "statistics") &&
+            strcmp(words[3], "granular-entry-heal")) {
+            ret = -1;
+            goto out;
+        }
+
+        if (!strcmp(words[3], "info")) {
+            if (!strcmp(words[4], "split-brain")) {
+                ret = dict_set_int32(dict, "heal-op",
+                                     GF_SHD_OP_SPLIT_BRAIN_FILES);
+                goto done;
+            }
+            if (!strcmp(words[4], "summary")) {
+                ret = dict_set_int32(dict, "heal-op", GF_SHD_OP_HEAL_SUMMARY);
+                goto done;
+            }
         }
-        if (i == 3)
-                goto out;
 
+        if (!strcmp(words[3], "statistics")) {
+            if (!strcmp(words[4], "heal-count")) {
+                ret = dict_set_int32(dict, "heal-op",
+                                     GF_SHD_OP_STATISTICS_HEAL_COUNT);
+                goto done;
+            }
+        }
+
+        if (!strcmp(words[3], "granular-entry-heal")) {
+            if (!strcmp(words[4], "enable")) {
+                ret = dict_set_int32(dict, "heal-op",
+                                     GF_SHD_OP_GRANULAR_ENTRY_HEAL_ENABLE);
+                goto done;
+            } else if (!strcmp(words[4], "disable")) {
+                ret = dict_set_int32(dict, "heal-op",
+                                     GF_SHD_OP_GRANULAR_ENTRY_HEAL_DISABLE);
+                goto done;
+            }
+        }
+
+        ret = -1;
+        goto out;
+    }
+    if (wordcount == 6) {
+        if (strcmp(words[3], "split-brain")) {
+            ret = -1;
+            goto out;
+        }
+        if (!strcmp(words[4], "bigger-file")) {
+            ret = dict_set_int32(dict, "heal-op",
+                                 GF_SHD_OP_SBRAIN_HEAL_FROM_BIGGER_FILE);
+            if (ret)
+                goto out;
+            ret = dict_set_str(dict, "file", (char *)words[5]);
+            if (ret)
+                goto out;
+            goto done;
+        }
+        if (!strcmp(words[4], "latest-mtime")) {
+            ret = dict_set_int32(dict, "heal-op",
+                                 GF_SHD_OP_SBRAIN_HEAL_FROM_LATEST_MTIME);
+            if (ret)
+                goto out;
+            ret = dict_set_str(dict, "file", (char *)words[5]);
+            if (ret)
+                goto out;
+            goto done;
+        }
+        if (!strcmp(words[4], "source-brick")) {
+            ret = dict_set_int32(dict, "heal-op",
+                                 GF_SHD_OP_SBRAIN_HEAL_FROM_BRICK);
+            if (ret)
+                goto out;
+            ret = set_hostname_path_in_dict(words[5], dict,
+                                            GF_SHD_OP_SBRAIN_HEAL_FROM_BRICK);
+            if (ret)
+                goto out;
+            goto done;
+        }
         ret = -1;
-        for (i = 0; types[i]; i++) {
-               if (!strcmp (words[6], types[i])) {
-                       free_ptr = gf_strdup (words[6]);
-                       ret = dict_set_dynstr (dict, "type", free_ptr);
-                       if (ret)
-                               goto out;
-                       free_ptr = NULL;
-                       break;
-               }
+        goto out;
+    }
+    if (wordcount == 7) {
+        if (!strcmp(words[3], "statistics") &&
+            !strcmp(words[4], "heal-count") && !strcmp(words[5], "replica")) {
+            ret = dict_set_int32(dict, "heal-op",
+                                 GF_SHD_OP_STATISTICS_HEAL_COUNT_PER_REPLICA);
+            if (ret)
+                goto out;
+            ret = set_hostname_path_in_dict(
+                words[6], dict, GF_SHD_OP_STATISTICS_HEAL_COUNT_PER_REPLICA);
+            if (ret)
+                goto out;
+            goto done;
         }
-        if (i == 3)
+        if (!strcmp(words[3], "split-brain") &&
+            !strcmp(words[4], "source-brick")) {
+            ret = dict_set_int32(dict, "heal-op",
+                                 GF_SHD_OP_SBRAIN_HEAL_FROM_BRICK);
+            ret = set_hostname_path_in_dict(words[5], dict,
+                                            GF_SHD_OP_SBRAIN_HEAL_FROM_BRICK);
+            if (ret)
+                goto out;
+            ret = dict_set_str(dict, "file", (char *)words[6]);
+            if (ret)
                 goto out;
+            goto done;
+        }
+    }
+    ret = -1;
+    goto out;
+done:
+    *options = dict;
+out:
+    if (ret && dict) {
+        dict_unref(dict);
+        *options = NULL;
+    }
 
-        if (wordcount == 8) {
-                free_ptr = gf_strdup (words[7]);
-                ret = dict_set_dynstr (dict, "opts", free_ptr);
-                if (ret)
-                        goto out;
-                free_ptr = NULL;
+    return ret;
+}
+
+int
+cli_cmd_volume_defrag_parse(const char **words, int wordcount, dict_t **options)
+{
+    dict_t *dict = NULL;
+    int ret = -1;
+    char *option = NULL;
+    char *volname = NULL;
+    char *command = NULL;
+    gf_cli_defrag_type cmd = 0;
+
+    GF_ASSERT(words);
+    GF_ASSERT(options);
+
+    dict = dict_new();
+    if (!dict)
+        goto out;
+
+    if (!((wordcount == 4) || (wordcount == 5)))
+        goto out;
+
+    if (wordcount == 4) {
+        if (strcmp(words[3], "start") && strcmp(words[3], "stop") &&
+            strcmp(words[3], "status"))
+            goto out;
+    } else {
+        if (strcmp(words[3], "fix-layout") && strcmp(words[3], "start"))
+            goto out;
+    }
+
+    volname = (char *)words[2];
+
+    if (wordcount == 4) {
+        command = (char *)words[3];
+    }
+    if (wordcount == 5) {
+        if ((strcmp(words[3], "fix-layout") || strcmp(words[4], "start")) &&
+            (strcmp(words[3], "start") || strcmp(words[4], "force"))) {
+            ret = -1;
+            goto out;
         }
+        command = (char *)words[3];
+        option = (char *)words[4];
+    }
+
+    if (strcmp(command, "start") == 0) {
+        cmd = GF_DEFRAG_CMD_START;
+        if (option && strcmp(option, "force") == 0) {
+            cmd = GF_DEFRAG_CMD_START_FORCE;
+        }
+        goto done;
+    }
+
+    if (strcmp(command, "fix-layout") == 0) {
+        cmd = GF_DEFRAG_CMD_START_LAYOUT_FIX;
+        goto done;
+    }
+    if (strcmp(command, "stop") == 0) {
+        cmd = GF_DEFRAG_CMD_STOP;
+        goto done;
+    }
+    if (strcmp(command, "status") == 0) {
+        cmd = GF_DEFRAG_CMD_STATUS;
+    }
+
+done:
+    ret = dict_set_str(dict, "volname", volname);
+
+    if (ret) {
+        gf_log(THIS->name, GF_LOG_ERROR, "failed to set dict");
+        goto out;
+    }
+
+    ret = dict_set_int32(dict, "rebalance-command", (int32_t)cmd);
+
+    if (ret) {
+        gf_log(THIS->name, GF_LOG_ERROR, "failed to set dict");
+        goto out;
+    }
+
+    *options = dict;
 
-        ret = 0;
-        *options = dict;
 out:
-       if (ret) {
-               GF_FREE (free_ptr);
-               dict_unref (dict);
-       }
+    if (ret && dict)
+        dict_unref(dict);
 
-       return ret;
+    return ret;
 }
 
-static int
-extract_hostname_path_from_token (const char *tmp_words, char **hostname,
-                                  char **path)
+int32_t
+cli_snap_create_desc_parse(dict_t *dict, const char **words, size_t wordcount,
+                           int32_t desc_opt_loc)
 {
-        int ret = 0;
-        char *delimiter = NULL;
-        char *tmp_host = NULL;
-        char *host_name = NULL;
-        char *words = NULL;
+    int32_t ret = -1;
+    char *desc = NULL;
+    int32_t desc_len = 0;
+    int len;
 
-        *hostname = NULL;
-        *path = NULL;
+    desc = GF_MALLOC(MAX_SNAP_DESCRIPTION_LEN + 1, gf_common_mt_char);
+    if (!desc) {
+        ret = -1;
+        goto out;
+    }
+
+    len = strlen(words[desc_opt_loc]);
+    if (len >= MAX_SNAP_DESCRIPTION_LEN) {
+        cli_out(
+            "snapshot create: description truncated: "
+            "Description provided is longer than 1024 characters");
+        desc_len = MAX_SNAP_DESCRIPTION_LEN;
+    } else {
+        desc_len = len;
+    }
+
+    snprintf(desc, desc_len + 1, "%s", words[desc_opt_loc]);
+    /* Calculating the size of the description as given by the user */
+
+    ret = dict_set_dynstr(dict, "description", desc);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR,
+               "Unable to save snap "
+               "description");
+        goto out;
+    }
 
-        words = GF_CALLOC (1, strlen (tmp_words) + 1, gf_common_mt_char);
-        if (!words){
-                ret = -1;
-                goto out;
+    ret = 0;
+out:
+    if (ret && desc)
+        GF_FREE(desc);
+
+    return ret;
+}
+
+/* Function to check whether the Volume name is repeated */
+int
+cli_check_if_volname_repeated(const char **words, unsigned int start_index,
+                              uint64_t cur_index)
+{
+    uint64_t i = -1;
+    int ret = 0;
+
+    GF_ASSERT(words);
+
+    for (i = start_index; i < cur_index; i++) {
+        if (strcmp(words[i], words[cur_index]) == 0) {
+            ret = -1;
+            goto out;
         }
+    }
+out:
+    return ret;
+}
 
-        strncpy (words, tmp_words, strlen (tmp_words) + 1);
+/* snapshot clone <clonename> <snapname>
+ * @arg-0, dict     : Request Dictionary to be sent to server side.
+ * @arg-1, words    : Contains individual words of CLI command.
+ * @arg-2, wordcount: Contains number of words present in the CLI command.
+ *
+ * return value : -1 on failure
+ *                 0 on success
+ */
+int
+cli_snap_clone_parse(dict_t *dict, const char **words, int wordcount)
+{
+    uint64_t i = 0;
+    int ret = -1;
+    char *clonename = NULL;
+    unsigned int cmdi = 2;
+    /* cmdi is command index, here cmdi is "2" (gluster snapshot clone)*/
+
+    GF_ASSERT(words);
+    GF_ASSERT(dict);
+
+    if (wordcount == cmdi + 1) {
+        cli_err("Invalid Syntax.");
+        gf_log("cli", GF_LOG_ERROR,
+               "Invalid number of  words for snap clone command");
+        goto out;
+    }
 
-        if (validate_brick_name (words)) {
-                cli_err ("Wrong brick type: %s, use <HOSTNAME>:"
-                        "<export-dir-abs-path>", words);
-                ret = -1;
-                goto out;
-        } else {
-                delimiter = strrchr (words, ':');
-                ret = gf_canonicalize_path (delimiter + 1);
-                if (ret) {
-                        goto out;
-                } else {
-                        *path = GF_CALLOC (1, strlen (delimiter+1) +1,
-                                           gf_common_mt_char);
-                        if (!*path) {
-                           ret = -1;
-                                goto out;
+    if (strlen(words[cmdi]) >= GLUSTERD_MAX_SNAP_NAME) {
+        cli_err(
+            "snapshot clone: failed: clonename cannot exceed "
+            "255 characters.");
+        gf_log("cli", GF_LOG_ERROR, "Clone name too long");
 
-                        }
-                        strncpy (*path, delimiter +1,
-                                 strlen(delimiter + 1) + 1);
-                }
+        goto out;
+    }
+
+    clonename = (char *)words[cmdi];
+    for (i = 0; i < strlen(clonename); i++) {
+        /* Following volume name convention */
+        if (!isalnum(clonename[i]) &&
+            (clonename[i] != '_' && (clonename[i] != '-'))) {
+            /* TODO : Is this message enough?? */
+            cli_err(
+                "Clonename can contain only alphanumeric, "
+                "\"-\" and \"_\" characters");
+            goto out;
         }
+    }
 
-        tmp_host = gf_strdup (words);
-        if (!tmp_host) {
-                gf_log ("cli", GF_LOG_ERROR, "Out of memory");
-                ret = -1;
-                goto out;
+    ret = dict_set_int32(dict, "volcount", 1);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Could not save volcount");
+        goto out;
+    }
+
+    ret = dict_set_str(dict, "clonename", (char *)words[cmdi]);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR,
+               "Could not save clone "
+               "name(%s)",
+               (char *)words[cmdi]);
+        goto out;
+    }
+
+    /* Filling snap name in the dictionary */
+    ret = dict_set_str(dict, "snapname", (char *)words[cmdi + 1]);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR,
+               "Could not "
+               "save snap name(%s)",
+               (char *)words[cmdi + 1]);
+        goto out;
+    }
+
+    ret = 0;
+
+out:
+    return ret;
+}
+
+/* snapshot create <snapname> <vol-name(s)> [description <description>]
+ *                                           [force]
+ * @arg-0, dict     : Request Dictionary to be sent to server side.
+ * @arg-1, words    : Contains individual words of CLI command.
+ * @arg-2, wordcount: Contains number of words present in the CLI command.
+ *
+ * return value : -1 on failure
+ *                 0 on success
+ */
+int
+cli_snap_create_parse(dict_t *dict, const char **words, int wordcount)
+{
+    uint64_t i = 0;
+    int ret = -1;
+    uint64_t volcount = 0;
+    char key[PATH_MAX] = "";
+    char *snapname = NULL;
+    unsigned int cmdi = 2;
+    int flags = 0;
+    /* cmdi is command index, here cmdi is "2" (gluster snapshot create)*/
+
+    GF_ASSERT(words);
+    GF_ASSERT(dict);
+
+    if (wordcount <= cmdi + 1) {
+        cli_err("Invalid Syntax.");
+        gf_log("cli", GF_LOG_ERROR, "Too less words for snap create command");
+        goto out;
+    }
+
+    if (strlen(words[cmdi]) >= GLUSTERD_MAX_SNAP_NAME) {
+        cli_err(
+            "snapshot create: failed: snapname cannot exceed "
+            "255 characters.");
+        gf_log("cli", GF_LOG_ERROR, "Snapname too long");
+
+        goto out;
+    }
+
+    snapname = (char *)words[cmdi];
+    for (i = 0; i < strlen(snapname); i++) {
+        /* Following volume name convention */
+        if (!isalnum(snapname[i]) &&
+            (snapname[i] != '_' && (snapname[i] != '-'))) {
+            /* TODO : Is this message enough?? */
+            cli_err(
+                "Snapname can contain only alphanumeric, "
+                "\"-\" and \"_\" characters");
+            goto out;
         }
-        get_host_name (tmp_host, &host_name);
-        if (!host_name) {
-                ret = -1;
-                gf_log("cli",GF_LOG_ERROR, "Unable to allocate "
-                        "memory");
-                goto out;
+    }
+
+    ret = dict_set_str(dict, "snapname", (char *)words[cmdi]);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR,
+               "Could not save snap "
+               "name(%s)",
+               (char *)words[cmdi]);
+        goto out;
+    }
+
+    /* Filling volume name in the dictionary */
+    for (i = cmdi + 1;
+         i < wordcount && (strcmp(words[i], "description")) != 0 &&
+         (strcmp(words[i], "force") != 0) &&
+         (strcmp(words[i], "no-timestamp") != 0);
+         i++) {
+        volcount++;
+        /* volume index starts from 1 */
+        ret = snprintf(key, sizeof(key), "volname%" PRIu64, volcount);
+        if (ret < 0) {
+            goto out;
         }
-        if (!(strcmp (host_name, "localhost") &&
-            strcmp (host_name, "127.0.0.1") &&
-            strncmp (host_name, "0.", 2))) {
-                cli_err ("Please provide a valid hostname/ip other "
-                         "than localhost, 127.0.0.1 or loopback "
-                         "address (0.0.0.0 to 0.255.255.255).");
-                ret = -1;
-                goto out;
+
+        ret = dict_set_str(dict, key, (char *)words[i]);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Could not "
+                   "save volume name(%s)",
+                   (char *)words[i]);
+            goto out;
         }
-        if (!valid_internet_address (host_name, _gf_false)) {
-                cli_err ("internet address '%s' does not conform to "
-                          "standards", host_name);
-                ret = -1;
-                goto out;
+
+        if (i >= cmdi + 2) {
+            ret = -1;
+            cli_err(
+                "Creating multiple volume snapshot is not "
+                "supported as of now");
+            goto out;
         }
+        /* TODO : remove this above condition check once
+         * multiple volume snapshot is supported */
+    }
 
-        *hostname = GF_CALLOC (1, strlen (host_name) + 1,
-                                       gf_common_mt_char);
-        if (!*hostname) {
-                ret = -1;
-                goto out;
+    if (volcount == 0) {
+        ret = -1;
+        cli_err("Please provide the volume name");
+        gf_log("cli", GF_LOG_ERROR, "Invalid Syntax");
+        goto out;
+    }
+
+    ret = dict_set_int32(dict, "volcount", volcount);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Could not save volcount");
+        goto out;
+    }
+
+    /* Verify how we got out of "for" loop,
+     * if it is by reaching wordcount limit then goto "out",
+     * because we need not parse for "description","force" and
+     * "no-timestamp" after this.
+     */
+    if (i == wordcount) {
+        goto out;
+    }
+
+    if (strcmp(words[i], "no-timestamp") == 0) {
+        ret = dict_set_int32n(dict, "no-timestamp", SLEN("no-timestamp"), 1);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Could not save "
+                   "time-stamp option");
         }
-        strncpy (*hostname, host_name, strlen (host_name) + 1);
-        ret = 0;
+        if (i == (wordcount - 1))
+            goto out;
+        i++;
+    }
+
+    if ((strcmp(words[i], "description")) == 0) {
+        ++i;
+        if (i > (wordcount - 1)) {
+            ret = -1;
+            cli_err("Please provide a description");
+            gf_log("cli", GF_LOG_ERROR, "Description not provided");
+            goto out;
+        }
+
+        ret = cli_snap_create_desc_parse(dict, words, wordcount, i);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Could not save snap "
+                   "description");
+            goto out;
+        }
+
+        if (i == (wordcount - 1))
+            goto out;
+        i++;
+        /* point the index to next word.
+         * As description might be follwed by force option.
+         * Before that, check if wordcount limit is reached
+         */
+    }
+
+    if (strcmp(words[i], "force") == 0) {
+        flags = GF_CLI_FLAG_OP_FORCE;
+
+    } else {
+        ret = -1;
+        cli_err("Invalid Syntax.");
+        gf_log("cli", GF_LOG_ERROR, "Invalid Syntax");
+        goto out;
+    }
+
+    /* Check if the command has anything after "force" keyword */
+    if (++i < wordcount) {
+        ret = -1;
+        gf_log("cli", GF_LOG_ERROR, "Invalid Syntax");
+        goto out;
+    }
+
+    ret = 0;
 
 out:
-        GF_FREE (words);
-        GF_FREE (tmp_host);
-        return ret;
+    if (ret == 0) {
+        /*Adding force flag in either of the case i.e force set
+         * or unset*/
+        ret = dict_set_int32(dict, "flags", flags);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Could not save "
+                   "snap force option");
+        }
+    }
+    return ret;
 }
 
+/* snapshot list [volname]
+ * @arg-0, dict     : Request Dictionary to be sent to server side.
+ * @arg-1, words    : Contains individual words of CLI command.
+ * @arg-2, wordcount: Contains number of words present in the CLI command.
+ *
+ * return value : -1 on failure
+ *                 0 on success
+ */
+int
+cli_snap_list_parse(dict_t *dict, const char **words, int wordcount)
+{
+    int ret = -1;
+
+    GF_ASSERT(words);
+    GF_ASSERT(dict);
+
+    if (wordcount < 2 || wordcount > 3) {
+        gf_log("cli", GF_LOG_ERROR, "Invalid Syntax");
+        goto out;
+    }
+
+    if (wordcount == 2) {
+        ret = 0;
+        goto out;
+    }
+
+    ret = dict_set_str(dict, "volname", (char *)words[2]);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to save volname in dictionary");
+        goto out;
+    }
+out:
+    return ret;
+}
 
+/* snapshot info [(snapname |  volume <volname>)]
+ * @arg-0, dict     : Request Dictionary to be sent to server side.
+ * @arg-1, words    : Contains individual words of CLI command.
+ * @arg-2, wordcount: Contains number of words present in the CLI command.
+ *
+ * return value : -1 on failure
+ *                 0 on success
+ */
 int
-cli_cmd_volume_heal_options_parse (const char **words, int wordcount,
-                                   dict_t **options)
+cli_snap_info_parse(dict_t *dict, const char **words, int wordcount)
 {
-        int     ret = 0;
-        dict_t  *dict = NULL;
-        char    *hostname = NULL;
-        char    *path = NULL;
+    int ret = -1;
+    int32_t cmd = GF_SNAP_INFO_TYPE_ALL;
+    unsigned int cmdi = 2;
+    /* cmdi is command index, here cmdi is "2" (gluster snapshot info)*/
 
-        dict = dict_new ();
-        if (!dict)
-                goto out;
+    GF_ASSERT(words);
+    GF_ASSERT(dict);
+
+    if (wordcount > 4 || wordcount < cmdi) {
+        gf_log("cli", GF_LOG_ERROR, "Invalid syntax");
+        goto out;
+    }
 
-        ret = dict_set_str (dict, "volname", (char *) words[2]);
+    if (wordcount == cmdi) {
+        ret = 0;
+        goto out;
+    }
+
+    /* If 3rd word is not "volume", then it must
+     * be snapname.
+     */
+    if (strcmp(words[cmdi], "volume") != 0) {
+        ret = dict_set_str(dict, "snapname", (char *)words[cmdi]);
         if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "failed to set volname");
-                goto out;
+            gf_log("cli", GF_LOG_ERROR,
+                   "Unable to save "
+                   "snapname %s",
+                   words[cmdi]);
+            goto out;
         }
 
-        if (wordcount == 3) {
-                ret = dict_set_int32 (dict, "heal-op", GF_AFR_OP_HEAL_INDEX);
-                goto done;
+        /* Once snap name is parsed, if we encounter any other
+         * word then fail it. Invalid Syntax.
+         * example : snapshot info <snapname> word
+         */
+        if ((cmdi + 1) != wordcount) {
+            ret = -1;
+            gf_log("cli", GF_LOG_ERROR, "Invalid Syntax");
+            goto out;
         }
 
-        if (wordcount == 4) {
-                if (!strcmp (words[3], "full")) {
-                        ret = dict_set_int32 (dict, "heal-op",
-                                              GF_AFR_OP_HEAL_FULL);
-                        goto done;
-                } else if (!strcmp (words[3], "statistics")) {
-                        ret = dict_set_int32 (dict, "heal-op",
-                                              GF_AFR_OP_STATISTICS);
-                        goto done;
-
-                } else if (!strcmp (words[3], "info")) {
-                        ret = dict_set_int32 (dict, "heal-op",
-                                              GF_AFR_OP_INDEX_SUMMARY);
-                        goto done;
-                } else {
-                        ret = -1;
-                        goto out;
-                }
+        cmd = GF_SNAP_INFO_TYPE_SNAP;
+        ret = 0;
+        goto out;
+        /* No need to continue the parsing once we
+         * get the snapname
+         */
+    }
+
+    /* If 3rd word is "volume", then check if next word
+     * is present. As, "snapshot info volume" is an
+     * invalid command.
+     */
+    if ((cmdi + 1) == wordcount) {
+        ret = -1;
+        gf_log("cli", GF_LOG_ERROR, "Invalid Syntax");
+        goto out;
+    }
+
+    ret = dict_set_str(dict, "volname", (char *)words[wordcount - 1]);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR,
+               "Could not save "
+               "volume name %s",
+               words[wordcount - 1]);
+        goto out;
+    }
+    cmd = GF_SNAP_INFO_TYPE_VOL;
+out:
+    if (ret == 0) {
+        ret = dict_set_int32(dict, "sub-cmd", cmd);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Could not save "
+                   "type of snapshot info");
         }
-        if (wordcount == 5) {
-                if (strcmp (words[3], "info") &&
-                    strcmp (words[3], "statistics")) {
-                        ret = -1;
-                        goto out;
-                }
+    }
+    return ret;
+}
 
-                if (!strcmp (words[3], "info")) {
-                        if (!strcmp (words[4], "healed")) {
-                                ret = dict_set_int32 (dict, "heal-op",
-                                                      GF_AFR_OP_HEALED_FILES);
-                                goto done;
-                        }
-                        if (!strcmp (words[4], "heal-failed")) {
-                                ret = dict_set_int32 (dict, "heal-op",
-                                                   GF_AFR_OP_HEAL_FAILED_FILES);
-                                goto done;
-                        }
-                        if (!strcmp (words[4], "split-brain")) {
-                                ret = dict_set_int32 (dict, "heal-op",
-                                                   GF_AFR_OP_SPLIT_BRAIN_FILES);
-                                goto done;
-                        }
-                }
+/* snapshot restore <snapname>
+ * @arg-0, dict     : Request Dictionary to be sent to server side.
+ * @arg-1, words    : Contains individual words of CLI command.
+ * @arg-2, wordcount: Contains number of words present in the CLI command.
+ *
+ * return value : -1 on failure
+ *                 0 on success
+ */
+int
+cli_snap_restore_parse(dict_t *dict, const char **words, int wordcount,
+                       struct cli_state *state)
+{
+    int ret = -1;
+    const char *question = NULL;
+    gf_answer_t answer = GF_ANSWER_NO;
 
-                if (!strcmp (words[3], "statistics")) {
-                        if (!strcmp (words[4], "heal-count")) {
-                                ret = dict_set_int32 (dict, "heal-op",
-                                               GF_AFR_OP_STATISTICS_HEAL_COUNT);
-                                goto done;
-                        }
-                }
-                ret = -1;
-                goto out;
+    GF_ASSERT(words);
+    GF_ASSERT(dict);
+
+    if (wordcount != 3) {
+        gf_log("cli", GF_LOG_ERROR, "Invalid Syntax");
+        goto out;
+    }
+
+    ret = dict_set_str(dict, "snapname", (char *)words[2]);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Unable to save snap-name %s", words[2]);
+        goto out;
+    }
+
+    question =
+        "Restore operation will replace the "
+        "original volume with the snapshotted volume. "
+        "Do you still want to continue?";
+
+    answer = cli_cmd_get_confirmation(state, question);
+    if (GF_ANSWER_NO == answer) {
+        ret = 1;
+        gf_log("cli", GF_LOG_ERROR,
+               "User cancelled a snapshot "
+               "restore operation for snap %s",
+               (char *)words[2]);
+        goto out;
+    }
+out:
+    return ret;
+}
+
+/* snapshot activate <snapname> [force]
+ * @arg-0, dict     : Request Dictionary to be sent to server side.
+ * @arg-1, words    : Contains individual words of CLI command.
+ * @arg-2, wordcount: Contains number of words present in the CLI command.
+ *
+ * return value : -1 on failure
+ *                 0 on success
+ */
+int
+cli_snap_activate_parse(dict_t *dict, const char **words, int wordcount)
+{
+    int ret = -1;
+    int flags = 0;
+
+    GF_ASSERT(words);
+    GF_ASSERT(dict);
+
+    if ((wordcount < 3) || (wordcount > 4)) {
+        gf_log("cli", GF_LOG_ERROR, "Invalid Syntax");
+        goto out;
+    }
+
+    ret = dict_set_str(dict, "snapname", (char *)words[2]);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Unable to save snap-name %s", words[2]);
+        goto out;
+    }
+
+    if (wordcount == 4) {
+        if (!strcmp("force", (char *)words[3])) {
+            flags = GF_CLI_FLAG_OP_FORCE;
+        } else {
+            gf_log("cli", GF_LOG_ERROR, "Invalid option");
+            ret = -1;
+            goto out;
         }
-        if (wordcount == 7) {
-                if (!strcmp (words[3], "statistics")
-                    && !strcmp (words[4], "heal-count")
-                    && !strcmp (words[5], "replica")) {
-
-                        ret = dict_set_int32 (dict, "heal-op",
-                                   GF_AFR_OP_STATISTICS_HEAL_COUNT_PER_REPLICA);
-                        if (ret)
-                                goto out;
-                        ret = extract_hostname_path_from_token (words[6],
-                                                              &hostname, &path);
-                        if (ret)
-                                goto out;
-                        ret = dict_set_dynstr (dict, "per-replica-cmd-hostname",
-                                               hostname);
-                        if (ret)
-                                goto out;
-                        ret = dict_set_dynstr (dict, "per-replica-cmd-path",
-                                               path);
-                        if (ret)
-                                goto out;
-                        else
-                                goto done;
+    }
+    ret = dict_set_int32(dict, "flags", flags);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Unable to save force option");
+        goto out;
+    }
+out:
+    return ret;
+}
 
-                }
+/* snapshot deactivate <snapname>
+ * @arg-0, dict     : Request Dictionary to be sent to server side.
+ * @arg-1, words    : Contains individual words of CLI command.
+ * @arg-2, wordcount: Contains number of words present in the CLI command.
+ *
+ * return value : -1 on failure
+ *                 0 on success
+ *                 1 if user cancelled the request
+ */
+int
+cli_snap_deactivate_parse(dict_t *dict, const char **words, int wordcount,
+                          struct cli_state *state)
+{
+    int ret = -1;
+    gf_answer_t answer = GF_ANSWER_NO;
+    const char *question =
+        "Deactivating snap will make its "
+        "data inaccessible. Do you want to "
+        "continue?";
+
+    GF_ASSERT(words);
+    GF_ASSERT(dict);
+
+    if ((wordcount != 3)) {
+        gf_log("cli", GF_LOG_ERROR, "Invalid Syntax");
+        goto out;
+    }
+
+    ret = dict_set_str(dict, "snapname", (char *)words[2]);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Unable to save snap-name %s", words[2]);
+        goto out;
+    }
+
+    answer = cli_cmd_get_confirmation(state, question);
+    if (GF_ANSWER_NO == answer) {
+        ret = 1;
+        gf_log("cli", GF_LOG_DEBUG,
+               "User cancelled "
+               "snapshot deactivate operation");
+        goto out;
+    }
+
+out:
+    return ret;
+}
+
+/* snapshot delete (all | snapname | volume <volname>)
+ * @arg-0, dict     : Request Dictionary to be sent to server side.
+ * @arg-1, words    : Contains individual words of CLI command.
+ * @arg-2, wordcount: Contains number of words present in the CLI command.
+ *
+ * return value : -1 on failure
+ *                 0 on success
+ *                 1 if user cancel the operation
+ */
+int
+cli_snap_delete_parse(dict_t *dict, const char **words, int wordcount,
+                      struct cli_state *state)
+{
+    int ret = -1;
+    const char *question = NULL;
+    int32_t cmd = -1;
+    unsigned int cmdi = 2;
+    gf_answer_t answer = GF_ANSWER_NO;
+
+    GF_ASSERT(words);
+    GF_ASSERT(dict);
+
+    if (wordcount > 4 || wordcount <= cmdi) {
+        gf_log("cli", GF_LOG_ERROR, "Invalid Syntax");
+        goto out;
+    }
+
+    question =
+        "Deleting snap will erase all the information about "
+        "the snap. Do you still want to continue?";
+
+    if (strcmp(words[cmdi], "all") == 0) {
+        ret = 0;
+        cmd = GF_SNAP_DELETE_TYPE_ALL;
+    } else if (strcmp(words[cmdi], "volume") == 0) {
+        if (++cmdi == wordcount) {
+            ret = -1;
+            gf_log("cli", GF_LOG_ERROR, "Invalid Syntax");
+            goto out;
         }
+
+        ret = dict_set_str(dict, "volname", (char *)words[cmdi]);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Could not save "
+                   "volume name %s",
+                   words[wordcount - 1]);
+            goto out;
+        }
+        cmd = GF_SNAP_DELETE_TYPE_VOL;
+    } else {
+        ret = dict_set_str(dict, "snapname", (char *)words[cmdi]);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Unable to save "
+                   "snapname %s",
+                   words[2]);
+            goto out;
+        }
+        cmd = GF_SNAP_DELETE_TYPE_SNAP;
+    }
+
+    if ((cmdi + 1) != wordcount) {
         ret = -1;
+        gf_log("cli", GF_LOG_ERROR, "Invalid Syntax");
         goto out;
-done:
-        *options = dict;
+    }
+
+    if (cmd == GF_SNAP_DELETE_TYPE_SNAP) {
+        answer = cli_cmd_get_confirmation(state, question);
+        if (GF_ANSWER_NO == answer) {
+            ret = 1;
+            gf_log("cli", GF_LOG_DEBUG,
+                   "User cancelled "
+                   "snapshot delete operation for snap %s",
+                   (char *)words[2]);
+            goto out;
+        }
+    }
+
+    ret = dict_set_int32(dict, "sub-cmd", cmd);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR,
+               "Could not save "
+               "type of snapshot delete");
+    }
+out:
+    return ret;
+}
+
+/* snapshot status [(snapname | volume <volname>)]
+ * @arg-0, dict     : Request Dictionary to be sent to server side.
+ * @arg-1, words    : Contains individual words of CLI command.
+ * @arg-2, wordcount: Contains number of words present in the CLI command.
+ *
+ * return value : -1 on failure
+ *                 0 on success
+ */
+int
+cli_snap_status_parse(dict_t *dict, const char **words, int wordcount)
+{
+    int ret = -1;
+    int32_t cmd = GF_SNAP_STATUS_TYPE_ALL;
+    unsigned int cmdi = 2;
+    /* cmdi is command index, here cmdi is "2" (gluster snapshot status)*/
+
+    GF_ASSERT(words);
+    GF_ASSERT(dict);
+
+    if (wordcount > 4 || wordcount < cmdi) {
+        gf_log("cli", GF_LOG_ERROR, "Invalid Syntax");
+        goto out;
+    }
+
+    if (wordcount == cmdi) {
+        ret = 0;
+        goto out;
+    }
+
+    /* if 3rd word is not "volume", then it must be "snapname"
+     */
+    if (strcmp(words[cmdi], "volume") != 0) {
+        ret = dict_set_str(dict, "snapname", (char *)words[cmdi]);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Count not save "
+                   "snap name %s",
+                   words[cmdi]);
+            goto out;
+        }
+
+        if ((cmdi + 1) != wordcount) {
+            ret = -1;
+            gf_log("cli", GF_LOG_ERROR, "Invalid Syntax");
+            goto out;
+        }
+
+        ret = 0;
+        cmd = GF_SNAP_STATUS_TYPE_SNAP;
+        goto out;
+    }
+
+    /* If 3rd word is "volume", then check if next word is present.
+     * As, "snapshot info volume" is an invalid command
+     */
+    if ((cmdi + 1) == wordcount) {
+        ret = -1;
+        gf_log("cli", GF_LOG_ERROR, "Invalid Syntax");
+        goto out;
+    }
+
+    ret = dict_set_str(dict, "volname", (char *)words[wordcount - 1]);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR,
+               "Count not save "
+               "volume name %s",
+               words[wordcount - 1]);
+        goto out;
+    }
+    cmd = GF_SNAP_STATUS_TYPE_VOL;
+
 out:
-        if (ret && dict) {
-                dict_unref (dict);
-                *options = NULL;
+    if (ret == 0) {
+        ret = dict_set_int32(dict, "sub-cmd", cmd);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Could not save cmd "
+                   "of snapshot status");
         }
+    }
 
-        return ret;
+    return ret;
+}
+
+/* return value:
+ *      -1  in case of failure.
+ *       0  in case of success.
+ */
+int32_t
+cli_snap_config_limit_parse(const char **words, dict_t *dict,
+                            unsigned int wordcount, unsigned int index,
+                            char *key)
+{
+    int ret = -1;
+    int limit = 0;
+    char *end_ptr = NULL;
+
+    GF_ASSERT(words);
+    GF_ASSERT(dict);
+    GF_ASSERT(key);
+
+    if (index >= wordcount) {
+        ret = -1;
+        cli_err("Please provide a value for %s.", key);
+        gf_log("cli", GF_LOG_ERROR, "Value not provided for %s", key);
+        goto out;
+    }
+
+    limit = strtol(words[index], &end_ptr, 10);
+
+    if (limit <= 0 || strcmp(end_ptr, "") != 0) {
+        ret = -1;
+        cli_err(
+            "Please enter an integer value "
+            "greater than zero for %s",
+            key);
+        goto out;
+    }
+
+    ret = dict_set_int32(dict, key, limit);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR,
+               "Could not set "
+               "%s in dictionary",
+               key);
+        goto out;
+    }
+
+    ret = dict_set_dynstr_with_alloc(dict, "globalname", "All");
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Could not set global key");
+        goto out;
+    }
+    ret = dict_set_int32(dict, "hold_global_locks", _gf_true);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Could not set global locks");
+        goto out;
+    }
+
+out:
+    return ret;
+}
+
+/* function cli_snap_config_parse
+ * Config Syntax : gluster snapshot config [volname]
+ *                                         [snap-max-hard-limit <count>]
+ *                                         [snap-max-soft-limit <count>]
+ *
+   return value: <0  on failure
+                  1  if user cancels the operation, or limit value is out of
+                                                                      range
+                  0  on success
+
+  NOTE : snap-max-soft-limit can only be set for system.
+*/
+int32_t
+cli_snap_config_parse(const char **words, int wordcount, dict_t *dict,
+                      struct cli_state *state)
+{
+    int ret = -1;
+    gf_answer_t answer = GF_ANSWER_NO;
+    gf_boolean_t vol_presence = _gf_false;
+    struct snap_config_opt_vals_ *conf_vals = NULL;
+    int8_t hard_limit = 0;
+    int8_t soft_limit = 0;
+    int8_t config_type = -1;
+    const char *question = NULL;
+    unsigned int cmdi = 2;
+    /* cmdi is command index, here cmdi is "2" (gluster snapshot config)*/
+
+    GF_ASSERT(words);
+    GF_ASSERT(dict);
+    GF_ASSERT(state);
+
+    if ((wordcount < 2) || (wordcount > 7)) {
+        gf_log("cli", GF_LOG_ERROR, "Invalid wordcount(%d)", wordcount);
+        goto out;
+    }
+
+    if (wordcount == 2) {
+        config_type = GF_SNAP_CONFIG_DISPLAY;
+        ret = 0;
+        goto set;
+    }
+
+    /* auto-delete cannot be a volume name */
+    /* Check whether the 3rd word is volname */
+    if (strcmp(words[cmdi], "snap-max-hard-limit") != 0 &&
+        strcmp(words[cmdi], "snap-max-soft-limit") != 0 &&
+        strcmp(words[cmdi], "auto-delete") != 0 &&
+        strcmp(words[cmdi], "activate-on-create") != 0) {
+        ret = dict_set_str(dict, "volname", (char *)words[cmdi]);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR, "Failed to set volname");
+            goto out;
+        }
+        cmdi++;
+        vol_presence = _gf_true;
+
+        if (cmdi == wordcount) {
+            config_type = GF_SNAP_CONFIG_DISPLAY;
+            ret = 0;
+            goto set;
+        }
+    }
+
+    config_type = GF_SNAP_CONFIG_TYPE_SET;
+
+    if (strcmp(words[cmdi], "snap-max-hard-limit") == 0) {
+        ret = cli_snap_config_limit_parse(words, dict, wordcount, ++cmdi,
+                                          "snap-max-hard-limit");
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Failed to parse snap "
+                   "config hard limit");
+            goto out;
+        }
+        hard_limit = 1;
+
+        if (++cmdi == wordcount) {
+            ret = 0;
+            goto set;
+        }
+    }
+
+    if (strcmp(words[cmdi], "snap-max-soft-limit") == 0) {
+        if (vol_presence == 1) {
+            ret = -1;
+            cli_err(
+                "Soft limit cannot be set to individual "
+                "volumes.");
+            gf_log("cli", GF_LOG_ERROR,
+                   "Soft limit cannot be "
+                   "set to volumes");
+            goto out;
+        }
+
+        ret = cli_snap_config_limit_parse(words, dict, wordcount, ++cmdi,
+                                          "snap-max-soft-limit");
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Failed to parse snap "
+                   "config soft limit");
+            goto out;
+        }
+
+        if (++cmdi != wordcount) {
+            ret = -1;
+            gf_log("cli", GF_LOG_ERROR, "Invalid Syntax");
+            goto out;
+        }
+        soft_limit = 1;
+    }
+
+    if (hard_limit || soft_limit)
+        goto set;
+
+    if (strcmp(words[cmdi], "auto-delete") == 0) {
+        if (vol_presence == 1) {
+            ret = -1;
+            cli_err(
+                "As of now, auto-delete option cannot be set "
+                "to volumes");
+            gf_log("cli", GF_LOG_ERROR,
+                   "auto-delete option "
+                   "cannot be set to volumes");
+            goto out;
+        }
+
+        if (++cmdi >= wordcount) {
+            ret = -1;
+            gf_log("cli", GF_LOG_ERROR, "Invalid Syntax");
+            goto out;
+        }
+
+        ret = dict_set_str(dict, "auto-delete", (char *)words[cmdi]);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Failed to set "
+                   "value of auto-delete in request "
+                   "dictionary");
+            goto out;
+        }
+
+        if (++cmdi != wordcount) {
+            ret = -1;
+            gf_log("cli", GF_LOG_ERROR, "Invalid Syntax");
+            goto out;
+        }
+    } else if (strcmp(words[cmdi], "activate-on-create") == 0) {
+        if (vol_presence == 1) {
+            ret = -1;
+            cli_err(
+                "As of now, activate-on-create option "
+                "cannot be set to volumes");
+            gf_log("cli", GF_LOG_ERROR,
+                   "activate-on-create "
+                   "option cannot be set to volumes");
+            goto out;
+        }
+
+        if (++cmdi >= wordcount) {
+            ret = -1;
+            gf_log("cli", GF_LOG_ERROR, "Invalid Syntax");
+            goto out;
+        }
+
+        ret = dict_set_str(dict, "snap-activate-on-create",
+                           (char *)words[cmdi]);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Failed to set value "
+                   "of activate-on-create in request dictionary");
+            goto out;
+        }
+
+        if (++cmdi != wordcount) {
+            ret = -1;
+            gf_log("cli", GF_LOG_ERROR, "Invalid Syntax");
+            goto out;
+        }
+    } else {
+        ret = -1;
+        gf_log("cli", GF_LOG_ERROR, "Invalid Syntax");
+        goto out;
+    }
+
+    ret = 0; /* Success */
+
+set:
+    ret = dict_set_int32(dict, "config-command", config_type);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR,
+               "Unable to set "
+               "config-command");
+        goto out;
+    }
+
+    if (config_type == GF_SNAP_CONFIG_TYPE_SET && (hard_limit || soft_limit)) {
+        conf_vals = snap_confopt_vals;
+        if (hard_limit && soft_limit) {
+            question = conf_vals[GF_SNAP_CONFIG_SET_BOTH].question;
+        } else if (soft_limit) {
+            question = conf_vals[GF_SNAP_CONFIG_SET_SOFT].question;
+        } else if (hard_limit) {
+            question = conf_vals[GF_SNAP_CONFIG_SET_HARD].question;
+        }
+
+        answer = cli_cmd_get_confirmation(state, question);
+        if (GF_ANSWER_NO == answer) {
+            ret = 1;
+            gf_log("cli", GF_LOG_DEBUG,
+                   "User cancelled "
+                   "snapshot config operation");
+        }
+    }
+
+out:
+    return ret;
 }
 
 int
-cli_cmd_volume_defrag_parse (const char **words, int wordcount,
-                             dict_t **options)
+validate_op_name(const char *op, const char *opname, char **opwords)
 {
-        dict_t                 *dict = NULL;
-        int                      ret = -1;
-        char                *option  = NULL;
-        char                *volname = NULL;
-        char                *command = NULL;
-        gf_cli_defrag_type       cmd = 0;
+    int ret = -1;
+    int i = 0;
+
+    GF_ASSERT(opname);
+    GF_ASSERT(opwords);
+
+    for (i = 0; opwords[i] != NULL; i++) {
+        if (strcmp(opwords[i], opname) == 0) {
+            cli_out("\"%s\" cannot be a %s", opname, op);
+            goto out;
+        }
+    }
+    ret = 0;
+out:
+    return ret;
+}
 
-        GF_ASSERT (words);
-        GF_ASSERT (options);
+int32_t
+cli_cmd_snapshot_parse(const char **words, int wordcount, dict_t **options,
+                       struct cli_state *state)
+{
+    int32_t ret = -1;
+    dict_t *dict = NULL;
+    gf1_cli_snapshot type = GF_SNAP_OPTION_TYPE_NONE;
+    char *w = NULL;
+    static char *opwords[] = {"create",     "delete", "restore", "activate",
+                              "deactivate", "list",   "status",  "config",
+                              "info",       "clone",  NULL};
+    static char *invalid_snapnames[] = {"description", "force", "volume", "all",
+                                        NULL};
+    static char *invalid_volnames[] = {"volume",
+                                       "type",
+                                       "subvolumes",
+                                       "option",
+                                       "end-volume",
+                                       "all",
+                                       "volume_not_in_ring",
+                                       "description",
+                                       "force",
+                                       "snap-max-hard-limit",
+                                       "snap-max-soft-limit",
+                                       "auto-delete",
+                                       "activate-on-create",
+                                       NULL};
+
+    GF_ASSERT(words);
+    GF_ASSERT(options);
+    GF_ASSERT(state);
+
+    dict = dict_new();
+    if (!dict)
+        goto out;
 
-        dict = dict_new ();
-        if (!dict)
+    /* Lowest wordcount possible */
+    if (wordcount < 2) {
+        gf_log("", GF_LOG_ERROR, "Invalid command: Not enough arguments");
+        goto out;
+    }
+
+    w = str_getunamb(words[1], opwords);
+    if (!w) {
+        /* Checks if the operation is a valid operation */
+        gf_log("", GF_LOG_ERROR, "Opword Mismatch");
+        goto out;
+    }
+
+    if (!strcmp(w, "create")) {
+        type = GF_SNAP_OPTION_TYPE_CREATE;
+    } else if (!strcmp(w, "list")) {
+        type = GF_SNAP_OPTION_TYPE_LIST;
+    } else if (!strcmp(w, "info")) {
+        type = GF_SNAP_OPTION_TYPE_INFO;
+    } else if (!strcmp(w, "delete")) {
+        type = GF_SNAP_OPTION_TYPE_DELETE;
+    } else if (!strcmp(w, "config")) {
+        type = GF_SNAP_OPTION_TYPE_CONFIG;
+    } else if (!strcmp(w, "restore")) {
+        type = GF_SNAP_OPTION_TYPE_RESTORE;
+    } else if (!strcmp(w, "status")) {
+        type = GF_SNAP_OPTION_TYPE_STATUS;
+    } else if (!strcmp(w, "activate")) {
+        type = GF_SNAP_OPTION_TYPE_ACTIVATE;
+    } else if (!strcmp(w, "deactivate")) {
+        type = GF_SNAP_OPTION_TYPE_DEACTIVATE;
+    } else if (!strcmp(w, "clone")) {
+        type = GF_SNAP_OPTION_TYPE_CLONE;
+    }
+
+    if (type != GF_SNAP_OPTION_TYPE_CONFIG &&
+        type != GF_SNAP_OPTION_TYPE_STATUS) {
+        ret = dict_set_int32(dict, "hold_snap_locks", _gf_true);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Unable to set hold-snap-locks value "
+                   "as _gf_true");
+            goto out;
+        }
+    }
+
+    /* Following commands does not require volume locks  */
+    if (type == GF_SNAP_OPTION_TYPE_STATUS ||
+        type == GF_SNAP_OPTION_TYPE_ACTIVATE ||
+        type == GF_SNAP_OPTION_TYPE_DEACTIVATE) {
+        ret = dict_set_int32(dict, "hold_vol_locks", _gf_false);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Setting volume lock "
+                   "flag failed");
+            goto out;
+        }
+    }
+
+    /* Check which op is intended */
+    switch (type) {
+        case GF_SNAP_OPTION_TYPE_CREATE:
+            /* Syntax :
+             * gluster snapshot create <snapname> <vol-name(s)>
+             *                         [no-timestamp]
+             *                         [description <description>]
+             *                         [force]
+             */
+            /* In cases where the snapname is not given then
+             * parsing fails & snapname cannot be "description",
+             * "force" and "volume", that check is made here
+             */
+            if (wordcount == 2) {
+                ret = -1;
+                gf_log("cli", GF_LOG_ERROR, "Invalid Syntax");
+                goto out;
+            }
+
+            ret = validate_op_name("snapname", words[2], invalid_snapnames);
+            if (ret) {
+                goto out;
+            }
+
+            ret = cli_snap_create_parse(dict, words, wordcount);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR, "create command parsing failed.");
+                goto out;
+            }
+            break;
+
+        case GF_SNAP_OPTION_TYPE_CLONE:
+            /* Syntax :
+             * gluster snapshot clone <clonename> <snapname>
+             */
+            /* In cases where the clonename is not given then
+             * parsing fails & snapname cannot be "description",
+             * "force" and "volume", that check is made here
+             */
+            if (wordcount == 2) {
+                ret = -1;
+                gf_log("cli", GF_LOG_ERROR, "Invalid Syntax");
+                goto out;
+            }
+
+            ret = validate_op_name("clonename", words[2], invalid_volnames);
+            if (ret) {
+                goto out;
+            }
+
+            ret = cli_snap_clone_parse(dict, words, wordcount);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR, "clone command parsing failed.");
+                goto out;
+            }
+            break;
+
+        case GF_SNAP_OPTION_TYPE_INFO:
+            /* Syntax :
+             * gluster snapshot info [(snapname] | [vol <volname>)]
+             */
+            ret = cli_snap_info_parse(dict, words, wordcount);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR,
+                       "Failed to parse "
+                       "snapshot info command");
+                goto out;
+            }
+            break;
+
+        case GF_SNAP_OPTION_TYPE_LIST:
+            /* Syntax :
+             * gluster snaphsot list [volname]
+             */
+
+            ret = cli_snap_list_parse(dict, words, wordcount);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR,
+                       "Failed to parse "
+                       "snapshot list command");
+                goto out;
+            }
+            break;
+
+        case GF_SNAP_OPTION_TYPE_DELETE:
+            /* Syntax :
+             * snapshot delete (all | snapname | volume <volname>)
+             */
+            ret = cli_snap_delete_parse(dict, words, wordcount, state);
+            if (ret) {
+                /* A positive ret value means user cancelled
+                 * the command */
+                if (ret < 0) {
+                    gf_log("cli", GF_LOG_ERROR,
+                           "Failed to parse "
+                           "snapshot delete command");
+                }
+                goto out;
+            }
+            break;
+
+        case GF_SNAP_OPTION_TYPE_CONFIG:
+            /* snapshot config [volname]  [snap-max-hard-limit <count>]
+             *                            [snap-max-soft-limit <percent>] */
+            ret = cli_snap_config_parse(words, wordcount, dict, state);
+            if (ret) {
+                if (ret < 0)
+                    gf_log("cli", GF_LOG_ERROR,
+                           "config command parsing failed.");
+                goto out;
+            }
+
+            ret = dict_set_int32(dict, "type", GF_SNAP_OPTION_TYPE_CONFIG);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR,
+                       "Unable to set "
+                       "config type");
+                ret = -1;
                 goto out;
+            }
+            break;
+
+        case GF_SNAP_OPTION_TYPE_STATUS: {
+            /* Syntax :
+             * gluster snapshot status [(snapname |
+             *                         volume <volname>)]
+             */
+            ret = cli_snap_status_parse(dict, words, wordcount);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR,
+                       "Failed to parse "
+                       "snapshot status command");
+                goto out;
+            }
+            break;
+        }
 
-        if (!((wordcount == 4) || (wordcount == 5)))
+        case GF_SNAP_OPTION_TYPE_RESTORE:
+            /* Syntax:
+             * snapshot restore <snapname>
+             */
+            ret = cli_snap_restore_parse(dict, words, wordcount, state);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR,
+                       "Failed to parse "
+                       "restore command");
+                goto out;
+            }
+            break;
+
+        case GF_SNAP_OPTION_TYPE_ACTIVATE:
+            /* Syntax:
+             * snapshot activate <snapname> [force]
+             */
+            ret = cli_snap_activate_parse(dict, words, wordcount);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR,
+                       "Failed to parse "
+                       "start command");
+                goto out;
+            }
+            break;
+        case GF_SNAP_OPTION_TYPE_DEACTIVATE:
+            /* Syntax:
+             * snapshot deactivate <snapname>
+             */
+            ret = cli_snap_deactivate_parse(dict, words, wordcount, state);
+            if (ret) {
+                /* A positive ret value means user cancelled
+                 * the command */
+                if (ret < 0) {
+                    gf_log("cli", GF_LOG_ERROR,
+                           "Failed to parse deactivate "
+                           "command");
+                }
                 goto out;
+            }
+            break;
+
+        default:
+            ret = -1;
+            gf_log("", GF_LOG_ERROR, "Opword Mismatch");
+            goto out;
+    }
+
+    ret = dict_set_int32(dict, "type", type);
+    if (ret) {
+        gf_log("", GF_LOG_ERROR, "Failed to set type.");
+        goto out;
+    }
+    /* If you got so far, input is valid */
+    ret = 0;
+out:
+    if (ret) {
+        if (dict)
+            dict_unref(dict);
+    } else
+        *options = dict;
+
+    return ret;
+}
+
+int
+cli_cmd_validate_volume(char *volname)
+{
+    int i = 0;
+    int ret = -1;
+    int volname_len;
+
+    if (volname[0] == '-')
+        return ret;
+
+    if (!strcmp(volname, "all")) {
+        cli_err("\"all\" cannot be the name of a volume.");
+        return ret;
+    }
+
+    if (strchr(volname, '/')) {
+        cli_err("Volume name should not contain \"/\" character.");
+        return ret;
+    }
+
+    volname_len = strlen(volname);
+    if (volname_len > GD_VOLUME_NAME_MAX) {
+        cli_err("Volname can not exceed %d characters.", GD_VOLUME_NAME_MAX);
+        return ret;
+    }
+
+    for (i = 0; i < volname_len; i++)
+        if (!isalnum(volname[i]) && (volname[i] != '_') &&
+            (volname[i] != '-')) {
+            cli_err(
+                "Volume name should not contain \"%c\""
+                " character.\nVolume names can only"
+                "contain alphanumeric, '-' and '_' "
+                "characters.",
+                volname[i]);
+            return ret;
+        }
 
+    ret = 0;
+
+    return ret;
+}
+
+int32_t
+cli_cmd_bitrot_parse(const char **words, int wordcount, dict_t **options)
+{
+    int32_t ret = -1;
+    char *w = NULL;
+    char *volname = NULL;
+    static char *opwords[] = {"enable",          "disable", "scrub-throttle",
+                              "scrub-frequency", "scrub",   "signing-time",
+                              "signer-threads",  NULL};
+    static char *scrub_throt_values[] = {"lazy", "normal", "aggressive", NULL};
+    static char *scrub_freq_values[] = {
+        "hourly", "daily", "weekly", "biweekly", "monthly", "minute", NULL};
+    static char *scrub_values[] = {"pause", "resume", "status", "ondemand",
+                                   NULL};
+    dict_t *dict = NULL;
+    gf_bitrot_type type = GF_BITROT_OPTION_TYPE_NONE;
+    int32_t expiry_time = 0;
+    int32_t signer_th_count = 0;
+
+    GF_ASSERT(words);
+    GF_ASSERT(options);
+
+    /* Hack to print out bitrot help properly */
+    if ((wordcount == 3) && !(strcmp(words[2], "help"))) {
+        ret = 1;
+        return ret;
+    }
+
+    if (wordcount < 4 || wordcount > 5) {
+        gf_log("cli", GF_LOG_ERROR, "Invalid syntax");
+        goto out;
+    }
+
+    dict = dict_new();
+    if (!dict)
+        goto out;
+
+    volname = (char *)words[2];
+    if (!volname) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = cli_cmd_validate_volume(volname);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to validate volume name");
+        goto out;
+    }
+
+    ret = dict_set_str(dict, "volname", volname);
+    if (ret) {
+        cli_out("Failed to set volume name in dictionary ");
+        goto out;
+    }
+
+    w = str_getunamb(words[3], opwords);
+    if (!w) {
+        cli_out("Invalid bit rot option : %s", words[3]);
+        ret = -1;
+        goto out;
+    }
+
+    if (strcmp(w, "enable") == 0) {
         if (wordcount == 4) {
-                if (strcmp (words[3], "start") && strcmp (words[3], "stop") &&
-                    strcmp (words[3], "status"))
-                            goto out;
+            type = GF_BITROT_OPTION_TYPE_ENABLE;
+            ret = 0;
+            goto set_type;
         } else {
-                if (strcmp (words[3], "fix-layout") &&
-                    strcmp (words[3], "start"))
-                        goto out;
+            ret = -1;
+            goto out;
         }
+    }
 
-        volname = (char *) words[2];
-
+    if (strcmp(w, "disable") == 0) {
         if (wordcount == 4) {
-                command = (char *) words[3];
+            type = GF_BITROT_OPTION_TYPE_DISABLE;
+            ret = 0;
+            goto set_type;
+        } else {
+            ret = -1;
+            goto out;
         }
-        if (wordcount == 5) {
-               if ((strcmp (words[3], "fix-layout") ||
-                    strcmp (words[4], "start")) &&
-                    (strcmp (words[3], "start") ||
-                    strcmp (words[4], "force"))) {
-                        ret = -1;
-                        goto out;
+    }
+
+    if (!strcmp(w, "scrub-throttle")) {
+        if (!words[4]) {
+            cli_err(
+                "Missing scrub-throttle value for bitrot "
+                "option");
+            ret = -1;
+            goto out;
+        } else {
+            w = str_getunamb(words[4], scrub_throt_values);
+            if (!w) {
+                cli_err(
+                    "Invalid scrub-throttle option for "
+                    "bitrot");
+                ret = -1;
+                goto out;
+            } else {
+                type = GF_BITROT_OPTION_TYPE_SCRUB_THROTTLE;
+                ret = dict_set_str(dict, "scrub-throttle-value",
+                                   (char *)words[4]);
+                if (ret) {
+                    cli_out(
+                        "Failed to set scrub-throttle "
+                        "value in the dict");
+                    goto out;
                 }
-                command = (char *) words[3];
-                option = (char *) words[4];
+                goto set_type;
+            }
         }
+    }
 
-        if (strcmp (command, "start") == 0) {
-                cmd = GF_DEFRAG_CMD_START;
-                if (option && strcmp (option, "force") == 0) {
-                                cmd = GF_DEFRAG_CMD_START_FORCE;
-                        }
-                goto done;
+    if (!strcmp(words[3], "scrub-frequency")) {
+        if (!words[4]) {
+            cli_err("Missing scrub-frequency value");
+            ret = -1;
+            goto out;
+        } else {
+            w = str_getunamb(words[4], scrub_freq_values);
+            if (!w) {
+                cli_err("Invalid frequency option for bitrot");
+                ret = -1;
+                goto out;
+            } else {
+                type = GF_BITROT_OPTION_TYPE_SCRUB_FREQ;
+                ret = dict_set_str(dict, "scrub-frequency-value",
+                                   (char *)words[4]);
+                if (ret) {
+                    cli_out(
+                        "Failed to set dict for "
+                        "bitrot");
+                    goto out;
+                }
+                goto set_type;
+            }
         }
+    }
 
-        if (strcmp (command, "fix-layout") == 0) {
-                cmd = GF_DEFRAG_CMD_START_LAYOUT_FIX;
-                goto done;
-        }
-        if (strcmp (command, "stop") == 0) {
-                cmd = GF_DEFRAG_CMD_STOP;
-                goto done;
-        }
-        if (strcmp (command, "status") == 0) {
-                cmd = GF_DEFRAG_CMD_STATUS;
+    if (!strcmp(words[3], "scrub")) {
+        if (!words[4]) {
+            cli_err("Missing scrub value for bitrot option");
+            ret = -1;
+            goto out;
+        } else {
+            w = str_getunamb(words[4], scrub_values);
+            if (!w) {
+                cli_err("Invalid scrub option for bitrot");
+                ret = -1;
+                goto out;
+            } else {
+                if (strcmp(words[4], "status") == 0) {
+                    type = GF_BITROT_CMD_SCRUB_STATUS;
+                } else if (strcmp(words[4], "ondemand") == 0) {
+                    type = GF_BITROT_CMD_SCRUB_ONDEMAND;
+                } else {
+                    type = GF_BITROT_OPTION_TYPE_SCRUB;
+                }
+                ret = dict_set_str(dict, "scrub-value", (char *)words[4]);
+                if (ret) {
+                    cli_out(
+                        "Failed to set dict for "
+                        "bitrot");
+                    goto out;
+                }
+                goto set_type;
+            }
         }
+    }
+
+    if (!strcmp(words[3], "signing-time")) {
+        if (!words[4]) {
+            cli_err(
+                "Missing signing-time value for bitrot "
+                "option");
+            ret = -1;
+            goto out;
+        } else {
+            type = GF_BITROT_OPTION_TYPE_EXPIRY_TIME;
 
-done:
-        ret = dict_set_str (dict, "volname", volname);
+            expiry_time = strtol(words[4], NULL, 0);
+            if (expiry_time < 1) {
+                cli_err(
+                    "Expiry time  value should not be less"
+                    " than 1");
+                ret = -1;
+                goto out;
+            }
 
-        if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "failed to set dict");
+            ret = dict_set_uint32(dict, "expiry-time",
+                                  (unsigned int)expiry_time);
+            if (ret) {
+                cli_out("Failed to set dict for bitrot");
                 goto out;
+            }
+            goto set_type;
         }
+    } else if (!strcmp(words[3], "signer-threads")) {
+        if (!words[4]) {
+            cli_err(
+                "Missing signer-thread value for bitrot "
+                "option");
+            ret = -1;
+            goto out;
+        } else {
+            type = GF_BITROT_OPTION_TYPE_SIGNER_THREADS;
 
-        ret = dict_set_int32 (dict, "rebalance-command", (int32_t) cmd);
+            signer_th_count = strtol(words[4], NULL, 0);
+            if (signer_th_count < 1) {
+                cli_err("signer-thread count should not be less than 1");
+                ret = -1;
+                goto out;
+            }
 
-        if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "failed to set dict");
+            ret = dict_set_uint32(dict, "signer-threads",
+                                  (unsigned int)signer_th_count);
+            if (ret) {
+                cli_out("Failed to set dict for bitrot");
                 goto out;
+            }
+            goto set_type;
         }
+    } else {
+        cli_err(
+            "Invalid option %s for bitrot. Please enter valid "
+            "bitrot option",
+            words[3]);
+        ret = -1;
+        goto out;
+    }
+set_type:
+    ret = dict_set_int32(dict, "type", type);
+    if (ret < 0)
+        goto out;
 
-        *options = dict;
+    *options = dict;
 
 out:
-        if (ret && dict)
-                dict_destroy (dict);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Unable to parse bitrot command");
+        if (dict)
+            dict_unref(dict);
+    }
 
-        return ret;
+    return ret;
+}
+
+/* Parsing global option for NFS-Ganesha config
+ *  gluster nfs-ganesha enable/disable */
+
+int32_t
+cli_cmd_ganesha_parse(struct cli_state *state, const char **words,
+                      int wordcount, dict_t **options, char **op_errstr)
+{
+    dict_t *dict = NULL;
+    int ret = -1;
+    char *key = NULL;
+    char *value = NULL;
+    char *w = NULL;
+    static char *opwords[] = {"enable", "disable", NULL};
+    const char *question = NULL;
+    gf_answer_t answer = GF_ANSWER_NO;
+
+    GF_ASSERT(words);
+    GF_ASSERT(options);
+
+    dict = dict_new();
+
+    if (!dict)
+        goto out;
+
+    if (wordcount != 2)
+        goto out;
+
+    key = (char *)words[0];
+    value = (char *)words[1];
+
+    if (!key || !value) {
+        cli_out("Usage : nfs-ganesha <enable/disable>");
+        ret = -1;
+        goto out;
+    }
+
+    ret = gf_strip_whitespace(value, strlen(value));
+    if (ret == -1)
+        goto out;
+
+    if (strcmp(key, "nfs-ganesha")) {
+        gf_asprintf(op_errstr,
+                    "Global option: error: ' %s '"
+                    "is not a valid global option.",
+                    key);
+        ret = -1;
+        goto out;
+    }
+
+    w = str_getunamb(value, opwords);
+    if (!w) {
+        cli_out(
+            "Invalid global option \n"
+            "Usage : nfs-ganesha <enable/disable>");
+        ret = -1;
+        goto out;
+    }
+
+    if (strcmp(value, "enable") == 0) {
+        question =
+            "Enabling NFS-Ganesha requires Gluster-NFS to be "
+            "disabled across the trusted pool. Do you "
+            "still want to continue?\n";
+    } else if (strcmp(value, "disable") == 0) {
+        question =
+            "Disabling NFS-Ganesha will tear down the entire "
+            "ganesha cluster across the trusted pool. Do you "
+            "still want to continue?\n";
+    } else {
+        ret = -1;
+        goto out;
+    }
+    answer = cli_cmd_get_confirmation(state, question);
+    if (GF_ANSWER_NO == answer) {
+        gf_log("cli", GF_LOG_ERROR,
+               "Global operation "
+               "cancelled, exiting");
+        ret = -1;
+        goto out;
+    }
+    cli_out("This will take a few minutes to complete. Please wait ..");
+
+    ret = dict_set_str(dict, "key", key);
+    if (ret) {
+        gf_log(THIS->name, GF_LOG_ERROR, "dict set on key failed");
+        goto out;
+    }
+
+    ret = dict_set_str(dict, "value", value);
+    if (ret) {
+        gf_log(THIS->name, GF_LOG_ERROR, "dict set on value failed");
+        goto out;
+    }
+
+    ret = dict_set_str(dict, "globalname", "All");
+    if (ret) {
+        gf_log(THIS->name, GF_LOG_ERROR,
+               "dict set on global"
+               " key failed.");
+        goto out;
+    }
+
+    ret = dict_set_int32(dict, "hold_global_locks", _gf_true);
+    if (ret) {
+        gf_log(THIS->name, GF_LOG_ERROR,
+               "dict set on global key "
+               "failed.");
+        goto out;
+    }
+
+    *options = dict;
+out:
+    if (ret)
+        dict_unref(dict);
+
+    return ret;
 }
diff --git a/cli/src/cli-cmd-peer.c b/cli/src/cli-cmd-peer.c
index 551312411a0..084998701d8 100644
--- a/cli/src/cli-cmd-peer.c
+++ b/cli/src/cli-cmd-peer.c
@@ -13,288 +13,305 @@
 #include <stdint.h>
 #include <pthread.h>
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 #include "cli.h"
 #include "cli-cmd.h"
 #include "cli-mem-types.h"
 #include "cli1-xdr.h"
 #include "protocol-common.h"
+#include <glusterfs/events.h>
 
-extern struct rpc_clnt *global_rpc;
-
-extern rpc_clnt_prog_t *cli_rpc_prog;
-
-int cli_cmd_peer_help_cbk (struct cli_state *state, struct cli_cmd_word *in_word,
+int
+cli_cmd_peer_help_cbk(struct cli_state *state, struct cli_cmd_word *in_word,
                       const char **words, int wordcount);
 
 int
-cli_cmd_peer_probe_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                   const char **words, int wordcount)
+cli_cmd_peer_probe_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                       const char **words, int wordcount)
 {
-        int                     ret = -1;
-        rpc_clnt_procedure_t    *proc = NULL;
-        call_frame_t            *frame = NULL;
-        dict_t                  *dict = NULL;
-        int                     sent = 0;
-        int                     parse_error = 0;
-        cli_local_t             *local = NULL;
-
-        if (!(wordcount == 3)) {
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-                goto out;
-        }
-
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_PROBE];
-
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
-
-        dict = dict_new ();
-        if (!dict)
-                goto out;
-
-        ret = dict_set_str (dict, "hostname", (char *)words[2]);
-        if (ret)
-                goto out;
-
-        ret = valid_internet_address ((char *) words[2], _gf_false);
-        if (ret == 1) {
-                ret = 0;
-        } else {
-                cli_out ("%s is an invalid address", words[2]);
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-                ret = -1;
-                goto out;
-        }
-/*        if (words[3]) {
-                ret = dict_set_str (dict, "port", (char *)words[3]);
-                if (ret)
-                        goto out;
-        }
-*/
-
-        CLI_LOCAL_INIT (local, words, frame, dict);
-
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, dict);
-        }
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *dict = NULL;
+    int sent = 0;
+    int parse_error = 0;
+    cli_local_t *local = NULL;
+
+    if (!(wordcount == 3)) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
+
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_PROBE];
+
+    dict = dict_new();
+    if (!dict)
+        goto out;
+
+    ret = dict_set_str(dict, "hostname", (char *)words[2]);
+    if (ret)
+        goto out;
+
+    ret = valid_internet_address((char *)words[2], _gf_false, _gf_false);
+    if (ret == 1) {
+        ret = 0;
+    } else {
+        cli_out("%s is an invalid address", words[2]);
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        ret = -1;
+        goto out;
+    }
+    /*        if (words[3]) {
+                    ret = dict_set_str (dict, "port", (char *)words[3]);
+                    if (ret)
+                            goto out;
+            }
+    */
+
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
+
+    CLI_LOCAL_INIT(local, words, frame, dict);
+
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, dict);
+    }
 
 out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_out ("Peer probe failed");
-        }
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("Peer probe failed");
+    }
 
-        CLI_STACK_DESTROY (frame);
+    CLI_STACK_DESTROY(frame);
 
-        return ret;
-}
+    if (ret == 0) {
+        gf_event(EVENT_PEER_ATTACH, "host=%s", (char *)words[2]);
+    }
 
+    return ret;
+}
 
 int
-cli_cmd_peer_deprobe_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                          const char **words, int wordcount)
+cli_cmd_peer_deprobe_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                         const char **words, int wordcount)
 {
-        int                   ret   = -1;
-        rpc_clnt_procedure_t *proc  = NULL;
-        call_frame_t         *frame = NULL;
-        dict_t               *dict  = NULL;
-        int                  flags = 0;
-        int                  sent = 0;
-        int                  parse_error = 0;
-        cli_local_t          *local = NULL;
-
-        if ((wordcount < 3) || (wordcount > 4)) {
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-                goto out;
-        }
-
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_DEPROBE];
-
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
-
-        dict = dict_new ();
-
-        ret = dict_set_str (dict, "hostname", (char *)words[2]);
-        if (ret)
-                goto out;
-
-/*        if (words[3]) {
-                ret = dict_set_str (dict, "port", (char *)words[3]);
-                if (ret)
-                        goto out;
-        }
-*/
-        if (wordcount == 4) {
-                if (!strcmp("force", words[3]))
-                        flags |= GF_CLI_FLAG_OP_FORCE;
-                else {
-                        ret = -1;
-                        cli_usage_out (word->pattern);
-                        parse_error = 1;
-                        goto out;
-                }
-        }
-        ret = dict_set_int32 (dict, "flags", flags);
-        if (ret)
-                goto out;
-
-        CLI_LOCAL_INIT (local, words, frame, dict);
-
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, dict);
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *dict = NULL;
+    int flags = 0;
+    int sent = 0;
+    int parse_error = 0;
+    cli_local_t *local = NULL;
+    gf_answer_t answer = GF_ANSWER_NO;
+    const char *question = NULL;
+
+    if ((wordcount < 3) || (wordcount > 4)) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
+    question =
+        "All clients mounted through the peer which is getting detached need "
+        "to be remounted using one of the other active peers in the trusted "
+        "storage pool to ensure client gets notification on any changes done "
+        "on the gluster configuration and if the same has been done do you "
+        "want to proceed?";
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_DEPROBE];
+
+    dict = dict_new();
+
+    ret = dict_set_str(dict, "hostname", (char *)words[2]);
+    if (ret)
+        goto out;
+
+    /*        if (words[3]) {
+                    ret = dict_set_str (dict, "port", (char *)words[3]);
+                    if (ret)
+                            goto out;
+            }
+    */
+    if (wordcount == 4) {
+        if (!strcmp("force", words[3]))
+            flags |= GF_CLI_FLAG_OP_FORCE;
+        else {
+            ret = -1;
+            cli_usage_out(word->pattern);
+            parse_error = 1;
+            goto out;
         }
+    }
+    ret = dict_set_int32(dict, "flags", flags);
+    if (ret)
+        goto out;
+    answer = cli_cmd_get_confirmation(state, question);
+    if (GF_ANSWER_NO == answer) {
+        ret = 0;
+        goto out;
+    }
+
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
+
+    CLI_LOCAL_INIT(local, words, frame, dict);
+
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, dict);
+    }
 
 out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_out ("Peer detach failed");
-        }
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("Peer detach failed");
+    }
 
-        CLI_STACK_DESTROY (frame);
+    CLI_STACK_DESTROY(frame);
 
-        return ret;
+    if (ret == 0) {
+        gf_event(EVENT_PEER_DETACH, "host=%s", (char *)words[2]);
+    }
+
+    return ret;
 }
 
 int
-cli_cmd_peer_status_cbk (struct cli_state *state, struct cli_cmd_word *word,
+cli_cmd_peer_status_cbk(struct cli_state *state, struct cli_cmd_word *word,
                         const char **words, int wordcount)
 {
-        int                     ret = -1;
-        rpc_clnt_procedure_t    *proc = NULL;
-        call_frame_t            *frame = NULL;
-        int                     sent = 0;
-        int                     parse_error = 0;
-
-        if (wordcount != 2) {
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-                goto out;
-        }
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    int sent = 0;
+    int parse_error = 0;
 
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_LIST_FRIENDS];
+    if (wordcount != 2) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
 
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_LIST_FRIENDS];
 
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, (void *)GF_CLI_LIST_PEERS);
-        }
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame)
+        goto out;
+
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, (void *)GF_CLI_LIST_PEERS);
+    }
 
 out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_out ("Peer status failed");
-        }
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("Peer status failed");
+    }
 
-        CLI_STACK_DESTROY (frame);
+    CLI_STACK_DESTROY(frame);
 
-        return ret;
+    return ret;
 }
 
 int
-cli_cmd_pool_list_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                       const char **words, int wordcount)
+cli_cmd_pool_list_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                      const char **words, int wordcount)
 {
-        int                     ret = -1;
-        rpc_clnt_procedure_t    *proc = NULL;
-        call_frame_t            *frame = NULL;
-        int                     sent = 0;
-        int                     parse_error = 0;
-
-        if (wordcount != 2) {
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-                goto out;
-        }
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    int sent = 0;
+    int parse_error = 0;
 
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_LIST_FRIENDS];
+    if (wordcount != 2) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
 
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_LIST_FRIENDS];
 
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS,
-                                (void *)GF_CLI_LIST_POOL_NODES);
-        }
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame)
+        goto out;
+
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, (void *)GF_CLI_LIST_POOL_NODES);
+    }
 
 out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_err ("pool list: command execution failed");
-        }
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_err("pool list: command execution failed");
+    }
 
-        CLI_STACK_DESTROY (frame);
+    CLI_STACK_DESTROY(frame);
 
-        return ret;
+    return ret;
 }
 
 struct cli_cmd cli_probe_cmds[] = {
-        { "peer probe <HOSTNAME>",
-          cli_cmd_peer_probe_cbk,
-          "probe peer specified by <HOSTNAME>"},
+    {"peer probe { <HOSTNAME> | <IP-address> }", cli_cmd_peer_probe_cbk,
+     "probe peer specified by <HOSTNAME>"},
 
-        { "peer detach <HOSTNAME> [force]",
-          cli_cmd_peer_deprobe_cbk,
-          "detach peer specified by <HOSTNAME>"},
+    {"peer detach { <HOSTNAME> | <IP-address> } [force]",
+     cli_cmd_peer_deprobe_cbk, "detach peer specified by <HOSTNAME>"},
 
-        { "peer status",
-          cli_cmd_peer_status_cbk,
-          "list status of peers"},
+    {"peer status", cli_cmd_peer_status_cbk, "list status of peers"},
 
-	{ "peer help",
-           cli_cmd_peer_help_cbk,
-           "Help command for peer "},
+    {"peer help", cli_cmd_peer_help_cbk, "display help for peer commands"},
 
-        { "pool list",
-          cli_cmd_pool_list_cbk,
-          "list all the nodes in the pool (including localhost)"},
+    {"pool list", cli_cmd_pool_list_cbk,
+     "list all the nodes in the pool (including localhost)"},
 
-        { NULL, NULL, NULL }
-};
+    {NULL, NULL, NULL}};
 
 int
-cli_cmd_peer_help_cbk (struct cli_state *state, struct cli_cmd_word *in_word,
+cli_cmd_peer_help_cbk(struct cli_state *state, struct cli_cmd_word *in_word,
                       const char **words, int wordcount)
 {
-        struct cli_cmd        *cmd = NULL;
+    struct cli_cmd *cmd = NULL;
+    struct cli_cmd *probe_cmd = NULL;
+    int count = 0;
+
+    cli_out("\ngluster peer commands");
+    cli_out("======================\n");
 
+    cmd = GF_MALLOC(sizeof(cli_probe_cmds), cli_mt_cli_cmd);
+    memcpy(cmd, cli_probe_cmds, sizeof(cli_probe_cmds));
+    count = (sizeof(cli_probe_cmds) / sizeof(struct cli_cmd));
+    cli_cmd_sort(cmd, count);
 
+    for (probe_cmd = cmd; probe_cmd->pattern; probe_cmd++)
+        cli_out("%s - %s", probe_cmd->pattern, probe_cmd->desc);
 
-        for (cmd = cli_probe_cmds; cmd->pattern; cmd++)
-                cli_out ("%s - %s", cmd->pattern, cmd->desc);
+    GF_FREE(cmd);
 
-        return 0;
+    cli_out("\n");
+    return 0;
 }
 
 int
-cli_cmd_probe_register (struct cli_state *state)
+cli_cmd_probe_register(struct cli_state *state)
 {
-        int  ret = 0;
-        struct cli_cmd *cmd = NULL;
+    int ret = 0;
+    struct cli_cmd *cmd = NULL;
 
-        for (cmd = cli_probe_cmds; cmd->pattern; cmd++) {
-
-                ret = cli_cmd_register (&state->tree, cmd);
-                if (ret)
-                        goto out;
-        }
+    for (cmd = cli_probe_cmds; cmd->pattern; cmd++) {
+        ret = cli_cmd_register(&state->tree, cmd);
+        if (ret)
+            goto out;
+    }
 out:
-        return ret;
+    return ret;
 }
diff --git a/cli/src/cli-cmd-snapshot.c b/cli/src/cli-cmd-snapshot.c
new file mode 100644
index 00000000000..859d6b2e40d
--- /dev/null
+++ b/cli/src/cli-cmd-snapshot.c
@@ -0,0 +1,135 @@
+/*
+   Copyright (c) 2013-2014 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#include <stdio.h>
+#include <string.h>
+#include <stdlib.h>
+#include <stdint.h>
+#include <pthread.h>
+
+#include "cli.h"
+#include "cli-cmd.h"
+#include "cli-mem-types.h"
+
+int
+cli_cmd_snapshot_help_cbk(struct cli_state *state, struct cli_cmd_word *in_word,
+                          const char **words, int wordcount);
+
+int
+cli_cmd_snapshot_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                     const char **words, int wordcount)
+{
+    int ret = 0;
+    int parse_err = 0;
+    dict_t *options = NULL;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    cli_local_t *local = NULL;
+
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_SNAP];
+
+    /* Parses the command entered by the user */
+    ret = cli_cmd_snapshot_parse(words, wordcount, &options, state);
+    if (ret) {
+        if (ret < 0) {
+            cli_usage_out(word->pattern);
+            parse_err = 1;
+        } else {
+            /* User might have cancelled the snapshot operation */
+            ret = 0;
+        }
+        goto out;
+    }
+
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (frame == NULL) {
+        ret = -1;
+        goto out;
+    }
+
+    CLI_LOCAL_INIT(local, words, frame, options);
+
+    if (proc->fn)
+        ret = proc->fn(frame, THIS, options);
+
+out:
+    if (ret && parse_err == 0)
+        cli_out("Snapshot command failed");
+
+    CLI_STACK_DESTROY(frame);
+
+    return ret;
+}
+
+struct cli_cmd snapshot_cmds[] = {
+    {"snapshot help", cli_cmd_snapshot_help_cbk,
+     "display help for snapshot commands"},
+    {"snapshot create <snapname> <volname> [no-timestamp] "
+     "[description <description>] [force]",
+     cli_cmd_snapshot_cbk, "Snapshot Create."},
+    {"snapshot clone <clonename> <snapname>", cli_cmd_snapshot_cbk,
+     "Snapshot Clone."},
+    {"snapshot restore <snapname>", cli_cmd_snapshot_cbk, "Snapshot Restore."},
+    {"snapshot status [(snapname | volume <volname>)]", cli_cmd_snapshot_cbk,
+     "Snapshot Status."},
+    {"snapshot info [(snapname | volume <volname>)]", cli_cmd_snapshot_cbk,
+     "Snapshot Info."},
+    {"snapshot list [volname]", cli_cmd_snapshot_cbk, "Snapshot List."},
+    {"snapshot config [volname] ([snap-max-hard-limit <count>] "
+     "[snap-max-soft-limit <percent>]) "
+     "| ([auto-delete <enable|disable>])"
+     "| ([activate-on-create <enable|disable>])",
+     cli_cmd_snapshot_cbk, "Snapshot Config."},
+    {"snapshot delete (all | snapname | volume <volname>)",
+     cli_cmd_snapshot_cbk, "Snapshot Delete."},
+    {"snapshot activate <snapname> [force]", cli_cmd_snapshot_cbk,
+     "Activate snapshot volume."},
+    {"snapshot deactivate <snapname>", cli_cmd_snapshot_cbk,
+     "Deactivate snapshot volume."},
+    {NULL, NULL, NULL}};
+
+int
+cli_cmd_snapshot_help_cbk(struct cli_state *state, struct cli_cmd_word *in_word,
+                          const char **words, int wordcount)
+{
+    struct cli_cmd *cmd = NULL;
+    struct cli_cmd *snap_cmd = NULL;
+    int count = 0;
+
+    cmd = GF_MALLOC(sizeof(snapshot_cmds), cli_mt_cli_cmd);
+    memcpy(cmd, snapshot_cmds, sizeof(snapshot_cmds));
+    count = (sizeof(snapshot_cmds) / sizeof(struct cli_cmd));
+    cli_cmd_sort(cmd, count);
+
+    cli_out("\ngluster snapshot commands");
+    cli_out("=========================\n");
+
+    for (snap_cmd = cmd; snap_cmd->pattern; snap_cmd++)
+        if (_gf_false == snap_cmd->disable)
+            cli_out("%s - %s", snap_cmd->pattern, snap_cmd->desc);
+    cli_out("\n");
+
+    GF_FREE(cmd);
+    return 0;
+}
+
+int
+cli_cmd_snapshot_register(struct cli_state *state)
+{
+    int ret = 0;
+    struct cli_cmd *cmd = NULL;
+
+    for (cmd = snapshot_cmds; cmd->pattern; cmd++) {
+        ret = cli_cmd_register(&state->tree, cmd);
+        if (ret)
+            goto out;
+    }
+out:
+    return ret;
+}
diff --git a/cli/src/cli-cmd-system.c b/cli/src/cli-cmd-system.c
index 8cfa5e70c3c..801e8f4efed 100644
--- a/cli/src/cli-cmd-system.c
+++ b/cli/src/cli-cmd-system.c
@@ -13,590 +13,612 @@
 #include <stdint.h>
 #include <pthread.h>
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 #include "cli.h"
 #include "cli-cmd.h"
 #include "cli-mem-types.h"
 #include "protocol-common.h"
 
+int
+cli_cmd_system_help_cbk(struct cli_state *state, struct cli_cmd_word *in_word,
+                        const char **words, int wordcount);
 
-extern struct rpc_clnt *global_rpc;
-
-extern rpc_clnt_prog_t *cli_rpc_prog;
-
-int cli_cmd_system_help_cbk (struct cli_state *state, struct cli_cmd_word *in_word,
-                             const char **words, int wordcount);
-
-int cli_cmd_copy_file_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                           const char **words, int wordcount);
+int
+cli_cmd_copy_file_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                      const char **words, int wordcount);
 
-int cli_cmd_sys_exec_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                          const char **words, int wordcount);
+int
+cli_cmd_sys_exec_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                     const char **words, int wordcount);
 
 int
-cli_cmd_getspec_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                     const char **words, int wordcount)
+cli_cmd_getspec_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                    const char **words, int wordcount)
 {
-        int                     ret = -1;
-        rpc_clnt_procedure_t    *proc = NULL;
-        call_frame_t            *frame = NULL;
-        dict_t                  *dict = NULL;
-
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
-
-        dict = dict_new ();
-        if (!dict)
-                goto out;
-
-        if (wordcount != 3) {
-                cli_usage_out (word->pattern);
-                goto out;
-        }
-
-        ret = dict_set_str (dict, "volid", (char *)words[2]);
-        if (ret)
-                goto out;
-
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_GETSPEC];
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, dict);
-        }
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *dict = NULL;
+
+    if (wordcount != 3) {
+        cli_usage_out(word->pattern);
+        goto out;
+    }
+
+    dict = dict_new();
+    if (!dict)
+        goto out;
+
+    ret = dict_set_str(dict, "volid", (char *)words[2]);
+    if (ret)
+        goto out;
+
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_GETSPEC];
+    if (proc->fn) {
+        frame = create_frame(THIS, THIS->ctx->pool);
+        if (!frame) {
+            ret = -1;
+            goto out;
+        }
+        ret = proc->fn(frame, THIS, dict);
+    }
 
 out:
-        if (!proc && ret) {
-                if (dict)
-                        dict_destroy (dict);
-                if (wordcount > 1)
-                        cli_out ("Fetching spec for volume %s failed",
-                                 (char *)words[2]);
-        }
+    if (!proc && ret) {
+        if (wordcount > 1)
+            cli_out("Fetching spec for volume %s failed", (char *)words[2]);
+    }
+
+    if (dict)
+        dict_unref(dict);
 
-        return ret;
+    CLI_STACK_DESTROY(frame);
+    return ret;
 }
 
 int
-cli_cmd_pmap_b2p_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                 const char **words, int wordcount)
+cli_cmd_pmap_b2p_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                     const char **words, int wordcount)
 {
-        int                     ret = -1;
-        rpc_clnt_procedure_t    *proc = NULL;
-        call_frame_t            *frame = NULL;
-        dict_t                  *dict = NULL;
-
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
-
-        dict = dict_new ();
-        if (!dict)
-                goto out;
-
-        if (wordcount != 4) {
-                cli_usage_out (word->pattern);
-                goto out;
-        }
-
-        ret = dict_set_str (dict, "brick", (char *)words[3]);
-        if (ret)
-                goto out;
-
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_PMAP_PORTBYBRICK];
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, dict);
-        }
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *dict = NULL;
+
+    if (wordcount != 4) {
+        cli_usage_out(word->pattern);
+        goto out;
+    }
+
+    dict = dict_new();
+    if (!dict)
+        goto out;
+
+    ret = dict_set_str(dict, "brick", (char *)words[3]);
+    if (ret)
+        goto out;
+
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_PMAP_PORTBYBRICK];
+    if (proc->fn) {
+        frame = create_frame(THIS, THIS->ctx->pool);
+        if (!frame) {
+            ret = -1;
+            goto out;
+        }
+        ret = proc->fn(frame, THIS, dict);
+    }
 
 out:
-        if (!proc && ret) {
-                if (dict)
-                        dict_destroy (dict);
-                if (wordcount > 1)
-                        cli_out ("Fetching spec for volume %s failed",
-                                 (char *)words[3]);
-        }
+    if (!proc && ret) {
+        if (wordcount > 1)
+            cli_out("Fetching spec for volume %s failed", (char *)words[3]);
+    }
+
+    if (dict)
+        dict_unref(dict);
 
-        return ret;
+    CLI_STACK_DESTROY(frame);
+    return ret;
 }
 
 int
-cli_cmd_fsm_log_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                     const char **words, int wordcount)
+cli_cmd_fsm_log_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                    const char **words, int wordcount)
 {
-        int                             ret = -1;
-        rpc_clnt_procedure_t            *proc = NULL;
-        call_frame_t                    *frame = NULL;
-        char                            *name = "";
-
-        if ((wordcount != 4) && (wordcount != 3)) {
-                cli_usage_out (word->pattern);
-                goto out;
-        }
-
-        if (wordcount == 4)
-                name = (char*)words[3];
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_FSM_LOG];
-        if (proc && proc->fn) {
-                frame = create_frame (THIS, THIS->ctx->pool);
-                if (!frame)
-                        goto out;
-                ret = proc->fn (frame, THIS, (void*)name);
-        }
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    char *name = "";
+
+    if ((wordcount != 4) && (wordcount != 3)) {
+        cli_usage_out(word->pattern);
+        goto out;
+    }
+
+    if (wordcount == 4)
+        name = (char *)words[3];
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_FSM_LOG];
+    if (proc && proc->fn) {
+        frame = create_frame(THIS, THIS->ctx->pool);
+        if (!frame)
+            goto out;
+        ret = proc->fn(frame, THIS, (void *)name);
+    }
 out:
-        return ret;
+    return ret;
 }
 
 int
-cli_cmd_getwd_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                   const char **words, int wordcount)
+cli_cmd_getwd_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                  const char **words, int wordcount)
 {
-        int                             ret = -1;
-        rpc_clnt_procedure_t            *proc = NULL;
-        call_frame_t                    *frame = NULL;
-
-        if (wordcount != 2) {
-                cli_usage_out (word->pattern);
-                goto out;
-        }
-
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_GETWD];
-        if (proc && proc->fn) {
-                frame = create_frame (THIS, THIS->ctx->pool);
-                if (!frame)
-                        goto out;
-                ret = proc->fn (frame, THIS, NULL);
-        }
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+
+    if (wordcount != 2) {
+        cli_usage_out(word->pattern);
+        goto out;
+    }
+
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_GETWD];
+    if (proc && proc->fn) {
+        frame = create_frame(THIS, THIS->ctx->pool);
+        if (!frame)
+            goto out;
+        ret = proc->fn(frame, THIS, NULL);
+    }
 out:
-        return ret;
+    return ret;
 }
 
 static dict_t *
-make_seq_dict (int argc, char **argv)
+make_seq_dict(int argc, char **argv)
 {
-        char index[] = "4294967296"; // 1<<32
-        int i        = 0;
-        int ret      = 0;
-        dict_t *dict = dict_new ();
-
-        if (!dict)
-                return NULL;
-
-        for (i = 0; i < argc; i++) {
-                snprintf(index, sizeof(index), "%d", i);
-                ret = dict_set_str (dict, index, argv[i]);
-                if (ret == -1)
-                        break;
-        }
-
-        if (ret) {
-                dict_destroy (dict);
-                dict = NULL;
-        }
-
-        return dict;
+    char index[] = "4294967296";  // 1<<32
+    int i = 0;
+    int len;
+    int ret = 0;
+    dict_t *dict = dict_new();
+
+    if (!dict)
+        return NULL;
+
+    for (i = 0; i < argc; i++) {
+        len = snprintf(index, sizeof(index), "%d", i);
+        ret = dict_set_strn(dict, index, len, argv[i]);
+        if (ret == -1)
+            break;
+    }
+
+    if (ret) {
+        dict_unref(dict);
+        dict = NULL;
+    }
+
+    return dict;
 }
 
 int
-cli_cmd_mount_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                   const char **words, int wordcount)
+cli_cmd_mount_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                  const char **words, int wordcount)
 {
-        rpc_clnt_procedure_t *proc = NULL;
-        call_frame_t *frame        = NULL;
-        int ret                    = -1;
-        dict_t *dict               = NULL;
-        void *dataa[]              = {NULL, NULL};
-
-        if (wordcount < 4) {
-                cli_usage_out (word->pattern);
-                goto out;
-        }
-
-        dict = make_seq_dict (wordcount - 3, (char **)words + 3);
-        if (!dict)
-                goto out;
-
-        dataa[0] = (void *)words[2];
-        dataa[1] = dict;
-
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_MOUNT];
-        if (proc && proc->fn) {
-                frame = create_frame (THIS, THIS->ctx->pool);
-                if (!frame)
-                        goto out;
-                ret = proc->fn (frame, THIS, dataa);
-        }
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    int ret = -1;
+    dict_t *dict = NULL;
+    void *dataa[] = {NULL, NULL};
+
+    if (wordcount < 4) {
+        cli_usage_out(word->pattern);
+        goto out;
+    }
+
+    dict = make_seq_dict(wordcount - 3, (char **)words + 3);
+    if (!dict)
+        goto out;
+
+    dataa[0] = (void *)words[2];
+    dataa[1] = dict;
+
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_MOUNT];
+    if (proc && proc->fn) {
+        frame = create_frame(THIS, THIS->ctx->pool);
+        if (!frame)
+            goto out;
+        ret = proc->fn(frame, THIS, dataa);
+    }
 
- out:
-        if (dict)
-                dict_unref (dict);
+out:
+    if (dict)
+        dict_unref(dict);
 
-        if (!proc && ret)
-                cli_out ("Mount command failed");
+    if (!proc && ret)
+        cli_out("Mount command failed");
 
-        return ret;
+    return ret;
 }
 
 int
-cli_cmd_umount_cbk (struct cli_state *state, struct cli_cmd_word *word,
+cli_cmd_umount_cbk(struct cli_state *state, struct cli_cmd_word *word,
                    const char **words, int wordcount)
 {
-        rpc_clnt_procedure_t *proc = NULL;
-        call_frame_t *frame        = NULL;
-        int ret                    = -1;
-        dict_t *dict               = NULL;
-
-        if (!(wordcount == 3 ||
-              (wordcount == 4 && strcmp (words[3], "lazy") == 0))) {
-                cli_usage_out (word->pattern);
-                goto out;
-        }
-
-        dict = dict_new ();
-        if (!dict)
-                goto out;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    int ret = -1;
+    dict_t *dict = NULL;
+
+    if (!(wordcount == 3 ||
+          (wordcount == 4 && strcmp(words[3], "lazy") == 0))) {
+        cli_usage_out(word->pattern);
+        goto out;
+    }
+
+    dict = dict_new();
+    if (!dict)
+        goto out;
+
+    ret = dict_set_str(dict, "path", (char *)words[2]);
+    if (ret != 0)
+        goto out;
+    ret = dict_set_int32(dict, "lazy", wordcount == 4);
+    if (ret != 0)
+        goto out;
+
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_UMOUNT];
+    if (proc && proc->fn) {
+        frame = create_frame(THIS, THIS->ctx->pool);
+        if (!frame) {
+            gf_log(THIS->name, GF_LOG_ERROR, "failed to create frame");
+            ret = -1;
+            goto out;
+        }
+        ret = proc->fn(frame, THIS, dict);
+    }
 
-        ret = dict_set_str (dict, "path", (char *)words[2]);
-        if (ret != 0)
-                goto out;
-        ret = dict_set_int32 (dict, "lazy", wordcount == 4);
-        if (ret != 0)
-                goto out;
-
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_UMOUNT];
-        if (proc && proc->fn) {
-                frame = create_frame (THIS, THIS->ctx->pool);
-                if (!frame)
-                        goto out;
-                ret = proc->fn (frame, THIS, dict);
-        }
-
- out:
-        if (dict)
-                dict_unref (dict);
+out:
+    if (dict)
+        dict_unref(dict);
 
-        if (!proc && ret)
-                cli_out ("Umount command failed");
+    if (!proc && ret)
+        cli_out("Umount command failed");
 
-        return ret;
+    return ret;
 }
 
 int
-cli_cmd_uuid_get_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                      const char **words, int wordcount)
+cli_cmd_uuid_get_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                     const char **words, int wordcount)
 {
-        int                     ret = -1;
-        int                     sent = 0;
-        int                     parse_error = 0;
-        dict_t                  *dict  = NULL;
-        rpc_clnt_procedure_t    *proc = NULL;
-        call_frame_t            *frame = NULL;
-        cli_local_t             *local = NULL;
-        xlator_t                *this  = NULL;
-
-        this = THIS;
-        if (wordcount != 3) {
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-                goto out;
-        }
-
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_UUID_GET];
-        frame = create_frame (this, this->ctx->pool);
-        if (!frame)
-                goto out;
-
-        dict = dict_new ();
-        if (!dict)
-                goto out;
-
-        CLI_LOCAL_INIT (local, words, frame, dict);
-        if (proc->fn)
-                ret = proc->fn (frame, this, dict);
+    int ret = -1;
+    int sent = 0;
+    int parse_error = 0;
+    dict_t *dict = NULL;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    cli_local_t *local = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    if (wordcount != 3) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
+
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_UUID_GET];
+    frame = create_frame(this, this->ctx->pool);
+    if (!frame)
+        goto out;
+
+    dict = dict_new();
+    if (!dict)
+        goto out;
+
+    CLI_LOCAL_INIT(local, words, frame, dict);
+    if (proc->fn)
+        ret = proc->fn(frame, this, dict);
 
 out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_out ("uuid get failed");
-        }
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("uuid get failed");
+    }
 
-        if (dict)
-                dict_unref (dict);
+    if (dict)
+        dict_unref(dict);
 
-        CLI_STACK_DESTROY (frame);
-        return ret;
+    CLI_STACK_DESTROY(frame);
+    return ret;
 }
 
 int
-cli_cmd_uuid_reset_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                        const char **words, int wordcount)
+cli_cmd_uuid_reset_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                       const char **words, int wordcount)
 {
-        int                     ret = -1;
-        rpc_clnt_procedure_t    *proc = NULL;
-        call_frame_t            *frame = NULL;
-        int                     sent = 0;
-        int                     parse_error = 0;
-        gf_answer_t             answer = GF_ANSWER_NO;
-        char                    *question = NULL;
-        cli_local_t             *local = NULL;
-        dict_t                  *dict  = NULL;
-        xlator_t                *this  = NULL;
-
-        question = "Resetting uuid changes the uuid of local glusterd. "
-                   "Do you want to continue?";
-
-        if (wordcount != 3) {
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-                goto out;
-        }
-
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_UUID_RESET];
-
-        this = THIS;
-        frame = create_frame (this, this->ctx->pool);
-        if (!frame)
-                goto out;
-
-        dict = dict_new ();
-        if (!dict) {
-                ret = -1;
-                goto out;
-        }
-        CLI_LOCAL_INIT (local, words, frame, dict);
-        answer = cli_cmd_get_confirmation (state, question);
-
-        if (GF_ANSWER_NO == answer) {
-                ret = 0;
-                goto out;
-        }
-
-        //send NULL as argument since no dictionary is sent to glusterd
-        if (proc->fn) {
-                ret = proc->fn (frame, this, dict);
-        }
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    int sent = 0;
+    int parse_error = 0;
+    gf_answer_t answer = GF_ANSWER_NO;
+    char *question = NULL;
+    cli_local_t *local = NULL;
+    dict_t *dict = NULL;
+    xlator_t *this = NULL;
+
+    question =
+        "Resetting uuid changes the uuid of local glusterd. "
+        "Do you want to continue?";
+
+    if (wordcount != 3) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
+
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_UUID_RESET];
+
+    this = THIS;
+    frame = create_frame(this, this->ctx->pool);
+    if (!frame)
+        goto out;
+
+    dict = dict_new();
+    if (!dict) {
+        ret = -1;
+        goto out;
+    }
+    CLI_LOCAL_INIT(local, words, frame, dict);
+    answer = cli_cmd_get_confirmation(state, question);
+
+    if (GF_ANSWER_NO == answer) {
+        ret = 0;
+        goto out;
+    }
+
+    // send NULL as argument since no dictionary is sent to glusterd
+    if (proc->fn) {
+        ret = proc->fn(frame, this, dict);
+    }
 
 out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_out ("uuid reset failed");
-        }
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("uuid reset failed");
+    }
+
+    if (dict)
+        dict_unref(dict);
 
-        CLI_STACK_DESTROY (frame);
+    CLI_STACK_DESTROY(frame);
 
-        return ret;
+    return ret;
 }
 
-struct cli_cmd cli_system_cmds[] = {
-        { "system:: getspec <VOLID>",
-          cli_cmd_getspec_cbk,
-          "fetch spec for volume <VOLID>"},
+static struct cli_cmd cli_system_cmds[] = {
+    {"system:: getspec <VOLNAME>", cli_cmd_getspec_cbk,
+     "fetch the volume file for the volume <VOLNAME>"},
 
-        { "system:: portmap brick2port <BRICK>",
-          cli_cmd_pmap_b2p_cbk,
-          "query which port <BRICK> listens on"},
+    {"system:: portmap brick2port <BRICK>", cli_cmd_pmap_b2p_cbk,
+     "query which port <BRICK> listens on"},
 
-        { "system:: fsm log [<peer-name>]",
-          cli_cmd_fsm_log_cbk,
-          "display fsm transitions"},
+    {"system:: fsm log [<peer-name>]", cli_cmd_fsm_log_cbk,
+     "display fsm transitions"},
 
-        { "system:: getwd",
-          cli_cmd_getwd_cbk,
-          "query glusterd work directory"},
+    {"system:: getwd", cli_cmd_getwd_cbk, "query glusterd work directory"},
 
-        { "system:: mount <label> <args...>",
-          cli_cmd_mount_cbk,
-          "request a mount"},
+    {"system:: mount <label> <args...>", cli_cmd_mount_cbk, "request a mount"},
 
-        { "system:: umount <path> [lazy]",
-          cli_cmd_umount_cbk,
-          "request an umount"},
+    {"system:: umount <path> [lazy]", cli_cmd_umount_cbk, "request an umount"},
 
-        { "system:: uuid get",
-          cli_cmd_uuid_get_cbk,
-          "get uuid of glusterd"},
+    {"system:: uuid get", cli_cmd_uuid_get_cbk, "get uuid of glusterd"},
 
-        { "system:: uuid reset",
-          cli_cmd_uuid_reset_cbk,
-          "reset the uuid of glusterd"},
+    {"system:: uuid reset", cli_cmd_uuid_reset_cbk,
+     "reset the uuid of glusterd"},
 
-        { "system:: help",
-           cli_cmd_system_help_cbk,
-           "display help for system commands"},
+    {"system:: help", cli_cmd_system_help_cbk,
+     "display help for system commands"},
 
-        { "system:: copy file [<filename>]",
-           cli_cmd_copy_file_cbk,
-           "Copy file from current node's $working_dir to "
-           "$working_dir of all cluster nodes"},
+    {"system:: copy file [<filename>]", cli_cmd_copy_file_cbk,
+     "Copy file from current node's $working_dir to "
+     "$working_dir of all cluster nodes"},
 
-        { "system:: execute <command> <args>",
-           cli_cmd_sys_exec_cbk,
-           "Execute the command on all the nodes "
-           "in the cluster and display their output."},
+    {"system:: execute <command> <args>", cli_cmd_sys_exec_cbk,
+     "Execute the command on all the nodes "
+     "in the cluster and display their output."},
 
-        { NULL, NULL, NULL }
-};
+    {NULL, NULL, NULL}};
 
 int
-cli_cmd_sys_exec_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                      const char **words, int wordcount)
+cli_cmd_sys_exec_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                     const char **words, int wordcount)
 {
-        char                   cmd_arg_name[PATH_MAX] = "";
-        char                  *command                = NULL;
-        char                  *saveptr                = NULL;
-        char                  *tmp                    = NULL;
-        int                    ret                    = -1;
-        int                    i                      = -1;
-        int                    cmd_args_count         = 0;
-        int                    in_cmd_args_count      = 0;
-        rpc_clnt_procedure_t  *proc                   = NULL;
-        call_frame_t          *frame                  = NULL;
-        dict_t                *dict                   = NULL;
-        cli_local_t           *local                  = NULL;
-
-        if (wordcount < 3) {
-                cli_usage_out (word->pattern);
-                goto out;
-        }
-
-        dict = dict_new ();
-        if (!dict)
-                goto out;
-
-        command = strtok_r ((char *)words[2], " ", &saveptr);
-        do {
-                tmp = strtok_r (NULL, " ", &saveptr);
-                if (tmp) {
-                        in_cmd_args_count++;
-                        memset (cmd_arg_name, '\0', sizeof(cmd_arg_name));
-                        snprintf (cmd_arg_name, sizeof(cmd_arg_name),
-                                  "cmd_arg_%d", in_cmd_args_count);
-                        ret = dict_set_str (dict, cmd_arg_name, tmp);
-                        if (ret) {
-                                gf_log ("", GF_LOG_ERROR, "Unable to set "
-                                        "%s in dict", cmd_arg_name);
-                                goto out;
-                        }
-                }
-        } while (tmp);
-
-        cmd_args_count = wordcount - 3;
-
-        ret = dict_set_str (dict, "command", command);
-        if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to set command in dict");
+    char cmd_arg_name[PATH_MAX] = "";
+    char *command = NULL;
+    char *saveptr = NULL;
+    char *tmp = NULL;
+    int ret = -1;
+    int i = -1;
+    int len;
+    int cmd_args_count = 0;
+    int in_cmd_args_count = 0;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *dict = NULL;
+    cli_local_t *local = NULL;
+
+    if ((wordcount < 3) || (words[2] == NULL)) {
+        cli_usage_out(word->pattern);
+        goto out;
+    }
+
+    command = strtok_r((char *)words[2], " ", &saveptr);
+    if (command == NULL) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to parse command");
+        goto out;
+    }
+
+    dict = dict_new();
+    if (!dict)
+        goto out;
+
+    do {
+        tmp = strtok_r(NULL, " ", &saveptr);
+        if (tmp) {
+            in_cmd_args_count++;
+            snprintf(cmd_arg_name, sizeof(cmd_arg_name), "cmd_arg_%d",
+                     in_cmd_args_count);
+            ret = dict_set_str(dict, cmd_arg_name, tmp);
+            if (ret) {
+                gf_log("", GF_LOG_ERROR,
+                       "Unable to set "
+                       "%s in dict",
+                       cmd_arg_name);
                 goto out;
+            }
         }
+    } while (tmp);
 
-        for (i=1; i <= cmd_args_count; i++) {
-                in_cmd_args_count++;
-                memset (cmd_arg_name, '\0', sizeof(cmd_arg_name));
-                snprintf (cmd_arg_name, sizeof(cmd_arg_name),
-                          "cmd_arg_%d", in_cmd_args_count);
-                ret = dict_set_str (dict, cmd_arg_name,
-                                    (char *)words[2+i]);
-                if (ret) {
-                        gf_log ("", GF_LOG_ERROR, "Unable to set %s in dict",
-                               cmd_arg_name);
-                        goto out;
-                }
-        }
+    cmd_args_count = wordcount - 3;
 
-        ret = dict_set_int32 (dict, "cmd_args_count", in_cmd_args_count);
-        if (ret) {
-                gf_log ("", GF_LOG_ERROR,
-                        "Unable to set cmd_args_count in dict");
-                goto out;
-        }
+    ret = dict_set_str(dict, "command", command);
+    if (ret) {
+        gf_log("", GF_LOG_ERROR, "Unable to set command in dict");
+        goto out;
+    }
 
-        ret = dict_set_str (dict, "volname", "N/A");
+    for (i = 1; i <= cmd_args_count; i++) {
+        in_cmd_args_count++;
+        len = snprintf(cmd_arg_name, sizeof(cmd_arg_name), "cmd_arg_%d",
+                       in_cmd_args_count);
+        ret = dict_set_strn(dict, cmd_arg_name, len, (char *)words[2 + i]);
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to set volname in dict");
-                goto out;
-        }
-
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_SYS_EXEC];
-        if (proc && proc->fn) {
-                frame = create_frame (THIS, THIS->ctx->pool);
-                if (!frame)
-                        goto out;
-                CLI_LOCAL_INIT (local, words, frame, dict);
-                ret = proc->fn (frame, THIS, (void*)dict);
-        }
+            gf_log("", GF_LOG_ERROR, "Unable to set %s in dict", cmd_arg_name);
+            goto out;
+        }
+    }
+
+    ret = dict_set_int32(dict, "cmd_args_count", in_cmd_args_count);
+    if (ret) {
+        gf_log("", GF_LOG_ERROR, "Unable to set cmd_args_count in dict");
+        goto out;
+    }
+
+    ret = dict_set_str(dict, "volname", "N/A");
+    if (ret) {
+        gf_log("", GF_LOG_ERROR, "Unable to set volname in dict");
+        goto out;
+    }
+
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_SYS_EXEC];
+    if (proc->fn) {
+        frame = create_frame(THIS, THIS->ctx->pool);
+        if (!frame) {
+            gf_log(THIS->name, GF_LOG_ERROR, "failed to create frame");
+            ret = -1;
+            goto out;
+        }
+        CLI_LOCAL_INIT(local, words, frame, dict);
+        ret = proc->fn(frame, THIS, (void *)dict);
+
+        /* proc->fn is processed synchronously, which means that the
+         * execution flow won't return here until the operation is
+         * fully processed, including any related callback. For this
+         * reason, it's safe to destroy the stack here, since no one
+         * can still be using it. Additionally, it's not easy to move
+         * the stack destroy to the callback executed after completion
+         * of the operation because there are multiple things than can
+         * fail even before having queued the callback, so we would
+         * still need to destroy the stack if proc->fn returns an
+         * error. */
+        CLI_STACK_DESTROY(frame);
+        dict = NULL;
+    }
 out:
-        return ret;
+    if (dict != NULL) {
+        dict_unref(dict);
+    }
+
+    return ret;
 }
 
 int
-cli_cmd_copy_file_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                       const char **words, int wordcount)
+cli_cmd_copy_file_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                      const char **words, int wordcount)
 {
-        int                    ret      = -1;
-        rpc_clnt_procedure_t  *proc     = NULL;
-        call_frame_t          *frame    = NULL;
-        char                  *filename = "";
-        dict_t                *dict     = NULL;
-        cli_local_t           *local    = NULL;
-
-        if (wordcount != 4) {
-                cli_usage_out (word->pattern);
-                goto out;
-        }
-
-        dict = dict_new ();
-        if (!dict)
-                goto out;
-
-        filename = (char*)words[3];
-        ret = dict_set_str (dict, "source", filename);
-        if (ret)
-                 gf_log ("", GF_LOG_ERROR, "Unable to set filename in dict");
-
-        ret = dict_set_str (dict, "volname", "N/A");
-        if (ret)
-                 gf_log ("", GF_LOG_ERROR, "Unable to set volname in dict");
-
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_COPY_FILE];
-        if (proc && proc->fn) {
-                frame = create_frame (THIS, THIS->ctx->pool);
-                if (!frame)
-                        goto out;
-                CLI_LOCAL_INIT (local, words, frame, dict);
-                ret = proc->fn (frame, THIS, (void*)dict);
-        }
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    char *filename = "";
+    dict_t *dict = NULL;
+    cli_local_t *local = NULL;
+
+    if (wordcount != 4) {
+        cli_usage_out(word->pattern);
+        goto out;
+    }
+
+    dict = dict_new();
+    if (!dict)
+        goto out;
+
+    filename = (char *)words[3];
+    ret = dict_set_str(dict, "source", filename);
+    if (ret)
+        gf_log("", GF_LOG_ERROR, "Unable to set filename in dict");
+
+    ret = dict_set_str(dict, "volname", "N/A");
+    if (ret)
+        gf_log("", GF_LOG_ERROR, "Unable to set volname in dict");
+
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_COPY_FILE];
+    if (proc && proc->fn) {
+        frame = create_frame(THIS, THIS->ctx->pool);
+        if (!frame) {
+            gf_log(THIS->name, GF_LOG_ERROR, "failed to create frame");
+            ret = -1;
+            goto out;
+        }
+        CLI_LOCAL_INIT(local, words, frame, dict);
+        ret = proc->fn(frame, THIS, (void *)dict);
+    }
 out:
-        return ret;
+    return ret;
 }
 
 int
-cli_cmd_system_help_cbk (struct cli_state *state, struct cli_cmd_word *in_word,
-                         const char **words, int wordcount)
+cli_cmd_system_help_cbk(struct cli_state *state, struct cli_cmd_word *in_word,
+                        const char **words, int wordcount)
 {
-        struct cli_cmd *cmd = NULL;
+    struct cli_cmd *cmd = NULL;
+    struct cli_cmd *system_cmd = NULL;
+    int count = 0;
 
-        for (cmd = cli_system_cmds; cmd->pattern; cmd++)
-                cli_out ("%s - %s", cmd->pattern, cmd->desc);
+    cmd = GF_MALLOC(sizeof(cli_system_cmds), cli_mt_cli_cmd);
+    memcpy(cmd, cli_system_cmds, sizeof(cli_system_cmds));
+    count = (sizeof(cli_system_cmds) / sizeof(struct cli_cmd));
+    cli_cmd_sort(cmd, count);
 
-        return 0;
+    for (system_cmd = cmd; system_cmd->pattern; system_cmd++)
+        cli_out("%s - %s", system_cmd->pattern, system_cmd->desc);
+
+    GF_FREE(cmd);
+    return 0;
 }
 
 int
-cli_cmd_system_register (struct cli_state *state)
+cli_cmd_system_register(struct cli_state *state)
 {
-        int  ret = 0;
-        struct cli_cmd *cmd = NULL;
-
-        for (cmd = cli_system_cmds; cmd->pattern; cmd++) {
+    int ret = 0;
+    struct cli_cmd *cmd = NULL;
 
-                ret = cli_cmd_register (&state->tree, cmd);
-                if (ret)
-                        goto out;
-        }
+    for (cmd = cli_system_cmds; cmd->pattern; cmd++) {
+        ret = cli_cmd_register(&state->tree, cmd);
+        if (ret)
+            goto out;
+    }
 out:
-        return ret;
+    return ret;
 }
diff --git a/cli/src/cli-cmd-volume.c b/cli/src/cli-cmd-volume.c
index 9a50789b6b3..f238851586e 100644
--- a/cli/src/cli-cmd-volume.c
+++ b/cli/src/cli-cmd-volume.c
@@ -19,2415 +19,3260 @@
 #include <sys/wait.h>
 #include <netinet/in.h>
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 #include "cli.h"
 #include "cli-cmd.h"
 #include "cli-mem-types.h"
 #include "cli1-xdr.h"
-#include "run.h"
-#include "syscall.h"
+#include <glusterfs/run.h>
+#include <glusterfs/syscall.h>
+#include <glusterfs/common-utils.h>
+#include <glusterfs/events.h>
 
-extern struct rpc_clnt *global_rpc;
-extern struct rpc_clnt *global_quotad_rpc;
-
-extern rpc_clnt_prog_t *cli_rpc_prog;
 extern rpc_clnt_prog_t cli_quotad_clnt;
 
-int
-cli_cmd_volume_help_cbk (struct cli_state *state, struct cli_cmd_word *in_word,
-                      const char **words, int wordcount);
+static int
+gf_asprintf_append(char **string_ptr, const char *format, ...);
 
 int
-cli_cmd_volume_info_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                         const char **words, int wordcount)
-{
-        int                             ret = -1;
-        rpc_clnt_procedure_t            *proc = NULL;
-        call_frame_t                    *frame = NULL;
-        cli_cmd_volume_get_ctx_t        ctx = {0,};
-        cli_local_t                     *local = NULL;
-        int                             sent = 0;
-        int                             parse_error = 0;
+cli_cmd_volume_help_cbk(struct cli_state *state, struct cli_cmd_word *in_word,
+                        const char **words, int wordcount);
 
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_GET_VOLUME];
+int
+cli_cmd_bitrot_help_cbk(struct cli_state *state, struct cli_cmd_word *in_word,
+                        const char **words, int wordcount);
 
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
+int
+cli_cmd_quota_help_cbk(struct cli_state *state, struct cli_cmd_word *in_word,
+                       const char **words, int wordcount);
 
-        if ((wordcount == 2)  || (wordcount == 3 &&
-                                  !strcmp (words[2], "all"))) {
-                ctx.flags = GF_CLI_GET_NEXT_VOLUME;
-                proc = &cli_rpc_prog->proctable[GLUSTER_CLI_GET_NEXT_VOLUME];
-        } else if (wordcount == 3) {
-                ctx.flags = GF_CLI_GET_VOLUME;
-                ctx.volname = (char *)words[2];
-                if (strlen (ctx.volname) > 1024) {
-                        cli_out ("Invalid volume name");
-                        goto out;
-                }
-                proc = &cli_rpc_prog->proctable[GLUSTER_CLI_GET_VOLUME];
-        } else {
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-                return -1;
+int
+cli_cmd_volume_info_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                        const char **words, int wordcount)
+{
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    cli_cmd_volume_get_ctx_t ctx = {
+        0,
+    };
+    cli_local_t *local = NULL;
+    int sent = 0;
+    int parse_error = 0;
+
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_GET_VOLUME];
+
+    if ((wordcount == 2) || (wordcount == 3 && !strcmp(words[2], "all"))) {
+        ctx.flags = GF_CLI_GET_NEXT_VOLUME;
+        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_GET_NEXT_VOLUME];
+    } else if (wordcount == 3) {
+        ctx.flags = GF_CLI_GET_VOLUME;
+        ctx.volname = (char *)words[2];
+        if (strlen(ctx.volname) > GD_VOLUME_NAME_MAX) {
+            cli_out("Invalid volume name");
+            goto out;
         }
+        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_GET_VOLUME];
+    } else {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        return -1;
+    }
 
-        local = cli_local_get ();
+    local = cli_local_get();
 
-        if (!local)
-                goto out;
+    if (!local)
+        goto out;
 
-        local->get_vol.flags = ctx.flags;
-        if (ctx.volname)
-                local->get_vol.volname = gf_strdup (ctx.volname);
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame)
+        goto out;
 
-        frame->local = local;
+    local->get_vol.flags = ctx.flags;
+    if (ctx.volname)
+        local->get_vol.volname = gf_strdup(ctx.volname);
 
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, &ctx);
-        }
+    frame->local = local;
 
-out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_out ("Getting Volume information failed!");
-        }
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, &ctx);
+    }
 
-        CLI_STACK_DESTROY (frame);
+out:
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("Getting Volume information failed!");
+    }
 
-        return ret;
+    CLI_STACK_DESTROY(frame);
 
+    return ret;
 }
 
 int
-cli_cmd_sync_volume_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                         const char **words, int wordcount)
+cli_cmd_sync_volume_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                        const char **words, int wordcount)
 {
-        int                     ret = -1;
-        rpc_clnt_procedure_t    *proc = NULL;
-        call_frame_t            *frame = NULL;
-        int                     sent = 0;
-        int                     parse_error = 0;
-        dict_t                  *dict = NULL;
-        cli_local_t             *local = NULL;
-        gf_answer_t             answer = GF_ANSWER_NO;
-        const char              *question = "Sync volume may make data "
-                                            "inaccessible while the sync "
-                                            "is in progress. Do you want "
-                                            "to continue?";
-
-        if ((wordcount < 3) || (wordcount > 4)) {
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-                goto out;
-        }
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    int sent = 0;
+    int parse_error = 0;
+    dict_t *dict = NULL;
+    cli_local_t *local = NULL;
+    gf_answer_t answer = GF_ANSWER_NO;
+    const char *question =
+        "Sync volume may make data "
+        "inaccessible while the sync "
+        "is in progress. Do you want "
+        "to continue?";
+
+    if ((wordcount < 3) || (wordcount > 4)) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
 
-        dict = dict_new ();
-        if (!dict)
-                goto out;
+    dict = dict_new();
+    if (!dict)
+        goto out;
 
-        if ((wordcount == 3) || !strcmp(words[3], "all")) {
-                ret = dict_set_int32 (dict, "flags", (int32_t)
-                                      GF_CLI_SYNC_ALL);
-                if (ret) {
-                        gf_log (THIS->name, GF_LOG_ERROR, "failed to set"
-                                "flag");
-                        goto out;
-                }
-        } else {
-                ret = dict_set_str (dict, "volname", (char *) words[3]);
-                if (ret) {
-                        gf_log (THIS->name, GF_LOG_ERROR, "failed to set "
-                                "volume");
-                        goto out;
-                }
+    if ((wordcount == 3) || !strcmp(words[3], "all")) {
+        ret = dict_set_int32(dict, "flags", (int32_t)GF_CLI_SYNC_ALL);
+        if (ret) {
+            gf_log(THIS->name, GF_LOG_ERROR,
+                   "failed to set"
+                   "flag");
+            goto out;
         }
-
-        ret = dict_set_str (dict, "hostname", (char *) words[2]);
+    } else {
+        ret = dict_set_str(dict, "volname", (char *)words[3]);
         if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "failed to set hostname");
-                goto out;
+            gf_log(THIS->name, GF_LOG_ERROR,
+                   "failed to set "
+                   "volume");
+            goto out;
         }
+    }
 
-        if (!(state->mode & GLUSTER_MODE_SCRIPT)) {
-                answer = cli_cmd_get_confirmation (state, question);
-                if (GF_ANSWER_NO == answer) {
-                        ret = 0;
-                        goto out;
-                }
+    ret = dict_set_str(dict, "hostname", (char *)words[2]);
+    if (ret) {
+        gf_log(THIS->name, GF_LOG_ERROR, "failed to set hostname");
+        goto out;
+    }
+
+    if (!(state->mode & GLUSTER_MODE_SCRIPT)) {
+        answer = cli_cmd_get_confirmation(state, question);
+        if (GF_ANSWER_NO == answer) {
+            ret = 0;
+            goto out;
         }
+    }
 
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_SYNC_VOLUME];
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_SYNC_VOLUME];
 
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        gf_log(THIS->name, GF_LOG_ERROR, "failed to create frame");
+        ret = -1;
+        goto out;
+    }
 
-        CLI_LOCAL_INIT (local, words, frame, dict);
+    CLI_LOCAL_INIT(local, words, frame, dict);
 
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, dict);
-        }
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, dict);
+    }
 
 out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_out ("Volume sync failed");
-        }
-
-        CLI_STACK_DESTROY (frame);
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("Volume sync failed");
+    }
 
-        return ret;
-}
+    CLI_STACK_DESTROY(frame);
 
-gf_ai_compare_t
-cli_cmd_compare_addrinfo (struct addrinfo *first, struct addrinfo *next)
-{
-        int             ret = -1;
-        struct addrinfo *tmp1 = NULL;
-        struct addrinfo *tmp2 = NULL;
-        char            firstip[NI_MAXHOST] = {0.};
-        char            nextip[NI_MAXHOST] = {0,};
-
-        for (tmp1 = first; tmp1 != NULL; tmp1 = tmp1->ai_next) {
-                ret = getnameinfo (tmp1->ai_addr, tmp1->ai_addrlen, firstip,
-                                   NI_MAXHOST, NULL, 0, NI_NUMERICHOST);
-                if (ret)
-                        return GF_AI_COMPARE_ERROR;
-                for (tmp2 = next; tmp2 != NULL; tmp2 = tmp2->ai_next) {
-                        ret = getnameinfo (tmp2->ai_addr, tmp2->ai_addrlen, nextip,
-                                           NI_MAXHOST, NULL, 0, NI_NUMERICHOST);
-                        if (ret)
-                                return GF_AI_COMPARE_ERROR;
-                        if (!strcmp (firstip, nextip)) {
-                                return GF_AI_COMPARE_MATCH;
-                        }
-                }
-        }
-        return GF_AI_COMPARE_NO_MATCH;
+    return ret;
 }
 
-/* Check for non optimal brick order for replicate :
- * Checks if bricks belonging to a replicate volume
- * are present on the same server
- */
-int32_t
-cli_cmd_check_brick_order (struct cli_state *state, const char *bricks,
-                           int brick_count, int sub_count)
+int
+cli_cmd_volume_create_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                          const char **words, int wordcount)
 {
-        int             ret = -1;
-        int             i = 0;
-        int             j = 0;
-        int             k = 0;
-        addrinfo_list_t *ai_list = NULL;
-        addrinfo_list_t *ai_list_tmp1 = NULL;
-        addrinfo_list_t *ai_list_tmp2 = NULL;
-        char            *brick = NULL;
-        char            *brick_list = NULL;
-        char            *brick_list_dup = NULL;
-        char            *tmpptr = NULL;
-        struct addrinfo *ai_info = NULL;
-        gf_answer_t     answer = GF_ANSWER_NO;
-        const char      *failed_question = NULL;
-        const char      *found_question = NULL;
-        failed_question = "Failed to perform brick order check. "
-                          "Do you want to continue creating the volume? ";
-        found_question =  "Multiple bricks of a replicate volume are present"
-                          " on the same server. This setup is not optimal.\n"
-                          "Do you still want to continue creating the volume? ";
-
-        GF_ASSERT (bricks);
-        GF_ASSERT (brick_count > 0);
-        GF_ASSERT (sub_count > 0);
-
-        ai_list = malloc (sizeof (addrinfo_list_t));
-        ai_list->info = NULL;
-        INIT_LIST_HEAD (&ai_list->list);
-        brick_list = gf_strdup (bricks);
-        if (brick_list == NULL) {
-                gf_log ("cli", GF_LOG_DEBUG, "failed to allocate memory");
-                goto check_failed;
-        }
-        brick_list_dup = brick_list;
-        /* Resolve hostnames and get addrinfo */
-        while (i < brick_count) {
-                ++i;
-                brick = strtok_r (brick_list, " \n", &tmpptr);
-                brick_list = tmpptr;
-                if (brick == NULL)
-                        goto check_failed;
-                brick = strtok_r (brick, ":", &tmpptr);
-                if (brick == NULL)
-                        goto check_failed;
-                ret = getaddrinfo (brick, NULL, NULL, &ai_info);
-                if (ret)
-                        goto check_failed;
-                ai_list_tmp1 = malloc (sizeof (addrinfo_list_t));
-                if (ai_list_tmp1 == NULL)
-                        goto check_failed;
-                ai_list_tmp1->info = ai_info;
-                list_add_tail (&ai_list_tmp1->list, &ai_list->list);
-                ai_list_tmp1 = NULL;
-        }
-
-        i = 0;
-        ai_list_tmp1 = list_entry (ai_list->list.next, addrinfo_list_t, list);
-
-        /* Check for bad brick order */
-        while (i < brick_count) {
-                ++i;
-                ai_info = ai_list_tmp1->info;
-                ai_list_tmp1 = list_entry (ai_list_tmp1->list.next,
-                                           addrinfo_list_t, list);
-                if ( 0 == i % sub_count) {
-                        j = 0;
-                        continue;
-                }
-                ai_list_tmp2 = ai_list_tmp1;
-                k = j;
-                while (k < sub_count - 1) {
-                        ++k;
-                        ret = cli_cmd_compare_addrinfo (ai_info,
-                                                        ai_list_tmp2->info);
-                        if (GF_AI_COMPARE_ERROR == ret)
-                                goto check_failed;
-                        if (GF_AI_COMPARE_MATCH == ret)
-                                goto found_bad_brick_order;
-                        ai_list_tmp2 = list_entry (ai_list_tmp2->list.next,
-                                                   addrinfo_list_t, list);
-                }
-                ++j;
-        }
-        gf_log ("cli", GF_LOG_INFO, "Brick order okay");
-        ret = 0;
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *options = NULL;
+    int sent = 0;
+    int parse_error = 0;
+    cli_local_t *local = NULL;
+    char *trans_type = NULL;
+    char *bricks = NULL;
+
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_CREATE_VOLUME];
+
+    ret = cli_cmd_volume_create_parse(state, words, wordcount, &options,
+                                      &bricks);
+
+    if (ret) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
         goto out;
+    }
 
-check_failed:
-        gf_log ("cli", GF_LOG_INFO, "Failed bad brick order check");
-        answer = cli_cmd_get_confirmation(state, failed_question);
-        if (GF_ANSWER_YES == answer)
-                ret = 0;
+    ret = dict_get_str(options, "transport", &trans_type);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Unable to get transport type");
         goto out;
+    }
 
-found_bad_brick_order:
-        gf_log ("cli", GF_LOG_INFO, "Bad brick order found");
-        answer = cli_cmd_get_confirmation (state, found_question);
-        if (GF_ANSWER_YES == answer)
-                ret = 0;
-out:
-        ai_list_tmp2 = NULL;
-        i = 0;
-        GF_FREE (brick_list_dup);
-        list_for_each_entry (ai_list_tmp1, &ai_list->list, list) {
-                if (ai_list_tmp1->info)
-                        freeaddrinfo (ai_list_tmp1->info);
-                free (ai_list_tmp2);
-                ai_list_tmp2 = ai_list_tmp1;
-        }
-        free (ai_list_tmp2);
-        return ret;
-}
-
-int
-cli_cmd_volume_create_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                           const char **words, int wordcount)
-{
-        int                     ret = -1;
-        rpc_clnt_procedure_t    *proc = NULL;
-        call_frame_t            *frame = NULL;
-        dict_t                  *options = NULL;
-        int                     sent = 0;
-        int                     parse_error = 0;
-        char                    *brick_list = NULL;
-        int32_t                 brick_count = 0;
-        int32_t                 sub_count = 0;
-        int32_t                 type = GF_CLUSTER_TYPE_NONE;
-        cli_local_t             *local = NULL;
-
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_CREATE_VOLUME];
-
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
-
-        ret = cli_cmd_volume_create_parse (words, wordcount, &options);
-
+    if (state->mode & GLUSTER_MODE_WIGNORE) {
+        ret = dict_set_int32(options, "force", _gf_true);
         if (ret) {
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-                goto out;
-        }
-        /*Check brick order if type is replicate*/
-        ret = dict_get_int32 (options, "type", &type);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Could not get brick type");
-                goto out;
-        }
-        if ((type == GF_CLUSTER_TYPE_REPLICATE) ||
-            (type == GF_CLUSTER_TYPE_STRIPE_REPLICATE)) {
-                if ((ret = dict_get_str (options, "bricks", &brick_list)) != 0) {
-                        gf_log ("cli", GF_LOG_ERROR, "Replica bricks check : "
-                                                     "Could not retrieve bricks list");
-                        goto out;
-                }
-                if ((ret = dict_get_int32 (options, "count", &brick_count)) != 0) {
-                        gf_log ("cli", GF_LOG_ERROR, "Replica bricks check : "
-                                                     "Could not retrieve brick count");
-                        goto out;
-                }
-                if ((ret = dict_get_int32 (options, "replica-count", &sub_count)) != 0) {
-                        gf_log ("cli", GF_LOG_ERROR, "Replica bricks check : "
-                                                    "Could not retrieve replica count");
-                        goto out;
-                }
-                gf_log ("cli", GF_LOG_INFO, "Replicate cluster type found."
-                                            " Checking brick order.");
-                ret = cli_cmd_check_brick_order (state, brick_list, brick_count, sub_count);
-                if (ret) {
-                        gf_log("cli", GF_LOG_INFO, "Not creating volume because of bad brick order");
-                        goto out;
-                }
+            gf_log("cli", GF_LOG_ERROR,
+                   "Failed to set force "
+                   "option");
+            goto out;
         }
+    }
 
-        if (state->mode & GLUSTER_MODE_SCRIPT) {
-                ret = dict_set_int32 (options, "force", _gf_true);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Failed to set force "
-                                "option");
-                        goto out;
-                }
-        }
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
 
-        CLI_LOCAL_INIT (local, words, frame, options);
+    CLI_LOCAL_INIT(local, words, frame, options);
 
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, options);
-        }
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, options);
+    }
 
 out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_out ("Volume create failed");
-        }
-
-        CLI_STACK_DESTROY (frame);
-
-        return ret;
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("Volume create failed");
+    }
+
+    if (ret == 0) {
+        gf_event(EVENT_VOLUME_CREATE, "name=%s;bricks=%s", (char *)words[2],
+                 bricks);
+    }
+
+    CLI_STACK_DESTROY(frame);
+    return ret;
 }
 
-
 int
-cli_cmd_volume_delete_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                           const char **words, int wordcount)
+cli_cmd_volume_delete_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                          const char **words, int wordcount)
 {
-        int                     ret = -1;
-        rpc_clnt_procedure_t    *proc = NULL;
-        call_frame_t            *frame = NULL;
-        char                    *volname = NULL;
-        gf_answer_t             answer = GF_ANSWER_NO;
-        const char              *question = NULL;
-        int                     sent = 0;
-        int                     parse_error = 0;
-        cli_local_t             *local = NULL;
-        dict_t                  *dict = NULL;
-
-        question = "Deleting volume will erase all information about the volume. "
-                   "Do you want to continue?";
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_DELETE_VOLUME];
-
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
-
-        dict = dict_new ();
-        if (!dict)
-                goto out;
-
-        if (wordcount != 3) {
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-                goto out;
-        }
-
-        answer = cli_cmd_get_confirmation (state, question);
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    char *volname = NULL;
+    gf_answer_t answer = GF_ANSWER_NO;
+    const char *question = NULL;
+    int sent = 0;
+    int parse_error = 0;
+    cli_local_t *local = NULL;
+    dict_t *dict = NULL;
+
+    question =
+        "Deleting volume will erase all information about the volume. "
+        "Do you want to continue?";
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_DELETE_VOLUME];
+
+    if (wordcount != 3) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
 
-        if (GF_ANSWER_NO == answer) {
-                ret = 0;
-                goto out;
-        }
+    volname = (char *)words[2];
 
-        volname = (char *)words[2];
+    dict = dict_new();
+    if (!dict)
+        goto out;
 
-        ret = dict_set_str (dict, "volname", volname);
+    ret = dict_set_str(dict, "volname", volname);
+    if (ret) {
+        gf_log(THIS->name, GF_LOG_WARNING, "dict set failed");
+        goto out;
+    }
+
+    if (!strcmp(volname, GLUSTER_SHARED_STORAGE)) {
+        question =
+            "Deleting the shared storage volume"
+            "(gluster_shared_storage), will affect features "
+            "like snapshot scheduler, geo-replication "
+            "and NFS-Ganesha. Do you still want to "
+            "continue?";
+    }
+
+    answer = cli_cmd_get_confirmation(state, question);
+    if (GF_ANSWER_NO == answer) {
+        ret = 0;
+        goto out;
+    }
 
-        if (ret) {
-                gf_log (THIS->name, GF_LOG_WARNING, "dict set failed");
-                goto out;
-        }
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
 
-        CLI_LOCAL_INIT (local, words, frame, dict);
+    CLI_LOCAL_INIT(local, words, frame, dict);
 
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, dict);
-        }
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, dict);
+    }
 
 out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_out ("Volume delete failed");
-        }
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("Volume delete failed");
+    }
+
+    CLI_STACK_DESTROY(frame);
 
-        CLI_STACK_DESTROY (frame);
+    if (ret == 0 && GF_ANSWER_YES == answer) {
+        gf_event(EVENT_VOLUME_DELETE, "name=%s", (char *)words[2]);
+    }
 
-        return ret;
+    return ret;
 }
 
 int
-cli_cmd_volume_start_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                          const char **words, int wordcount)
+cli_cmd_volume_start_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                         const char **words, int wordcount)
 {
-        int                     ret = -1;
-        rpc_clnt_procedure_t    *proc = NULL;
-        call_frame_t            *frame = NULL;
-        int                     sent = 0;
-        int                     parse_error = 0;
-        dict_t                  *dict = NULL;
-        int                     flags = 0;
-        cli_local_t             *local = NULL;
-
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    int sent = 0;
+    int parse_error = 0;
+    dict_t *dict = NULL;
+    int flags = 0;
+    cli_local_t *local = NULL;
+
+    if (wordcount < 3 || wordcount > 4) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
 
-        if (wordcount < 3 || wordcount > 4) {
-               cli_usage_out (word->pattern);
-                parse_error = 1;
-               goto out;
-        }
+    if (!words[2])
+        goto out;
 
-        dict = dict_new ();
-        if (!dict) {
-                goto out;
+    if (wordcount == 4) {
+        if (!strcmp("force", words[3])) {
+            flags |= GF_CLI_FLAG_OP_FORCE;
+        } else {
+            ret = -1;
+            cli_usage_out(word->pattern);
+            parse_error = 1;
+            goto out;
         }
+    }
 
-        if (!words[2])
-                goto out;
+    dict = dict_new();
+    if (!dict) {
+        goto out;
+    }
 
-        ret = dict_set_str (dict, "volname", (char *)words[2]);
-        if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "dict set failed");
-                goto out;
-        }
+    ret = dict_set_str(dict, "volname", (char *)words[2]);
+    if (ret) {
+        gf_log(THIS->name, GF_LOG_ERROR, "dict set failed");
+        goto out;
+    }
 
-        if (wordcount == 4) {
-                if (!strcmp("force", words[3])) {
-                        flags |= GF_CLI_FLAG_OP_FORCE;
-                } else {
-                        ret = -1;
-                        cli_usage_out (word->pattern);
-                        parse_error = 1;
-                        goto out;
-                }
-        }
-        ret = dict_set_int32 (dict, "flags", flags);
-        if (ret) {
-                 gf_log (THIS->name, GF_LOG_ERROR,
-                         "dict set failed");
-                 goto out;
-        }
+    ret = dict_set_int32(dict, "flags", flags);
+    if (ret) {
+        gf_log(THIS->name, GF_LOG_ERROR, "dict set failed");
+        goto out;
+    }
 
-        if (ret < 0) {
-                gf_log (THIS->name, GF_LOG_ERROR,
-                        "failed to serialize dict");
-                goto out;
-        }
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_START_VOLUME];
 
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_START_VOLUME];
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
 
-        CLI_LOCAL_INIT (local, words, frame, dict);
+    CLI_LOCAL_INIT(local, words, frame, dict);
 
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, dict);
-        }
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, dict);
+    }
 
 out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_out ("Volume start failed");
-        }
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("Volume start failed");
+    }
+
+    CLI_STACK_DESTROY(frame);
 
-        CLI_STACK_DESTROY (frame);
+    if (ret == 0) {
+        gf_event(EVENT_VOLUME_START, "name=%s;force=%d", (char *)words[2],
+                 (flags & GF_CLI_FLAG_OP_FORCE));
+    }
 
-        return ret;
+    return ret;
 }
 
 gf_answer_t
-cli_cmd_get_confirmation (struct cli_state *state, const char *question)
+cli_cmd_get_confirmation(struct cli_state *state, const char *question)
 {
-        char                    answer[5] = {'\0', };
-        char                    flush = '\0';
-        size_t			len;
+    char answer[5] = {
+        '\0',
+    };
+    int flush = '\0';
+    size_t len;
 
-        if (state->mode & GLUSTER_MODE_SCRIPT)
-                return GF_ANSWER_YES;
+    if (state->mode & GLUSTER_MODE_SCRIPT)
+        return GF_ANSWER_YES;
 
-	printf ("%s (y/n) ", question);
+    printf("%s (y/n) ", question);
 
-        if (fgets (answer, 4, stdin) == NULL) {
-                cli_out("gluster cli read error");
-                goto out;
-        }
+    if (fgets(answer, 4, stdin) == NULL) {
+        cli_out("gluster cli read error");
+        goto out;
+    }
 
-	len = strlen (answer);
+    len = strlen(answer);
 
-	if (len && answer [len - 1] == '\n'){
-		answer [--len] = '\0';
-	} else {
-		do{
-			flush = getchar ();
-		}while (flush != '\n');
-	}
+    if (len && answer[len - 1] == '\n') {
+        answer[--len] = '\0';
+    } else {
+        do {
+            flush = getchar();
+        } while (flush != '\n');
+    }
 
-	if (len > 3)
-		goto out;
+    if (len > 3)
+        goto out;
 
-	if (!strcasecmp (answer, "y") || !strcasecmp (answer, "yes"))
-		return GF_ANSWER_YES;
+    if (!strcasecmp(answer, "y") || !strcasecmp(answer, "yes"))
+        return GF_ANSWER_YES;
 
-	else if (!strcasecmp (answer, "n") || !strcasecmp (answer, "no"))
-		return GF_ANSWER_NO;
+    else if (!strcasecmp(answer, "n") || !strcasecmp(answer, "no"))
+        return GF_ANSWER_NO;
 
 out:
-	cli_out ("Invalid input, please enter y/n");
+    cli_out("Invalid input, please enter y/n");
 
-	return GF_ANSWER_NO;
+    return GF_ANSWER_NO;
 }
 
 int
-cli_cmd_volume_stop_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                           const char **words, int wordcount)
+cli_cmd_volume_stop_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                        const char **words, int wordcount)
 {
-        int                     ret = -1;
-        rpc_clnt_procedure_t    *proc = NULL;
-        call_frame_t            *frame = NULL;
-        int                     flags   = 0;
-        gf_answer_t             answer = GF_ANSWER_NO;
-        int                     sent = 0;
-        int                     parse_error = 0;
-        dict_t                  *dict = NULL;
-        char                    *volname = NULL;
-        cli_local_t             *local = NULL;
-
-        const char *question = "Stopping volume will make its data inaccessible. "
-                               "Do you want to continue?";
-
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
-
-        if (wordcount < 3 || wordcount > 4) {
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-                goto out;
-        }
-
-        volname = (char*) words[2];
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    int flags = 0;
+    gf_answer_t answer = GF_ANSWER_NO;
+    int sent = 0;
+    int parse_error = 0;
+    dict_t *dict = NULL;
+    char *volname = NULL;
+    cli_local_t *local = NULL;
+
+    const char *question =
+        "Stopping volume will make its data inaccessible. "
+        "Do you want to continue?";
+
+    if (wordcount < 3 || wordcount > 4) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
 
-        dict = dict_new ();
-        ret = dict_set_str (dict, "volname", volname);
-        if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "dict set failed");
-                goto out;
-        }
+    volname = (char *)words[2];
 
-        if (wordcount == 4) {
-                if (!strcmp("force", words[3])) {
-                        flags |= GF_CLI_FLAG_OP_FORCE;
-                } else {
-                        ret = -1;
-                        cli_usage_out (word->pattern);
-                        parse_error = 1;
-                        goto out;
-                }
-        }
-        ret = dict_set_int32 (dict, "flags", flags);
-        if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR,
-                        "dict set failed");
-                goto out;
+    dict = dict_new();
+    ret = dict_set_str(dict, "volname", volname);
+    if (ret) {
+        gf_log(THIS->name, GF_LOG_ERROR, "dict set failed");
+        goto out;
+    }
+
+    if (!strcmp(volname, GLUSTER_SHARED_STORAGE)) {
+        question =
+            "Stopping the shared storage volume"
+            "(gluster_shared_storage), will affect features "
+            "like snapshot scheduler, geo-replication "
+            "and NFS-Ganesha. Do you still want to "
+            "continue?";
+    }
+
+    if (wordcount == 4) {
+        if (!strcmp("force", words[3])) {
+            flags |= GF_CLI_FLAG_OP_FORCE;
+        } else {
+            ret = -1;
+            cli_usage_out(word->pattern);
+            parse_error = 1;
+            goto out;
         }
+    }
 
-        answer = cli_cmd_get_confirmation (state, question);
+    ret = dict_set_int32(dict, "flags", flags);
+    if (ret) {
+        gf_log(THIS->name, GF_LOG_ERROR, "dict set failed");
+        goto out;
+    }
 
-        if (GF_ANSWER_NO == answer) {
-                ret = 0;
-                goto out;
-        }
+    answer = cli_cmd_get_confirmation(state, question);
 
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_STOP_VOLUME];
+    if (GF_ANSWER_NO == answer) {
+        ret = 0;
+        goto out;
+    }
 
-        CLI_LOCAL_INIT (local, words, frame, dict);
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_STOP_VOLUME];
 
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, dict);
-        }
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
 
-out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_out ("Volume stop on '%s' failed", volname);
-        }
+    CLI_LOCAL_INIT(local, words, frame, dict);
 
-        CLI_STACK_DESTROY (frame);
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, dict);
+    }
 
-        return ret;
+out:
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("Volume stop on '%s' failed", volname);
+    }
+
+    CLI_STACK_DESTROY(frame);
+    if (dict)
+        dict_unref(dict);
+
+    if (ret == 0 && GF_ANSWER_YES == answer) {
+        gf_event(EVENT_VOLUME_STOP, "name=%s;force=%d", (char *)words[2],
+                 (flags & GF_CLI_FLAG_OP_FORCE));
+    }
+
+    return ret;
 }
 
-
 int
-cli_cmd_volume_rename_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                           const char **words, int wordcount)
+cli_cmd_volume_rename_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                          const char **words, int wordcount)
 {
-        int                     ret = -1;
-        rpc_clnt_procedure_t    *proc = NULL;
-        call_frame_t            *frame = NULL;
-        dict_t                  *dict = NULL;
-        int                     sent = 0;
-        int                     parse_error = 0;
-
-
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
-
-        dict = dict_new ();
-        if (!dict)
-                goto out;
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *dict = NULL;
+    int sent = 0;
+    int parse_error = 0;
+
+    if (wordcount != 4) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
 
-        if (wordcount != 4) {
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-                goto out;
-        }
+    dict = dict_new();
+    if (!dict)
+        goto out;
 
-        ret = dict_set_str (dict, "old-volname", (char *)words[2]);
+    ret = dict_set_str(dict, "old-volname", (char *)words[2]);
 
-        if (ret)
-                goto out;
+    if (ret)
+        goto out;
 
-        ret = dict_set_str (dict, "new-volname", (char *)words[3]);
+    ret = dict_set_str(dict, "new-volname", (char *)words[3]);
 
-        if (ret)
-                goto out;
+    if (ret)
+        goto out;
 
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_RENAME_VOLUME];
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_RENAME_VOLUME];
 
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, dict);
+    if (proc->fn) {
+        frame = create_frame(THIS, THIS->ctx->pool);
+        if (!frame) {
+            ret = -1;
+            goto out;
         }
+        ret = proc->fn(frame, THIS, dict);
+    }
 
 out:
-        if (dict)
-                dict_destroy (dict);
+    if (dict)
+        dict_unref(dict);
 
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_out ("Volume rename on '%s' failed", (char *)words[2]);
-        }
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("Volume rename on '%s' failed", (char *)words[2]);
+    }
 
-        CLI_STACK_DESTROY (frame);
+    CLI_STACK_DESTROY(frame);
 
-        return ret;
+    return ret;
 }
 
 int
-cli_cmd_volume_defrag_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                           const char **words, int wordcount)
+cli_cmd_volume_defrag_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                          const char **words, int wordcount)
 {
-        int                   ret     = -1;
-        rpc_clnt_procedure_t *proc    = NULL;
-        call_frame_t         *frame   = NULL;
-        dict_t               *dict = NULL;
-        int                     sent = 0;
-        int                     parse_error = 0;
-        cli_local_t          *local = NULL;
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *dict = NULL;
+    int sent = 0;
+    int parse_error = 0;
+    cli_local_t *local = NULL;
+#if (USE_EVENTS)
+    eventtypes_t event = EVENT_LAST;
+#endif
+
 #ifdef GF_SOLARIS_HOST_OS
-        cli_out ("Command not supported on Solaris");
-        goto out;
+    cli_out("Command not supported on Solaris");
+    goto out;
 #endif
 
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
+    ret = cli_cmd_volume_defrag_parse(words, wordcount, &dict);
 
-        ret = cli_cmd_volume_defrag_parse (words, wordcount, &dict);
+    if (ret) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+    }
 
-        if (ret) {
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-        }
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_DEFRAG_VOLUME];
 
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_DEFRAG_VOLUME];
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
 
-        CLI_LOCAL_INIT (local, words, frame, dict);
+    CLI_LOCAL_INIT(local, words, frame, dict);
 
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, dict);
-        }
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, dict);
+    }
 
 out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_out ("Volume rebalance failed");
-        }
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("Volume rebalance failed");
+    } else {
+#if (USE_EVENTS)
+        if (!(strcmp(words[wordcount - 1], "start")) ||
+            !(strcmp(words[wordcount - 1], "force"))) {
+            event = EVENT_VOLUME_REBALANCE_START;
+        } else if (!strcmp(words[wordcount - 1], "stop")) {
+            event = EVENT_VOLUME_REBALANCE_STOP;
+        }
+
+        if (event != EVENT_LAST)
+            gf_event(event, "volume=%s", (char *)words[2]);
+#endif
+    }
 
-        CLI_STACK_DESTROY (frame);
+    CLI_STACK_DESTROY(frame);
 
-        return ret;
+    return ret;
 }
 
 int
-cli_cmd_volume_reset_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                          const char **words, int wordcount)
+cli_cmd_volume_reset_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                         const char **words, int wordcount)
 {
-        int                     sent = 0;
-        int                     parse_error = 0;
-        int                     ret = -1;
-        rpc_clnt_procedure_t    *proc = NULL;
-        call_frame_t            *frame = NULL;
-        dict_t                  *options = NULL;
-        cli_local_t             *local = NULL;
+    int sent = 0;
+    int parse_error = 0;
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *options = NULL;
+    cli_local_t *local = NULL;
+#if (USE_EVENTS)
+    int ret1 = -1;
+    char *tmp_opt = NULL;
+#endif
 
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_RESET_VOLUME];
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_RESET_VOLUME];
 
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
+    ret = cli_cmd_volume_reset_parse(words, wordcount, &options);
+    if (ret) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
 
-        ret = cli_cmd_volume_reset_parse (words, wordcount, &options);
-        if (ret) {
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-                goto out;
-        }
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
 
-        CLI_LOCAL_INIT (local, words, frame, options);
+    CLI_LOCAL_INIT(local, words, frame, options);
 
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, options);
-        }
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, options);
+    }
 
 out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_out ("Volume reset failed");
-        }
-
-        CLI_STACK_DESTROY (frame);
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("Volume reset failed");
+    }
+
+#if (USE_EVENTS)
+    if (ret == 0) {
+        ret1 = dict_get_str(options, "key", &tmp_opt);
+        if (ret1)
+            tmp_opt = "";
+
+        gf_event(EVENT_VOLUME_RESET, "name=%s;option=%s", (char *)words[2],
+                 tmp_opt);
+    }
+#endif
 
-        return ret;
+    CLI_STACK_DESTROY(frame);
 
+    return ret;
 }
 
 int
-cli_cmd_volume_profile_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                          const char **words, int wordcount)
+cli_cmd_volume_profile_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                           const char **words, int wordcount)
 {
-        int                     sent = 0;
-        int                     parse_error = 0;
+    int sent = 0;
+    int parse_error = 0;
 
-        int                     ret      = -1;
-        rpc_clnt_procedure_t    *proc    = NULL;
-        call_frame_t            *frame   = NULL;
-        dict_t                  *options = NULL;
-        cli_local_t             *local = NULL;
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *options = NULL;
+    cli_local_t *local = NULL;
 
-        ret = cli_cmd_volume_profile_parse (words, wordcount, &options);
+    ret = cli_cmd_volume_profile_parse(words, wordcount, &options);
 
-        if (ret) {
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-                goto out;
-        }
+    if (ret) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
 
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_PROFILE_VOLUME];
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_PROFILE_VOLUME];
 
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        gf_log(THIS->name, GF_LOG_ERROR, "failed to create frame");
+        ret = -1;
+        goto out;
+    }
 
-        CLI_LOCAL_INIT (local, words, frame, options);
+    CLI_LOCAL_INIT(local, words, frame, options);
 
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, options);
-        }
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, options);
+    }
 
 out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_out ("Volume profile failed");
-        }
-
-        CLI_STACK_DESTROY (frame);
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("Volume profile failed");
+    }
 
-        return ret;
+    CLI_STACK_DESTROY(frame);
 
+    return ret;
 }
 
 int
-cli_cmd_volume_set_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                        const char **words, int wordcount)
+cli_cmd_volume_set_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                       const char **words, int wordcount)
 {
-        int                     sent = 0;
-        int                     parse_error = 0;
+    int sent = 0;
+    int parse_error = 0;
+
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *options = NULL;
+    cli_local_t *local = NULL;
+    char *op_errstr = NULL;
+
+#if (USE_EVENTS)
+    int ret1 = -1;
+    int i = 1;
+    char dict_key[50] = {
+        0,
+    };
+    char *tmp_opt = NULL;
+    char *opts_str = NULL;
+    int num_options = 0;
+#endif
 
-	int                     ret = -1;
-        rpc_clnt_procedure_t    *proc = NULL;
-        call_frame_t            *frame = NULL;
-        dict_t                  *options = NULL;
-        cli_local_t             *local = NULL;
-        char                    *op_errstr = NULL;
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_SET_VOLUME];
 
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_SET_VOLUME];
+    ret = cli_cmd_volume_set_parse(state, words, wordcount, &options,
+                                   &op_errstr);
+    if (ret) {
+        if (op_errstr) {
+            cli_err("%s", op_errstr);
+            GF_FREE(op_errstr);
+        } else
+            cli_usage_out(word->pattern);
 
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
-
-        ret = cli_cmd_volume_set_parse (words, wordcount, &options, &op_errstr);
-        if (ret) {
-                if (op_errstr) {
-                    cli_err ("%s", op_errstr);
-                    GF_FREE (op_errstr);
-                } else
-                    cli_usage_out (word->pattern);
+        parse_error = 1;
+        goto out;
+    }
 
-                parse_error = 1;
-                goto out;
-        }
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
 
-        CLI_LOCAL_INIT (local, words, frame, options);
+    CLI_LOCAL_INIT(local, words, frame, options);
 
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, options);
-        }
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, options);
+    }
 
 out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_out ("Volume set failed");
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("Volume set failed");
+    }
+
+#if (USE_EVENTS)
+    if (ret == 0 && strcmp(words[2], "help") != 0) {
+        ret1 = dict_get_int32(options, "count", &num_options);
+        if (ret1) {
+            num_options = 0;
+            goto end;
+        } else {
+            num_options = num_options / 2;
         }
 
-        CLI_STACK_DESTROY (frame);
-
-        return ret;
-
-}
-
-int
-cli_cmd_volume_add_brick_cbk (struct cli_state *state,
-                              struct cli_cmd_word *word, const char **words,
-                              int wordcount)
-{
-        int                     ret = -1;
-        rpc_clnt_procedure_t    *proc = NULL;
-        call_frame_t            *frame = NULL;
-        dict_t                  *options = NULL;
-        int                     sent = 0;
-        int                     parse_error = 0;
-        gf_answer_t             answer = GF_ANSWER_NO;
-        cli_local_t             *local = NULL;
-
-        const char *question = "Changing the 'stripe count' of the volume is "
-                "not a supported feature. In some cases it may result in data "
-                "loss on the volume. Also there may be issues with regular "
-                "filesystem operations on the volume after the change. Do you "
-                "really want to continue with 'stripe' count option ? ";
-
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
-
-        ret = cli_cmd_volume_add_brick_parse (words, wordcount, &options);
-        if (ret) {
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-                goto out;
+        char *free_list_key[num_options];
+        char *free_list_val[num_options];
+        for (i = 0; i < num_options; i++) {
+            free_list_key[i] = NULL;
+            free_list_val[i] = NULL;
         }
+        /* Initialize opts_str */
+        opts_str = "";
 
-        /* TODO: there are challenges in supporting changing of
-           stripe-count, untill it is properly supported give warning to user */
-        if (dict_get (options, "stripe-count")) {
-                answer = cli_cmd_get_confirmation (state, question);
+        /* Prepare String in format options=KEY1,VALUE1,KEY2,VALUE2 */
+        for (i = 1; i <= num_options; i++) {
+            sprintf(dict_key, "key%d", i);
+            ret1 = dict_get_str(options, dict_key, &tmp_opt);
+            if (ret1)
+                tmp_opt = "";
 
-                if (GF_ANSWER_NO == answer) {
-                        ret = 0;
-                        goto out;
-                }
-        }
+            gf_asprintf(&opts_str, "%s,%s", opts_str, tmp_opt);
+            free_list_key[i - 1] = opts_str;
 
-        if (state->mode & GLUSTER_MODE_SCRIPT) {
-                ret = dict_set_int32 (options, "force", _gf_true);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Failed to set force "
-                                "option");
-                        goto out;
-                }
-        }
-
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_ADD_BRICK];
-
-        CLI_LOCAL_INIT (local, words, frame, options);
+            sprintf(dict_key, "value%d", i);
+            ret1 = dict_get_str(options, dict_key, &tmp_opt);
+            if (ret1)
+                tmp_opt = "";
 
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, options);
+            gf_asprintf(&opts_str, "%s,%s", opts_str, tmp_opt);
+            free_list_val[i - 1] = opts_str;
         }
 
-out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_out ("Volume add-brick failed");
+        gf_event(EVENT_VOLUME_SET, "name=%s;options=%s", (char *)words[2],
+                 opts_str);
+
+        /* Allocated by gf_strdup and gf_asprintf */
+        for (i = 0; i < num_options; i++) {
+            GF_FREE(free_list_key[i]);
+            GF_FREE(free_list_val[i]);
         }
+    }
+#endif
 
-        CLI_STACK_DESTROY (frame);
+end:
+    CLI_STACK_DESTROY(frame);
 
-        return ret;
+    return ret;
 }
 
 static int
-gf_cli_create_auxiliary_mount (char *volname)
+cli_event_remove_brick_str(dict_t *options, char **event_str,
+                           eventtypes_t *event)
 {
-        int      ret                     = -1;
-        char     mountdir[PATH_MAX]      = {0,};
-        char     pidfile_path[PATH_MAX]  = {0,};
-        char     logfile[PATH_MAX]       = {0,};
+    int ret = -1;
+    char *bricklist = NULL;
+    char *brick = NULL;
+    char *volname = NULL;
+    char key[256] = {
+        0,
+    };
+    const char *eventstrformat = "volume=%s;bricks=%s";
+    int32_t command = 0;
+    int32_t i = 1;
+    int32_t count = 0;
+    int32_t eventstrlen = 1;
+    int bricklen = 0;
+    char *tmp_ptr = NULL;
+
+    if (!options || !event_str || !event)
+        goto out;
 
-        GLUSTERFS_GET_AUX_MOUNT_PIDFILE (pidfile_path, volname);
+    ret = dict_get_str(options, "volname", &volname);
+    if (ret || !volname) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to fetch volname");
+        ret = -1;
+        goto out;
+    }
+    /* Get the list of bricks for the event */
+    ret = dict_get_int32(options, "command", &command);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to fetch command");
+        ret = -1;
+        goto out;
+    }
+
+    switch (command) {
+        case GF_OP_CMD_START:
+            *event = EVENT_VOLUME_REMOVE_BRICK_START;
+            break;
+        case GF_OP_CMD_COMMIT:
+            *event = EVENT_VOLUME_REMOVE_BRICK_COMMIT;
+            break;
+        case GF_OP_CMD_COMMIT_FORCE:
+            *event = EVENT_VOLUME_REMOVE_BRICK_FORCE;
+            break;
+        case GF_OP_CMD_STOP:
+            *event = EVENT_VOLUME_REMOVE_BRICK_STOP;
+            break;
+        default:
+            *event = EVENT_LAST;
+            break;
+    }
 
-        if (gf_is_service_running (pidfile_path, NULL)) {
-                gf_log ("cli", GF_LOG_DEBUG, "Aux mount of volume %s is running"
-                        " already", volname);
-                ret = 0;
-                goto out;
-        }
+    ret = -1;
 
-        GLUSTERD_GET_QUOTA_AUX_MOUNT_PATH (mountdir, volname, "/");
-        ret = mkdir_p (mountdir, 0700, _gf_true);
-        if (ret && errno != EEXIST) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to create auxiliary mount "
-                        "directory %s. Reason : %s", mountdir,
-                        strerror (errno));
-                goto out;
+    if (*event == EVENT_LAST) {
+        goto out;
+    }
+
+    /* I could just get this from words[] but this is cleaner in case the
+     * format changes  */
+    while (i) {
+        snprintf(key, sizeof(key), "brick%d", i);
+        ret = dict_get_str(options, key, &brick);
+        if (ret) {
+            break;
         }
+        eventstrlen += strlen(brick) + 1;
+        i++;
+    }
 
-        snprintf (logfile, PATH_MAX-1, "%s/quota-mount-%s.log",
-                  DEFAULT_LOG_FILE_DIRECTORY, volname);
+    count = --i;
 
-        ret = runcmd (SBIN_DIR"/glusterfs",
-                      "-s", "localhost",
-                      "--volfile-id", volname,
-                      "-l", logfile,
-                      "-p", pidfile_path,
-                      mountdir,
-                      "--client-pid", "-42", NULL);
+    eventstrlen += 1;
 
-        if (ret) {
-                gf_log ("cli", GF_LOG_WARNING, "failed to mount glusterfs "
-                        "client. Please check the log file %s for more details",
-                        logfile);
-                ret = -1;
-                goto out;
-        }
+    bricklist = GF_CALLOC(eventstrlen, sizeof(char), gf_common_mt_char);
+    if (!bricklist) {
+        gf_log(THIS->name, GF_LOG_ERROR,
+               "memory allocation failed for"
+               "bricklist");
+        ret = -1;
+        goto out;
+    }
 
-        ret = 0;
+    tmp_ptr = bricklist;
 
+    i = 1;
+    while (i <= count) {
+        snprintf(key, sizeof(key), "brick%d", i);
+        ret = dict_get_str(options, key, &brick);
+        if (ret) {
+            break;
+        }
+        snprintf(tmp_ptr, eventstrlen, "%s ", brick);
+        bricklen = strlen(brick);
+        eventstrlen -= (bricklen + 1);
+        tmp_ptr += (bricklen + 1);
+        i++;
+    }
+
+    if (!ret) {
+        gf_asprintf(event_str, eventstrformat, volname, bricklist);
+    } else {
+        gf_asprintf(event_str, eventstrformat, volname, "<unavailable>");
+    }
+
+    ret = 0;
 out:
-        return ret;
+    GF_FREE(bricklist);
+    return ret;
 }
 
-static int
-cli_stage_quota_op (char *volname, int op_code)
+int
+cli_cmd_volume_add_brick_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                             const char **words, int wordcount)
 {
-        int ret = -1;
-
-        switch (op_code) {
-                case GF_QUOTA_OPTION_TYPE_ENABLE:
-                case GF_QUOTA_OPTION_TYPE_LIMIT_USAGE:
-                case GF_QUOTA_OPTION_TYPE_REMOVE:
-                case GF_QUOTA_OPTION_TYPE_LIST:
-                        ret = gf_cli_create_auxiliary_mount (volname);
-                        if (ret) {
-                                cli_err ("quota: Could not start quota "
-                                         "auxiliary mount");
-                                goto out;
-                        }
-                        ret = 0;
-                        break;
-
-                default:
-                        ret = 0;
-                        break;
-        }
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *options = NULL;
+    int sent = 0;
+    int parse_error = 0;
+    gf_answer_t answer = GF_ANSWER_NO;
+    cli_local_t *local = NULL;
+
+#if (USE_EVENTS)
+    char *event_str = NULL;
+    char *bricks = NULL;
+    const char *eventstrformat = "volume=%s;bricks=%s";
+#endif
 
-out:
-        return ret;
-}
+    const char *question =
+        "Changing the 'stripe count' of the volume is "
+        "not a supported feature. In some cases it may result in data "
+        "loss on the volume. Also there may be issues with regular "
+        "filesystem operations on the volume after the change. Do you "
+        "really want to continue with 'stripe' count option ? ";
+
+    ret = cli_cmd_volume_add_brick_parse(state, words, wordcount, &options, 0);
+    if (ret) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
 
-static void
-print_quota_list_header (void)
-{
-        //Header
-        cli_out ("                  Path                   Hard-limit "
-                 "Soft-limit   Used  Available");
-        cli_out ("-----------------------------------------------------"
-                 "---------------------------");
-}
+    /* TODO: there are challenges in supporting changing of
+       stripe-count, until it is properly supported give warning to user */
+    if (dict_get(options, "stripe-count")) {
+        answer = cli_cmd_get_confirmation(state, question);
 
-int
-cli_get_soft_limit (dict_t *options, const char **words, dict_t *xdata)
-{
-        call_frame_t            *frame          = NULL;
-        cli_local_t             *local          = NULL;
-        rpc_clnt_procedure_t    *proc           = NULL;
-        char                    *default_sl     = NULL;
-        char                    *default_sl_dup = NULL;
-        int                      ret  = -1;
-
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame) {
-                ret = -1;
-                goto out;
+        if (GF_ANSWER_NO == answer) {
+            ret = 0;
+            goto out;
         }
+    }
 
-        //We need a ref on @options to prevent CLI_STACK_DESTROY
-        //from destroying it prematurely.
-        dict_ref (options);
-        CLI_LOCAL_INIT (local, words, frame, options);
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_QUOTA];
-        ret = proc->fn (frame, THIS, options);
+#if (USE_EVENTS)
+    /* Get the list of bricks for the event */
 
-        ret = dict_get_str (options, "default-soft-limit", &default_sl);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to get default soft limit");
-                goto out;
-        }
+    ret = dict_get_str(options, "bricks", &bricks);
 
-        default_sl_dup = gf_strdup (default_sl);
-        if (!default_sl_dup) {
-                ret = -1;
-                goto out;
-        }
+    if (!ret) {
+        gf_asprintf(&event_str, eventstrformat, (char *)words[2],
+                    &bricks[1] /*Skip leading space*/);
+    } else {
+        gf_asprintf(&event_str, eventstrformat, (char *)words[2],
+                    "<unavailable>");
+    }
+#endif
 
-        ret = dict_set_dynstr (xdata, "default-soft-limit", default_sl_dup);
+    if (state->mode & GLUSTER_MODE_WIGNORE) {
+        ret = dict_set_int32(options, "force", _gf_true);
         if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to set default soft limit");
-                GF_FREE (default_sl_dup);
-                goto out;
+            gf_log("cli", GF_LOG_ERROR,
+                   "Failed to set force "
+                   "option");
+            goto out;
         }
+    }
+
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_ADD_BRICK];
+
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
+
+    CLI_LOCAL_INIT(local, words, frame, options);
+
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, options);
+    }
 
 out:
-        CLI_STACK_DESTROY (frame);
-        return ret;
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("Volume add-brick failed");
+    } else {
+#if (USE_EVENTS)
+        gf_event(EVENT_VOLUME_ADD_BRICK, "%s", event_str);
+#endif
+    }
+#if (USE_EVENTS)
+    GF_FREE(event_str);
+#endif
+
+    CLI_STACK_DESTROY(frame);
+    return ret;
 }
 
-#define QUOTA_CONF_HEADER                                                \
-        "GlusterFS Quota conf | version: v%d.%d\n"
 int
-cli_cmd_quota_conf_skip_header (int fd)
+cli_get_soft_limit(dict_t *options, const char **words, dict_t *xdata)
 {
-        char buf[PATH_MAX] = {0,};
-
-        snprintf (buf, sizeof(buf)-1, QUOTA_CONF_HEADER, 1, 1);
-        return gf_skip_header_section (fd, strlen (buf));
-}
+    call_frame_t *frame = NULL;
+    cli_local_t *local = NULL;
+    rpc_clnt_procedure_t *proc = NULL;
+    char *default_sl = NULL;
+    char *default_sl_dup = NULL;
+    int ret = -1;
+
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
+
+    // We need a ref on @options to prevent CLI_STACK_DESTROY
+    // from destroying it prematurely.
+    dict_ref(options);
+    CLI_LOCAL_INIT(local, words, frame, options);
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_QUOTA];
+    ret = proc->fn(frame, THIS, options);
+
+    ret = dict_get_str(options, "default-soft-limit", &default_sl);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to get default soft limit");
+        goto out;
+    }
 
-/* Checks if at least one limit has been set on the volume
- *
- * Returns true if at least one limit is set. Returns false otherwise.
- */
-gf_boolean_t
-_limits_set_on_volume (char *volname) {
-        gf_boolean_t    limits_set = _gf_false;
-        int             ret = -1;
-        char            quota_conf_file[PATH_MAX] = {0,};
-        int             fd = -1;
-        char            buf[16] = {0,};
-
-        /* TODO: fix hardcoding; Need to perform an RPC call to glusterd
-         * to fetch working directory
-         */
-        sprintf (quota_conf_file, "/var/lib/glusterd/vols/%s/quota.conf",
-                 volname);
-        fd = open (quota_conf_file, O_RDONLY);
-        if (fd == -1)
-                goto out;
+    default_sl_dup = gf_strdup(default_sl);
+    if (!default_sl_dup) {
+        ret = -1;
+        goto out;
+    }
 
-        ret = cli_cmd_quota_conf_skip_header (fd);
-        if (ret)
-                goto out;
+    ret = dict_set_dynstr(xdata, "default-soft-limit", default_sl_dup);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to set default soft limit");
+        GF_FREE(default_sl_dup);
+        goto out;
+    }
 
-        /* Try to read atleast one gfid */
-        ret = read (fd, (void *)buf, 16);
-        if (ret == 16)
-                limits_set = _gf_true;
 out:
-        if (fd != -1)
-                close (fd);
-        return limits_set;
+    CLI_STACK_DESTROY(frame);
+    return ret;
 }
 
-/* Checks if the mount is connected to the bricks
+/* Checks if at least one limit has been set on the volume
  *
- * Returns true if connected and false if not
+ * Returns true if at least one limit is set. Returns false otherwise.
  */
 gf_boolean_t
-_quota_aux_mount_online (char *volname)
+_limits_set_on_volume(char *volname, int type)
 {
-        int         ret = 0;
-        char        mount_path[PATH_MAX + 1] = {0,};
-        struct stat buf = {0,};
+    gf_boolean_t limits_set = _gf_false;
+    int ret = -1;
+    char quota_conf_file[PATH_MAX] = {
+        0,
+    };
+    int fd = -1;
+    char buf[16] = {
+        0,
+    };
+    float version = 0.0f;
+    char gfid_type_stored = 0;
+    char gfid_type = 0;
+
+    /* TODO: fix hardcoding; Need to perform an RPC call to glusterd
+     * to fetch working directory
+     */
+    snprintf(quota_conf_file, sizeof quota_conf_file, "%s/vols/%s/quota.conf",
+             GLUSTERD_DEFAULT_WORKDIR, volname);
+    fd = open(quota_conf_file, O_RDONLY);
+    if (fd == -1)
+        goto out;
 
-        GF_ASSERT (volname);
+    ret = quota_conf_read_version(fd, &version);
+    if (ret)
+        goto out;
 
-        /* Try to create the aux mount before checking if bricks are online */
-        ret = gf_cli_create_auxiliary_mount (volname);
-        if (ret) {
-                cli_err ("quota: Could not start quota auxiliary mount");
-                return _gf_false;
-        }
+    if (type == GF_QUOTA_OPTION_TYPE_LIST)
+        gfid_type = GF_QUOTA_CONF_TYPE_USAGE;
+    else
+        gfid_type = GF_QUOTA_CONF_TYPE_OBJECTS;
 
-        GLUSTERD_GET_QUOTA_AUX_MOUNT_PATH (mount_path, volname, "/");
+    /* Try to read at least one gfid  of type 'gfid_type' */
+    while (1) {
+        ret = quota_conf_read_gfid(fd, buf, &gfid_type_stored, version);
+        if (ret <= 0)
+            break;
 
-        ret = sys_stat (mount_path, &buf);
-        if (ret) {
-                if (ENOTCONN == errno) {
-                        cli_err ("quota: Cannot connect to bricks. Check if "
-                                 "bricks are online.");
-                } else {
-                        cli_err ("quota: Error on quota auxiliary mount (%s).",
-                                 strerror (errno));
-                }
-                return _gf_false;
+        if (gfid_type_stored == gfid_type) {
+            limits_set = _gf_true;
+            break;
         }
-        return _gf_true;
+    }
+out:
+    if (fd != -1)
+        sys_close(fd);
+
+    return limits_set;
 }
 
 int
-cli_cmd_quota_handle_list_all (const char **words, dict_t *options)
+cli_cmd_quota_handle_list_all(const char **words, dict_t *options)
 {
-        int                      all_failed = 1;
-        int                      count      = 0;
-        int                      ret        = -1;
-        rpc_clnt_procedure_t    *proc       = NULL;
-        cli_local_t             *local      = NULL;
-        call_frame_t            *frame      = NULL;
-        dict_t                  *xdata      = NULL;
-        char                    *gfid_str   = NULL;
-        char                    *volname    = NULL;
-        char                    *volname_dup = NULL;
-        unsigned char            buf[16]   = {0};
-        int                      fd        = -1;
-        char                     quota_conf_file[PATH_MAX] = {0};
-        gf_boolean_t             xml_err_flag   = _gf_false;
-        char                     err_str[NAME_MAX] = {0,};
-
-        xdata = dict_new ();
-        if (!xdata) {
-                ret = -1;
-                goto out;
-        }
+    int all_failed = 1;
+    int count = 0;
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    cli_local_t *local = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *xdata = NULL;
+    char gfid_str[UUID_CANONICAL_FORM_LEN + 1];
+    char *volname = NULL;
+    char *volname_dup = NULL;
+    unsigned char buf[16] = {0};
+    int fd = -1;
+    char quota_conf_file[PATH_MAX] = {0};
+    gf_boolean_t xml_err_flag = _gf_false;
+    char err_str[NAME_MAX] = {
+        0,
+    };
+    int32_t type = 0;
+    char gfid_type = 0;
+    float version = 0.0f;
+    int32_t max_count = 0;
+
+    xdata = dict_new();
+    if (!xdata) {
+        ret = -1;
+        goto out;
+    }
 
-        ret = dict_get_str (options, "volname", &volname);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to get volume name");
-                goto out;
-        }
+    ret = dict_get_str(options, "volname", &volname);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to get volume name");
+        goto out;
+    }
 
-        ret = cli_get_soft_limit (options, words, xdata);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to fetch default "
-                        "soft-limit");
-                goto out;
-        }
+    ret = dict_get_int32(options, "type", &type);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to get quota option type");
+        goto out;
+    }
 
-        /* Check if at least one limit is set on volume. No need to check for
-         * quota enabled as cli_get_soft_limit() handles that
-         */
-        if (!_limits_set_on_volume (volname)) {
-                snprintf (err_str, sizeof (err_str), "No quota configured on "
-                          "volume %s", volname);
-                if (global_state->mode & GLUSTER_MODE_XML) {
-                        xml_err_flag = _gf_true;
-                } else {
-                        cli_out ("quota: %s", err_str);
-                }
-                ret = 0;
-                goto out;
-        }
+    ret = dict_set_int32(xdata, "type", type);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to set type in xdata");
+        goto out;
+    }
 
-        /* Check if the mount is online before doing any listing */
-        if (!_quota_aux_mount_online (volname)) {
-                ret = -1;
-                goto out;
+    ret = cli_get_soft_limit(options, words, xdata);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR,
+               "Failed to fetch default "
+               "soft-limit");
+        goto out;
+    }
+
+    /* Check if at least one limit is set on volume. No need to check for
+     * quota enabled as cli_get_soft_limit() handles that
+     */
+    if (!_limits_set_on_volume(volname, type)) {
+        snprintf(err_str, sizeof(err_str),
+                 "No%s quota configured on"
+                 " volume %s",
+                 (type == GF_QUOTA_OPTION_TYPE_LIST) ? "" : " inode", volname);
+        if (global_state->mode & GLUSTER_MODE_XML) {
+            xml_err_flag = _gf_true;
+        } else {
+            cli_out("quota: %s", err_str);
         }
+        ret = 0;
+        goto out;
+    }
 
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame) {
-                ret = -1;
-                goto out;
-        }
+    volname_dup = gf_strdup(volname);
+    if (!volname_dup) {
+        ret = -1;
+        goto out;
+    }
 
-        volname_dup = gf_strdup (volname);
-        if (!volname_dup) {
-                ret = -1;
-                goto out;
+    ret = dict_set_dynstr(xdata, "volume-uuid", volname_dup);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to set volume-uuid");
+        GF_FREE(volname_dup);
+        goto out;
+    }
+
+    // TODO: fix hardcoding; Need to perform an RPC call to glusterd
+    // to fetch working directory
+    snprintf(quota_conf_file, sizeof quota_conf_file, "%s/vols/%s/quota.conf",
+             GLUSTERD_DEFAULT_WORKDIR, volname);
+    fd = open(quota_conf_file, O_RDONLY);
+    if (fd == -1) {
+        // This may because no limits were yet set on the volume
+        gf_log("cli", GF_LOG_TRACE,
+               "Unable to open "
+               "quota.conf");
+        ret = 0;
+        goto out;
+    }
+
+    ret = quota_conf_read_version(fd, &version);
+    if (ret)
+        goto out;
+
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
+
+    CLI_LOCAL_INIT(local, words, frame, xdata);
+    proc = &cli_quotad_clnt.proctable[GF_AGGREGATOR_GETLIMIT];
+
+    for (count = 0;; count++) {
+        ret = quota_conf_read_gfid(fd, buf, &gfid_type, version);
+        if (ret == 0) {
+            break;
+        } else if (ret < 0) {
+            gf_log(THIS->name, GF_LOG_CRITICAL,
+                   "Quota "
+                   "configuration store may be corrupt.");
+            goto out;
+        }
+
+        if ((type == GF_QUOTA_OPTION_TYPE_LIST &&
+             gfid_type == GF_QUOTA_CONF_TYPE_OBJECTS) ||
+            (type == GF_QUOTA_OPTION_TYPE_LIST_OBJECTS &&
+             gfid_type == GF_QUOTA_CONF_TYPE_USAGE))
+            continue;
+
+        max_count++;
+    }
+    ret = dict_set_int32(xdata, "max_count", max_count);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to set max_count");
+        goto out;
+    }
+
+    ret = sys_lseek(fd, 0L, SEEK_SET);
+    if (ret < 0) {
+        gf_log(THIS->name, GF_LOG_ERROR,
+               "failed to move offset to "
+               "the beginning: %s",
+               strerror(errno));
+        goto out;
+    }
+    ret = quota_conf_read_version(fd, &version);
+    if (ret)
+        goto out;
+
+    for (count = 0;; count++) {
+        ret = quota_conf_read_gfid(fd, buf, &gfid_type, version);
+        if (ret == 0) {
+            break;
+        } else if (ret < 0) {
+            gf_log(THIS->name, GF_LOG_CRITICAL,
+                   "Quota "
+                   "configuration store may be corrupt.");
+            goto out;
+        }
+
+        if ((type == GF_QUOTA_OPTION_TYPE_LIST &&
+             gfid_type == GF_QUOTA_CONF_TYPE_OBJECTS) ||
+            (type == GF_QUOTA_OPTION_TYPE_LIST_OBJECTS &&
+             gfid_type == GF_QUOTA_CONF_TYPE_USAGE))
+            continue;
+
+        uuid_utoa_r(buf, gfid_str);
+        ret = dict_set_str(xdata, "gfid", gfid_str);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR, "Failed to set gfid");
+            goto out;
         }
 
-        ret = dict_set_dynstr (xdata, "volume-uuid", volname_dup);
+        ret = proc->fn(frame, THIS, xdata);
         if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to set volume-uuid");
-                GF_FREE (volname_dup);
-                goto out;
+            gf_log("cli", GF_LOG_ERROR,
+                   "Failed to get quota "
+                   "limits for %s",
+                   uuid_utoa((unsigned char *)buf));
         }
 
-        //TODO: fix hardcoding; Need to perform an RPC call to glusterd
-        //to fetch working directory
-        sprintf (quota_conf_file, "/var/lib/glusterd/vols/%s/quota.conf",
-                 volname);
-        fd = open (quota_conf_file, O_RDONLY);
-        if (fd == -1) {
-                //This may because no limits were yet set on the volume
-                gf_log ("cli", GF_LOG_TRACE, "Unable to open "
-                        "quota.conf");
-                ret = 0;
-                goto out;
-         }
+        dict_del(xdata, "gfid");
+        all_failed = all_failed && ret;
+    }
 
-        ret = cli_cmd_quota_conf_skip_header (fd);
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        ret = cli_xml_output_vol_quota_limit_list_end(local);
         if (ret) {
-                goto out;
+            gf_log("cli", GF_LOG_ERROR,
+                   "Error in printing "
+                   "xml output");
+            goto out;
         }
-        CLI_LOCAL_INIT (local, words, frame, xdata);
-        proc = &cli_quotad_clnt.proctable[GF_AGGREGATOR_GETLIMIT];
+    }
 
-        if (!(global_state->mode & GLUSTER_MODE_XML)) {
-                print_quota_list_header ();
-        } else {
-                ret = cli_xml_output_vol_quota_limit_list_begin
-                        (local, 0, 0, NULL);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Error in printing "
-                                "xml output");
-                        goto out;
-                }
-        }
+    if (count > 0) {
+        ret = all_failed ? -1 : 0;
+    } else {
+        ret = 0;
+    }
 
-        gfid_str = GF_CALLOC (1, gf_common_mt_char, 64);
-        if (!gfid_str) {
-                ret = -1;
-                goto out;
-        }
-        for (count = 0;; count++) {
-                ret = read (fd, (void*) buf, 16);
-                if (ret <= 0) {
-                        //Finished reading all entries in the conf file
-                        break;
-                }
-                if (ret < 16) {
-                        //This should never happen. We must have a multiple of
-                        //entry_sz bytes in our configuration file.
-                        gf_log (THIS->name, GF_LOG_CRITICAL, "Quota "
-                                "configuration store may be corrupt.");
-                        goto out;
-                }
-                uuid_utoa_r (buf, gfid_str);
-                ret = dict_set_str (xdata, "gfid", gfid_str);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Failed to set gfid");
-                        goto out;
-                }
+out:
+    if (xml_err_flag) {
+        ret = cli_xml_output_str("volQuota", NULL, -1, 0, err_str);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Error outputting in "
+                   "xml format");
+        }
+    }
+    if (xdata)
+        dict_unref(xdata);
+
+    if (fd != -1) {
+        sys_close(fd);
+    }
+
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR,
+               "Could not fetch and display quota"
+               " limits");
+    }
+    CLI_STACK_DESTROY(frame);
+    return ret;
+}
 
-                ret = proc->fn (frame, THIS, xdata);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Failed to get quota "
-                                "limits for %s", uuid_utoa ((unsigned char*)buf));
-                }
+int
+cli_cmd_bitrot_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                   const char **words, int wordcount)
+{
+    int ret = -1;
+    int parse_err = 0;
+    call_frame_t *frame = NULL;
+    dict_t *options = NULL;
+    cli_local_t *local = NULL;
+    rpc_clnt_procedure_t *proc = NULL;
+    int sent = 0;
+#if (USE_EVENTS)
+    int cmd_type = -1;
+    int ret1 = -1;
+    int event_type = -1;
+    char *tmp = NULL;
+    char *events_str = NULL;
+    char *volname = NULL;
+#endif
 
-                dict_del (xdata, "gfid");
-                all_failed = all_failed && ret;
-        }
+    ret = cli_cmd_bitrot_parse(words, wordcount, &options);
+    if (ret < 0) {
+        cli_usage_out(word->pattern);
+        parse_err = 1;
+        goto out;
+    }
 
-        if (global_state->mode & GLUSTER_MODE_XML) {
-                ret = cli_xml_output_vol_quota_limit_list_end (local);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Error in printing "
-                                "xml output");
-                        goto out;
-                }
-        }
+    if (ret == 1) {
+        /* this is 'volume bitrot help' */
+        cli_cmd_bitrot_help_cbk(state, word, words, wordcount);
+        ret = 0;
+        goto out2;
+    }
 
-        if (count > 0) {
-                ret = all_failed? -1: 0;
-        } else {
-                ret = 0;
-        }
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
+
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_BITROT];
+
+    CLI_LOCAL_INIT(local, words, frame, options);
 
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, options);
+    }
 
 out:
-        if (xml_err_flag) {
-                ret = cli_xml_output_str ("volQuota", NULL, -1, 0, err_str);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Error outputting in "
-                                "xml format");
-                }
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_err == 0))
+            cli_err(
+                "Bit rot command failed. Please check the cli "
+                "logs for more details");
+    }
+
+#if (USE_EVENTS)
+    if (ret == 0) {
+        ret1 = dict_get_int32(options, "type", &cmd_type);
+        if (ret1)
+            cmd_type = -1;
+        else {
+            ret1 = dict_get_str(options, "volname", &volname);
+            if (ret1)
+                volname = "";
+        }
+
+        switch (cmd_type) {
+            case GF_BITROT_OPTION_TYPE_ENABLE:
+                event_type = EVENT_BITROT_ENABLE;
+                break;
+            case GF_BITROT_OPTION_TYPE_DISABLE:
+                event_type = EVENT_BITROT_DISABLE;
+                break;
+            case GF_BITROT_CMD_SCRUB_ONDEMAND:
+                event_type = EVENT_BITROT_SCRUB_ONDEMAND;
+                break;
+            case GF_BITROT_OPTION_TYPE_SCRUB_THROTTLE:
+                event_type = EVENT_BITROT_SCRUB_THROTTLE;
+                ret1 = dict_get_str(options, "scrub-throttle-value", &tmp);
+                if (ret1)
+                    tmp = "";
+                gf_asprintf(&events_str, "name=%s;value=%s", volname, tmp);
+                break;
+            case GF_BITROT_OPTION_TYPE_SCRUB_FREQ:
+                event_type = EVENT_BITROT_SCRUB_FREQ;
+                ret1 = dict_get_str(options, "scrub-frequency-value", &tmp);
+                if (ret1)
+                    tmp = "";
+                gf_asprintf(&events_str, "name=%s;value=%s", volname, tmp);
+                break;
+            case GF_BITROT_OPTION_TYPE_SCRUB:
+                event_type = EVENT_BITROT_SCRUB_OPTION;
+                ret1 = dict_get_str(options, "scrub-value", &tmp);
+                if (ret1)
+                    tmp = "";
+                gf_asprintf(&events_str, "name=%s;value=%s", volname, tmp);
+                break;
+            default:
+                break;
         }
 
-        if (fd != -1) {
-                close (fd);
-        }
+        if (event_type > -1)
+            gf_event(event_type, "%s", events_str);
 
-        GF_FREE (gfid_str);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Could not fetch and display quota"
-                        " limits");
-        }
-        CLI_STACK_DESTROY (frame);
-        return ret;
+        if (events_str)
+            GF_FREE(events_str);
+    }
+#endif
+
+    CLI_STACK_DESTROY(frame);
+out2:
+    return ret;
 }
 
 int
-cli_cmd_quota_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                   const char **words, int wordcount)
+cli_cmd_quota_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                  const char **words, int wordcount)
 {
-
-        int                      ret       = 0;
-        int                      parse_err = 0;
-        int32_t                  type      = 0;
-        rpc_clnt_procedure_t    *proc      = NULL;
-        call_frame_t            *frame     = NULL;
-        dict_t                  *options   = NULL;
-        gf_answer_t              answer    = GF_ANSWER_NO;
-        cli_local_t             *local     = NULL;
-        int                      sent      = 0;
-        char                    *volname   = NULL;
-        const char *question = "Disabling quota will delete all the quota "
-                               "configuration. Do you want to continue?";
-
-        //parse **words into options dictionary
-        ret = cli_cmd_quota_parse (words, wordcount, &options);
+    int ret = 0;
+    int parse_err = 0;
+    int32_t type = 0;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *options = NULL;
+    gf_answer_t answer = GF_ANSWER_NO;
+    cli_local_t *local = NULL;
+    int sent = 0;
+    char *volname = NULL;
+    const char *question =
+        "Disabling quota will delete all the quota "
+        "configuration. Do you want to continue?";
+
+    // parse **words into options dictionary
+    if (strcmp(words[1], "inode-quota") == 0) {
+        ret = cli_cmd_inode_quota_parse(words, wordcount, &options);
         if (ret < 0) {
-                cli_usage_out (word->pattern);
-                parse_err = 1;
-                goto out;
+            cli_usage_out(word->pattern);
+            parse_err = 1;
+            goto out;
         }
+    } else {
+        ret = cli_cmd_quota_parse(words, wordcount, &options);
 
-        ret = dict_get_int32 (options, "type", &type);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to get opcode");
-                goto out;
+        if (ret == 1) {
+            cli_cmd_quota_help_cbk(state, word, words, wordcount);
+            ret = 0;
+            goto out;
+        }
+        if (ret < 0) {
+            cli_usage_out(word->pattern);
+            parse_err = 1;
+            goto out;
         }
+    }
 
-        //handle quota-disable and quota-list-all different from others
-        switch (type) {
+    ret = dict_get_int32(options, "type", &type);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to get opcode");
+        goto out;
+    }
+
+    // handle quota-disable and quota-list-all different from others
+    switch (type) {
         case GF_QUOTA_OPTION_TYPE_DISABLE:
-                answer = cli_cmd_get_confirmation (state, question);
-                if (answer == GF_ANSWER_NO)
-                        goto out;
-                break;
-        case GF_QUOTA_OPTION_TYPE_LIST:
-                if (wordcount != 4)
-                        break;
-                ret = cli_cmd_quota_handle_list_all (words, options);
+            answer = cli_cmd_get_confirmation(state, question);
+            if (answer == GF_ANSWER_NO)
                 goto out;
-        default:
+            break;
+        case GF_QUOTA_OPTION_TYPE_LIST:
+        case GF_QUOTA_OPTION_TYPE_LIST_OBJECTS:
+            if (wordcount != 4)
                 break;
-        }
+            ret = cli_cmd_quota_handle_list_all(words, options);
+            goto out;
+        default:
+            break;
+    }
 
-        ret = dict_get_str (options, "volname", &volname);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to get volume name");
-                goto out;
-        }
+    ret = dict_get_str(options, "volname", &volname);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to get volume name");
+        goto out;
+    }
 
-        //create auxillary mount need for quota commands that operate on path
-        ret = cli_stage_quota_op (volname, type);
-        if (ret)
-                goto out;
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
 
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame) {
-                ret = -1;
-                goto out;
-        }
+    CLI_LOCAL_INIT(local, words, frame, options);
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_QUOTA];
 
-        CLI_LOCAL_INIT (local, words, frame, options);
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_QUOTA];
-        if (proc == NULL) {
-                ret = -1;
-                goto out;
-        }
-
-        if (proc->fn)
-                ret = proc->fn (frame, THIS, options);
+    if (proc->fn)
+        ret = proc->fn(frame, THIS, options);
 
 out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if (sent == 0 && parse_err == 0)
-                        cli_out ("Quota command failed. Please check the cli "
-                                 "logs for more details");
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if (sent == 0 && parse_err == 0)
+            cli_out(
+                "Quota command failed. Please check the cli "
+                "logs for more details");
+    }
+    if (options)
+        dict_unref(options);
+
+    /* Events for Quota */
+    if (ret == 0) {
+        switch (type) {
+            case GF_QUOTA_OPTION_TYPE_ENABLE:
+                gf_event(EVENT_QUOTA_ENABLE, "volume=%s", volname);
+                break;
+            case GF_QUOTA_OPTION_TYPE_DISABLE:
+                gf_event(EVENT_QUOTA_DISABLE, "volume=%s", volname);
+                break;
+            case GF_QUOTA_OPTION_TYPE_LIMIT_USAGE:
+                gf_event(EVENT_QUOTA_SET_USAGE_LIMIT,
+                         "volume=%s;"
+                         "path=%s;limit=%s",
+                         volname, words[4], words[5]);
+                break;
+            case GF_QUOTA_OPTION_TYPE_LIMIT_OBJECTS:
+                gf_event(EVENT_QUOTA_SET_OBJECTS_LIMIT,
+                         "volume=%s;"
+                         "path=%s;limit=%s",
+                         volname, words[4], words[5]);
+                break;
+            case GF_QUOTA_OPTION_TYPE_REMOVE:
+                gf_event(EVENT_QUOTA_REMOVE_USAGE_LIMIT,
+                         "volume=%s;"
+                         "path=%s",
+                         volname, words[4]);
+                break;
+            case GF_QUOTA_OPTION_TYPE_REMOVE_OBJECTS:
+                gf_event(EVENT_QUOTA_REMOVE_OBJECTS_LIMIT,
+                         "volume=%s;"
+                         "path=%s",
+                         volname, words[4]);
+                break;
+            case GF_QUOTA_OPTION_TYPE_ALERT_TIME:
+                gf_event(EVENT_QUOTA_ALERT_TIME, "volume=%s;time=%s", volname,
+                         words[4]);
+                break;
+            case GF_QUOTA_OPTION_TYPE_SOFT_TIMEOUT:
+                gf_event(EVENT_QUOTA_SOFT_TIMEOUT,
+                         "volume=%s;"
+                         "soft-timeout=%s",
+                         volname, words[4]);
+                break;
+            case GF_QUOTA_OPTION_TYPE_HARD_TIMEOUT:
+                gf_event(EVENT_QUOTA_HARD_TIMEOUT,
+                         "volume=%s;"
+                         "hard-timeout=%s",
+                         volname, words[4]);
+                break;
+            case GF_QUOTA_OPTION_TYPE_DEFAULT_SOFT_LIMIT:
+                gf_event(EVENT_QUOTA_DEFAULT_SOFT_LIMIT,
+                         "volume=%s;"
+                         "default-soft-limit=%s",
+                         volname, words[4]);
+                break;
         }
+    }
 
-        CLI_STACK_DESTROY (frame);
-        return ret;
+    CLI_STACK_DESTROY(frame);
+    return ret;
 }
 
 int
-cli_cmd_volume_remove_brick_cbk (struct cli_state *state,
-                                 struct cli_cmd_word *word, const char **words,
-                                 int wordcount)
+cli_cmd_volume_remove_brick_cbk(struct cli_state *state,
+                                struct cli_cmd_word *word, const char **words,
+                                int wordcount)
 {
-        int                     ret = -1;
-        rpc_clnt_procedure_t    *proc = NULL;
-        call_frame_t            *frame = NULL;
-        dict_t                  *options = NULL;
-        gf_answer_t             answer = GF_ANSWER_NO;
-        int                     sent = 0;
-        int                     parse_error = 0;
-        int                     need_question = 0;
-        cli_local_t             *local = NULL;
-
-        const char *question = "Removing brick(s) can result in data loss. "
-                               "Do you want to Continue?";
-
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *options = NULL;
+    gf_answer_t answer = GF_ANSWER_NO;
+    int brick_count = 0;
+    int sent = 0;
+    int parse_error = 0;
+    int need_question = 0;
+    cli_local_t *local = NULL;
+    char *volname = NULL;
+#if (USE_EVENTS)
+    eventtypes_t event = EVENT_LAST;
+    char *event_str = NULL;
+    int event_ret = -1;
+#endif
+    int32_t command = GF_OP_CMD_NONE;
+    char *question = NULL;
+
+    ret = cli_cmd_volume_remove_brick_parse(state, words, wordcount, &options,
+                                            &need_question, &brick_count,
+                                            &command);
+    if (ret) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
+
+    if (command == GF_OP_CMD_COMMIT_FORCE) {
+        question =
+            "Remove-brick force will not migrate files from the "
+            "removed bricks, so they will no longer be available"
+            " on the volume.\nDo you want to continue?";
+    } else if (command == GF_OP_CMD_START) {
+        question =
+            "It is recommended that remove-brick be run with"
+            " cluster.force-migration option disabled to prevent"
+            " possible data corruption. Doing so will ensure that"
+            " files that receive writes during migration will not"
+            " be migrated and will need to be manually copied"
+            " after the remove-brick commit operation. Please"
+            " check the value of the option and update accordingly."
+            " \nDo you want to continue with your current"
+            " cluster.force-migration settings?";
+    }
+
+    if (!brick_count) {
+        cli_err("No bricks specified");
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        ret = -1;
+        goto out;
+    }
+    ret = dict_get_str(options, "volname", &volname);
+    if (ret || !volname) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to fetch volname");
+        ret = -1;
+        goto out;
+    }
 
-        ret = cli_cmd_volume_remove_brick_parse (words, wordcount, &options,
-                                                 &need_question);
-        if (ret) {
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-                goto out;
-        }
+#if (USE_EVENTS)
+    event_ret = cli_event_remove_brick_str(options, &event_str, &event);
+#endif
 
-        if (!(state->mode & GLUSTER_MODE_SCRIPT) && need_question) {
-                /* we need to ask question only in case of 'commit or force' */
-                answer = cli_cmd_get_confirmation (state, question);
-                if (GF_ANSWER_NO == answer) {
-                        ret = 0;
-                        goto out;
-                }
+    if (!strcmp(volname, GLUSTER_SHARED_STORAGE)) {
+        question =
+            "Removing brick from the shared storage volume"
+            "(gluster_shared_storage), will affect features "
+            "like snapshot scheduler, geo-replication "
+            "and NFS-Ganesha. Do you still want to "
+            "continue?";
+        need_question = _gf_true;
+    }
+
+    if (!(state->mode & GLUSTER_MODE_SCRIPT) && need_question) {
+        answer = cli_cmd_get_confirmation(state, question);
+        if (GF_ANSWER_NO == answer) {
+            ret = 0;
+            goto out;
         }
+    }
 
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_REMOVE_BRICK];
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_REMOVE_BRICK];
 
-        CLI_LOCAL_INIT (local, words, frame, options);
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
 
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, options);
-        }
+    CLI_LOCAL_INIT(local, words, frame, options);
+
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, options);
+    }
 
 out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_out ("Volume remove-brick failed");
-        }
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("Volume remove-brick failed");
+    }
+#if (USE_EVENTS)
+    if (!ret && !event_ret)
+        gf_event(event, "%s", event_str);
+    if (event_str)
+        GF_FREE(event_str);
 
-        CLI_STACK_DESTROY (frame);
+#endif
 
-        return ret;
+    CLI_STACK_DESTROY(frame);
+    if (options)
+        dict_unref(options);
 
+    return ret;
 }
 
 int
-cli_cmd_volume_replace_brick_cbk (struct cli_state *state,
-                                  struct cli_cmd_word *word,
-                                  const char **words,
-                                  int wordcount)
+cli_cmd_volume_reset_brick_cbk(struct cli_state *state,
+                               struct cli_cmd_word *word, const char **words,
+                               int wordcount)
 {
-        int                     ret = -1;
-        rpc_clnt_procedure_t    *proc = NULL;
-        call_frame_t            *frame = NULL;
-        dict_t                  *options = NULL;
-        int                     sent = 0;
-        int                     parse_error = 0;
-        cli_local_t             *local = NULL;
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *options = NULL;
+    int sent = 0;
+    int parse_error = 0;
+    cli_local_t *local = NULL;
 
 #ifdef GF_SOLARIS_HOST_OS
-        cli_out ("Command not supported on Solaris");
-        goto out;
+    cli_out("Command not supported on Solaris");
+    goto out;
 #endif
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_REPLACE_BRICK];
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_RESET_BRICK];
 
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
+    ret = cli_cmd_volume_reset_brick_parse(words, wordcount, &options);
 
-        ret = cli_cmd_volume_replace_brick_parse (words, wordcount, &options);
+    if (ret) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
 
+    if (state->mode & GLUSTER_MODE_WIGNORE_PARTITION) {
+        ret = dict_set_int32(options, "ignore-partition", _gf_true);
         if (ret) {
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-                goto out;
+            gf_log("cli", GF_LOG_ERROR,
+                   "Failed to set ignore-"
+                   "partition option");
+            goto out;
         }
+    }
 
-        if (state->mode & GLUSTER_MODE_SCRIPT) {
-                ret = dict_set_int32 (options, "force", _gf_true);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Failed to set force"
-                                "option");
-                        goto out;
-                }
-        }
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
 
-        CLI_LOCAL_INIT (local, words, frame, options);
+    CLI_LOCAL_INIT(local, words, frame, options);
 
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, options);
-        }
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, options);
+    }
 
 out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_out ("Volume replace-brick failed");
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("Volume reset-brick failed");
+    } else {
+        if (wordcount > 5) {
+            gf_event(EVENT_BRICK_RESET_COMMIT,
+                     "Volume=%s;source-brick=%s;"
+                     "destination-brick=%s",
+                     (char *)words[2], (char *)words[3], (char *)words[4]);
+        } else {
+            gf_event(EVENT_BRICK_RESET_START, "Volume=%s;source-brick=%s",
+                     (char *)words[2], (char *)words[3]);
         }
+    }
+    CLI_STACK_DESTROY(frame);
 
-        CLI_STACK_DESTROY (frame);
-
-        return ret;
+    return ret;
 }
 
-
 int
-cli_cmd_volume_set_transport_cbk (struct cli_state *state,
-                                  struct cli_cmd_word *word,
-                                  const char **words, int wordcount)
+cli_cmd_volume_replace_brick_cbk(struct cli_state *state,
+                                 struct cli_cmd_word *word, const char **words,
+                                 int wordcount)
 {
-        cli_cmd_broadcast_response (0);
-        return 0;
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *options = NULL;
+    int sent = 0;
+    int parse_error = 0;
+    cli_local_t *local = NULL;
+
+#ifdef GF_SOLARIS_HOST_OS
+    cli_out("Command not supported on Solaris");
+    goto out;
+#endif
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_REPLACE_BRICK];
+
+    ret = cli_cmd_volume_replace_brick_parse(words, wordcount, &options);
+
+    if (ret) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
+
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
+
+    CLI_LOCAL_INIT(local, words, frame, options);
+
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, options);
+    }
+
+out:
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("Volume replace-brick failed");
+    } else {
+        gf_event(EVENT_BRICK_REPLACE,
+                 "Volume=%s;source-brick=%s;destination-brick=%s",
+                 (char *)words[2], (char *)words[3], (char *)words[4]);
+    }
+    CLI_STACK_DESTROY(frame);
+
+    return ret;
 }
 
 int
-cli_cmd_volume_top_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                          const char **words, int wordcount)
+cli_cmd_volume_set_transport_cbk(struct cli_state *state,
+                                 struct cli_cmd_word *word, const char **words,
+                                 int wordcount)
 {
+    cli_cmd_broadcast_response(0);
+    return 0;
+}
 
-        int                     ret      = -1;
-        rpc_clnt_procedure_t    *proc    = NULL;
-        call_frame_t            *frame   = NULL;
-        dict_t                  *options = NULL;
-        int                     sent     = 0;
-        int                     parse_error = 0;
-        cli_local_t             *local = NULL;
-
-        ret = cli_cmd_volume_top_parse (words, wordcount, &options);
-
-        if (ret) {
-                parse_error = 1;
-                cli_usage_out (word->pattern);
-                goto out;
-        }
+int
+cli_cmd_volume_top_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                       const char **words, int wordcount)
+{
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *options = NULL;
+    int sent = 0;
+    int parse_error = 0;
+    cli_local_t *local = NULL;
+
+    ret = cli_cmd_volume_top_parse(words, wordcount, &options);
+
+    if (ret) {
+        parse_error = 1;
+        cli_usage_out(word->pattern);
+        goto out;
+    }
 
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_TOP_VOLUME];
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_TOP_VOLUME];
 
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        gf_log(THIS->name, GF_LOG_ERROR, "failed to create frame");
+        ret = -1;
+        goto out;
+    }
 
-        CLI_LOCAL_INIT (local, words, frame, options);
+    CLI_LOCAL_INIT(local, words, frame, options);
 
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, options);
-        }
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, options);
+    }
 
 out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_out ("Volume top failed");
-        }
-
-        CLI_STACK_DESTROY (frame);
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("Volume top failed");
+    }
 
-        return ret;
+    CLI_STACK_DESTROY(frame);
 
+    return ret;
 }
 
-
 int
-cli_cmd_log_rotate_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                        const char **words, int wordcount)
+cli_cmd_log_rotate_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                       const char **words, int wordcount)
 {
-        int                     ret = -1;
-        rpc_clnt_procedure_t    *proc = NULL;
-        call_frame_t            *frame = NULL;
-        dict_t                  *options = NULL;
-        int                     sent = 0;
-        int                     parse_error = 0;
-        cli_local_t             *local = NULL;
-
-        if (!((wordcount == 4) || (wordcount == 5))) {
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-                goto out;
-        }
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *options = NULL;
+    int sent = 0;
+    int parse_error = 0;
+    cli_local_t *local = NULL;
+
+    if (!((wordcount == 4) || (wordcount == 5))) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
 
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_LOG_ROTATE];
+    if (!(strcmp("rotate", words[3]) == 0)) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
 
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_LOG_ROTATE];
 
-        ret = cli_cmd_log_rotate_parse (words, wordcount, &options);
-        if (ret)
-                goto out;
+    ret = cli_cmd_log_rotate_parse(words, wordcount, &options);
+    if (ret)
+        goto out;
 
-        CLI_LOCAL_INIT (local, words, frame, options);
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        gf_log(THIS->name, GF_LOG_ERROR, "failed to create frame");
+        ret = -1;
+        goto out;
+    }
 
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, options);
-        }
+    CLI_LOCAL_INIT(local, words, frame, options);
 
-out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_out ("Volume log rotate failed");
-        }
-        CLI_STACK_DESTROY (frame);
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, options);
+    }
 
-        return ret;
+out:
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("Volume log rotate failed");
+    }
+    CLI_STACK_DESTROY(frame);
+
+    return ret;
 }
 
 #if (SYNCDAEMON_COMPILE)
 static int
-cli_check_gsync_present ()
+cli_check_gsync_present()
 {
-        char                buff[PATH_MAX] = {0, };
-        runner_t            runner = {0,};
-        char                *ptr = NULL;
-        int                 ret = 0;
-
-        ret = setenv ("_GLUSTERD_CALLED_", "1", 1);
-        if (-1 == ret) {
-                gf_log ("", GF_LOG_WARNING, "setenv syscall failed, hence could"
-                        "not assert if geo-replication is installed");
-                goto out;
-        }
-
-        runinit (&runner);
-        runner_add_args (&runner, GSYNCD_PREFIX"/gsyncd", "--version", NULL);
-        runner_redir (&runner, STDOUT_FILENO, RUN_PIPE);
-        ret = runner_start (&runner);
-        if (ret == -1) {
-                gf_log ("", GF_LOG_INFO, "geo-replication not installed");
-                goto out;
-        }
+    char buff[PATH_MAX] = {
+        0,
+    };
+    runner_t runner = {
+        0,
+    };
+    char *ptr = NULL;
+    int ret = 0;
+
+    ret = setenv("_GLUSTERD_CALLED_", "1", 1);
+    if (-1 == ret) {
+        gf_log("", GF_LOG_WARNING,
+               "setenv syscall failed, hence could"
+               "not assert if geo-replication is installed");
+        goto out;
+    }
+
+    runinit(&runner);
+    runner_add_args(&runner, GSYNCD_PREFIX "/gsyncd", "--version", NULL);
+    runner_redir(&runner, STDOUT_FILENO, RUN_PIPE);
+    ret = runner_start(&runner);
+    if (ret == -1) {
+        gf_log("", GF_LOG_INFO, "geo-replication not installed");
+        goto out;
+    }
 
-        ptr = fgets(buff, sizeof(buff), runner_chio (&runner, STDOUT_FILENO));
-        if (ptr) {
-                if (!strstr (buff, "gsyncd")) {
-                        ret  = -1;
-                        goto out;
-                }
-        } else {
-                ret = -1;
-                goto out;
+    ptr = fgets(buff, sizeof(buff), runner_chio(&runner, STDOUT_FILENO));
+    if (ptr) {
+        if (!strstr(buff, "gsyncd")) {
+            ret = -1;
+            goto out;
         }
+    } else {
+        ret = -1;
+        goto out;
+    }
 
-        ret = runner_end (&runner);
+    ret = runner_end(&runner);
 
-        if (ret)
-                gf_log ("", GF_LOG_ERROR, "geo-replication not installed");
+    if (ret)
+        gf_log("", GF_LOG_ERROR, "geo-replication not installed");
 
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret ? -1 : 0;
-
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret ? -1 : 0;
 }
 
 void
-cli_cmd_check_gsync_exists_cbk (struct cli_cmd *this)
+cli_cmd_check_gsync_exists_cbk(struct cli_cmd *this)
 {
+    int ret = 0;
 
-        int             ret = 0;
-
-        ret = cli_check_gsync_present ();
-        if (ret)
-                this->disable = _gf_true;
-
+    ret = cli_check_gsync_present();
+    if (ret)
+        this->disable = _gf_true;
 }
 #endif
 
 int
-cli_cmd_volume_gsync_set_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                              const char **words, int wordcount)
+cli_cmd_volume_gsync_set_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                             const char **words, int wordcount)
 {
-        int                      ret     = 0;
-        int                      parse_err = 0;
-        dict_t                  *options = NULL;
-        rpc_clnt_procedure_t    *proc    = NULL;
-        call_frame_t            *frame   = NULL;
-        cli_local_t             *local   = NULL;
-
-        proc = &cli_rpc_prog->proctable [GLUSTER_CLI_GSYNC_SET];
-        if (proc == NULL) {
-                ret = -1;
-                goto out;
-        }
+    int ret = 0;
+    int parse_err = 0;
+    dict_t *options = NULL;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    cli_local_t *local = NULL;
+    char *errstr = NULL;
+#if (USE_EVENTS)
+    int ret1 = -1;
+    int cmd_type = -1;
+    int tmpi = 0;
+    char *tmp = NULL;
+    char *events_str = NULL;
+    int event_type = -1;
+#endif
 
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (frame == NULL) {
-                ret = -1;
-                goto out;
-        }
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_GSYNC_SET];
 
-        ret = cli_cmd_gsync_set_parse (words, wordcount, &options);
-        if (ret) {
-                cli_usage_out (word->pattern);
-                parse_err = 1;
-                goto out;
+    ret = cli_cmd_gsync_set_parse(state, words, wordcount, &options, &errstr);
+    if (ret) {
+        if (errstr) {
+            cli_err("%s", errstr);
+            GF_FREE(errstr);
+        } else {
+            cli_usage_out(word->pattern);
         }
+        parse_err = 1;
+        goto out;
+    }
+
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (frame == NULL) {
+        ret = -1;
+        goto out;
+    }
 
-        CLI_LOCAL_INIT (local, words, frame, options);
+    CLI_LOCAL_INIT(local, words, frame, options);
 
-        if (proc->fn)
-                ret = proc->fn (frame, THIS, options);
+    if (proc->fn)
+        ret = proc->fn(frame, THIS, options);
 
 out:
-        if (ret && parse_err == 0)
-                cli_out (GEOREP" command failed");
+    if (ret && parse_err == 0)
+        cli_out(GEOREP " command failed");
+
+#if (USE_EVENTS)
+    if (ret == 0) {
+        events_str = gf_strdup("");
+
+        /* Type of Geo-rep Action - Create, Start etc */
+        ret1 = dict_get_int32(options, "type", &cmd_type);
+        if (ret1)
+            cmd_type = -1;
+
+        /* Only capture Events for modification commands */
+        switch (cmd_type) {
+            case GF_GSYNC_OPTION_TYPE_CREATE:
+                event_type = EVENT_GEOREP_CREATE;
+                break;
+            case GF_GSYNC_OPTION_TYPE_START:
+                event_type = EVENT_GEOREP_START;
+                break;
+            case GF_GSYNC_OPTION_TYPE_STOP:
+                event_type = EVENT_GEOREP_STOP;
+                break;
+            case GF_GSYNC_OPTION_TYPE_PAUSE:
+                event_type = EVENT_GEOREP_PAUSE;
+                break;
+            case GF_GSYNC_OPTION_TYPE_RESUME:
+                event_type = EVENT_GEOREP_RESUME;
+                break;
+            case GF_GSYNC_OPTION_TYPE_DELETE:
+                event_type = EVENT_GEOREP_DELETE;
+                break;
+            case GF_GSYNC_OPTION_TYPE_CONFIG:
+                ret1 = dict_get_str(options, "subop", &tmp);
+                if (ret1)
+                    tmp = "";
+
+                /* For Config Set additionally capture key and value */
+                /* For Config Reset capture key */
+                if (strcmp(tmp, "set") == 0) {
+                    event_type = EVENT_GEOREP_CONFIG_SET;
+
+                    ret1 = dict_get_str(options, "op_name", &tmp);
+                    if (ret1)
+                        tmp = "";
+
+                    gf_asprintf_append(&events_str, "%soption=%s;", events_str,
+                                       tmp);
+
+                    ret1 = dict_get_str(options, "op_value", &tmp);
+                    if (ret1)
+                        tmp = "";
+
+                    gf_asprintf_append(&events_str, "%svalue=%s;", events_str,
+                                       tmp);
+                } else if (strcmp(tmp, "del") == 0) {
+                    event_type = EVENT_GEOREP_CONFIG_RESET;
+
+                    ret1 = dict_get_str(options, "op_name", &tmp);
+                    if (ret1)
+                        tmp = "";
+
+                    gf_asprintf_append(&events_str, "%soption=%s;", events_str,
+                                       tmp);
+                }
+                break;
+            default:
+                break;
+        }
+
+        if (event_type > -1) {
+            /* Capture all optional arguments used */
+            ret1 = dict_get_int32(options, "force", &tmpi);
+            if (ret1 == 0) {
+                gf_asprintf_append(&events_str, "%sforce=%d;", events_str,
+                                   tmpi);
+            }
+            ret1 = dict_get_int32(options, "push_pem", &tmpi);
+            if (ret1 == 0) {
+                gf_asprintf_append(&events_str, "%spush_pem=%d;", events_str,
+                                   tmpi);
+            }
+            ret1 = dict_get_int32(options, "no_verify", &tmpi);
+            if (ret1 == 0) {
+                gf_asprintf_append(&events_str, "%sno_verify=%d;", events_str,
+                                   tmpi);
+            }
+
+            ret1 = dict_get_int32(options, "ssh_port", &tmpi);
+            if (ret1 == 0) {
+                gf_asprintf_append(&events_str, "%sssh_port=%d;", events_str,
+                                   tmpi);
+            }
+
+            ret1 = dict_get_int32(options, "reset-sync-time", &tmpi);
+            if (ret1 == 0) {
+                gf_asprintf_append(&events_str, "%sreset_sync_time=%d;",
+                                   events_str, tmpi);
+            }
+            /* Capture Master and Slave Info */
+            ret1 = dict_get_str(options, "master", &tmp);
+            if (ret1)
+                tmp = "";
+            gf_asprintf_append(&events_str, "%smaster=%s;", events_str, tmp);
+
+            ret1 = dict_get_str(options, "slave", &tmp);
+            if (ret1)
+                tmp = "";
+            gf_asprintf_append(&events_str, "%sslave=%s", events_str, tmp);
+
+            gf_event(event_type, "%s", events_str);
+        }
+
+        /* Allocated by gf_strdup and gf_asprintf */
+        if (events_str)
+            GF_FREE(events_str);
+    }
+#endif
 
-        CLI_STACK_DESTROY (frame);
+    CLI_STACK_DESTROY(frame);
 
-        return ret;
+    return ret;
 }
 
 int
-cli_cmd_volume_status_cbk (struct cli_state *state,
-                           struct cli_cmd_word *word,
-                           const char **words, int wordcount)
+cli_cmd_volume_status_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                          const char **words, int wordcount)
 {
-        int                   ret         = -1;
-        rpc_clnt_procedure_t *proc        = NULL;
-        call_frame_t         *frame       = NULL;
-        dict_t               *dict        = NULL;
-        uint32_t              cmd         = 0;
-        cli_local_t          *local       = NULL;
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *dict = NULL;
+    uint32_t cmd = 0;
+    cli_local_t *local = NULL;
 
-        ret = cli_cmd_volume_status_parse (words, wordcount, &dict);
+    ret = cli_cmd_volume_status_parse(words, wordcount, &dict);
 
-        if (ret) {
-                cli_usage_out (word->pattern);
-                goto out;
-        }
+    if (ret) {
+        cli_usage_out(word->pattern);
+        goto out;
+    }
 
-        ret = dict_get_uint32 (dict, "cmd", &cmd);
-        if (ret)
-                goto out;
+    ret = dict_get_uint32(dict, "cmd", &cmd);
+    if (ret)
+        goto out;
 
-        if (!(cmd & GF_CLI_STATUS_ALL)) {
-                /* for one volume or brick */
-                proc = &cli_rpc_prog->proctable[GLUSTER_CLI_STATUS_VOLUME];
-        } else {
-                /* volume status all or all detail */
-                proc = &cli_rpc_prog->proctable[GLUSTER_CLI_STATUS_ALL];
-        }
+    if (!(cmd & GF_CLI_STATUS_ALL)) {
+        /* for one volume or brick */
+        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_STATUS_VOLUME];
+    } else {
+        /* volume status all or all detail */
+        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_STATUS_ALL];
+    }
 
-        if (!proc->fn)
-                goto out;
+    if (!proc->fn) {
+        ret = -1;
+        goto out;
+    }
 
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        gf_log(THIS->name, GF_LOG_ERROR, "failed to create frame");
+        ret = -1;
+        goto out;
+    }
 
-        CLI_LOCAL_INIT (local, words, frame, dict);
+    CLI_LOCAL_INIT(local, words, frame, dict);
 
-        ret = proc->fn (frame, THIS, dict);
+    ret = proc->fn(frame, THIS, dict);
 
 out:
-        CLI_STACK_DESTROY (frame);
+    CLI_STACK_DESTROY(frame);
 
-        return ret;
+    return ret;
 }
 
-
 int
-cli_get_detail_status (dict_t *dict, int i, cli_volume_status_t *status)
+cli_get_detail_status(dict_t *dict, int i, cli_volume_status_t *status)
 {
-        uint64_t                   free            = 0;
-        uint64_t                   total           = 0;
-        char                       key[1024]       = {0};
-        int                        ret             = 0;
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "brick%d.free", i);
-        ret = dict_get_uint64 (dict, key, &free);
-
-        status->free = gf_uint64_2human_readable (free);
-        if (!status->free)
-                goto out;
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "brick%d.total", i);
-        ret = dict_get_uint64 (dict, key, &total);
-
-        status->total = gf_uint64_2human_readable (total);
-        if (!status->total)
-                goto out;
+    uint64_t free = 0;
+    uint64_t total = 0;
+    char key[1024] = {0};
+    int ret = 0;
 
-#ifdef GF_LINUX_HOST_OS
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "brick%d.device", i);
-        ret = dict_get_str (dict, key, &(status->device));
-        if (ret)
-                status->device = NULL;
-#endif
+    snprintf(key, sizeof(key), "brick%d.free", i);
+    ret = dict_get_uint64(dict, key, &free);
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "brick%d.block_size", i);
-        ret = dict_get_uint64 (dict, key, &(status->block_size));
-        if (ret) {
-                ret = 0;
-                status->block_size = 0;
-        }
+    status->free = gf_uint64_2human_readable(free);
+    if (!status->free)
+        goto out;
 
-#ifdef GF_LINUX_HOST_OS
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "brick%d.mnt_options", i);
-        ret = dict_get_str (dict, key, &(status->mount_options));
-        if (ret)
-                status->mount_options = NULL;
+    snprintf(key, sizeof(key), "brick%d.total", i);
+    ret = dict_get_uint64(dict, key, &total);
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "brick%d.fs_name", i);
-        ret = dict_get_str (dict, key, &(status->fs_name));
-        if (ret) {
-                ret = 0;
-                status->fs_name = NULL;
-        }
+    status->total = gf_uint64_2human_readable(total);
+    if (!status->total)
+        goto out;
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "brick%d.inode_size", i);
-        ret = dict_get_str (dict, key, &(status->inode_size));
-        if (ret)
-                status->inode_size = NULL;
-#endif /* GF_LINUX_HOST_OS */
+    snprintf(key, sizeof(key), "brick%d.device", i);
+    ret = dict_get_str(dict, key, &(status->device));
+    if (ret)
+        status->device = NULL;
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "brick%d.total_inodes", i);
-        ret = dict_get_uint64 (dict, key,
-                        &(status->total_inodes));
-        if (ret)
-                status->total_inodes = 0;
+    snprintf(key, sizeof(key), "brick%d.block_size", i);
+    ret = dict_get_uint64(dict, key, &(status->block_size));
+    if (ret) {
+        ret = 0;
+        status->block_size = 0;
+    }
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "brick%d.free_inodes", i);
-        ret = dict_get_uint64 (dict, key, &(status->free_inodes));
-        if (ret) {
-                ret = 0;
-                status->free_inodes = 0;
-        }
+    snprintf(key, sizeof(key), "brick%d.mnt_options", i);
+    ret = dict_get_str(dict, key, &(status->mount_options));
+    if (ret)
+        status->mount_options = NULL;
 
+    snprintf(key, sizeof(key), "brick%d.fs_name", i);
+    ret = dict_get_str(dict, key, &(status->fs_name));
+    if (ret) {
+        ret = 0;
+        status->fs_name = NULL;
+    }
+
+    snprintf(key, sizeof(key), "brick%d.inode_size", i);
+    ret = dict_get_str(dict, key, &(status->inode_size));
+    if (ret)
+        status->inode_size = NULL;
+
+    snprintf(key, sizeof(key), "brick%d.total_inodes", i);
+    ret = dict_get_uint64(dict, key, &(status->total_inodes));
+    if (ret)
+        status->total_inodes = 0;
+
+    snprintf(key, sizeof(key), "brick%d.free_inodes", i);
+    ret = dict_get_uint64(dict, key, &(status->free_inodes));
+    if (ret) {
+        ret = 0;
+        status->free_inodes = 0;
+    }
 
- out:
-        return ret;
+out:
+    return ret;
 }
 
 void
-cli_print_detailed_status (cli_volume_status_t *status)
+cli_print_detailed_status(cli_volume_status_t *status)
 {
-        cli_out ("%-20s : %-20s", "Brick", status->brick);
-        if (status->online)
-                cli_out ("%-20s : %-20d", "Port", status->port);
-        else
-                cli_out ("%-20s : %-20s", "Port", "N/A");
-        cli_out ("%-20s : %-20c", "Online", (status->online) ? 'Y' : 'N');
-        cli_out ("%-20s : %-20s", "Pid", status->pid_str);
-
-#ifdef GF_LINUX_HOST_OS
-        if (status->fs_name)
-                cli_out ("%-20s : %-20s", "File System", status->fs_name);
-        else
-                cli_out ("%-20s : %-20s", "File System", "N/A");
-
-        if (status->device)
-                cli_out ("%-20s : %-20s", "Device", status->device);
-        else
-                cli_out ("%-20s : %-20s", "Device", "N/A");
-
-        if (status->mount_options) {
-                cli_out ("%-20s : %-20s", "Mount Options",
-                         status->mount_options);
-        } else {
-                cli_out ("%-20s : %-20s", "Mount Options", "N/A");
-        }
+    cli_out("%-20s : %-20s", "Brick", status->brick);
+
+    if (status->online) {
+        cli_out("%-20s : %-20d", "TCP Port", status->port);
+        cli_out("%-20s : %-20d", "RDMA Port", status->rdma_port);
+    } else {
+        cli_out("%-20s : %-20s", "TCP Port", "N/A");
+        cli_out("%-20s : %-20s", "RDMA Port", "N/A");
+    }
+
+    cli_out("%-20s : %-20c", "Online", (status->online) ? 'Y' : 'N');
+    cli_out("%-20s : %-20s", "Pid", status->pid_str);
+
+    if (status->fs_name)
+        cli_out("%-20s : %-20s", "File System", status->fs_name);
+    else
+        cli_out("%-20s : %-20s", "File System", "N/A");
+
+    if (status->device)
+        cli_out("%-20s : %-20s", "Device", status->device);
+    else
+        cli_out("%-20s : %-20s", "Device", "N/A");
+
+    if (status->mount_options) {
+        cli_out("%-20s : %-20s", "Mount Options", status->mount_options);
+    } else {
+        cli_out("%-20s : %-20s", "Mount Options", "N/A");
+    }
+
+    if (status->inode_size) {
+        cli_out("%-20s : %-20s", "Inode Size", status->inode_size);
+    } else {
+        cli_out("%-20s : %-20s", "Inode Size", "N/A");
+    }
+    if (status->free)
+        cli_out("%-20s : %-20s", "Disk Space Free", status->free);
+    else
+        cli_out("%-20s : %-20s", "Disk Space Free", "N/A");
+
+    if (status->total)
+        cli_out("%-20s : %-20s", "Total Disk Space", status->total);
+    else
+        cli_out("%-20s : %-20s", "Total Disk Space", "N/A");
+
+    if (status->total_inodes) {
+        cli_out("%-20s : %-20" GF_PRI_INODE, "Inode Count",
+                status->total_inodes);
+    } else {
+        cli_out("%-20s : %-20s", "Inode Count", "N/A");
+    }
+
+    if (status->free_inodes) {
+        cli_out("%-20s : %-20" GF_PRI_INODE, "Free Inodes",
+                status->free_inodes);
+    } else {
+        cli_out("%-20s : %-20s", "Free Inodes", "N/A");
+    }
+}
 
-        if (status->inode_size) {
-                cli_out ("%-20s : %-20s", "Inode Size",
-                         status->inode_size);
+int
+cli_print_brick_status(cli_volume_status_t *status)
+{
+    int fieldlen = CLI_VOL_STATUS_BRICK_LEN;
+    int bricklen = 0;
+    char *p = NULL;
+    int num_spaces = 0;
+
+    p = status->brick;
+    bricklen = strlen(p);
+    while (bricklen > 0) {
+        if (bricklen > fieldlen) {
+            cli_out("%.*s", fieldlen, p);
+            p += fieldlen;
+            bricklen -= fieldlen;
         } else {
-                cli_out ("%-20s : %-20s", "Inode Size", "N/A");
-        }
-#endif
-        if (status->free)
-                cli_out ("%-20s : %-20s", "Disk Space Free", status->free);
-        else
-                cli_out ("%-20s : %-20s", "Disk Space Free", "N/A");
-
-        if (status->total)
-                cli_out ("%-20s : %-20s", "Total Disk Space", status->total);
-        else
-                cli_out ("%-20s : %-20s", "Total Disk Space", "N/A");
+            num_spaces = (fieldlen - bricklen) + 1;
+            printf("%s", p);
+            while (num_spaces-- != 0)
+                printf(" ");
+            if (status->port || status->rdma_port) {
+                if (status->online)
+                    cli_out("%-10d%-11d%-8c%-5s", status->port,
+                            status->rdma_port, status->online ? 'Y' : 'N',
+                            status->pid_str);
+                else
+                    cli_out("%-10s%-11s%-8c%-5s", "N/A", "N/A",
+                            status->online ? 'Y' : 'N', status->pid_str);
+            } else
+                cli_out("%-10s%-11s%-8c%-5s", "N/A", "N/A",
+                        status->online ? 'Y' : 'N', status->pid_str);
+            bricklen = 0;
+        }
+    }
+
+    return 0;
+}
 
+#define NEEDS_GLFS_HEAL(op)                                                    \
+    ((op == GF_SHD_OP_SBRAIN_HEAL_FROM_BIGGER_FILE) ||                         \
+     (op == GF_SHD_OP_SBRAIN_HEAL_FROM_LATEST_MTIME) ||                        \
+     (op == GF_SHD_OP_SBRAIN_HEAL_FROM_BRICK) ||                               \
+     (op == GF_SHD_OP_INDEX_SUMMARY) || (op == GF_SHD_OP_SPLIT_BRAIN_FILES) || \
+     (op == GF_SHD_OP_GRANULAR_ENTRY_HEAL_ENABLE) ||                           \
+     (op == GF_SHD_OP_HEAL_SUMMARY))
 
-        if (status->total_inodes) {
-                cli_out ("%-20s : %-20ld", "Inode Count",
-                         status->total_inodes);
-        } else {
-                cli_out ("%-20s : %-20s", "Inode Count", "N/A");
-        }
+int
+cli_launch_glfs_heal(int heal_op, dict_t *options)
+{
+    char buff[PATH_MAX] = {0};
+    runner_t runner = {0};
+    char *filename = NULL;
+    char *hostname = NULL;
+    char *path = NULL;
+    char *volname = NULL;
+    char *out = NULL;
+    int ret = 0;
+
+    runinit(&runner);
+    ret = dict_get_str(options, "volname", &volname);
+    runner_add_args(&runner, GLFSHEAL_PREFIX "/glfsheal", volname, NULL);
+    runner_redir(&runner, STDOUT_FILENO, RUN_PIPE);
+
+    switch (heal_op) {
+        case GF_SHD_OP_INDEX_SUMMARY:
+            if (global_state->mode & GLUSTER_MODE_XML) {
+                runner_add_args(&runner, "--xml", NULL);
+            }
+            break;
+        case GF_SHD_OP_SBRAIN_HEAL_FROM_BIGGER_FILE:
+            ret = dict_get_str(options, "file", &filename);
+            runner_add_args(&runner, "bigger-file", filename, NULL);
+            break;
+        case GF_SHD_OP_SBRAIN_HEAL_FROM_LATEST_MTIME:
+            ret = dict_get_str(options, "file", &filename);
+            runner_add_args(&runner, "latest-mtime", filename, NULL);
+            break;
+        case GF_SHD_OP_SBRAIN_HEAL_FROM_BRICK:
+            ret = dict_get_str(options, "heal-source-hostname", &hostname);
+            ret = dict_get_str(options, "heal-source-brickpath", &path);
+            runner_add_args(&runner, "source-brick", NULL);
+            runner_argprintf(&runner, "%s:%s", hostname, path);
+            if (dict_get_str(options, "file", &filename) == 0)
+                runner_argprintf(&runner, "%s", filename);
+            break;
+        case GF_SHD_OP_SPLIT_BRAIN_FILES:
+            runner_add_args(&runner, "split-brain-info", NULL);
+            if (global_state->mode & GLUSTER_MODE_XML) {
+                runner_add_args(&runner, "--xml", NULL);
+            }
+            break;
+        case GF_SHD_OP_GRANULAR_ENTRY_HEAL_ENABLE:
+        case GF_SHD_OP_GRANULAR_ENTRY_HEAL_DISABLE:
+            runner_add_args(&runner, "granular-entry-heal-op", NULL);
+            break;
+        case GF_SHD_OP_HEAL_SUMMARY:
+            runner_add_args(&runner, "info-summary", NULL);
+            if (global_state->mode & GLUSTER_MODE_XML) {
+                runner_add_args(&runner, "--xml", NULL);
+            }
+            break;
+        default:
+            ret = -1;
+            goto out;
+    }
+    if (global_state->mode & GLUSTER_MODE_GLFSHEAL_NOLOG)
+        runner_add_args(&runner, "--nolog", NULL);
+    ret = runner_start(&runner);
+    if (ret == -1)
+        goto out;
+    while ((
+        out = fgets(buff, sizeof(buff), runner_chio(&runner, STDOUT_FILENO)))) {
+        printf("%s", out);
+    }
+    ret = runner_end(&runner);
 
-        if (status->free_inodes) {
-                cli_out ("%-20s : %-20ld", "Free Inodes",
-                         status->free_inodes);
-        } else {
-                cli_out ("%-20s : %-20s", "Free Inodes", "N/A");
-        }
+out:
+    return ret;
 }
 
 int
-cli_print_brick_status (cli_volume_status_t *status)
+cli_cmd_volume_heal_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                        const char **words, int wordcount)
 {
-        int  fieldlen = CLI_VOL_STATUS_BRICK_LEN;
-        int  bricklen = 0;
-        char *p = NULL;
-        int  num_tabs = 0;
-
-        p = status->brick;
-        bricklen = strlen (p);
-        while (bricklen > 0) {
-                if (bricklen > fieldlen) {
-                        cli_out ("%.*s", fieldlen, p);
-                        p += fieldlen;
-                        bricklen -= fieldlen;
-                } else {
-                        num_tabs = (fieldlen - bricklen) / CLI_TAB_LENGTH + 1;
-                        printf ("%s", p);
-                        while (num_tabs-- != 0)
-                                printf ("\t");
-                        if (status->port) {
-                                if (status->online)
-                                        cli_out ("%d\t%c\t%s",
-                                                 status->port,
-                                                 status->online?'Y':'N',
-                                                 status->pid_str);
-                                else
-                                        cli_out ("%s\t%c\t%s",
-                                                 "N/A",
-                                                 status->online?'Y':'N',
-                                                 status->pid_str);
-                        }
-                        else
-                                cli_out ("%s\t%c\t%s",
-                                         "N/A", status->online?'Y':'N',
-                                         status->pid_str);
-                        bricklen = 0;
-                }
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    int sent = 0;
+    int parse_error = 0;
+    dict_t *options = NULL;
+    xlator_t *this = NULL;
+    cli_local_t *local = NULL;
+    int heal_op = 0;
+
+    this = THIS;
+
+    if (wordcount < 3) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
+
+    ret = cli_cmd_volume_heal_options_parse(words, wordcount, &options);
+    if (ret) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
+    ret = dict_get_int32(options, "heal-op", &heal_op);
+    if (ret < 0)
+        goto out;
+    if (NEEDS_GLFS_HEAL(heal_op)) {
+        ret = cli_launch_glfs_heal(heal_op, options);
+        if (ret < 0)
+            goto out;
+        if (heal_op != GF_SHD_OP_GRANULAR_ENTRY_HEAL_ENABLE)
+            goto out;
+    }
+
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_HEAL_VOLUME];
+
+    frame = create_frame(this, this->ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
+
+    CLI_LOCAL_INIT(local, words, frame, options);
+
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, options);
+    }
+out:
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0) &&
+            !(global_state->mode & GLUSTER_MODE_XML)) {
+            cli_out("Volume heal failed.");
         }
+    }
+
+    if (options)
+        dict_unref(options);
 
-        return 0;
+    CLI_STACK_DESTROY(frame);
+
+    return ret;
 }
 
 int
-cli_cmd_volume_heal_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                          const char **words, int wordcount)
+cli_cmd_volume_statedump_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                             const char **words, int wordcount)
 {
-        int                     ret = -1;
-        rpc_clnt_procedure_t    *proc = NULL;
-        call_frame_t            *frame = NULL;
-        int                     sent = 0;
-        int                     parse_error = 0;
-        dict_t                  *options = NULL;
-        xlator_t                *this = NULL;
-        cli_local_t             *local = NULL;
-        int                     heal_op = 0;
-        runner_t                runner = {0};
-        char                    buff[PATH_MAX] = {0};
-        char                    *out = NULL;
-
-        this = THIS;
-        frame = create_frame (this, this->ctx->pool);
-        if (!frame)
-                goto out;
-
-        if (wordcount < 3) {
-               cli_usage_out (word->pattern);
-               parse_error = 1;
-               goto out;
-        }
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *options = NULL;
+    int sent = 0;
+    int parse_error = 0;
+    cli_local_t *local = NULL;
+
+    if (wordcount < 3) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
 
-        ret = cli_cmd_volume_heal_options_parse (words, wordcount, &options);
+    if (wordcount >= 3) {
+        ret = cli_cmd_volume_statedump_options_parse(words, wordcount,
+                                                     &options);
         if (ret) {
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-                goto out;
+            parse_error = 1;
+            gf_log("cli", GF_LOG_ERROR,
+                   "Error parsing "
+                   "statedump options");
+            cli_out("Error parsing options");
+            cli_usage_out(word->pattern);
         }
-        ret = dict_get_int32 (options, "heal-op", &heal_op);
-        if (ret < 0)
-                goto out;
+    }
 
-        if (heal_op == GF_AFR_OP_INDEX_SUMMARY) {
-                runinit (&runner);
-                runner_add_args (&runner, SBIN_DIR"/glfsheal", words[2], NULL);
-                runner_redir (&runner, STDOUT_FILENO, RUN_PIPE);
-                ret = runner_start (&runner);
-                if (ret == -1)
-                        goto out;
-                while ((out = fgets(buff, sizeof(buff),
-                                   runner_chio (&runner, STDOUT_FILENO)))) {
-                        printf ("%s", out);
-                }
+    ret = dict_set_str(options, "volname", (char *)words[2]);
+    if (ret)
+        goto out;
 
-                ret = runner_end (&runner);
-                ret = WEXITSTATUS (ret);
-        } else {
-                proc = &cli_rpc_prog->proctable[GLUSTER_CLI_HEAL_VOLUME];
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_STATEDUMP_VOLUME];
 
-                CLI_LOCAL_INIT (local, words, frame, options);
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
 
-                if (proc->fn) {
-                        ret = proc->fn (frame, THIS, options);
-                }
-        }
+    CLI_LOCAL_INIT(local, words, frame, options);
+
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, options);
+    }
 
 out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_out ("Volume heal failed");
-        }
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("Volume statedump failed");
+    }
 
-        CLI_STACK_DESTROY (frame);
+    CLI_STACK_DESTROY(frame);
 
-        return ret;
+    return ret;
 }
 
 int
-cli_cmd_volume_statedump_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                              const char **words, int wordcount)
+cli_cmd_volume_list_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                        const char **words, int wordcount)
 {
-        int                             ret = -1;
-        rpc_clnt_procedure_t            *proc = NULL;
-        call_frame_t                    *frame = NULL;
-        dict_t                          *options = NULL;
-        int                             sent = 0;
-        int                             parse_error = 0;
-        cli_local_t                     *local = NULL;
-
-        frame = create_frame (THIS, THIS->ctx->pool);
+    int ret = -1;
+    call_frame_t *frame = NULL;
+    rpc_clnt_procedure_t *proc = NULL;
+    int sent = 0;
+
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_LIST_VOLUME];
+    if (proc->fn) {
+        frame = create_frame(THIS, THIS->ctx->pool);
         if (!frame)
-                goto out;
+            goto out;
+        ret = proc->fn(frame, THIS, NULL);
+    }
 
-        if (wordcount < 3) {
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-                goto out;
-        }
+out:
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if (sent == 0)
+            cli_out("Volume list failed");
+    }
 
-        if (wordcount >= 3) {
-               ret = cli_cmd_volume_statedump_options_parse (words, wordcount,
-                                                              &options);
-               if (ret) {
-                       parse_error = 1;
-                       gf_log ("cli", GF_LOG_ERROR, "Error parsing "
-                               "statedump options");
-                       cli_out ("Error parsing options");
-                       cli_usage_out (word->pattern);
-               }
-        }
+    CLI_STACK_DESTROY(frame);
 
-        ret = dict_set_str (options, "volname", (char *)words[2]);
-        if (ret)
-                goto out;
+    return ret;
+}
 
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_STATEDUMP_VOLUME];
+int
+cli_cmd_volume_clearlocks_cbk(struct cli_state *state,
+                              struct cli_cmd_word *word, const char **words,
+                              int wordcount)
+{
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *options = NULL;
+    int sent = 0;
+    int parse_error = 0;
+    cli_local_t *local = NULL;
+
+    if (wordcount < 7 || wordcount > 8) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
+
+    ret = cli_cmd_volume_clrlks_opts_parse(words, wordcount, &options);
+    if (ret) {
+        parse_error = 1;
+        gf_log("cli", GF_LOG_ERROR,
+               "Error parsing "
+               "clear-locks options");
+        cli_out("Error parsing options");
+        cli_usage_out(word->pattern);
+    }
+
+    ret = dict_set_str(options, "volname", (char *)words[2]);
+    if (ret)
+        goto out;
 
-        CLI_LOCAL_INIT (local, words, frame, options);
+    ret = dict_set_str(options, "path", (char *)words[3]);
+    if (ret)
+        goto out;
 
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, options);
-        }
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_CLRLOCKS_VOLUME];
+
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
+
+    CLI_LOCAL_INIT(local, words, frame, options);
+
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, options);
+    }
 
 out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error = 0))
-                        cli_out ("Volume statedump failed");
-        }
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("Volume clear-locks failed");
+    }
 
-        CLI_STACK_DESTROY (frame);
+    CLI_STACK_DESTROY(frame);
 
-        return ret;
+    return ret;
 }
 
 int
-cli_cmd_volume_list_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                         const char **words, int wordcount)
+cli_cmd_volume_barrier_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                           const char **words, int wordcount)
 {
-        int                     ret = -1;
-        call_frame_t            *frame = NULL;
-        rpc_clnt_procedure_t    *proc = NULL;
-        int                     sent = 0;
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *options = NULL;
+    int sent = 0;
+    int parse_error = 0;
+    cli_local_t *local = NULL;
+
+    if (wordcount != 4) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
 
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
+    options = dict_new();
+    if (!options) {
+        ret = -1;
+        goto out;
+    }
+    ret = dict_set_str(options, "volname", (char *)words[2]);
+    if (ret)
+        goto out;
 
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_LIST_VOLUME];
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, NULL);
-        }
+    ret = dict_set_str(options, "barrier", (char *)words[3]);
+    if (ret)
+        goto out;
 
-out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if (sent == 0)
-                        cli_out ("Volume list failed");
-        }
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_BARRIER_VOLUME];
+
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
 
-        CLI_STACK_DESTROY (frame);
+    CLI_LOCAL_INIT(local, words, frame, options);
 
-        return ret;
+    if (proc->fn)
+        ret = proc->fn(frame, THIS, options);
+
+out:
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_err("Volume barrier failed");
+    }
+    CLI_STACK_DESTROY(frame);
+
+    return ret;
 }
 
 int
-cli_cmd_volume_clearlocks_cbk (struct cli_state *state,
-                               struct cli_cmd_word *word,
-                               const char **words, int wordcount)
+cli_cmd_volume_getopt_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                          const char **words, int wordcount)
 {
-        int                             ret = -1;
-        rpc_clnt_procedure_t            *proc = NULL;
-        call_frame_t                    *frame = NULL;
-        dict_t                          *options = NULL;
-        int                             sent = 0;
-        int                             parse_error = 0;
-        cli_local_t                     *local = NULL;
-
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *options = NULL;
+    int sent = 0;
+    int parse_err = 0;
+    cli_local_t *local = NULL;
+
+    if (wordcount != 4) {
+        cli_usage_out(word->pattern);
+        parse_err = 1;
+        goto out;
+    }
 
-        if (wordcount < 7 || wordcount > 8) {
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-                goto out;
-        }
+    options = dict_new();
+    if (!options)
+        goto out;
 
-       ret = cli_cmd_volume_clrlks_opts_parse (words, wordcount, &options);
-       if (ret) {
-               parse_error = 1;
-               gf_log ("cli", GF_LOG_ERROR, "Error parsing "
-                       "clear-locks options");
-               cli_out ("Error parsing options");
-               cli_usage_out (word->pattern);
-       }
+    ret = dict_set_str(options, "volname", (char *)words[2]);
+    if (ret)
+        goto out;
 
-        ret = dict_set_str (options, "volname", (char *)words[2]);
-        if (ret)
-                goto out;
+    ret = dict_set_str(options, "key", (char *)words[3]);
+    if (ret)
+        goto out;
 
-        ret = dict_set_str (options, "path", (char *)words[3]);
-        if (ret)
-                goto out;
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_GET_VOL_OPT];
 
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_CLRLOCKS_VOLUME];
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
 
-        CLI_LOCAL_INIT (local, words, frame, options);
+    CLI_LOCAL_INIT(local, words, frame, options);
 
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, options);
-        }
+    if (proc->fn)
+        ret = proc->fn(frame, THIS, options);
 
 out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error = 0))
-                        cli_out ("Volume clear-locks failed");
-        }
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_err == 0))
+            cli_err("Volume get option failed");
+    }
+    CLI_STACK_DESTROY(frame);
+    return ret;
+}
 
-        CLI_STACK_DESTROY (frame);
+/* This is a bit of a hack to display the help. The current bitrot cmd
+ * format does not work well when registering the cmds.
+ * Ideally the should have been of the form
+ * gluster volume bitrot <subcommand> <volumename> ...
+ */
 
-        return ret;
-}
+struct cli_cmd bitrot_cmds[] = {
+
+    {"volume bitrot help", cli_cmd_bitrot_help_cbk,
+     "display help for volume bitrot commands"},
+
+    {"volume bitrot <VOLNAME> {enable|disable}", NULL, /*cli_cmd_bitrot_cbk,*/
+     "Enable/disable bitrot for volume <VOLNAME>"},
+
+    {"volume bitrot <VOLNAME> signing-time <time-in-secs>",
+     NULL, /*cli_cmd_bitrot_cbk,*/
+     "Waiting time for an object after last fd is closed to start signing "
+     "process"},
+
+    {"volume bitrot <VOLNAME> signer-threads <count>",
+     NULL, /*cli_cmd_bitrot_cbk,*/
+     "Number of signing process threads. Usually set to number of available "
+     "cores"},
+
+    {"volume bitrot <VOLNAME> scrub-throttle {lazy|normal|aggressive}",
+     NULL, /*cli_cmd_bitrot_cbk,*/
+     "Set the speed of the scrubber for volume <VOLNAME>"},
+
+    {"volume bitrot <VOLNAME> scrub-frequency {hourly|daily|weekly|biweekly"
+     "|monthly}",
+     NULL, /*cli_cmd_bitrot_cbk,*/
+     "Set the frequency of the scrubber for volume <VOLNAME>"},
+
+    {"volume bitrot <VOLNAME> scrub {pause|resume|status|ondemand}",
+     NULL, /*cli_cmd_bitrot_cbk,*/
+     "Pause/resume the scrubber for <VOLNAME>. Status displays the status of "
+     "the scrubber. ondemand starts the scrubber immediately."},
+
+    {"volume bitrot <VOLNAME> {enable|disable}\n"
+     "volume bitrot <VOLNAME> signing-time <time-in-secs>\n"
+     "volume bitrot <VOLNAME> signer-threads <count>\n"
+     "volume bitrot <volname> scrub-throttle {lazy|normal|aggressive}\n"
+     "volume bitrot <volname> scrub-frequency {hourly|daily|weekly|biweekly"
+     "|monthly}\n"
+     "volume bitrot <volname> scrub {pause|resume|status|ondemand}",
+     cli_cmd_bitrot_cbk, NULL},
+
+    {NULL, NULL, NULL}};
+
+struct cli_cmd quota_cmds[] = {
+
+    /* Quota commands */
+    {"volume quota help", cli_cmd_quota_help_cbk,
+     "display help for volume quota commands"},
+
+    {"volume quota <VOLNAME> {enable|disable|list [<path> ...]| "
+     "list-objects [<path> ...] | remove <path>| remove-objects <path> | "
+     "default-soft-limit <percent>}",
+     cli_cmd_quota_cbk, "Enable/disable and configure quota for <VOLNAME>"},
+
+    {"volume quota <VOLNAME> {limit-usage <path> <size> [<percent>]}",
+     cli_cmd_quota_cbk, "Set maximum size for <path> for <VOLNAME>"},
+
+    {"volume quota <VOLNAME> {limit-objects <path> <number> [<percent>]}",
+     cli_cmd_quota_cbk,
+     "Set the maximum number of entries allowed in <path> for <VOLNAME>"},
+
+    {"volume quota <VOLNAME> {alert-time|soft-timeout|hard-timeout} {<time>}",
+     cli_cmd_quota_cbk, "Set quota timeout for <VOLNAME>"},
+
+    {"volume inode-quota <VOLNAME> enable", cli_cmd_quota_cbk,
+     "Enable/disable inode-quota for <VOLNAME>"},
+
+    {"volume quota <VOLNAME> {enable|disable|list [<path> ...]| "
+     "list-objects [<path> ...] | remove <path>| remove-objects <path> | "
+     "default-soft-limit <percent>}\n"
+     "volume quota <VOLNAME> {limit-usage <path> <size> [<percent>]}\n"
+     "volume quota <VOLNAME> {limit-objects <path> <number> [<percent>]}\n"
+     "volume quota <VOLNAME> {alert-time|soft-timeout|hard-timeout} {<time>}",
+     cli_cmd_quota_cbk, NULL},
+
+    {NULL, NULL, NULL}};
 
 struct cli_cmd volume_cmds[] = {
-        { "volume info [all|<VOLNAME>]",
-          cli_cmd_volume_info_cbk,
-          "list information of all volumes"},
-
-        { "volume create <NEW-VOLNAME> [stripe <COUNT>] [replica <COUNT>] "
-          "[transport <tcp|rdma|tcp,rdma>] <NEW-BRICK>"
-#ifdef HAVE_BD_XLATOR
-          "?<vg_name>"
-#endif
-          "... [force]",
+    {"volume help", cli_cmd_volume_help_cbk,
+     "display help for volume commands"},
+
+    {"volume info [all|<VOLNAME>]", cli_cmd_volume_info_cbk,
+     "list information of all volumes"},
+
+    {"volume create <NEW-VOLNAME> [stripe <COUNT>] "
+     "[[replica <COUNT> [arbiter <COUNT>]]|[replica 2 thin-arbiter 1]] "
+     "[disperse [<COUNT>]] [disperse-data <COUNT>] [redundancy <COUNT>] "
+     "[transport <tcp|rdma|tcp,rdma>] <NEW-BRICK> <TA-BRICK>"
+     "... [force]",
 
-          cli_cmd_volume_create_cbk,
-          "create a new volume of specified type with mentioned bricks"},
+     cli_cmd_volume_create_cbk,
+     "create a new volume of specified type with mentioned bricks"},
 
-        { "volume delete <VOLNAME>",
-          cli_cmd_volume_delete_cbk,
-          "delete volume specified by <VOLNAME>"},
+    {"volume delete <VOLNAME>", cli_cmd_volume_delete_cbk,
+     "delete volume specified by <VOLNAME>"},
 
-        { "volume start <VOLNAME> [force]",
-          cli_cmd_volume_start_cbk,
-          "start volume specified by <VOLNAME>"},
+    {"volume start <VOLNAME> [force]", cli_cmd_volume_start_cbk,
+     "start volume specified by <VOLNAME>"},
 
-        { "volume stop <VOLNAME> [force]",
-          cli_cmd_volume_stop_cbk,
-          "stop volume specified by <VOLNAME>"},
+    {"volume stop <VOLNAME> [force]", cli_cmd_volume_stop_cbk,
+     "stop volume specified by <VOLNAME>"},
 
-        /*{ "volume rename <VOLNAME> <NEW-VOLNAME>",
-          cli_cmd_volume_rename_cbk,
-          "rename volume <VOLNAME> to <NEW-VOLNAME>"},*/
+    /*{ "volume rename <VOLNAME> <NEW-VOLNAME>",
+      cli_cmd_volume_rename_cbk,
+      "rename volume <VOLNAME> to <NEW-VOLNAME>"},*/
 
-        { "volume add-brick <VOLNAME> [<stripe|replica> <COUNT>] <NEW-BRICK> ... [force]",
-          cli_cmd_volume_add_brick_cbk,
-          "add brick to volume <VOLNAME>"},
+    {"volume add-brick <VOLNAME> [<stripe|replica> <COUNT> "
+     "[arbiter <COUNT>]] <NEW-BRICK> ... [force]",
+     cli_cmd_volume_add_brick_cbk, "add brick to volume <VOLNAME>"},
 
-        { "volume remove-brick <VOLNAME> [replica <COUNT>] <BRICK> ... [start|stop|status|commit|force]",
-          cli_cmd_volume_remove_brick_cbk,
-          "remove brick from volume <VOLNAME>"},
+    {"volume remove-brick <VOLNAME> [replica <COUNT>] <BRICK> ..."
+     " <start|stop|status|commit|force>",
+     cli_cmd_volume_remove_brick_cbk, "remove brick from volume <VOLNAME>"},
 
-        { "volume rebalance <VOLNAME> [fix-layout] {start|stop|status} [force]",
-          cli_cmd_volume_defrag_cbk,
-          "rebalance operations"},
+    {"volume rebalance <VOLNAME> {{fix-layout start} | {start "
+     "[force]|stop|status}}",
+     cli_cmd_volume_defrag_cbk, "rebalance operations"},
 
-        { "volume replace-brick <VOLNAME> <BRICK> <NEW-BRICK> {start [force]|pause|abort|status|commit [force]}",
-          cli_cmd_volume_replace_brick_cbk,
-          "replace-brick operations"},
+    {"volume replace-brick <VOLNAME> <SOURCE-BRICK> <NEW-BRICK> "
+     "{commit force}",
+     cli_cmd_volume_replace_brick_cbk, "replace-brick operations"},
 
-        /*{ "volume set-transport <VOLNAME> <TRANSPORT-TYPE> [<TRANSPORT-TYPE>] ...",
-          cli_cmd_volume_set_transport_cbk,
-          "set transport type for volume <VOLNAME>"},*/
+    /*{ "volume set-transport <VOLNAME> <TRANSPORT-TYPE> [<TRANSPORT-TYPE>]
+      ...", cli_cmd_volume_set_transport_cbk, "set transport type for volume
+      <VOLNAME>"},*/
 
-        { "volume set <VOLNAME> <KEY> <VALUE>",
-          cli_cmd_volume_set_cbk,
-         "set options for volume <VOLNAME>"},
+    {"volume set <VOLNAME> <KEY> <VALUE>", cli_cmd_volume_set_cbk,
+     "set options for volume <VOLNAME>"},
 
-        { "volume help",
-          cli_cmd_volume_help_cbk,
-          "display help for the volume command"},
+    {"volume set <VOLNAME> group <GROUP>", cli_cmd_volume_set_cbk,
+     "This option can be used for setting multiple pre-defined volume options "
+     "where group_name is a file under /var/lib/glusterd/groups containing one "
+     "key value pair per line"},
 
-        { "volume log rotate <VOLNAME> [BRICK]",
-          cli_cmd_log_rotate_cbk,
-         "rotate the log file for corresponding volume/brick"},
+    {"volume log <VOLNAME> rotate [BRICK]", cli_cmd_log_rotate_cbk,
+     "rotate the log file for corresponding volume/brick"},
 
-        { "volume sync <HOSTNAME> [all|<VOLNAME>]",
-          cli_cmd_sync_volume_cbk,
-         "sync the volume information from a peer"},
+    {"volume sync <HOSTNAME> [all|<VOLNAME>]", cli_cmd_sync_volume_cbk,
+     "sync the volume information from a peer"},
 
-         { "volume reset <VOLNAME> [option] [force]",
-         cli_cmd_volume_reset_cbk,
-         "reset all the reconfigured options"},
+    {"volume reset <VOLNAME> [option] [force]", cli_cmd_volume_reset_cbk,
+     "reset all the reconfigured options"},
 
 #if (SYNCDAEMON_COMPILE)
-        {"volume "GEOREP" [<VOLNAME>] [<SLAVE-URL>] {create [push-pem] [force]"
-         "|start [force]|stop [force]|config|status [detail]|delete} [options...]",
-         cli_cmd_volume_gsync_set_cbk,
-         "Geo-sync operations",
-         cli_cmd_check_gsync_exists_cbk},
+    {"volume " GEOREP " [<MASTER-VOLNAME>] [<SLAVE-IP>]::[<SLAVE-VOLNAME>] {"
+     "\\\n create [[ssh-port n] [[no-verify] \\\n | [push-pem]]] [force] \\\n"
+     " | start [force] \\\n | stop [force] \\\n | pause [force] \\\n | resume "
+     "[force] \\\n"
+     " | config [[[\\!]<option>] [<value>]] \\\n | status "
+     "[detail] \\\n | delete [reset-sync-time]} ",
+     cli_cmd_volume_gsync_set_cbk, "Geo-sync operations",
+     cli_cmd_check_gsync_exists_cbk},
 #endif
 
-         { "volume profile <VOLNAME> {start|stop|info [nfs]}",
-           cli_cmd_volume_profile_cbk,
-           "volume profile operations"},
+    {"volume profile <VOLNAME> {start|info [peek|incremental "
+     "[peek]|cumulative|clear]|stop} [nfs]",
+     cli_cmd_volume_profile_cbk, "volume profile operations"},
+
+    {"volume top <VOLNAME> {open|read|write|opendir|readdir|clear} [nfs|brick "
+     "<brick>] [list-cnt <value>] | "
+     "{read-perf|write-perf} [bs <size> count <count>] "
+     "[brick <brick>] [list-cnt <value>]",
+     cli_cmd_volume_top_cbk, "volume top operations"},
+
+    {"volume status [all | <VOLNAME> [nfs|shd|<BRICK>|quotad]]"
+     " [detail|clients|mem|inode|fd|callpool|tasks|client-list]",
+     cli_cmd_volume_status_cbk,
+     "display status of all or specified volume(s)/brick"},
+
+    {"volume heal <VOLNAME> [enable | disable | full |"
+     "statistics [heal-count [replica <HOSTNAME:BRICKNAME>]] |"
+     "info [summary | split-brain] |"
+     "split-brain {bigger-file <FILE> | latest-mtime <FILE> |"
+     "source-brick <HOSTNAME:BRICKNAME> [<FILE>]} |"
+     "granular-entry-heal {enable | disable}]",
+     cli_cmd_volume_heal_cbk,
+     "self-heal commands on volume specified by <VOLNAME>"},
+
+    {"volume statedump <VOLNAME> [[nfs|quotad] [all|mem|iobuf|callpool|"
+     "priv|fd|inode|history]... | [client <hostname:process-id>]]",
+     cli_cmd_volume_statedump_cbk, "perform statedump on bricks"},
+
+    {"volume list", cli_cmd_volume_list_cbk, "list all volumes in cluster"},
+
+    {"volume clear-locks <VOLNAME> <path> kind {blocked|granted|all}"
+     "{inode [range]|entry [basename]|posix [range]}",
+     cli_cmd_volume_clearlocks_cbk, "Clear locks held on path"},
+    {"volume barrier <VOLNAME> {enable|disable}", cli_cmd_volume_barrier_cbk,
+     "Barrier/unbarrier file operations on a volume"},
+    {"volume get <VOLNAME|all> <key|all>", cli_cmd_volume_getopt_cbk,
+     "Get the value of the all options or given option for volume <VOLNAME>"
+     " or all option. gluster volume get all all is to get all global "
+     "options"},
+
+    {"volume reset-brick <VOLNAME> <SOURCE-BRICK> {{start} |"
+     " {<NEW-BRICK> commit}}",
+     cli_cmd_volume_reset_brick_cbk, "reset-brick operations"},
+
+    {NULL, NULL, NULL}};
 
-        { "volume quota <VOLNAME> {enable|disable|list [<path> ...]|remove <path>| default-soft-limit <percent>} |\n"
-          "volume quota <VOLNAME> {limit-usage <path> <size> [<percent>]} |\n"
-          "volume quota <VOLNAME> {alert-time|soft-timeout|hard-timeout} {<time>}",
-          cli_cmd_quota_cbk,
-          "quota translator specific operations"},
+int
+cli_cmd_quota_help_cbk(struct cli_state *state, struct cli_cmd_word *in_word,
+                       const char **words, int wordcount)
+{
+    struct cli_cmd *cmd = NULL;
+    struct cli_cmd *quota_cmd = NULL;
+    int count = 0;
 
-         { "volume top <VOLNAME> {open|read|write|opendir|readdir|clear} [nfs|brick <brick>] [list-cnt <value>] |\n"
-           "volume top <VOLNAME> {read-perf|write-perf} [bs <size> count <count>] [brick <brick>] [list-cnt <value>]",
-           cli_cmd_volume_top_cbk,
-           "volume top operations"},
+    cmd = GF_MALLOC(sizeof(quota_cmds), cli_mt_cli_cmd);
+    memcpy(cmd, quota_cmds, sizeof(quota_cmds));
+    count = (sizeof(quota_cmds) / sizeof(struct cli_cmd));
+    cli_cmd_sort(cmd, count);
 
-        { "volume status [all | <VOLNAME> [nfs|shd|<BRICK>|quotad]]"
-          " [detail|clients|mem|inode|fd|callpool|tasks]",
-          cli_cmd_volume_status_cbk,
-          "display status of all or specified volume(s)/brick"},
+    cli_out("\ngluster quota commands");
+    cli_out("=======================\n");
 
-        { "volume heal <VOLNAME> [{full | statistics {heal-count {replica <hostname:brickname>}} |info {healed | heal-failed | split-brain}}]",
-          cli_cmd_volume_heal_cbk,
-          "self-heal commands on volume specified by <VOLNAME>"},
+    for (quota_cmd = cmd; quota_cmd->pattern; quota_cmd++)
+        if ((_gf_false == quota_cmd->disable) && (quota_cmd->desc))
+            cli_out("%s - %s", quota_cmd->pattern, quota_cmd->desc);
 
-        {"volume statedump <VOLNAME> [nfs|quotad] [all|mem|iobuf|callpool|priv|fd|"
-         "inode|history]...",
-         cli_cmd_volume_statedump_cbk,
-         "perform statedump on bricks"},
+    cli_out("\n");
+    GF_FREE(cmd);
 
-        {"volume list",
-         cli_cmd_volume_list_cbk,
-         "list all volumes in cluster"},
+    return 0;
+}
 
-        {"volume clear-locks <VOLNAME> <path> kind {blocked|granted|all}"
-          "{inode [range]|entry [basename]|posix [range]}",
-          cli_cmd_volume_clearlocks_cbk,
-          "Clear locks held on path"
-        },
+int
+cli_cmd_bitrot_help_cbk(struct cli_state *state, struct cli_cmd_word *in_word,
+                        const char **words, int wordcount)
+{
+    struct cli_cmd *cmd = NULL;
+    struct cli_cmd *bitrot_cmd = NULL;
+    int count = 0;
+
+    cmd = GF_MALLOC(sizeof(bitrot_cmds), cli_mt_cli_cmd);
+    memcpy(cmd, bitrot_cmds, sizeof(bitrot_cmds));
+    count = (sizeof(bitrot_cmds) / sizeof(struct cli_cmd));
+    cli_cmd_sort(cmd, count);
+
+    cli_out("\ngluster bitrot commands");
+    cli_out("========================\n");
+
+    for (bitrot_cmd = cmd; bitrot_cmd->pattern; bitrot_cmd++)
+        if ((_gf_false == bitrot_cmd->disable) && (bitrot_cmd->desc))
+            cli_out("%s - %s", bitrot_cmd->pattern, bitrot_cmd->desc);
 
-        { NULL, NULL, NULL }
-};
+    cli_out("\n");
+    GF_FREE(cmd);
+
+    return 0;
+}
 
 int
-cli_cmd_volume_help_cbk (struct cli_state *state, struct cli_cmd_word *in_word,
-                      const char **words, int wordcount)
+cli_cmd_volume_help_cbk(struct cli_state *state, struct cli_cmd_word *in_word,
+                        const char **words, int wordcount)
 {
-        struct cli_cmd        *cmd = NULL;
+    struct cli_cmd *cmd = NULL;
+    struct cli_cmd *vol_cmd = NULL;
+    int count = 0;
 
-        for (cmd = volume_cmds; cmd->pattern; cmd++)
-                if (_gf_false == cmd->disable)
-                        cli_out ("%s - %s", cmd->pattern, cmd->desc);
+    cmd = GF_MALLOC(sizeof(volume_cmds), cli_mt_cli_cmd);
+    memcpy(cmd, volume_cmds, sizeof(volume_cmds));
+    count = (sizeof(volume_cmds) / sizeof(struct cli_cmd));
+    cli_cmd_sort(cmd, count);
 
-        return 0;
+    cli_out("\ngluster volume commands");
+    cli_out("========================\n");
+
+    for (vol_cmd = cmd; vol_cmd->pattern; vol_cmd++)
+        if (_gf_false == vol_cmd->disable)
+            cli_out("%s - %s", vol_cmd->pattern, vol_cmd->desc);
+
+    cli_out("\n");
+    GF_FREE(cmd);
+    return 0;
 }
 
 int
-cli_cmd_volume_register (struct cli_state *state)
+cli_cmd_volume_register(struct cli_state *state)
 {
-        int  ret = 0;
-        struct cli_cmd *cmd = NULL;
+    int ret = 0;
+    struct cli_cmd *cmd = NULL;
+
+    for (cmd = volume_cmds; cmd->pattern; cmd++) {
+        ret = cli_cmd_register(&state->tree, cmd);
+        if (ret)
+            goto out;
+    }
 
-        for (cmd = volume_cmds; cmd->pattern; cmd++) {
+    for (cmd = bitrot_cmds; cmd->pattern; cmd++) {
+        ret = cli_cmd_register(&state->tree, cmd);
+        if (ret)
+            goto out;
+    }
+
+    for (cmd = quota_cmds; cmd->pattern; cmd++) {
+        ret = cli_cmd_register(&state->tree, cmd);
+        if (ret)
+            goto out;
+    }
 
-                ret = cli_cmd_register (&state->tree, cmd);
-                if (ret)
-                        goto out;
-        }
 out:
-        return ret;
+    return ret;
+}
+
+static int
+gf_asprintf_append(char **string_ptr, const char *format, ...)
+{
+    va_list arg;
+    int rv = 0;
+    char *tmp = *string_ptr;
+
+    va_start(arg, format);
+    rv = gf_vasprintf(string_ptr, format, arg);
+    va_end(arg);
+
+    if (tmp)
+        GF_FREE(tmp);
+
+    return rv;
 }
diff --git a/cli/src/cli-cmd.c b/cli/src/cli-cmd.c
index 63b939282fa..2d458b16a56 100644
--- a/cli/src/cli-cmd.c
+++ b/cli/src/cli-cmd.c
@@ -13,11 +13,6 @@
 #include <stdint.h>
 #include <pthread.h>
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 #include "cli.h"
 #include "cli-cmd.h"
 #include "cli-mem-types.h"
@@ -27,357 +22,390 @@
 
 static int cmd_done;
 static int cmd_sent;
-static pthread_cond_t      cond  = PTHREAD_COND_INITIALIZER;
-static pthread_mutex_t     cond_mutex = PTHREAD_MUTEX_INITIALIZER;
-static pthread_cond_t      conn  = PTHREAD_COND_INITIALIZER;
-static pthread_mutex_t     conn_mutex = PTHREAD_MUTEX_INITIALIZER;
-
-int    cli_op_ret = 0;
-int    connected = 0;
+static pthread_cond_t cond = PTHREAD_COND_INITIALIZER;
+static pthread_mutex_t cond_mutex = PTHREAD_MUTEX_INITIALIZER;
+static pthread_cond_t conn = PTHREAD_COND_INITIALIZER;
+static pthread_mutex_t conn_mutex = PTHREAD_MUTEX_INITIALIZER;
 
-int cli_cmd_log_help_cbk (struct cli_state *state, struct cli_cmd_word *in_word,
-                      const char **words, int wordcount);
+int cli_op_ret = 0;
+static gf_boolean_t connected = _gf_false;
 
 static unsigned
-cli_cmd_needs_connection (struct cli_cmd_word *word)
+cli_cmd_needs_connection(struct cli_cmd_word *word)
 {
-        if (!strcasecmp ("quit", word->word))
-                return 0;
+    if (!strcasecmp("quit", word->word))
+        return 0;
 
-        if (!strcasecmp ("help", word->word))
-                return 0;
+    if (!strcasecmp("help", word->word))
+        return 0;
 
-        if (!strcasecmp ("getwd", word->word))
-                return 1;
+    if (!strcasecmp("getwd", word->word))
+        return 1;
 
-        if (!strcasecmp ("exit", word->word))
-                return 0;
+    if (!strcasecmp("exit", word->word))
+        return 0;
 
-        return CLI_DEFAULT_CONN_TIMEOUT;
+    return cli_default_conn_timeout;
 }
 
 int
-cli_cmd_status_reset (void)
+cli_cmd_status_reset(void)
 {
-        int ret = 0;
-
-        ret = cli_cmd_lock ();
-        {
-                if (ret == 0) {
-                        cmd_sent = 0;
-                        cmd_done = 0;
-                }
-        }
-        ret = cli_cmd_unlock ();
-        return ret;
+    int ret = 0;
 
+    ret = cli_cmd_lock();
+    {
+        if (ret == 0) {
+            cmd_sent = 0;
+            cmd_done = 0;
+        }
+    }
+    ret = cli_cmd_unlock();
+    return ret;
 }
 
 int
-cli_cmd_sent_status_get (int *status)
+cli_cmd_sent_status_get(int *status)
 {
-        int ret = 0;
-        GF_ASSERT (status);
-
-        ret = cli_cmd_lock ();
-        {
-                if (ret == 0)
-                        *status = cmd_sent;
-        }
-        ret = cli_cmd_unlock ();
-        return ret;
+    int ret = 0;
+    GF_ASSERT(status);
+
+    ret = cli_cmd_lock();
+    {
+        if (ret == 0)
+            *status = cmd_sent;
+    }
+    ret = cli_cmd_unlock();
+    return ret;
 }
 
 int
-cli_cmd_process (struct cli_state *state, int argc, char **argv)
+cli_cmd_process(struct cli_state *state, int argc, char **argv)
 {
-        int                  ret = 0;
-        struct cli_cmd_word *word = NULL;
-        struct cli_cmd_word *next = NULL;
-        int                  i = 0;
+    int ret = 0;
+    struct cli_cmd_word *word = NULL;
+    struct cli_cmd_word *next = NULL;
+    int i = 0;
 
-        word = &state->tree.root;
+    word = &state->tree.root;
 
-        if (!argc)
-                return 0;
+    if (!argc)
+        return 0;
 
-        for (i = 0; i < argc; i++) {
-                next = cli_cmd_nextword (word, argv[i]);
+    for (i = 0; i < argc; i++) {
+        next = cli_cmd_nextword(word, argv[i]);
 
-                word = next;
-                if (!word)
-                        break;
+        word = next;
+        if (!word)
+            break;
 
-                if (word->cbkfn)
-                        break;
-        }
+        if (word->cbkfn)
+            break;
+    }
 
-        if (!word) {
-                cli_out ("unrecognized word: %s (position %d)",
-                         argv[i], i);
-                return -1;
-        }
+    if (!word) {
+        cli_out("unrecognized word: %s (position %d)\n", argv[i], i);
+        usage();
+        return -1;
+    }
 
-        if (!word->cbkfn) {
-                cli_out ("unrecognized command");
-                return -1;
-        }
+    if (!word->cbkfn) {
+        cli_out("unrecognized command\n");
+        usage();
+        return -1;
+    }
 
-	if ( strcmp (word->word,"help")==0 )
-		goto callback;
+    if (strcmp(word->word, "help") == 0)
+        goto callback;
 
-        state->await_connected = cli_cmd_needs_connection (word);
+    state->await_connected = cli_cmd_needs_connection(word);
 
-        ret = cli_cmd_await_connected (state->await_connected);
-        if (ret) {
-                cli_out ("Connection failed. Please check if gluster "
-                          "daemon is operational.");
-                gf_log ("", GF_LOG_INFO, "Exiting with: %d", ret);
-                exit (ret);
-        }
+    ret = cli_cmd_await_connected(state->await_connected);
+    if (ret) {
+        cli_out(
+            "Connection failed. Please check if gluster "
+            "daemon is operational.");
+        gf_log("", GF_LOG_INFO, "Exiting with: %d", ret);
+        exit(ret);
+    }
 
 callback:
-        ret = word->cbkfn (state, word, (const char **)argv, argc);
-        (void) cli_cmd_status_reset ();
-        return ret;
+    ret = word->cbkfn(state, word, (const char **)argv, argc);
+    (void)cli_cmd_status_reset();
+    return ret;
 }
 
 int
-cli_cmd_input_token_count (const char *text)
+cli_cmd_input_token_count(const char *text)
 {
-        int          count = 0;
-        const char  *trav = NULL;
-        int          is_spc = 1;
-
-        for (trav = text; *trav; trav++) {
-                if (*trav == ' ') {
-                        is_spc = 1;
-                } else {
-                        if (is_spc) {
-                                count++;
-                                is_spc = 0;
-                        }
-                }
+    int count = 0;
+    const char *trav = NULL;
+    int is_spc = 1;
+
+    for (trav = text; *trav; trav++) {
+        if (*trav == ' ') {
+            is_spc = 1;
+        } else {
+            if (is_spc) {
+                count++;
+                is_spc = 0;
+            }
         }
+    }
 
-        return count;
+    return count;
 }
 
-
 int
-cli_cmd_process_line (struct cli_state *state, const char *text)
+cli_cmd_process_line(struct cli_state *state, const char *text)
 {
-        int     count = 0;
-        char  **tokens = NULL;
-        char  **tokenp = NULL;
-        char   *token = NULL;
-        char   *copy = NULL;
-        char   *saveptr = NULL;
-        int     i = 0;
-        int     ret = -1;
-
-        count = cli_cmd_input_token_count (text);
-
-        tokens = calloc (count + 1, sizeof (*tokens));
-        if (!tokens)
-                return -1;
-
-        copy = strdup (text);
-        if (!copy)
-                goto out;
-
-        tokenp = tokens;
-
-        for (token = strtok_r (copy, " \t\r\n", &saveptr); token;
-             token = strtok_r (NULL, " \t\r\n", &saveptr)) {
-                *tokenp = strdup (token);
-
-                if (!*tokenp)
-                        goto out;
-                tokenp++;
-                i++;
-
-        }
-
-        ret = cli_cmd_process (state, count, tokens);
+    int count = 0;
+    char **tokens = NULL;
+    char **tokenp = NULL;
+    char *token = NULL;
+    char *copy = NULL;
+    char *saveptr = NULL;
+    int i = 0;
+    int ret = -1;
+
+    count = cli_cmd_input_token_count(text);
+
+    tokens = calloc(count + 1, sizeof(*tokens));
+    if (!tokens)
+        return -1;
+
+    copy = strdup(text);
+    if (!copy)
+        goto out;
+
+    tokenp = tokens;
+
+    for (token = strtok_r(copy, " \t\r\n", &saveptr); token;
+         token = strtok_r(NULL, " \t\r\n", &saveptr)) {
+        *tokenp = strdup(token);
+
+        if (!*tokenp)
+            goto out;
+        tokenp++;
+        i++;
+    }
+
+    ret = cli_cmd_process(state, count, tokens);
 out:
-        free (copy);
+    free(copy);
 
-        if (tokens)
-                cli_cmd_tokens_destroy (tokens);
+    if (tokens)
+        cli_cmd_tokens_destroy(tokens);
 
-        return ret;
+    return ret;
 }
 
-
 int
-cli_cmds_register (struct cli_state *state)
+cli_cmds_register(struct cli_state *state)
 {
-        int  ret = 0;
-
-        ret = cli_cmd_volume_register (state);
-        if (ret)
-                goto out;
-
-        ret = cli_cmd_probe_register (state);
-        if (ret)
-                goto out;
-
-        ret = cli_cmd_system_register (state);
-        if (ret)
-                goto out;
-
-        ret = cli_cmd_misc_register (state);
-        if (ret)
-                goto out;
-
+    int ret = 0;
+
+    ret = cli_cmd_volume_register(state);
+    if (ret)
+        goto out;
+
+    ret = cli_cmd_probe_register(state);
+    if (ret)
+        goto out;
+
+    ret = cli_cmd_system_register(state);
+    if (ret)
+        goto out;
+
+    ret = cli_cmd_misc_register(state);
+    if (ret)
+        goto out;
+
+    ret = cli_cmd_snapshot_register(state);
+    if (ret)
+        goto out;
+    ret = cli_cmd_global_register(state);
+    if (ret)
+        goto out;
 out:
-        return ret;
-}
-
-int
-cli_cmd_cond_init ()
-{
-
-       pthread_mutex_init (&cond_mutex, NULL);
-       pthread_cond_init (&cond, NULL);
-
-       pthread_mutex_init (&conn_mutex, NULL);
-       pthread_cond_init (&conn, NULL);
-
-       return 0;
+    return ret;
 }
 
 int
-cli_cmd_lock ()
+cli_cmd_lock()
 {
-       pthread_mutex_lock (&cond_mutex);
-       return 0;
+    pthread_mutex_lock(&cond_mutex);
+    return 0;
 }
 
 int
-cli_cmd_unlock ()
+cli_cmd_unlock()
 {
-        pthread_mutex_unlock (&cond_mutex);
-        return 0;
+    pthread_mutex_unlock(&cond_mutex);
+    return 0;
 }
 
 static void
-seconds_from_now (unsigned secs, struct timespec *ts)
+seconds_from_now(unsigned secs, struct timespec *ts)
 {
-        struct timeval tv = {0,};
+    struct timeval tv = {
+        0,
+    };
 
-        gettimeofday (&tv, NULL);
+    gettimeofday(&tv, NULL);
 
-        ts->tv_sec = tv.tv_sec + secs;
-        ts->tv_nsec = tv.tv_usec * 1000;
+    ts->tv_sec = tv.tv_sec + secs;
+    ts->tv_nsec = tv.tv_usec * 1000;
 }
 
 int
-cli_cmd_await_response (unsigned time)
+cli_cmd_await_response(unsigned time)
 {
-        struct  timespec        ts = {0,};
-        int                     ret = 0;
+    struct timespec ts = {
+        0,
+    };
+    int ret = 0;
 
-        cli_op_ret = -1;
-
-        seconds_from_now (time, &ts);
-        while (!cmd_done && !ret) {
-                ret = pthread_cond_timedwait (&cond, &cond_mutex,
-                                        &ts);
-        }
+    cli_op_ret = -1;
 
-        cmd_done = 0;
+    seconds_from_now(time, &ts);
+    while (!cmd_done && !ret) {
+        ret = pthread_cond_timedwait(&cond, &cond_mutex, &ts);
+    }
 
-        if (ret)
-                return ret;
+    if (!cmd_done) {
+        if (ret == ETIMEDOUT)
+            cli_out("Error : Request timed out");
+        else
+            cli_out("Error : Command returned with error code:%d", ret);
+    }
+    cmd_done = 0;
 
-        return cli_op_ret;
+    return cli_op_ret;
 }
 
+/* This function must be called _only_ after all actions associated with
+ * command processing is complete. Otherwise, gluster process may exit before
+ * reporting results to stdout/stderr. */
 int
-cli_cmd_broadcast_response (int32_t status)
+cli_cmd_broadcast_response(int32_t status)
 {
-
-        pthread_mutex_lock (&cond_mutex);
-        {
-                if (!cmd_sent)
-                        goto out;
-                cmd_done = 1;
-                cli_op_ret = status;
-                pthread_cond_broadcast (&cond);
-        }
-
+    pthread_mutex_lock(&cond_mutex);
+    {
+        if (!cmd_sent)
+            goto out;
+        cmd_done = 1;
+        cli_op_ret = status;
+        pthread_cond_broadcast(&cond);
+    }
 
 out:
-        pthread_mutex_unlock (&cond_mutex);
-        return 0;
+    pthread_mutex_unlock(&cond_mutex);
+    return 0;
 }
 
 int32_t
-cli_cmd_await_connected (unsigned conn_timo)
+cli_cmd_await_connected(unsigned conn_timo)
 {
-        int32_t                 ret = 0;
-        struct  timespec        ts = {0,};
-
-        if (!conn_timo)
-                return 0;
-
-        pthread_mutex_lock (&conn_mutex);
-        {
-                seconds_from_now (conn_timo, &ts);
-                while (!connected && !ret) {
-                        ret = pthread_cond_timedwait (&conn, &conn_mutex,
-                                                      &ts);
-                }
-        }
-        pthread_mutex_unlock (&conn_mutex);
+    int32_t ret = 0;
+    struct timespec ts = {
+        0,
+    };
+
+    if (!conn_timo)
+        return 0;
 
+    pthread_mutex_lock(&conn_mutex);
+    {
+        seconds_from_now(conn_timo, &ts);
+        while (!connected && !ret) {
+            ret = pthread_cond_timedwait(&conn, &conn_mutex, &ts);
+        }
+    }
+    pthread_mutex_unlock(&conn_mutex);
 
-        return ret;
+    return ret;
 }
 
 int32_t
-cli_cmd_broadcast_connected ()
+cli_cmd_broadcast_connected(gf_boolean_t status)
 {
-        pthread_mutex_lock (&conn_mutex);
-        {
-                connected = 1;
-                pthread_cond_broadcast (&conn);
-        }
-
-        pthread_mutex_unlock (&conn_mutex);
-
-        return 0;
+    pthread_mutex_lock(&conn_mutex);
+    {
+        connected = status;
+        pthread_cond_broadcast(&conn);
+    }
+    pthread_mutex_unlock(&conn_mutex);
+
+    return 0;
 }
 
-int
-cli_cmd_submit (struct rpc_clnt* rpc, void *req, call_frame_t *frame,
-                rpc_clnt_prog_t *prog,
-                int procnum, struct iobref *iobref,
-                xlator_t *this, fop_cbk_fn_t cbkfn, xdrproc_t xdrproc)
+gf_boolean_t
+cli_cmd_connected(void)
 {
-        int             ret = -1;
-        unsigned        timeout = 0;
+    gf_boolean_t status;
 
-        if ((GLUSTER_CLI_PROFILE_VOLUME == procnum) ||
-            (GLUSTER_CLI_HEAL_VOLUME == procnum))
-                timeout = CLI_TEN_MINUTES_TIMEOUT;
-        else
-                timeout = CLI_DEFAULT_CMD_TIMEOUT;
+    pthread_mutex_lock(&conn_mutex);
+    {
+        status = connected;
+    }
+    pthread_mutex_unlock(&conn_mutex);
 
-        cli_cmd_lock ();
-        cmd_sent = 0;
-        ret = cli_submit_request (rpc, req, frame, prog,
-                                  procnum, NULL, this, cbkfn, xdrproc);
+    return status;
+}
 
-        if (!ret) {
-                cmd_sent = 1;
-                ret = cli_cmd_await_response (timeout);
-        }
+int
+cli_cmd_submit(struct rpc_clnt *rpc, void *req, call_frame_t *frame,
+               rpc_clnt_prog_t *prog, int procnum, struct iobref *iobref,
+               xlator_t *this, fop_cbk_fn_t cbkfn, xdrproc_t xdrproc)
+{
+    int ret = -1;
+    unsigned timeout = 0;
+
+    if ((GLUSTER_CLI_PROFILE_VOLUME == procnum) ||
+        (GLUSTER_CLI_HEAL_VOLUME == procnum) ||
+        (GLUSTER_CLI_GANESHA == procnum))
+        timeout = cli_ten_minutes_timeout;
+    else
+        timeout = cli_default_conn_timeout;
+
+    cli_cmd_lock();
+    cmd_sent = 0;
+    ret = cli_submit_request(rpc, req, frame, prog, procnum, NULL, this, cbkfn,
+                             xdrproc);
+
+    if (!ret) {
+        cmd_sent = 1;
+        ret = cli_cmd_await_response(timeout);
+    }
+
+    cli_cmd_unlock();
+
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
+}
 
-        cli_cmd_unlock ();
+int
+cli_cmd_pattern_cmp(void *a, void *b)
+{
+    struct cli_cmd *ia = NULL;
+    struct cli_cmd *ib = NULL;
+    int ret = 0;
+
+    ia = a;
+    ib = b;
+    if (strcmp(ia->pattern, ib->pattern) > 0)
+        ret = 1;
+    else if (strcmp(ia->pattern, ib->pattern) < 0)
+        ret = -1;
+    else
+        ret = 0;
+    return ret;
+}
 
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+void
+cli_cmd_sort(struct cli_cmd *cmd, int count)
+{
+    gf_array_insertionsort(cmd, 1, count - 2, sizeof(struct cli_cmd),
+                           cli_cmd_pattern_cmp);
 }
diff --git a/cli/src/cli-cmd.h b/cli/src/cli-cmd.h
index 541b4ff7332..c1c068c7085 100644
--- a/cli/src/cli-cmd.h
+++ b/cli/src/cli-cmd.h
@@ -10,119 +10,120 @@
 #ifndef __CLI_CMD_H__
 #define __CLI_CMD_H__
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 #include <netdb.h>
 
 #include "cli.h"
-#include "list.h"
-
-#define CLI_LOCAL_INIT(local, words, frame, dictionary) \
-        do {                                                 \
-                local = cli_local_get ();                    \
-                                                             \
-                if (local) {                                 \
-                        local->words = words;                \
-                        if (dictionary)                      \
-                                local->dict = dictionary;    \
-                        if (frame)                           \
-                                frame->local = local;        \
-                }                                            \
-        } while (0)
-
-#define CLI_STACK_DESTROY(_frame)                                       \
-        do {                                                            \
-                if (_frame) {                                           \
-                        if (_frame->local) {                            \
-                                gf_log ("cli", GF_LOG_DEBUG, "frame->local " \
-                                        "is not NULL (%p)", _frame->local); \
-                                cli_local_wipe (_frame->local);         \
-                                _frame->local = NULL;                   \
-                        }                                               \
-                        STACK_DESTROY (_frame->root);                   \
-                }                                                       \
-        } while (0);
-
-typedef enum {
-        GF_ANSWER_YES = 1,
-        GF_ANSWER_NO  = 2
-} gf_answer_t;
+#include <glusterfs/list.h>
+
+#define GLUSTER_SHARED_STORAGE "gluster_shared_storage"
+
+#define CLI_LOCAL_INIT(local, words, frame, dictionary)                        \
+    do {                                                                       \
+        local = cli_local_get();                                               \
+                                                                               \
+        if (local) {                                                           \
+            local->words = words;                                              \
+            if (dictionary)                                                    \
+                local->dict = dictionary;                                      \
+            if (frame)                                                         \
+                frame->local = local;                                          \
+        }                                                                      \
+    } while (0)
+
+#define CLI_STACK_DESTROY(_frame)                                              \
+    do {                                                                       \
+        if (_frame) {                                                          \
+            if (_frame->local) {                                               \
+                gf_log("cli", GF_LOG_DEBUG,                                    \
+                       "frame->local "                                         \
+                       "is not NULL (%p)",                                     \
+                       _frame->local);                                         \
+                cli_local_wipe(_frame->local);                                 \
+                _frame->local = NULL;                                          \
+            }                                                                  \
+            STACK_DESTROY(_frame->root);                                       \
+        }                                                                      \
+    } while (0);
+
+typedef enum { GF_ANSWER_YES = 1, GF_ANSWER_NO = 2 } gf_answer_t;
 
 struct cli_cmd {
-        const char         *pattern;
-        cli_cmd_cbk_t      *cbk;
-        const char         *desc;
-        cli_cmd_reg_cbk_t  *reg_cbk; /* callback to check in runtime if the   *
-                                      * command should be enabled or disabled */
-        gf_boolean_t       disable;
+    const char *pattern;
+    cli_cmd_cbk_t *cbk;
+    const char *desc;
+    cli_cmd_reg_cbk_t *reg_cbk; /* callback to check in runtime if the   *
+                                 * command should be enabled or disabled */
+    gf_boolean_t disable;
 };
 
 struct cli_cmd_volume_get_ctx_ {
-        char            *volname;
-        int             flags;
+    char *volname;
+    int flags;
 };
 
 typedef struct cli_profile_info_ {
-        uint64_t fop_hits;
-        double min_latency;
-        double max_latency;
-        double avg_latency;
-        char   *fop_name;
-        double percentage_avg_latency;
+    uint64_t fop_hits;
+    double min_latency;
+    double max_latency;
+    double avg_latency;
+    char *fop_name;
+    double percentage_avg_latency;
 } cli_profile_info_t;
 
-typedef struct addrinfo_list {
-        struct list_head list;
-        struct addrinfo *info;
-} addrinfo_list_t;
+typedef struct cli_cmd_volume_get_ctx_ cli_cmd_volume_get_ctx_t;
 
-typedef enum {
-        GF_AI_COMPARE_NO_MATCH     = 0,
-        GF_AI_COMPARE_MATCH        = 1,
-        GF_AI_COMPARE_ERROR        = 2
-} gf_ai_compare_t;
+int
+cli_cmd_volume_register(struct cli_state *state);
 
-typedef struct cli_cmd_volume_get_ctx_ cli_cmd_volume_get_ctx_t;
+int
+cli_cmd_probe_register(struct cli_state *state);
 
-int cli_cmd_volume_register (struct cli_state *state);
+int
+cli_cmd_system_register(struct cli_state *state);
 
-int cli_cmd_probe_register (struct cli_state *state);
+int
+cli_cmd_snapshot_register(struct cli_state *state);
 
-int cli_cmd_system_register (struct cli_state *state);
+int
+cli_cmd_global_register(struct cli_state *state);
 
-int cli_cmd_misc_register (struct cli_state *state);
+int
+cli_cmd_misc_register(struct cli_state *state);
 
-struct cli_cmd_word *cli_cmd_nextword (struct cli_cmd_word *word,
-                                       const char *text);
-void cli_cmd_tokens_destroy (char **tokens);
+struct cli_cmd_word *
+cli_cmd_nextword(struct cli_cmd_word *word, const char *text);
+void
+cli_cmd_tokens_destroy(char **tokens);
 
-int cli_cmd_await_response (unsigned time);
+int
+cli_cmd_await_response(unsigned time);
 
-int cli_cmd_broadcast_response (int32_t status);
+int
+cli_cmd_broadcast_response(int32_t status);
 
-int cli_cmd_cond_init ();
+int
+cli_cmd_lock();
 
-int cli_cmd_lock ();
+int
+cli_cmd_unlock();
 
-int cli_cmd_unlock ();
+int
+cli_cmd_submit(struct rpc_clnt *rpc, void *req, call_frame_t *frame,
+               rpc_clnt_prog_t *prog, int procnum, struct iobref *iobref,
+               xlator_t *this, fop_cbk_fn_t cbkfn, xdrproc_t xdrproc);
 
 int
-cli_cmd_submit (struct rpc_clnt *rpc, void *req, call_frame_t *frame,
-                rpc_clnt_prog_t *prog,
-                int procnum, struct iobref *iobref,
-                xlator_t *this, fop_cbk_fn_t cbkfn, xdrproc_t xdrproc);
+cli_cmd_pattern_cmp(void *a, void *b);
 
-gf_answer_t
-cli_cmd_get_confirmation (struct cli_state *state, const char *question);
-int cli_cmd_sent_status_get (int *status);
+void
+cli_cmd_sort(struct cli_cmd *cmd, int count);
 
-gf_boolean_t
-_limits_set_on_volume (char *volname);
+gf_answer_t
+cli_cmd_get_confirmation(struct cli_state *state, const char *question);
+int
+cli_cmd_sent_status_get(int *status);
 
 gf_boolean_t
-_quota_aux_mount_online (char *volname);
+_limits_set_on_volume(char *volname, int type);
 
 #endif /* __CLI_CMD_H__ */
diff --git a/cli/src/cli-mem-types.h b/cli/src/cli-mem-types.h
index 09fcb639bd0..b42b4dd86c2 100644
--- a/cli/src/cli-mem-types.h
+++ b/cli/src/cli-mem-types.h
@@ -10,20 +10,21 @@
 #ifndef __CLI_MEM_TYPES_H__
 #define __CLI_MEM_TYPES_H__
 
-#include "mem-types.h"
+#include <glusterfs/mem-types.h>
 
 #define CLI_MEM_TYPE_START (gf_common_mt_end + 1)
 
 enum cli_mem_types_ {
-        cli_mt_xlator_list_t = CLI_MEM_TYPE_START,
-        cli_mt_xlator_t,
-        cli_mt_xlator_cmdline_option_t,
-        cli_mt_char,
-        cli_mt_call_pool_t,
-        cli_mt_cli_local_t,
-        cli_mt_cli_get_vol_ctx_t,
-        cli_mt_append_str,
-        cli_mt_end
+    cli_mt_xlator_list_t = CLI_MEM_TYPE_START,
+    cli_mt_xlator_t,
+    cli_mt_xlator_cmdline_option_t,
+    cli_mt_char,
+    cli_mt_call_pool_t,
+    cli_mt_cli_local_t,
+    cli_mt_cli_get_vol_ctx_t,
+    cli_mt_append_str,
+    cli_mt_cli_cmd,
+    cli_mt_end
 
 };
 
diff --git a/cli/src/cli-quotad-client.c b/cli/src/cli-quotad-client.c
index f0efc8640bd..772b8f75bd9 100644
--- a/cli/src/cli-quotad-client.c
+++ b/cli/src/cli-quotad-client.c
@@ -7,148 +7,140 @@
    later), or the GNU General Public License, version 2 (GPLv2), in all
    cases as published by the Free Software Foundation.
 */
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 
 #include "cli-quotad-client.h"
 
-extern struct rpc_clnt global_quotad_rpc;
-extern struct rpc_clnt_program cli_quotad_clnt;
-
 int
-cli_quotad_submit_request (void *req, call_frame_t *frame,
-                           rpc_clnt_prog_t *prog,
-                           int procnum, struct iobref *iobref,
-                           xlator_t *this, fop_cbk_fn_t cbkfn,
-                           xdrproc_t xdrproc)
+cli_quotad_submit_request(void *req, call_frame_t *frame, rpc_clnt_prog_t *prog,
+                          int procnum, struct iobref *iobref, xlator_t *this,
+                          fop_cbk_fn_t cbkfn, xdrproc_t xdrproc)
 {
-        int           ret        = -1;
-        int           count      = 0;
-        struct iovec  iov        = {0, };
-        struct iobuf *iobuf      = NULL;
-        char          new_iobref = 0;
-        ssize_t       xdr_size   = 0;
-
-        GF_ASSERT (this);
-
-        if (req) {
-                xdr_size = xdr_sizeof (xdrproc, req);
-                iobuf = iobuf_get2 (this->ctx->iobuf_pool, xdr_size);
-                if (!iobuf) {
-                        goto out;
-                };
-
-                if (!iobref) {
-                        iobref = iobref_new ();
-                        if (!iobref) {
-                                goto out;
-                        }
-
-                        new_iobref = 1;
-                }
-
-                iobref_add (iobref, iobuf);
-
-                iov.iov_base = iobuf->ptr;
-                iov.iov_len  = iobuf_size (iobuf);
-
-                /* Create the xdr payload */
-                ret = xdr_serialize_generic (iov, req, xdrproc);
-                if (ret == -1) {
-                        goto out;
-                }
-                iov.iov_len = ret;
-                count = 1;
+    int ret = -1;
+    int count = 0;
+    struct iovec iov = {
+        0,
+    };
+    struct iobuf *iobuf = NULL;
+    char new_iobref = 0;
+    ssize_t xdr_size = 0;
+
+    GF_ASSERT(this);
+
+    if (req) {
+        xdr_size = xdr_sizeof(xdrproc, req);
+        iobuf = iobuf_get2(this->ctx->iobuf_pool, xdr_size);
+        if (!iobuf) {
+            goto out;
+        };
+
+        if (!iobref) {
+            iobref = iobref_new();
+            if (!iobref) {
+                goto out;
+            }
+
+            new_iobref = 1;
+        }
+
+        iobref_add(iobref, iobuf);
+
+        iov.iov_base = iobuf->ptr;
+        iov.iov_len = iobuf_size(iobuf);
+
+        /* Create the xdr payload */
+        ret = xdr_serialize_generic(iov, req, xdrproc);
+        if (ret == -1) {
+            goto out;
         }
+        iov.iov_len = ret;
+        count = 1;
+    }
 
-        /* Send the msg */
-        ret = rpc_clnt_submit (&global_quotad_rpc, prog, procnum, cbkfn,
-                               &iov, count,
-                               NULL, 0, iobref, frame, NULL, 0, NULL, 0, NULL);
-        ret = 0;
+    /* Send the msg */
+    ret = rpc_clnt_submit(global_quotad_rpc, prog, procnum, cbkfn, &iov, count,
+                          NULL, 0, iobref, frame, NULL, 0, NULL, 0, NULL);
+    ret = 0;
 
 out:
-        if (new_iobref)
-                iobref_unref (iobref);
-        if (iobuf)
-                iobuf_unref (iobuf);
+    if (new_iobref)
+        iobref_unref(iobref);
+    if (iobuf)
+        iobuf_unref(iobuf);
 
-        return ret;
+    return ret;
 }
 
 int
-cli_quotad_notify (struct rpc_clnt *rpc, void *mydata,
-                   rpc_clnt_event_t event, void *data)
+cli_quotad_notify(struct rpc_clnt *rpc, void *mydata, rpc_clnt_event_t event,
+                  void *data)
 {
-        xlator_t                *this = NULL;
-        int                     ret = 0;
+    xlator_t *this = NULL;
+    int ret = 0;
 
-        this = mydata;
+    this = mydata;
 
-        switch (event) {
-        case RPC_CLNT_CONNECT:
-        {
-                gf_log (this->name, GF_LOG_TRACE, "got RPC_CLNT_CONNECT");
-                break;
+    switch (event) {
+        case RPC_CLNT_CONNECT: {
+            gf_log(this->name, GF_LOG_TRACE, "got RPC_CLNT_CONNECT");
+            break;
         }
 
-        case RPC_CLNT_DISCONNECT:
-        {
-                gf_log (this->name, GF_LOG_TRACE, "got RPC_CLNT_DISCONNECT");
-                break;
+        case RPC_CLNT_DISCONNECT: {
+            gf_log(this->name, GF_LOG_TRACE, "got RPC_CLNT_DISCONNECT");
+            break;
         }
 
         default:
-                gf_log (this->name, GF_LOG_TRACE,
-                        "got some other RPC event %d", event);
-                ret = 0;
-                break;
-        }
+            gf_log(this->name, GF_LOG_TRACE, "got some other RPC event %d",
+                   event);
+            ret = 0;
+            break;
+    }
 
-        return ret;
+    return ret;
 }
 
 struct rpc_clnt *
-cli_quotad_clnt_init (xlator_t *this, dict_t *options)
+cli_quotad_clnt_init(xlator_t *this, dict_t *options)
 {
-        struct rpc_clnt *rpc  = NULL;
-        int              ret  = -1;
-
-
-        ret = dict_set_str (options, "transport.address-family", "unix");
-        if (ret)
-                goto out;
-
-        ret = dict_set_str (options, "transport-type", "socket");
-        if (ret)
-                goto out;
-
-        ret = dict_set_str (options, "transport.socket.connect-path",
-                            "/tmp/quotad.socket");
-        if (ret)
-                goto out;
-
-        rpc = rpc_clnt_new (options, this->ctx, this->name, 16);
-        if (!rpc)
-                goto out;
-
-        ret = rpc_clnt_register_notify (rpc, cli_quotad_notify, this);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "failed to register notify");
-                goto out;
-        }
-
-        rpc_clnt_start (rpc);
+    struct rpc_clnt *rpc = NULL;
+    int ret = -1;
+
+    ret = dict_set_nstrn(options, "transport.address-family",
+                         SLEN("transport.address-family"), "unix",
+                         SLEN("unix"));
+    if (ret)
+        goto out;
+
+    ret = dict_set_nstrn(options, "transport-type", SLEN("transport-type"),
+                         "socket", SLEN("socket"));
+    if (ret)
+        goto out;
+
+    ret = dict_set_nstrn(options, "transport.socket.connect-path",
+                         SLEN("transport.socket.connect-path"),
+                         "/var/run/gluster/quotad.socket",
+                         SLEN("/var/run/gluster/quotad.socket"));
+    if (ret)
+        goto out;
+
+    rpc = rpc_clnt_new(options, this, this->name, 16);
+    if (!rpc)
+        goto out;
+
+    ret = rpc_clnt_register_notify(rpc, cli_quotad_notify, this);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "failed to register notify");
+        goto out;
+    }
+
+    rpc_clnt_start(rpc);
 out:
-        if (ret) {
-                if (rpc)
-                        rpc_clnt_unref (rpc);
-                rpc = NULL;
-        }
+    if (ret) {
+        if (rpc)
+            rpc_clnt_unref(rpc);
+        rpc = NULL;
+    }
 
-        return rpc;
+    return rpc;
 }
-
diff --git a/cli/src/cli-quotad-client.h b/cli/src/cli-quotad-client.h
index aa0b42af38d..71a44e5916b 100644
--- a/cli/src/cli-quotad-client.h
+++ b/cli/src/cli-quotad-client.h
@@ -8,26 +8,22 @@
    cases as published by the Free Software Foundation.
 */
 #include "cli.h"
-#include "compat-errno.h"
-#include "compat.h"
+#include <glusterfs/compat-errno.h>
+#include <glusterfs/compat.h>
 #include "cli-cmd.h"
 #include "cli1-xdr.h"
 #include "xdr-generic.h"
 #include "protocol-common.h"
 #include "cli-mem-types.h"
 
-
 int
-cli_quotad_submit_request (void *req, call_frame_t *frame,
-                           rpc_clnt_prog_t *prog,
-                           int procnum, struct iobref *iobref,
-                           xlator_t *this, fop_cbk_fn_t cbkfn,
-                           xdrproc_t xdrproc);
+cli_quotad_submit_request(void *req, call_frame_t *frame, rpc_clnt_prog_t *prog,
+                          int procnum, struct iobref *iobref, xlator_t *this,
+                          fop_cbk_fn_t cbkfn, xdrproc_t xdrproc);
 
 struct rpc_clnt *
-cli_quotad_clnt_init (xlator_t *this, dict_t *options);
+cli_quotad_clnt_init(xlator_t *this, dict_t *options);
 
 int
-cli_quotad_notify (struct rpc_clnt *rpc, void *mydata,
-                   rpc_clnt_event_t event, void *data);
-
+cli_quotad_notify(struct rpc_clnt *rpc, void *mydata, rpc_clnt_event_t event,
+                  void *data);
diff --git a/cli/src/cli-rl.c b/cli/src/cli-rl.c
index ade1c8ebb37..7a38a0b882a 100644
--- a/cli/src/cli-rl.c
+++ b/cli/src/cli-rl.c
@@ -13,16 +13,11 @@
 #include <stdint.h>
 #include <pthread.h>
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 #include "cli.h"
 #include "cli-cmd.h"
 #include "cli-mem-types.h"
 
-#include "event.h"
+#include <glusterfs/gf-event.h>
 
 #include <fnmatch.h>
 
@@ -32,384 +27,376 @@
 #include <readline/readline.h>
 #include <readline/history.h>
 
-
 int
-cli_rl_out (struct cli_state *state, const char *fmt, va_list ap)
+cli_rl_out(struct cli_state *state, const char *fmt, va_list ap)
 {
-        int tmp_rl_point = rl_point;
-        int            n = rl_end;
-        int            ret = 0;
+    int tmp_rl_point = rl_point;
+    int n = rl_end;
+    int ret = 0;
 
-        if (rl_end >= 0 ) {
-                rl_kill_text (0, rl_end);
-                rl_redisplay ();
-        }
+    if (rl_end >= 0) {
+        rl_kill_text(0, rl_end);
+        rl_redisplay();
+    }
 
-        printf ("\r%*s\r", (int)strlen (state->prompt), "");
+    printf("\r%*s\r", (int)strlen(state->prompt), "");
 
-        ret = vprintf (fmt, ap);
+    ret = vprintf(fmt, ap);
 
-        printf ("\n");
-        fflush(stdout);
+    printf("\n");
+    fflush(stdout);
 
-        if (n) {
-                rl_do_undo ();
-                rl_point = tmp_rl_point;
-                rl_reset_line_state ();
-        }
+    if (n) {
+        rl_do_undo();
+        rl_point = tmp_rl_point;
+        rl_reset_line_state();
+    }
 
-        return ret;
+    return ret;
 }
 
 int
-cli_rl_err (struct cli_state *state, const char *fmt, va_list ap)
+cli_rl_err(struct cli_state *state, const char *fmt, va_list ap)
 {
-        int tmp_rl_point = rl_point;
-        int            n = rl_end;
-        int            ret = 0;
+    int tmp_rl_point = rl_point;
+    int n = rl_end;
+    int ret = 0;
 
-        if (rl_end >= 0 ) {
-                rl_kill_text (0, rl_end);
-                rl_redisplay ();
-        }
+    if (rl_end >= 0) {
+        rl_kill_text(0, rl_end);
+        rl_redisplay();
+    }
 
-        fprintf (stderr, "\r%*s\r", (int)strlen (state->prompt), "");
+    fprintf(stderr, "\r%*s\r", (int)strlen(state->prompt), "");
 
-        ret = vfprintf (stderr, fmt, ap);
+    ret = vfprintf(stderr, fmt, ap);
 
-        fprintf (stderr, "\n");
-        fflush(stderr);
+    fprintf(stderr, "\n");
+    fflush(stderr);
 
-        if (n) {
-                rl_do_undo ();
-                rl_point = tmp_rl_point;
-                rl_reset_line_state ();
-        }
+    if (n) {
+        rl_do_undo();
+        rl_point = tmp_rl_point;
+        rl_reset_line_state();
+    }
 
-        return ret;
+    return ret;
 }
 
-
 void
-cli_rl_process_line (char *line)
+cli_rl_process_line(char *line)
 {
-        struct cli_state *state = NULL;
-        int               ret = 0;
+    struct cli_state *state = NULL;
+    int ret = 0;
 
-        state = global_state;
+    state = global_state;
 
-        state->rl_processing = 1;
-        {
-                ret = cli_cmd_process_line (state, line);
-                if (ret)
-                        gf_log (THIS->name, GF_LOG_WARNING,
-                                "failed to process line");
-
-                add_history (line);
-        }
-        state->rl_processing = 0;
+    state->rl_processing = 1;
+    {
+        ret = cli_cmd_process_line(state, line);
+        if (ret)
+            gf_log(THIS->name, GF_LOG_WARNING, "failed to process line");
 
+        add_history(line);
+    }
+    state->rl_processing = 0;
 }
 
-
-int
-cli_rl_stdin (int fd, int idx, void *data,
-              int poll_out, int poll_in, int poll_err)
+void
+cli_rl_stdin(int fd, int idx, int gen, void *data, int poll_out, int poll_in,
+             int poll_err, char event_thread_died)
 {
-        rl_callback_read_char ();
+    struct cli_state *state = NULL;
 
-        return 0;
-}
+    state = data;
 
+    rl_callback_read_char();
+
+    gf_event_handled(state->ctx->event_pool, fd, idx, gen);
+
+    return;
+}
 
 char *
-cli_rl_autocomplete_entry (const char *text, int times)
+cli_rl_autocomplete_entry(const char *text, int times)
 {
-        struct cli_state  *state = NULL;
-        char              *retp = NULL;
+    struct cli_state *state = NULL;
+    char *retp = NULL;
 
-        state = global_state;
+    state = global_state;
 
-        if (!state->matchesp)
-                return NULL;
+    if (!state->matchesp)
+        return NULL;
 
-        retp = *state->matchesp;
+    retp = *state->matchesp;
 
-        state->matchesp++;
+    state->matchesp++;
 
-        return retp ? strdup (retp) : NULL;
+    return retp ? strdup(retp) : NULL;
 }
 
-
 int
-cli_rl_token_count (const char *text)
+cli_rl_token_count(const char *text)
 {
-        int          count = 0;
-        const char  *trav = NULL;
-        int          is_spc = 1;
-
-        for (trav = text; *trav; trav++) {
-                if (*trav == ' ') {
-                        is_spc = 1;
-                } else {
-                        if (is_spc) {
-                                count++;
-                                is_spc = 0;
-                        }
-                }
+    int count = 0;
+    const char *trav = NULL;
+    int is_spc = 1;
+
+    for (trav = text; *trav; trav++) {
+        if (*trav == ' ') {
+            is_spc = 1;
+        } else {
+            if (is_spc) {
+                count++;
+                is_spc = 0;
+            }
         }
+    }
 
-        if (is_spc)
-                /* what needs to be autocompleted is a full
-                   new word, and not extend the last word
-                */
-                count++;
+    if (is_spc)
+        /* what needs to be autocompleted is a full
+           new word, and not extend the last word
+        */
+        count++;
 
-        return count;
+    return count;
 }
 
-
 char **
-cli_rl_tokenize (const char *text)
+cli_rl_tokenize(const char *text)
 {
-        int     count = 0;
-        char  **tokens = NULL;
-        char  **tokenp = NULL;
-        char   *token = NULL;
-        char   *copy = NULL;
-        char   *saveptr = NULL;
-        int     i = 0;
-
-        count = cli_rl_token_count (text);
-
-        tokens = calloc (count + 1, sizeof (*tokens));
-        if (!tokens)
-                return NULL;
-
-        copy = strdup (text);
-        if (!copy)
-                goto out;
-
-        tokenp = tokens;
-
-        for (token = strtok_r (copy, " \t\r\n", &saveptr); token;
-             token = strtok_r (NULL, " \t\r\n", &saveptr)) {
-                *tokenp = strdup (token);
-
-                if (!*tokenp)
-                        goto out;
-                tokenp++;
-                i++;
-
-        }
+    int count = 0;
+    char **tokens = NULL;
+    char **tokenp = NULL;
+    char *token = NULL;
+    char *copy = NULL;
+    char *saveptr = NULL;
+    int i = 0;
+
+    count = cli_rl_token_count(text);
+
+    tokens = calloc(count + 1, sizeof(*tokens));
+    if (!tokens)
+        return NULL;
 
-        if (i < count) {
-                /* symbolize that what needs to be autocompleted is
-                   the full set of possible nextwords, and not extend
-                   the last word
-                */
-                *tokenp = strdup ("");
-                if (!*tokenp)
-                        goto out;
-                tokenp++;
-                i++;
-        }
+    copy = strdup(text);
+    if (!copy)
+        goto out;
+
+    tokenp = tokens;
+
+    for (token = strtok_r(copy, " \t\r\n", &saveptr); token;
+         token = strtok_r(NULL, " \t\r\n", &saveptr)) {
+        *tokenp = strdup(token);
+
+        if (!*tokenp)
+            goto out;
+        tokenp++;
+        i++;
+    }
+
+    if (i < count) {
+        /* symbolize that what needs to be autocompleted is
+           the full set of possible nextwords, and not extend
+           the last word
+        */
+        *tokenp = strdup("");
+        if (!*tokenp)
+            goto out;
+        tokenp++;
+        i++;
+    }
 
 out:
-        free (copy);
+    free(copy);
 
-        if (i < count) {
-                cli_cmd_tokens_destroy (tokens);
-                tokens = NULL;
-        }
+    if (i < count) {
+        cli_cmd_tokens_destroy(tokens);
+        tokens = NULL;
+    }
 
-        return tokens;
+    return tokens;
 }
 
-
 char **
-cli_rl_get_matches (struct cli_state *state, struct cli_cmd_word *word,
-                    const char *text)
+cli_rl_get_matches(struct cli_state *state, struct cli_cmd_word *word,
+                   const char *text)
 {
-        char                 **matches = NULL;
-        char                 **matchesp = NULL;
-        struct cli_cmd_word  **next = NULL;
-        int                    count = 0;
-        int                    len = 0;
+    char **matches = NULL;
+    char **matchesp = NULL;
+    struct cli_cmd_word **next = NULL;
+    int count = 0;
+    int len = 0;
 
-        len = strlen (text);
+    len = strlen(text);
 
-        if (!word->nextwords)
-                return NULL;
+    if (!word->nextwords)
+        return NULL;
 
-        for (next = word->nextwords; *next; next++)
-                count++;
+    for (next = word->nextwords; *next; next++)
+        count++;
 
-        matches = calloc (count + 1, sizeof (*matches));
-        matchesp = matches;
+    matches = calloc(count + 1, sizeof(*matches));
+    matchesp = matches;
 
-        for (next = word->nextwords; *next; next++) {
-                if ((*next)->match) {
-                        continue;
-                }
+    for (next = word->nextwords; *next; next++) {
+        if ((*next)->match) {
+            continue;
+        }
 
-                if (strncmp ((*next)->word, text, len) == 0) {
-                        *matchesp = strdup ((*next)->word);
-                        matchesp++;
-                }
+        if (strncmp((*next)->word, text, len) == 0) {
+            *matchesp = strdup((*next)->word);
+            matchesp++;
         }
+    }
 
-        return matches;
+    return matches;
 }
 
-
 int
-cli_rl_autocomplete_prepare (struct cli_state *state, const char *text)
+cli_rl_autocomplete_prepare(struct cli_state *state, const char *text)
 {
-        struct cli_cmd_word   *word = NULL;
-        struct cli_cmd_word   *next = NULL;
-        char                 **tokens = NULL;
-        char                 **tokenp = NULL;
-        char                  *token = NULL;
-        char                 **matches = NULL;
-
-        tokens = cli_rl_tokenize (text);
-        if (!tokens)
-                return 0;
-
-        word = &state->tree.root;
-
-        for (tokenp = tokens; (token = *tokenp); tokenp++) {
-                if (!*(tokenp+1)) {
-                        /* last word */
-                        break;
-                }
-
-                next = cli_cmd_nextword (word, token);
-                word = next;
-                if (!word)
-                        break;
+    struct cli_cmd_word *word = NULL;
+    struct cli_cmd_word *next = NULL;
+    char **tokens = NULL;
+    char **tokenp = NULL;
+    char *token = NULL;
+    char **matches = NULL;
+
+    tokens = cli_rl_tokenize(text);
+    if (!tokens)
+        return 0;
+
+    word = &state->tree.root;
+
+    for (tokenp = tokens; (token = *tokenp); tokenp++) {
+        if (!*(tokenp + 1)) {
+            /* last word */
+            break;
         }
 
+        next = cli_cmd_nextword(word, token);
+        word = next;
         if (!word)
-                goto out;
+            break;
+    }
+
+    if (!word || !token)
+        goto out;
 
-        matches = cli_rl_get_matches (state, word, token);
+    matches = cli_rl_get_matches(state, word, token);
 
-        state->matches = matches;
-        state->matchesp = matches;
+    state->matches = matches;
+    state->matchesp = matches;
 
 out:
-        cli_cmd_tokens_destroy (tokens);
-        return 0;
+    cli_cmd_tokens_destroy(tokens);
+    return 0;
 }
 
-
 int
-cli_rl_autocomplete_cleanup (struct cli_state *state)
+cli_rl_autocomplete_cleanup(struct cli_state *state)
 {
-        if (state->matches)
-                cli_cmd_tokens_destroy (state->matches);
+    if (state->matches)
+        cli_cmd_tokens_destroy(state->matches);
 
-        state->matches = NULL;
-        state->matchesp = NULL;
+    state->matches = NULL;
+    state->matchesp = NULL;
 
-        return 0;
+    return 0;
 }
 
-
 char **
-cli_rl_autocomplete (const char *text, int start, int end)
+cli_rl_autocomplete(const char *text, int start, int end)
 {
-        struct cli_state  *state = NULL;
-        char             **matches = NULL;
-        char               save = 0;
+    struct cli_state *state = NULL;
+    char **matches = NULL;
+    char save = 0;
 
-        state = global_state;
+    state = global_state;
 
-        /* hack to make the autocompletion code neater */
-        /* fake it as though the cursor is at the end of line */
+    /* hack to make the autocompletion code neater */
+    /* fake it as though the cursor is at the end of line */
 
-        save = rl_line_buffer[rl_point];
-        rl_line_buffer[rl_point] = 0;
+    save = rl_line_buffer[rl_point];
+    rl_line_buffer[rl_point] = 0;
 
-        cli_rl_autocomplete_prepare (state, rl_line_buffer);
+    cli_rl_autocomplete_prepare(state, rl_line_buffer);
 
-        matches = rl_completion_matches (text, cli_rl_autocomplete_entry);
+    matches = rl_completion_matches(text, cli_rl_autocomplete_entry);
 
-        cli_rl_autocomplete_cleanup (state);
+    cli_rl_autocomplete_cleanup(state);
 
-        rl_line_buffer[rl_point] = save;
+    rl_line_buffer[rl_point] = save;
 
-        return matches;
+    return matches;
 }
 
-
 static char *
-complete_none (const char *txt, int times)
+complete_none(const char *txt, int times)
 {
-        return NULL;
+    return NULL;
 }
 
-
 void *
-cli_rl_input (void *_data)
+cli_rl_input(void *_data)
 {
-        struct cli_state *state = NULL;
-        char             *line = NULL;
+    struct cli_state *state = NULL;
+    char *line = NULL;
 
-        state = _data;
+    state = _data;
 
-        for (;;) {
-                line = readline (state->prompt);
-                if (!line)
-                        exit(0);  //break;
+    fprintf(stderr,
+            "Welcome to gluster prompt, type 'help' to see the available "
+            "commands.\n");
+    for (;;) {
+        line = readline(state->prompt);
+        if (!line)
+            exit(0);  // break;
 
-                if (*line)
-                        cli_rl_process_line (line);
+        if (*line)
+            cli_rl_process_line(line);
 
-                free (line);
-        }
+        free(line);
+    }
 
-        return NULL;
+    return NULL;
 }
 
-
 int
-cli_rl_enable (struct cli_state *state)
+cli_rl_enable(struct cli_state *state)
 {
-        int ret = 0;
-
-        rl_pre_input_hook = NULL;
-        rl_attempted_completion_function = cli_rl_autocomplete;
-        rl_completion_entry_function = complete_none;
-
-        if (!state->rl_async) {
-                ret = pthread_create (&state->input, NULL,
-                                      cli_rl_input, state);
-                if (ret == 0)
-                        state->rl_enabled = 1;
-                goto out;
-        }
+    int ret = 0;
 
-        ret = event_register (state->ctx->event_pool, 0, cli_rl_stdin, state,
-                              1, 0);
-        if (ret == -1)
-                goto out;
+    rl_pre_input_hook = NULL;
+    rl_attempted_completion_function = cli_rl_autocomplete;
+    rl_completion_entry_function = complete_none;
 
-        state->rl_enabled = 1;
-        rl_callback_handler_install (state->prompt, cli_rl_process_line);
+    if (!state->rl_async) {
+        ret = pthread_create(&state->input, NULL, cli_rl_input, state);
+        if (ret == 0)
+            state->rl_enabled = 1;
+        goto out;
+    }
+
+    ret = gf_event_register(state->ctx->event_pool, 0, cli_rl_stdin, state, 1,
+                            0, 0);
+    if (ret == -1)
+        goto out;
+
+    state->rl_enabled = 1;
+    rl_callback_handler_install(state->prompt, cli_rl_process_line);
 
 out:
-        return state->rl_enabled;
+    return state->rl_enabled;
 }
 
 #else /* HAVE_READLINE */
 
 int
-cli_rl_enable (struct cli_state *state)
+cli_rl_enable(struct cli_state *state)
 {
-        return 0;
+    return 0;
 }
 
 #endif /* HAVE_READLINE */
diff --git a/cli/src/cli-rpc-ops.c b/cli/src/cli-rpc-ops.c
index 252f0b12279..9b6b0c7fa50 100644
--- a/cli/src/cli-rpc-ops.c
+++ b/cli/src/cli-rpc-ops.c