2640 files changed, 647878 insertions, 361468 deletions

diff --git a/.clang-format b/.clang-format
new file mode 100644
index 00000000000..84c2efe3fad
--- /dev/null
+++ b/.clang-format
@@ -0,0 +1,107 @@
+---
+Language:        Cpp
+# BasedOnStyle:  Chromium
+AccessModifierOffset: -1
+AlignAfterOpenBracket: Align
+AlignConsecutiveAssignments: false
+AlignConsecutiveDeclarations: false
+AlignEscapedNewlines: Right
+AlignOperands:   true
+AlignTrailingComments: true
+AllowAllParametersOfDeclarationOnNextLine: false
+AllowShortBlocksOnASingleLine: false
+AllowShortCaseLabelsOnASingleLine: false
+AllowShortFunctionsOnASingleLine: Inline
+AllowShortIfStatementsOnASingleLine: false
+AllowShortLoopsOnASingleLine: false
+AlwaysBreakAfterDefinitionReturnType: All
+AlwaysBreakAfterReturnType: All
+AlwaysBreakBeforeMultilineStrings: true
+AlwaysBreakTemplateDeclarations: true
+BinPackArguments: true
+BinPackParameters: true
+BraceWrapping:
+  AfterClass:      false
+  AfterControlStatement: false
+  AfterEnum:       false
+  AfterFunction:   true
+  AfterNamespace:  false
+  AfterObjCDeclaration: false
+  AfterStruct:     false
+  AfterUnion:      false
+  BeforeCatch:     false
+  BeforeElse:      false
+  IndentBraces:    false
+  SplitEmptyFunction: true
+  SplitEmptyRecord: true
+  SplitEmptyNamespace: true
+BreakBeforeBinaryOperators: None
+BreakBeforeBraces: Linux
+BreakBeforeInheritanceComma: false
+BreakBeforeTernaryOperators: true
+BreakConstructorInitializersBeforeComma: false
+BreakConstructorInitializers: BeforeColon
+BreakAfterJavaFieldAnnotations: false
+BreakStringLiterals: true
+ColumnLimit:     80
+CommentPragmas:  '^ IWYU pragma:'
+CompactNamespaces: false
+ConstructorInitializerAllOnOneLineOrOnePerLine: true
+ConstructorInitializerIndentWidth: 4
+ContinuationIndentWidth: 4
+Cpp11BracedListStyle: true
+DerivePointerAlignment: false
+DisableFormat:   false
+ExperimentalAutoDetectBinPacking: false
+FixNamespaceComments: true
+ForEachMacros:
+  - foreach
+  - Q_FOREACH
+  - BOOST_FOREACH
+IncludeCategories:
+  - Regex:           '^<.*\.h>'
+    Priority:        1
+  - Regex:           '^<.*'
+    Priority:        2
+  - Regex:           '.*'
+    Priority:        3
+IncludeIsMainRegex: '([-_](test|unittest))?$'
+IndentCaseLabels: true
+IndentWidth:     4
+IndentWrappedFunctionNames: false
+JavaScriptQuotes: Leave
+JavaScriptWrapImports: true
+KeepEmptyLinesAtTheStartOfBlocks: false
+MacroBlockBegin: ''
+MacroBlockEnd:   ''
+MaxEmptyLinesToKeep: 1
+NamespaceIndentation: None
+ObjCBlockIndentWidth: 2
+ObjCSpaceAfterProperty: false
+ObjCSpaceBeforeProtocolList: false
+PenaltyBreakAssignment: 200
+PenaltyBreakBeforeFirstCallParameter: 1
+PenaltyBreakComment: 300
+PenaltyBreakFirstLessLess: 120
+PenaltyBreakString: 1000
+PenaltyExcessCharacter: 1000000
+PenaltyReturnTypeOnItsOwnLine: 200
+PointerAlignment: Right
+ReflowComments:  true
+SortIncludes:    false
+SortUsingDeclarations: true
+SpaceAfterCStyleCast: false
+SpaceAfterTemplateKeyword: true
+SpaceBeforeAssignmentOperators: true
+SpaceBeforeParens: ControlStatements
+SpaceInEmptyParentheses: false
+SpacesBeforeTrailingComments: 2
+SpacesInAngles:  false
+SpacesInContainerLiterals: true
+SpacesInCStyleCastParentheses: false
+SpacesInParentheses: false
+SpacesInSquareBrackets: false
+Standard:        Auto
+TabWidth:        8
+UseTab:          Never
+...
diff --git a/.github/ISSUE_TEMPLATE b/.github/ISSUE_TEMPLATE
new file mode 100644
index 00000000000..386ed2d8dd5
--- /dev/null
+++ b/.github/ISSUE_TEMPLATE
@@ -0,0 +1,30 @@
+<!-- Please use this template while reporting an issue, providing as much information as possible. Failure to do so may result in a delayed response. Thank you! -->
+
+**Description of problem:**
+
+
+**The exact command to reproduce the issue**:
+
+
+**The full output of the command that failed**:
+<details>
+
+
+
+</details>
+
+**Expected results:**
+
+
+**Additional info:**
+
+
+**- The output of the `gluster volume info` command**:
+<details>
+
+
+
+</details>
+
+**- The operating system / glusterfs version**:
+
diff --git a/.github/PULL_REQUEST_TEMPLATE b/.github/PULL_REQUEST_TEMPLATE
new file mode 100644
index 00000000000..e69de29bb2d
--- /dev/null
+++ b/.github/PULL_REQUEST_TEMPLATE
diff --git a/.github/RELEASE_TRACKER_TEMPLATE b/.github/RELEASE_TRACKER_TEMPLATE
new file mode 100644
index 00000000000..502bbd5556c
--- /dev/null
+++ b/.github/RELEASE_TRACKER_TEMPLATE
@@ -0,0 +1,12 @@
+<!-- Please use this template while creating a tracker issue -->
+
+**Description of problem:**
+A tracker issue to track the issues that will be fixed as a part of this release
+
+
+**Major or minor release**:
+
+
+**Release version**:
+
+
diff --git a/.github/stale.yml b/.github/stale.yml
new file mode 100644
index 00000000000..460e327c6ea
--- /dev/null
+++ b/.github/stale.yml
@@ -0,0 +1,25 @@
+# Number of days of inactivity before an issue becomes stale
+daysUntilStale: 210
+# Number of days of inactivity before a stale issue is closed
+daysUntilClose: 15
+# Issues with these labels will never be considered stale
+exemptLabels:
+  - pinned
+  - security
+# Label to use when marking an issue as stale
+staleLabel: wontfix
+
+# Comment to post when marking an issue as stale. Set to `false` to disable
+markComment: >
+  Thank you for your contributions.
+
+  Noticed that this issue is not having any activity in last ~6 months! We
+  are marking this issue as stale because it has not had recent activity.
+
+  It will be closed in 2 weeks if no one responds with a comment here.
+  
+
+# Comment to post when closing a stale issue. Set to `false` to disable
+closeComment: >
+  Closing this issue as there was no update since my last update on issue.
+  If this is an issue which is still valid, feel free to open it.
diff --git a/.gitignore b/.gitignore
index 0e31d20dc19..fc5ba586f8e 100644
--- a/.gitignore
+++ b/.gitignore
@@ -3,12 +3,21 @@ autom4te.cache
 build
 config.*
 configure
+cscope.*
+tags
 depcomp
+INSTALL
 install-sh
 ltmain.sh
+Makefile
 Makefile.in
 missing
-py-compile
+stamp-h1
+stamp-h2
+test-driver
+*compile
+*.gcda
+*.gcno
 *.sw?
 *~
 *.lo
@@ -16,30 +25,101 @@ py-compile
 *.o
 *.tar.gz
 *.rpm
+*.diff
+*.patch
 .libs
 .deps
-Makefile
-stamp-h1
+.dirstamp
+
+# Softlinks to test and log
+log
+*.vol
+.clang-format
+
+# cmocka unit tests
+*.log
+*.trs
+*_unittest
 
 # Generated files
-ufo/.tox
-ufo/test/unit/.coverage
-contrib/uuid/uuid_types.h
-extras/init.d/glusterd.plist
+site.h
+*.py[co]
+api/examples/__init__.py
+api/examples/setup.py
+api/src/gfapi.map
+cli/src/gluster
+contrib/fuse-util/fusermount-glusterfs
+extras/geo-rep/gsync-sync-gfid
+extras/geo-rep/schedule_georep.py
+extras/snap_scheduler/conf.py
 extras/init.d/glusterd-Debian
+extras/init.d/glusterd-FreeBSD
 extras/init.d/glusterd-Redhat
 extras/init.d/glusterd-SuSE
+extras/init.d/glusterd.plist
 extras/ocf/glusterd
 extras/ocf/volume
-glusterfs.spec
+extras/run-gluster.tmpfiles
+extras/systemd/glusterd.service
+extras/systemd/gluster-ta-volume.service
+extras/systemd/glusterfssharedstorage.service
+extras/who-wrote-glusterfs/gitdm
+geo-replication/.tox
+geo-replication/gsyncd.conf
+geo-replication/src/gsyncd
+geo-replication/src/peer_gsec_create
+geo-replication/src/peer_mountbroker
+geo-replication/src/peer_mountbroker.py
+geo-replication/src/set_geo_rep_pem_keys.sh
+geo-replication/src/peer_georep-sshkey.py
+geo-replication/syncdaemon.egg-info
+geo-replication/syncdaemon/conf.py
+geo-replication/tests/unit/.coverage
+geo-replication/tests/unit/cover
+geo-replication/tests/unit/coverage.xml
+geo-replication/tests/unit/nosetests.xml
+geo-replication/tests/unit/results.html
 glusterfs-api.pc
-libtool
-xlators/mount/fuse/utils/mount.glusterfs
-xlators/mount/fuse/utils/mount_glusterfs
-argp-standalone/libargp.a
+glusterfs.spec
+glusterfsd/src/glusterd
+glusterfsd/src/glusterfs
 glusterfsd/src/glusterfsd
-libglusterfs/src/spec.lex.c
+glusterfsd/src/gf_attach
+heal/src/glfsheal
+libgfchangelog.pc
+libglusterfs/src/graph.lex.c
 libglusterfs/src/y.tab.c
 libglusterfs/src/y.tab.h
-libgfchangelog.pc
-run-tests.sh
\ No newline at end of file
+libglusterfs/src/defaults.c
+libglusterfs/src/cli1-xdr.h
+libglusterfs/src/protocol-common.h
+libtool
+# copied XDR for cyclic libglusterfs <-> rpc-header dependency
+run-tests.sh
+!tests/basic/fuse/Makefile
+!tests/basic/gfapi/Makefile
+tests/env.rc
+tests/utils/arequal-checksum
+xlators/features/glupy/src/__init__.py
+xlators/features/glupy/src/setup.py
+xlators/mount/fuse/utils/mount.glusterfs
+xlators/mount/fuse/utils/mount_glusterfs
+extras/peer_add_secret_pub
+tools/gfind_missing_files/gcrawler
+tools/glusterfind/glusterfind
+tools/glusterfind/src/tool.conf
+# Eventing
+events/src/eventsapiconf.py
+extras/systemd/glustereventsd.service
+events/src/eventtypes.py
+libglusterfs/src/eventtypes.h
+extras/init.d/glustereventsd-Debian
+extras/init.d/glustereventsd-FreeBSD
+extras/init.d/glustereventsd-Redhat
+tools/setgfid2path/src/gluster-setgfid2path
+xlators/features/cloudsync/src/cloudsync-autogen-fops.c
+xlators/features/cloudsync/src/cloudsync-autogen-fops.h
+xlators/features/utime/src/utime-autogen-fops.c
+xlators/features/utime/src/utime-autogen-fops.h
+tests/basic/metadisp/ftruncate
+xlators/features/metadisp/src/fops.c
diff --git a/.mailmap b/.mailmap
new file mode 100644
index 00000000000..141a1667ffa
--- /dev/null
+++ b/.mailmap
@@ -0,0 +1,43 @@
+# .mailmap, see 'git short-log --help' for details
+#
+# This file needs to match extras/who-wrote-glusterfs/gitdm.aliases.
+#
+# Listing of contributors that filed patches with different email addresses.
+# Format: <name> <main-email> <alias> [<alias> ...]
+#
+
+Amar Tumballi <amarts@redhat.com> <amar@gluster.com> <amar@del.gluster.com>
+Anand Avati <avati@redhat.com> <avati@gluster.com> <avati@dev.gluster.com> <avati@amp.gluster.com> <avati@blackhole.gluster.com>
+Anush Shetty <ashetty@redhat.com> <anush@gluster.com>
+Csaba Henk <csaba@redhat.com> <csaba@gluster.com> <csaba@lowlife.hu> <csaba@zresearch.com>
+Günther Deschner <gd@redhat.com> <gd@samba.org>
+Harshavardhana <fharshav@redhat.com> <harsha@gluster.com> <harsha@zresearch.com> <harsha@dev.gluster.com> <harsha@harshavardhana.net>
+Ji-Hyeon Gim <potatogim@gluesys.com> <potatogim@potatogim.net>
+Justin Clift <justin@gluster.org> <jclift@redhat.com>
+Kaleb S. KEITHLEY <kkeithle@redhat.com> <kkeithle@f16node1.kkeithle.usersys.redhat.com> <kkeithle@linux.keithley.org>
+Kaushal M <kaushal@redhat.com> <kaushal@gluster.com>
+Kaushik BV <kbudiger@redhat.com> <kaushikbv@gluster.com>
+Krishna Srinivas <ksriniva@redhat.com> <krishna@gluster.com> <krishna@zresearch.com> <krishna@guest-laptop>
+Krishnan Parthasarathi <kparthas@redhat.com> <kp@gluster.com>
+Louis Zuckerman <louiszuckerman@gmail.com> <me@louiszuckerman.com>
+M S Vishwanath Bhat <vbhat@redhat.com> <msvbhat@gmail.com> <vishwanath@gluster.com>
+Michael Adam <madam@redhat.com> <obnox@samba.org>
+Oleksandr Natalenko <oleksandr@natalenko.name> <o.natalenko@lanet.ua>
+Patrick Uiterwijk <puiterwijk@fedoraproject.org> <patrick@puiterwijk.org>
+Pavan Sondur <pavan@gluster.com> <pavan@dev.gluster.com>
+Pete Zaitcev <zaitcev@kotori.zaitcev.us> <zaitcev@yahoo.com>
+Pranith Kumar K <pkarampu@redhat.com> <pranithk@gluster.com>
+Prashanth Pai <ppai@redhat.com> <nullpai@gmail.com>
+Raghavendra Bhat <raghavendra@redhat.com> <raghavendrabhat@gluster.com>
+Raghavendra G <rgowdapp@redhat.com> <raghavendra@gluster.com> <raghavendra@zresearch.com>
+Rahul C S <rahulcs@redhat.com> <rahulcssjce@gmail.com>
+Rajesh Amaravathi <rajesh@redhat.com> <rajesh@gluster.com> <rajesh.amaravathi@gmail.com>
+Ravishankar N <ravishankar@redhat.com> <root@ravi2.(none)>
+Sakshi Bansal <sabansal@redhat.com> <sabansal@localhost.localdomain>
+Shehjar Tikoo <shehjart@gluster.com> <shehjart@zresearch.com>
+Venky Shankar <vshankar@redhat.com> <venky@gluster.com>
+Vijay Bellur <vbellur@redhat.com> <vijay@gluster.com> <vijay@dev.gluster.com>
+Vijaykumar Koppad <vkoppad@redhat.com> <vijaykumar.koppad@gmail.com>
+Vijaikumar Mallikarjuna <vmallika@redhat.com>
+Vikas Gorur <vikas@gluster.com> <vikas@zresearch.com>
+shishir gowda <sgowda@redhat.com> <shishirng@gluster.com>
diff --git a/.testignore b/.testignore
new file mode 100644
index 00000000000..fe8f838bf2b
--- /dev/null
+++ b/.testignore
@@ -0,0 +1,64 @@
+.github/ISSUE_TEMPLATE
+.github/PULL_REQUEST_TEMPLATE
+.github/stale.yml
+.gitignore
+.mailmap
+.testignore
+.clang-format
+rfc.sh
+submit-for-review.sh
+AUTHORS
+CONTRIBUTING.md
+COPYING-GPLV2
+COPYING-LGPLV3
+ChangeLog
+INSTALL
+MAINTAINERS
+NEWS
+README.md
+THANKS
+COMMITMENT
+api/examples/README
+api/examples/getvolfile.py
+api/src/README.Symbol_Versions
+build-aux/checkpatch.pl
+contrib/fuse-lib/COPYING.LIB
+contrib/fuse-util/COPYING
+contrib/macfuse/COPYING.txt
+doc/*
+extras/FreeBSD/README.FreeBSD
+extras/Solaris/README.solaris
+extras/Ubuntu/README.Ubuntu
+extras/benchmarking/README
+extras/cliutils/README.md
+extras/command-completion/README
+extras/create_new_xlator/README.md
+extras/glusterfs.vim
+extras/glusterfs-logrotate
+extras/glusterfs-georep-logrotate
+extras/init.d/glusterd-Debian.in
+extras/init.d/glusterd-FreeBSD.in
+extras/init.d/glusterd-Redhat.in
+extras/init.d/glusterd-SuSE.in
+extras/init.d/glusterd.plist.in
+extras/init.d/glustereventsd-Debian.in
+extras/init.d/glustereventsd-FreeBSD.in
+extras/init.d/glustereventsd-Redhat.in
+extras/init.d/rhel5-load-fuse.modules
+extras/logger.conf.example
+extras/snap_scheduler/README.md
+extras/test/ld-preload-test/README
+extras/who-wrote-glusterfs/*
+extras/distributed-testing/*
+geo-replication/syncdaemon/README.md
+geo-replication/test-requirements.txt
+rpc/xdr/src/.gitignore
+tests/README.md
+xlators/experimental/README.md
+xlators/experimental/dht2/README.md
+xlators/experimental/dht2/TODO.md
+xlators/experimental/posix2/README.md
+xlators/experimental/posix2/TODO.md
+xlators/features/glupy/doc/README.md
+xlators/features/glupy/doc/TESTING
+xlators/features/glupy/doc/test.vol
diff --git a/COMMITMENT b/COMMITMENT
new file mode 100644
index 00000000000..16b75efcf29
--- /dev/null
+++ b/COMMITMENT
@@ -0,0 +1,46 @@
+Common Cure Rights Commitment
+Version 1.0
+
+Before filing or continuing to prosecute any legal proceeding or claim
+(other than a Defensive Action) arising from termination of a Covered
+License, we commit to extend to the person or entity ('you') accused
+of violating the Covered License the following provisions regarding
+cure and reinstatement, taken from GPL version 3. As used here, the
+term 'this License' refers to the specific Covered License being
+enforced.
+
+    However, if you cease all violation of this License, then your
+    license from a particular copyright holder is reinstated (a)
+    provisionally, unless and until the copyright holder explicitly
+    and finally terminates your license, and (b) permanently, if the
+    copyright holder fails to notify you of the violation by some
+    reasonable means prior to 60 days after the cessation.
+
+    Moreover, your license from a particular copyright holder is
+    reinstated permanently if the copyright holder notifies you of the
+    violation by some reasonable means, this is the first time you
+    have received notice of violation of this License (for any work)
+    from that copyright holder, and you cure the violation prior to 30
+    days after your receipt of the notice.
+
+We intend this Commitment to be irrevocable, and binding and
+enforceable against us and assignees of or successors to our
+copyrights.
+
+Definitions
+
+'Covered License' means the GNU General Public License, version 2
+(GPLv2), the GNU Lesser General Public License, version 2.1
+(LGPLv2.1), or the GNU Library General Public License, version 2
+(LGPLv2), all as published by the Free Software Foundation.
+
+'Defensive Action' means a legal proceeding or claim that We bring
+against you in response to a prior proceeding or claim initiated by
+you or your affiliate.
+
+'We' means each contributor to this repository as of the date of
+inclusion of this file, including subsidiaries of a corporate
+contributor.
+
+This work is available under a Creative Commons Attribution-ShareAlike
+4.0 International license (https://creativecommons.org/licenses/by-sa/4.0/).
diff --git a/CONTRIBUTING b/CONTRIBUTING
deleted file mode 100644
index 7bccd88d7e5..00000000000
--- a/CONTRIBUTING
+++ /dev/null
@@ -1,25 +0,0 @@
-        Developer's Certificate of Origin 1.1
-
-        By making a contribution to this project, I certify that:
-
-        (a) The contribution was created in whole or in part by me and I
-            have the right to submit it under the open source license
-            indicated in the file; or
-
-        (b) The contribution is based upon previous work that, to the best
-            of my knowledge, is covered under an appropriate open source
-            license and I have the right under that license to submit that
-            work with modifications, whether created in whole or in part
-            by me, under the same open source license (unless I am
-            permitted to submit under a different license), as indicated
-            in the file; or
-
-        (c) The contribution was provided directly to me by some other
-            person who certified (a), (b) or (c) and I have not modified
-            it.
-
-        (d) I understand and agree that this project and the contribution
-            are public and that a record of the contribution (including all
-            personal information I submit with it, including my sign-off) is
-            maintained indefinitely and may be redistributed consistent with
-            this project or the open source license(s) involved.
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
new file mode 100644
index 00000000000..65fc3497104
--- /dev/null
+++ b/CONTRIBUTING.md
@@ -0,0 +1,114 @@
+# GlusterFS project Contribution guidelines
+
+## Development Workflow
+
+We follow most of the details as per the [document here](https://help.github.com/en/github/collaborating-with-issues-and-pull-requests). If you are not aware of the github workflow, it is recommended to go through them before continuing here.
+
+
+#### Get the Repository setup
+
+0. Fork Repository
+   - Fork [GlusterFS repository](https://github.com/gluster/glusterfs/fork).
+
+1. Clone Repository
+   - Clone the glusterfs repo freshly from github using below steps.
+
+```
+   git clone git@github.com:${username}/glusterfs.git
+   cd glusterfs/
+   git remote add upstream git@github.com:gluster/glusterfs.git
+```
+
+About two tasks are one time for the life time. You can continue to use the same repository for all the work in future.
+
+#### Development & Other flows
+
+0. Issue:
+   - Make sure there is an issue filed for the task you are working on.
+   - If it is not filed, open the issue with all the description.
+   - If it is a bug fix, add label "Type:Bug".
+   - If it is an RFC, provide all the documentation, and request for "DocApproved", and "SpecApproved" label.
+
+1. Code:
+   - Start coding
+   - Build and test locally
+   - Make sure clang-format is installed and is run on the patch.
+
+2. Keep up-to-date
+   - GlusterFS is a large project with many developers, so there would be one or the other patch everyday.
+   - It is critical for developer to be up-to-date with `devel` repo to be Conflict-Free when PR is opened.
+   - Git provides many options to keep up-to-date, below is one of them
+```
+   git fetch upstream
+   git rebase upstream/devel
+```
+   - It is recommended you keep pushing to your repo every day, so you don't loose any work.
+     - It can be done by `./rfc.sh` (or `git push origin HEAD:issueNNN`)
+
+2. Commit Message / PR description:
+   - The name of the branch on your personal fork can start with issueNNNN, followed by anything of your choice.
+   - PRs continue to have the title of format "component: \<title\>", like it is practiced now.
+   - When you open a PR, having a reference Issue for the commit is mandatory in GlusterFS.
+   - Commit message can have, either `Fixes: #NNNN` or `Updates: #NNNN` in a separate line in the commit message.
+     - Here, NNNN is the Issue ID in glusterfs repository.
+   - Each commit needs the author to have the "Signed-off-by: Name \<email\>" line.
+     - Can do this by `-s` option for `git commit`.
+   - If the PR is not ready for review, apply the label `work-in-progress`.
+     - Check the availability of "Draft PR" is present for you, if yes, use that instead.
+
+3. Tests:
+   - All the required smoke tests would be auto-triggered.
+     - Developers get a chance to retrigger the smoke tests using **"/recheck smoke"** as comment.
+   - The "regression" tests would be triggered by a comment **"/run regression"** from developers in the [@gluster-maintainers](https://github.com/orgs/gluster/teams/gluster-maintainers) group.
+     - Ask for help as comment in PR if you have any questions about the process!
+
+4. Review Process:
+   - `+2` : is equivalent to "Approve" from the people in the maintainer's group.
+   - `+1` : can be given by a maintainer/reviewer by explicitly stating that in the comment.
+   - `-1` : provide details on required changes and pick "Request Changes" while submitting your review.
+   - `-2` : done by adding the `DO-NOT-MERGE` label.
+
+   - Any further discussions can happen as comments in the PR.
+
+5. Making changes:
+   - There are 2 approaches to submit changes done after addressing review comments.
+     - Commit changes as a new commit on top of the original commits in the branch, and push the changes to same branch (issueNNNN)
+     - Commit changes into the same commit with `--amend` option, and do a push to the same branch with `--force` option.
+
+6. Merging:
+   - GlusterFS project follows 'Squash and Merge' method
+     - This is mainly to preserve the historic Gerrit method of one patch in `git log` for one URL link.
+     - This also makes every merge a complete patch, which has passed all tests.
+   - The merging of the patch is expected to be done by the maintainers.
+     - It can be done when all the tests (smoke and regression) pass.
+     - When the PR has 'Approved' flag from corresponding maintainer.
+   - If you feel there is delay, feel free to add a comment, discuss the same in Slack channel, or send email.
+
+## By contributing to this project, the contributor would need to agree to below.
+
+### Developer's Certificate of Origin 1.1
+
+By making a contribution to this project, I certify that:
+
+(a) The contribution was created in whole or in part by me and I
+    have the right to submit it under the open source license
+    indicated in the file; or
+
+(b) The contribution is based upon previous work that, to the best
+    of my knowledge, is covered under an appropriate open source
+    license and I have the right under that license to submit that
+    work with modifications, whether created in whole or in part
+    by me, under the same open source license (unless I am
+    permitted to submit under a different license), as indicated
+    in the file; or
+
+(c) The contribution was provided directly to me by some other
+    person who certified (a), (b) or (c) and I have not modified
+    it.
+
+(d) I understand and agree that this project and the contribution
+    are public and that a record of the contribution (including all
+    personal information I submit with it, including my sign-off) is
+    maintained indefinitely and may be redistributed consistent with
+    this project or the open source license(s) involved.
+
diff --git a/INSTALL b/INSTALL
index 88e28999df9..a56390e54fb 100644
--- a/INSTALL
+++ b/INSTALL
@@ -1,32 +1,47 @@
 Installation Instructions
 *************************
 
-Run ./configure after untaring the package.
+0. If you have cloned from git, run ./autogen.sh.
+
+1. Run ./configure.
 
  bash# ./configure
  GlusterFS configure summary
  ===========================
- FUSE client        : yes
- Infiniband verbs   : yes
- epoll IO multiplex : yes
- Berkeley-DB        : yes
- libglusterfsclient : yes
- mod_glusterfs      : yes
- argp-standalone    : no
+ GlusterFS configure summary
+  ===========================
+   FUSE client          : yes
+   Infiniband verbs     : yes
+   epoll IO multiplex   : yes
+   argp-standalone      : no
+   fusermount           : yes
+   readline             : yes
+   georeplication       : yes
+   Linux-AIO            : yes
+   Enable Debug         : no
+   Block Device xlator  : yes
+   glupy                : yes
+   Use syslog           : yes
+   XML output           : yes
+   QEMU Block formats   : yes
+   Encryption xlator    : yes
+
  
 The configure summary will tell you what all components will be built with 
 GlusterFS. Other than 'argp-standalone' if something else says 'no', that 
 feature in GlusterFS will not be built. 'argp-standalone' package will only 
 be used if the system doesn't have a proper argp package installed.
 
-Now just run 'make' and later run 'make install' to install the package.
+2. Now just run 'make' and later run 'make install' to install the package.
 
  bash# make 
  bash# make install
 
-Installation complete :-)
+Installation completed :-)
 
  bash# glusterfs --version
 
 Make sure your version is the latest from the release, and the one you 
 just installed :-)
+
+For more information on GlusterFS installation refer# http://docs.gluster.org/en/latest/Developer-guide/Building-GlusterFS/
diff --git a/MAINTAINERS b/MAINTAINERS
new file mode 100644
index 00000000000..953e8755fd9
--- /dev/null
+++ b/MAINTAINERS
@@ -0,0 +1,510 @@
+GlusterFS Maintainers
+=====================
+
+The intention of this file is not to establish who owns what portions of the
+code base, but to provide a set of names that developers can consult when they
+have a question about a particular subset and also to provide a set of names
+to be CC'd when submitting a patch to obtain appropriate review.
+
+In general, if you have a question about inclusion of a patch, you should
+consult gluster-devel@gluster.org and not any specific individual privately.
+
+Descriptions of section entries:
+
+	M: Main contact that knows and takes care of this area
+	L: Mailing list that is relevant to this area
+	W: Web-page with status/info
+	Q: Patchwork web based patch tracking system site
+	T: SCM tree type and location.  Type is one of: git, hg, quilt, stgit.
+	S: Status, one of the following:
+	   Supported:	Someone is actually paid to look after this.
+	   Maintained:	Someone actually looks after it.
+	   Odd Fixes:	It has a maintainer but they don't have time to do
+			much other than throw the odd patch in. See below.
+	   Orphan:	No current maintainer [but maybe you could take the
+			role as you write your new code].
+	   Obsolete:	Old code. Something tagged obsolete generally means
+			it has been replaced by a better system and you
+			should be using that.
+	F: Files and directories with wildcard patterns.
+	   A trailing slash includes all files and subdirectory files.
+	   F:	drivers/net/	all files in and below drivers/net
+	   F:	drivers/net/*	all files in drivers/net, but not below
+	   F:	*/net/*		all files in "any top level directory"/net
+	   One pattern per line.  Multiple F: lines acceptable.
+	X: Files and directories that are NOT maintained, same rules as F:
+	   Files exclusions are tested before file matches.
+	   Can be useful for excluding a specific subdirectory, for instance:
+	   F:	net/
+	   X:	net/ipv6/
+	   matches all files in and below net excluding net/ipv6/
+	K: Keyword perl extended regex pattern to match content in a
+	   patch or file.  For instance:
+	   K: of_get_profile
+	      matches patches or files that contain "of_get_profile"
+	   K: \b(printk|pr_(info|err))\b
+	      matches patches or files that contain one or more of the words
+	      printk, pr_info or pr_err
+	   One regex pattern per line.  Multiple K: lines acceptable.
+        P: Peer for a component
+
+
+General Project Architects
+--------------------------
+M: Amar Tumballi <amarts@gmail.com>
+M: Xavier Hernandez  <xhernandez@redhat.com>
+P: Pranith Karampuri <pranith.karampuri@phonepe.com>
+P: Atin Mukherjee <amukherj@redhat.com>
+
+xlators:
+--------
+Access Control List (ACL)
+M: Raghavendra Talur <rtalur@redhat.com>
+P: Jiffin Tony Thottan <jthottan@redhat.com>
+S: Maintained
+F: xlators/system/posix-acl/
+
+Arbiter
+M: Ravishankar N <ravishankar@redhat.com>
+P: Pranith Karampuri <pranith.karampuri@phonepe.com>
+S: Maintained
+F: xlators/features/arbiter/
+
+Automatic File Replication (AFR)
+M: Pranith Karampuri <pranith.karampuri@phonepe.com>
+M: Ravishankar N <ravishankar@redhat.com>
+P: Karthik US <ksubrahm@redhat.com>
+S: Maintained
+F: xlators/cluster/afr/
+
+Barrier
+M: Raghavendra Bhat <rabhat@redhat.com>
+P: Atin Mukherjee <amukherj@redhat.com>
+S: Maintained
+F: xlators/features/barrier
+
+BitRot
+M: Kotresh HR <khiremat@redhat.com>
+P: Raghavendra Bhat <rabhat@redhat.com>
+S: Maintained
+F: xlators/features/bit-rot/
+
+Changelog
+M: Aravinda V K <avishwan@redhat.com>
+P: Kotresh HR <khiremat@redhat.com>
+S: Maintained
+F: xlators/features/changelog/
+
+Distributed Hashing Table (DHT)
+P: Susant Palai <spalai@redhat.com>
+S: Maintained
+F: xlators/cluster/dht/
+
+Erasure Coding
+M: Pranith Karampuri <pranith.karampuri@phonepe.com>
+M: Xavier Hernandez  <xhernandez@redhat.com>
+P: Ashish Pandey <aspandey@redhat.com>
+S: Maintained
+F: xlators/cluster/ec/
+
+Error-gen
+M: Raghavendra Talur <rtalur@redhat.com>
+S: Maintained
+F: xlators/debug/error-gen/
+
+FUSE Bridge
+M: Csaba Henk <chenk@redhat.com>
+P: Niels de Vos <ndevos@redhat.com>
+S: Maintained
+F: xlators/mount/
+
+Index
+M: Pranith Karampuri <pranith.karampuri@phonepe.com>
+P: Ravishankar N <ravishankar@redhat.com>
+S: Maintained
+F: xlators/features/index/
+
+IO Cache
+P: Mohammed Rafi KC <rafi.kavungal@iternity.com>
+S: Maintained
+F: xlators/performance/io-cache/
+
+IO Statistics
+M: Krutika Dhananjay <kdhananj@redhat.com>
+M: Shyam Ranganathan <srangana@redhat.com>
+S: Maintained
+F: xlators/debug/io-stats/
+
+IO threads
+M: Pranith Karampuri <pranith.karampuri@phonepe.com>
+P: Ravishankar N <ravishankar@redhat.com>
+S: Maintained
+F: xlators/performance/io-threads/
+
+Leases
+M: Poornima G <pgurusid@redhat.com>
+P: Niels de Vos <ndevos@redhat.com>
+P: Soumya Koduri <skoduri@redhat.com>
+S: Maintained
+F: xlators/features/leases/
+
+Locks
+M: Krutika Dhananjay <kdhananj@redhat.com>
+P: Xavier Hernandez  <xhernandez@redhat.com>
+S: Maintained
+F: xlators/features/locks/
+
+Marker
+M: Kotresh HR <khiremat@redhat.com>
+S: Maintained
+F: xlators/features/marker/
+
+Meta
+M: Mohammed Rafi KC <rafi.kavungal@iternity.com>
+S: Maintained
+F: xlators/features/meta/
+
+Metadata-cache
+M: Poornima G <pgurusid@redhat.com>
+P: Soumya Koduri <skoduri@redhat.com>
+S: Maintained
+F: xlators/performance/md-cache/
+
+Negative-lookup Cache
+M: Poornima G <pgurusid@redhat.com>
+P: Pranith Karampuri <pranith.karampuri@phonepe.com>
+S: Maintained
+F: xlators/performance/nl-cache/
+
+gNFS
+M: Jiffin Tony Thottan <jthottan@redhat.com>
+P: Xie Changlong <xiechanglong@cmss.chinamobile.com>
+P: Amar Tumballi <amarts@gmail.com>
+S: Odd Fixes
+F: xlators/nfs/server/
+
+Open-behind
+S: Maintained
+F: xlators/performance/open-behind/
+
+Posix:
+M: Raghavendra Bhat <raghavendra@redhat.com>
+P: Kotresh HR <khiremat@redhat.com>
+P: Krutika Dhananjay <kdhananj@redhat.com>
+S: Maintained
+F: xlators/storage/posix/
+
+Quick-read
+S: Maintained
+F: xlators/performance/quick-read/
+
+Quota
+M: Shyamsundar Ranganathan <srangana@redhat.com>
+P: Hari Gowtham <hgowtham@redhat.com>
+S: Maintained
+F: xlators/features/quota/
+
+Read-ahead
+P: Csaba Henk <chenk@redhat.com>
+S: Maintained
+F: xlators/performance/read-ahead/
+
+Readdir-ahead
+S: Maintained
+F: xlators/performance/readdir-ahead/
+
+Sharding
+M: Krutika Dhananjay <kdhananj@redhat.com>
+P: Xavier Hernandez  <xhernandez@redhat.com>
+S: Maintained
+F: xlators/features/shard/
+
+Trash
+M: Anoop C S <anoopcs@redhat.com>
+M: Jiffin Tony Thottan <jthottan@redhat.com>
+S: Maintained
+F: xlators/features/trash/
+
+Upcall
+M: Poornima G <pgurusid@redhat.com>
+M: Soumya Koduri <skoduri@redhat.com>
+P: Niels de Vos <ndevos@redhat.com>
+S: Maintained
+F: xlators/features/upcall/
+
+Write-behind
+P: Csaba Henk <chenk@redhat.com>
+S: Maintained
+F: xlators/performance/write-behind/
+
+Write Once Read Many
+P: Karthik US <ksubrahm@redhat.com>
+S: Maintained
+F: xlators/features/read-only/
+
+Cloudsync
+M: Susant Kumar Palai <spalai@redhat.com>
+S: Maintained
+F: xlators/features/cloudsync/
+
+Other bits of code:
+-------------------
+
+Doc
+M: Humble Chirammal <hchiramm@redhat.com>
+M: Raghavendra Talur <rtalur@redhat.com>
+S: Maintained
+F: doc/
+
+Geo Replication
+M: Aravinda V K <avishwan@redhat.com>
+M: Kotresh HR <khiremat@redhat.com>
+M: Sunny Kumar <sunkumar@redhat.com>
+S: Maintained
+F: geo-replication/
+
+Glusterfind
+M: Aravinda VK <avishwan@redhat.com>
+S: Maintained
+F: tools/glusterfind/
+
+libgfapi
+M: Niels de Vos <ndevos@redhat.com>
+P: Poornima G <pgurusid@redhat.com>
+P: Shyamsundar Ranganathan <srangana@redhat.com>
+P: Soumya Koduri <skoduri@redhat.com>
+S: Maintained
+F: api/
+
+libglusterfs
+M: Amar Tumballi <amarts@gmail.com>
+M: Xavier Hernandez  <xhernandez@redhat.com>
+M: Jeff Darcy <jeff@pl.atyp.us>
+P: Kaleb Keithley <kkeithle@redhat.com>
+P: Niels de Vos <ndevos@redhat.com>
+P: Pranith Karampuri <pranith.karampuri@phonepe.com>
+P: Shyamsundar Ranganathan <srangana@redhat.com>
+S: Maintained
+F: libglusterfs/
+
+xxhash
+M: Aravinda VK <avishwan@redhat.com>
+M: Kotresh HR <khiremat@redhat.com>
+P: Yaniv Kaul <ykaul@redhat.com>
+S: Maintained
+F: contrib/xxhash/
+T: https://github.com/Cyan4973/xxHash.git
+
+Management Daemon - glusterd
+M: Atin Mukherjee <amukherj@redhat.com>
+M: Mohit Agrawal <moagrawa@redhat.com>
+M: Sanju Rakonde <srakonde@redhat.com>
+S: Maintained
+F: cli/
+F: xlators/mgmt/glusterd/
+
+Protocol
+M: Niels de Vos <ndevos@redhat.com>
+P: Mohammed Rafi KC <rafi.kavungal@iternity.com>
+S: Maintained
+F: xlators/protocol/
+
+Remote Procedure Call subsystem
+P: Mohit Agrawal <moagrawa@redhat.com>
+S: Maintained
+F: rpc/rpc-lib/
+F: rpc/xdr/
+
+Snapshot
+M: Raghavendra Bhat <raghavendra@redhat.com>
+P: Mohammed Rafi KC <rafi.kavungal@iternity.com>
+P: Sunny Kumar <sunkumar@redhat.com>
+S: Maintained
+F: xlators/mgmt/glusterd/src/glusterd-snap*
+F: extras/snap-scheduler.py
+
+Socket subsystem
+P: Krutika Dhananjay <kdhananj@redhat.com>
+P: Milind Changire <mchangir@redhat.com>
+P: Mohammed Rafi KC <rafi.kavungal@iternity.com>
+P: Mohit Agrawal <moagrawa@redhat.com>
+S: Maintained
+F: rpc/rpc-transport/socket/
+
+Testing - .t framework
+M: Raghavendra Talur <rtalur@redhat.com>
+S: Maintained
+F: tests/
+
+Utilities
+M: Aravinda VK <avishwan@redhat.com>
+P: Niels de Vos <ndevos@redhat.com>
+P: Raghavendra Talur <rtalur@redhat.com>
+P: Sachidanda Urs <surs@redhat.com>
+S: Maintained
+F: extras/
+
+Events APIs
+M: Aravinda VK <avishwan@redhat.com>
+S: Maintained
+F: events/
+F: libglusterfs/src/events*
+F: libglusterfs/src/eventtypes*
+F: extras/systemd/glustereventsd*
+
+Distribution Specific:
+----------------------
+Build:
+M: Niels de Vos <ndevos@redhat.com>
+M: Hari Gowtham <hgowtham@redhat.com>
+P: Anoop C S <anoopcs@redhat.com>
+P: Raghavendra Talur <rtalur@redhat.com>
+P: Rinku Kothiya <rkothiya@redhat.com>
+S: Maintained
+
+Debian packages on download.gluster.org
+M: packaging@gluster.org
+M: Kaleb Keithley <kkeithle@redhat.com>
+P: Sheetal Pamecha <spamecha@redhat.com>
+P: Shwetha Acharya <sacharya@redhat.com>
+S: Maintained
+W: http://download.gluster.org/pub/gluster/glusterfs/LATEST/Debian/Debian.README
+T: https://github.com/gluster/glusterfs-debian.git
+
+OpenSuSE
+M: packaging@gluster.org
+M: Kaleb Keithley <kkeithle@redhat.com>
+P: Sheetal Pamecha <spamecha@redhat.com>
+P: Shwetha Acharya <sacharya@redhat.com>
+S: Maintained
+W: https://build.opensuse.org/repositories/home:glusterfs
+W: https://download.gluster.org/pub/gluster/glusterfs/LATEST/SuSE/SuSE.README
+T: https://github.com/gluster/glusterfs-suse.git
+
+Packages for the CentOS Storage SIG
+M: centos-devel@centos.org
+M: Niels de Vos <ndevos@redhat.com>
+P: Kaleb Keithley <kkeithle@redhat.com>
+S: Maintained
+W: https://wiki.centos.org/SpecialInterestGroup/Storage/Gluster
+T: https://github.com/CentOS-Storage-SIG/glusterfs.git
+
+Ubuntu PPA
+M: packaging@gluster.org
+M: Kaleb Keithley <kkeithle@redhat.com>
+P: Sheetal Pamecha <spamecha@redhat.com>
+P: Shwetha Acharya <sacharya@redhat.com>
+S: Maintained
+W: https://launchpad.net/~gluster
+W: http://download.gluster.org/pub/gluster/glusterfs/LATEST/Ubuntu/Ubuntu.README
+T: https://github.com/gluster/glusterfs-debian.git
+
+Related projects
+----------------
+Gluster Block
+M: Prasanna Kumar Kalever <prasanna.kalever@redhat.com>
+M: Xiubo Li <xiubli@redhat.com>
+S: Maintained
+T: https://github.com/gluster/gluster-block.git
+
+GlusterFS core-utils
+M: Anoop C S <anoopcs@redhat.com>
+S: Maintained
+T: https://github.com/gluster/glusterfs-coreutils.git
+
+NFS-Ganesha FSAL plugin
+M: Jiffin Tony Thottan <jthottan@redhat.com>
+M: Kaleb Keithley <kkeithle@redhat.com>
+M: Soumya Koduri <skoduri@redhat.com>
+S: Maintained
+T: git://github.com/nfs-ganesha/nfs-ganesha.git
+F: src/nfs-ganesha~/src/FSAL/FSAL_GLUSTER/
+
+QEMU integration
+M: Niels de Vos <ndevos@redhat.com>
+M: Prasanna Kumar Kalever <prasanna.kalever@redhat.com>
+S: Maintained
+T: git://git.qemu.org/qemu.git
+F: block/gluster.c
+
+Samba VFS plugin
+M: Anoop C S <anoopcs@redhat.com>
+M: Raghavendra Talur <rtalur@redhat.com>
+M: Michael Adam <madam@redhat.com>
+M: Poornima G <pgurusid@redhat.com>
+S: Maintained
+T: git://git.samba.org/samba.git
+F: source3/modules/vfs_glusterfs.c
+
+Storhaug
+M: Jose A. Rivera <jarrpa@redhat.com>
+P: Kaleb Keithley <kkeithle@redhat.com>
+S: Maintained
+T: https://github.com/linux-ha-storage/storhaug.git
+
+Testing - Glusto-Tests
+M: Jonathan Holloway <jholloway@redhat.com>
+M: Vijay Bhaskar Reddy Avuthu <vavuthu@redhat.com>
+M: Akarsha Rai <akrai@redhat.com>
+S: Maintained
+T: https://github.com/gluster/glusto-tests.git
+
+Wireshark dissectors
+M: Niels de Vos <ndevos@redhat.com>
+S: Maintained
+W: https://forge.gluster.org/wireshark
+T: http://code.wireshark.org/git/wireshark
+F: epan/dissectors/packet-gluster*
+
+Infrastructure
+--------------
+
+Platform
+M: Michael Scherer <misc@redhat.com>
+P: Shyamsundar Ranganathan <srangana@redhat.com>
+P: Amar Tumballi <amarts@gmail.com>
+
+Continuous Integration
+M: Michael Scherer <misc@redhat.com>
+M: Deepshikha Khandelwal <dkhandel@redhat.com>
+P: Niels de Vos <ndevos@redhat.com>
+
+Special Thanks
+--------------
+
+GlusterFS would not be possible without the contributions of:
+
+
+M: Vijay Bellur <vbellur@redhat.com>
+M: Jeff Darcy <jeff@pl.atyp.us>
+M: Shreyas Siravara <sshreyas@fb.com>
+M: Kaushal M <kaushal@redhat.com>
+M: Nigel Babu
+M: Prashanth Pai
+P: Sanoj Unnikrishnan
+P: Milind Changire <mchangir@redhat.com>
+P: Sunil Kumar Acharya <sheggodu@redhat.com>
+M: Samikshan Bairagya <samikshan@gmail.com>
+M: Chris Hertel
+M: M. Mohan Kumar <mohan@in.ibm.com>
+M: Shishir Gowda <gowda.shishir@gmail.com>
+M: Brian Foster <bfoster@redhat.com>
+M: Anand Avati <avati@cs.stanford.edu>
+M: Dennis Schafroth <dennis@schafroth.com>
+M: Harshavardhana <harsha@harshavardhana.net>
+M: Krishnan Parthasarathi
+M: Justin Clift <justin@gluster.org>
+M: Venky Shankar <vshankar@redhat.com>
+M: Shravan Chandrashekar <shravantc99@gmail.com>
+M: Joseph Fernandes
+M: Vijaikumar Mallikarjuna
+M: Anand Subramanian
+M: Bharata B Rao <bharata@linux.vnet.ibm.com>
+M: Rajesh Joseph
+M: Dan Lambright
+M: Jay Vyas
+M: Luis Pabon
+M: Ira Cooper
+M: Shwetha Panduranga
+M: Nithya Balachandran
+M: Raghavendra Gowdappa
diff --git a/Makefile.am b/Makefile.am
index 598ebb4103c..98ea5c1038d 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -1,29 +1,55 @@
+SOURCES = site.h
+
 EXTRA_DIST = autogen.sh \
-	COPYING-GPLV2 COPYING-LGPLV3 \
-	INSTALL README AUTHORS THANKS NEWS \
+	COPYING-GPLV2 COPYING-LGPLV3 COMMITMENT \
+	INSTALL README.md AUTHORS THANKS NEWS \
 	glusterfs.spec glusterfs-api.pc.in libgfchangelog.pc.in \
-	error-codes.json gf-error-codes.h.template \
-	gen-headers.py run-tests.sh \
+	run-tests.sh \
+	build-aux/pkg-version \
+	contrib/umountd \
 	$(shell find $(top_srcdir)/tests -type f -print)
 
-SUBDIRS = argp-standalone libglusterfs rpc api xlators glusterfsd \
-	$(FUSERMOUNT_SUBDIR) doc extras cli @SYNCDAEMON_SUBDIR@
+
+SUBDIRS = $(ARGP_STANDALONE_DIR) libglusterfs rpc libglusterd api \
+	glusterfsd xlators $(FUSERMOUNT_SUBDIR) doc extras cli heal \
+	@SYNCDAEMON_SUBDIR@ @UMOUNTD_SUBDIR@ tools events
 
 pkgconfigdir = @pkgconfigdir@
 pkgconfig_DATA = glusterfs-api.pc libgfchangelog.pc
 
-CLEANFILES =
+CLEANFILES = glusterfs-api.pc libgfchangelog.pc contrib/umountd/Makefile
+
+clean-local:
+	find . -name '*.o' -o -name '*.lo' -o -name '.Po' | xargs rm -f
 
 gitclean: distclean
 	find . -name Makefile.in -exec rm -f {} \;
-	find . -name Makefile -exec rm -f {} \;
 	find . -name mount.glusterfs -exec rm -f {} \;
+	find . -name .deps -o -name .libs | xargs rm -rf
 	rm -fr autom4te.cache
 	rm -f missing aclocal.m4 config.h.in config.guess config.sub ltmain.sh install-sh configure depcomp
-	rm -fr argp-standalone/autom4te.cache
-	rm -f argp-standalone/aclocal.m4 argp-standalone/config.h.in
-	rm -f argp-standalone/configure argp-standalone/depcomp
-	rm -f argp-standalone/install-sh argp-standalone/missing
 
-dist-hook:
+# dist-hook gets executed with 'make dist', this is the only target getting
+# executed, a dist-hook in other Makefile.am files seem to get ignored.
+dist-hook: gen-VERSION gen-ChangeLog
+	-rm -fr $(distdir)/contrib/umountd/.deps
+	-rm -f $(distdir)/events/src/eventtypes.py
+	-rm -f $(distdir)/tests/env.rc
+	-cp -f $(top_srcdir)/build-aux/config.sub.dist $(distdir)/config.sub
+	-cp -f $(top_srcdir)/build-aux/config.guess.dist $(distdir)/config.guess
+
+.PHONY: gen-VERSION gen-ChangeLog clang-check
+
+clang-check:
+	@$(top_srcdir)/extras/clang-checker.sh
+
+gen-ChangeLog:
 	(cd $(srcdir) && git diff && echo ===== git log ==== && git log) > $(distdir)/ChangeLog
+
+.PHONY : gen-VERSION
+gen-VERSION:
+	if test -d $(top_srcdir)/.git; then             \
+	  cd $(top_srcdir);                             \
+	  ./build-aux/pkg-version --full                \
+	    > $(abs_top_builddir)/$(distdir)/VERSION;   \
+	fi
diff --git a/README b/README
deleted file mode 100644
index ce648743ee9..00000000000
--- a/README
+++ /dev/null
@@ -1 +0,0 @@
-For more info, please visit http://www.gluster.org/.
diff --git a/README.md b/README.md
new file mode 100644
index 00000000000..9d68e033782
--- /dev/null
+++ b/README.md
@@ -0,0 +1,46 @@
+# Gluster
+  Gluster is a software defined distributed storage that can scale to several
+  petabytes. It provides interfaces for object, block and file storage.
+
+## Development
+  The development workflow is documented in [Contributors guide](CONTRIBUTING.md)
+
+## Documentation
+  The Gluster documentation can be found at [Gluster Docs](http://docs.gluster.org).
+
+## Deployment
+  Quick instructions to build and install can be found in [INSTALL](INSTALL) file.
+
+## Testing
+
+  GlusterFS source contains some functional tests under `tests/` directory. All
+  these tests are run against every patch submitted for review. If you want your
+  patch to be tested, please add a `.t` test file as part of your patch submission.
+  You can also submit a patch to only add a `.t` file for the test case you are
+  aware of.
+
+  To run these tests, on your test-machine, just run `./run-tests.sh`. Don't run
+  this on a machine where you have 'production' glusterfs is running, as it would
+  blindly kill all gluster processes in each runs.
+
+  If you are sending a patch, and want to validate one or few specific tests, then
+  run a single test by running the below command.
+
+```
+  bash# /bin/bash ${path_to_gluster}/tests/basic/rpc-coverage.t
+```
+
+  You can also use `prove` tool if available in your machine, as follows.
+
+```
+  bash# prove -vmfe '/bin/bash' ${path_to_gluster}/tests/basic/rpc-coverage.t
+```
+
+
+## Maintainers
+  The list of Gluster maintainers is available in [MAINTAINERS](MAINTAINERS) file.
+
+## License
+  Gluster is dual licensed under [GPLV2](COPYING-GPLV2) and [LGPLV3+](COPYING-LGPLV3).
+
+  Please visit the [Gluster Home Page](http://www.gluster.org/) to find out more about Gluster.
diff --git a/THANKS b/THANKS
index e1ce5105d79..d056d00bb72 100644
--- a/THANKS
+++ b/THANKS
@@ -1 +1 @@
-For all of you, who use the product and help us making it more robust, useful, popular..
+For all of you, who use the product and help us make it more robust, useful, and popular.
diff --git a/api/Makefile.am b/api/Makefile.am
index af437a64d6d..f0ad1ee971c 100644
--- a/api/Makefile.am
+++ b/api/Makefile.am
@@ -1 +1 @@
-SUBDIRS = src
+SUBDIRS = src examples
diff --git a/api/examples/Makefile.am b/api/examples/Makefile.am
index 6048bb1c8b7..7112c81d6a7 100644
--- a/api/examples/Makefile.am
+++ b/api/examples/Makefile.am
@@ -1,4 +1,6 @@
-noinst_PROGRAMS = glfsxmp
+# The bits needed for glfsxmp
+EXTRA_PROGRAMS = glfsxmp
 glfsxmp_SOURCES = glfsxmp.c
 glfsxmp_CFLAGS = $(GLFS_CFLAGS) -Wall
-glfsxmp_LDADD = $(GLFS_LIBS)
\ No newline at end of file
+glfsxmp_LDADD = $(GLFS_LIBS) -lrt
+
diff --git a/api/examples/README b/api/examples/README
index 8f0bab697d3..4d2b521f779 100644
--- a/api/examples/README
+++ b/api/examples/README
@@ -17,3 +17,20 @@ install glusterfs-api RPM.
 Make sure your LDFLAGS includes -L/path/to/lib where libgfapi.so is
 installed and -I/path/to/include/glusterfs where the 'api' directory
 containing the headers are available.
+
+glfsxmp.c
+=========
+
+glfsxmp.c is an example application which uses libgfapi
+
+Compilation Steps For glfsxmp.c
+===============================
+
+1. $./autogen.sh
+2. $./configure
+
+Note: Before running ./configure , as mentioned above, you need to
+      take care of #1 or #2 i.e. pkg-config path or LDFLAGS and
+      -I/<path> with correct values.
+
+3. $make glfsxmp
diff --git a/api/examples/getvolfile.py b/api/examples/getvolfile.py
new file mode 100755
index 00000000000..3b2c8ab5a15
--- /dev/null
+++ b/api/examples/getvolfile.py
@@ -0,0 +1,45 @@
+#!/usr/bin/python3
+
+from __future__ import print_function
+import ctypes
+import ctypes.util
+
+api = ctypes.CDLL("libgfapi.so")
+api.glfs_get_volfile.argtypes = [ctypes.c_void_p,
+                                 ctypes.c_void_p,
+                                 ctypes.c_ulong]
+api.glfs_get_volfile.restype = ctypes.c_long
+
+
+def get_volfile(host, volume):
+    # This is set to a large value to exercise the "buffer not big enough"
+    # path.  More realistically, you'd just start with a huge buffer.
+    BUF_LEN = 0
+    fs = api.glfs_new(volume)
+    # api.glfs_set_logging(fs,"/dev/stderr",7)
+    api.glfs_set_volfile_server(fs, "tcp", host, 24007)
+    api.glfs_init(fs)
+    vbuf = ctypes.create_string_buffer(BUF_LEN)
+    vlen = api.glfs_get_volfile(fs, vbuf, BUF_LEN)
+    if vlen < 0:
+        vlen = BUF_LEN - vlen
+        vbuf = ctypes.create_string_buffer(vlen)
+        vlen = api.glfs_get_volfile(fs, vbuf, vlen)
+    api.glfs_fini(fs)
+    if vlen <= 0:
+        return vlen
+    return vbuf.value[:vlen]
+
+if __name__ == "__main__":
+    import sys
+
+    try:
+        res = get_volfile(*sys.argv[1:3])
+    except:
+        print("fetching volfile failed (volume not started?)")
+
+    try:
+        for line in res.split('\n'):
+            print(line)
+    except:
+        print("bad return value %s" % res)
diff --git a/api/examples/gfapi.py b/api/examples/gfapi.py
deleted file mode 100755
index 8d9ed732947..00000000000
--- a/api/examples/gfapi.py
+++ /dev/null
@@ -1,419 +0,0 @@
-#!/usr/bin/python
-
-from ctypes import *
-import os
-import sys
-import time
-import types
-
-# Looks like ctypes is having trouble with dependencies, so just force them to
-# load with RTLD_GLOBAL until I figure that out.
-glfs = CDLL("libglusterfs.so",RTLD_GLOBAL)
-xdr = CDLL("libgfxdr.so",RTLD_GLOBAL)
-api = CDLL("libgfapi.so",RTLD_GLOBAL)
-
-# Wow, the Linux kernel folks really play nasty games with this structure.  If
-# you look at the man page for stat(2) and then at this definition you'll note
-# two discrepancies.  First, we seem to have st_nlink and st_mode reversed.  In
-# fact that's exactly how they're defined *for 64-bit systems*; for 32-bit
-# they're in the man-page order.  Even uglier, the man page makes no mention of
-# the *nsec fields, but they are very much present and if they're not included
-# then we get memory corruption because libgfapi has a structure definition
-# that's longer than ours and they overwrite some random bit of memory after
-# the space we allocated.  Yes, that's all very disgusting, and I'm still not
-# sure this will really work on 32-bit because all of the field types are so
-# obfuscated behind macros and feature checks.
-class Stat (Structure):
-        _fields_ = [
-                ("st_dev",              c_ulong),
-                ("st_ino",              c_ulong),
-                ("st_nlink",            c_ulong),
-                ("st_mode",             c_uint),
-                ("st_uid",              c_uint),
-                ("st_gid",              c_uint),
-                ("st_rdev",             c_ulong),
-                ("st_size",             c_ulong),
-                ("st_blksize",          c_ulong),
-                ("st_blocks",           c_ulong),
-                ("st_atime",            c_ulong),
-                ("st_atimensec",        c_ulong),
-                ("st_mtime",            c_ulong),
-                ("st_mtimensec",        c_ulong),
-                ("st_ctime",            c_ulong),
-                ("st_ctimensec",        c_ulong),
-        ]
-api.glfs_creat.restype = c_void_p
-api.glfs_open.restype = c_void_p
-api.glfs_lstat.restype = c_int
-api.glfs_lstat.argtypes = [c_void_p, c_char_p, POINTER(Stat)]
-
-class Dirent (Structure):
-        _fields_ = [
-                ("d_ino",       c_ulong),
-                ("d_off",       c_ulong),
-                ("d_reclen",    c_ushort),
-                ("d_type",      c_char),
-                ("d_name",      c_char * 256),
-        ]
-api.glfs_opendir.restype = c_void_p
-api.glfs_readdir_r.restype = c_int
-api.glfs_readdir_r.argtypes = [c_void_p, POINTER(Dirent),
-                               POINTER(POINTER(Dirent))]
-
-# There's a bit of ctypes glitchiness around __del__ functions and module-level
-# variables.  If we unload the module while we still have references to File or
-# Volume objects, the module-level variables might have disappeared by the time
-# __del__ gets called.  Therefore the objects hold references which they
-# release when __del__ is done.  We only actually use the object-local values
-# in __del__; for clarity, we just use the simpler module-level form elsewhere.
-
-class File(object):
-
-        def __init__ (self, fd):
-                # Add a reference so the module-level variable "api" doesn't
-                # get yanked out from under us (see comment above File def'n).
-                self._api = api
-                self.fd = fd
-
-        def __del__ (self):
-                self._api.glfs_close(self.fd)
-                self._api = None
-
-        # File operations, in alphabetical order.
-
-        def fsync (self):
-                return api.glfs_fsync(self.fd)
-
-        def read (self, buflen, flags=0):
-                rbuf = create_string_buffer(buflen)
-                rc = api.glfs_read(self.fd,rbuf,buflen,flags)
-                if rc > 0:
-                        return rbuf.value[:rc]
-                else:
-                        return rc
-
-        def read_buffer (self, buf, flags=0):
-                return api.glfs_read(self.fd,buf,len(buf),flags)
-
-        def write (self, data, flags=0):
-                return api.glfs_write(self.fd,data,len(data),flags)
-
-	def fallocate (self, mode, offset, len):
-		return api.glfs_fallocate(self.fd, mode, offset, len)
-	def discard (self, offset, len):
-		return api.glfs_discard(self.fd, offset, len)
-
-class Dir(object):
-
-        def __init__ (self, fd):
-                # Add a reference so the module-level variable "api" doesn't
-                # get yanked out from under us (see comment above File def'n).
-                self._api = api
-                self.fd = fd
-                self.cursor = POINTER(Dirent)()
-
-        def __del__ (self):
-                self._api.glfs_closedir(self.fd)
-                self._api = None
-
-        def next (self):
-                entry = Dirent()
-                entry.d_reclen = 256
-                rc = api.glfs_readdir_r(self.fd,byref(entry),byref(self.cursor))
-                if (rc < 0) or (not self.cursor) or (not self.cursor.contents):
-                        return rc
-                return entry
-
-class Volume(object):
-
-        # Housekeeping functions.
-
-        def __init__ (self, host, volid, proto="tcp", port=24007):
-                # Add a reference so the module-level variable "api" doesn't
-                # get yanked out from under us (see comment above File def'n).
-                self._api = api
-                self.fs = api.glfs_new(volid)
-                api.glfs_set_volfile_server(self.fs,proto,host,port)
-
-        def __del__ (self):
-                self._api.glfs_fini(self.fs)
-                self._api = None
-
-        def set_logging (self, path, level):
-                api.glfs_set_logging(self.fs,path,level)
-
-        def mount (self):
-                api.glfs_init(self.fs)
-
-        # File operations, in alphabetical order.
-
-        def creat (self, path, flags, mode):
-                fd = api.glfs_creat(self.fs,path,flags,mode)
-                if not fd:
-                        return fd
-                return File(fd)
-
-        def getxattr (self, path, key, maxlen):
-                buf = create_string_buffer(maxlen)
-                rc = api.glfs_getxattr(self.fs,path,key,buf,maxlen)
-                if rc < 0:
-                        return rc
-                return buf.value[:rc]
-
-        def listxattr (self, path):
-                buf = create_string_buffer(512)
-                rc = api.glfs_listxattr(self.fs,path,buf,512)
-                if rc < 0:
-                        return rc
-                xattrs = []
-                # Parsing character by character is ugly, but it seems like the
-                # easiest way to deal with the "strings separated by NUL in one
-                # buffer" format.
-                i = 0
-                while i < rc:
-                        new_xa = buf.raw[i]
-                        i += 1
-                        while i < rc:
-                                next_char = buf.raw[i]
-                                i += 1
-                                if next_char == '\0':
-                                        xattrs.append(new_xa)
-                                        break
-                                new_xa += next_char
-                xattrs.sort()
-                return xattrs
-
-        def lstat (self, path):
-                x = Stat()
-                rc = api.glfs_lstat(self.fs,path,byref(x))
-                if rc >= 0:
-                        return x
-                else:
-                        return rc
-
-        def mkdir (self, path):
-                return api.glfs_mkdir(self.fs,path)
-
-        def open (self, path, flags):
-                fd = api.glfs_open(self.fs,path,flags)
-                if not fd:
-                        return fd
-                return File(fd)
-
-        def opendir (self, path):
-                fd = api.glfs_opendir(self.fs,path)
-                if not fd:
-                        return fd
-                return Dir(fd)
-
-        def rename (self, opath, npath):
-                return api.glfs_rename(self.fs,opath,npath)
-
-        def rmdir (self, path):
-                return api.glfs_rmdir(self.fs,path)
-
-        def setxattr (self, path, key, value, vlen):
-                return api.glfs_setxattr(self.fs,path,key,value,vlen,0)
-
-        def unlink (self, path):
-                return api.glfs_unlink(self.fs,path)
-
-if __name__ == "__main__":
-        def test_create_write (vol, path, data):
-                mypath = path + ".io"
-                fd = vol.creat(mypath,os.O_WRONLY|os.O_EXCL,0644)
-                if not fd:
-                        return False, "creat error"
-                rc = fd.write(data)
-                if rc != len(data):
-                        return False, "wrote %d/%d bytes" % (rc, len(data))
-                return True, "wrote %d bytes" % rc
-
-        # TBD: this test fails if we do create, open, write, read
-        def test_open_read (vol, path, data):
-                mypath = path + ".io"
-                fd = vol.open(mypath,os.O_RDONLY)
-                if not fd:
-                        return False, "open error"
-                dlen = len(data) * 2
-                buf = fd.read(dlen)
-                if type(buf) == types.IntType:
-                        return False, "read error %d" % buf
-                if len(buf) != len(data):
-                        return False, "read %d/%d bytes" % (len(buf), len(data))
-                return True, "read '%s'" % buf
-
-        def test_lstat (vol, path, data):
-                mypath = path + ".io"
-                sb = vol.lstat(mypath)
-                if type(sb) == types.IntType:
-                        return False, "lstat error %d" % sb
-                if sb.st_size != len(data):
-                        return False, "lstat size is %d, expected %d" % (
-                                sb.st_size, len(data))
-                return True, "lstat got correct size %d" % sb.st_size
-
-        def test_rename (vol, path, data):
-                opath = path + ".io"
-                npath = path + ".tmp"
-                rc = vol.rename(opath,npath)
-                if rc < 0:
-                        return False, "rename error %d" % rc
-                ofd = vol.open(opath,os.O_RDWR)
-                if isinstance(ofd,File):
-                        return False, "old path working after rename"
-                nfd = vol.open(npath,os.O_RDWR)
-                if isinstance(nfd,File):
-                        return False, "new path not working after rename"
-                return True, "rename worked"
-
-        def test_unlink (vol, path, data):
-                mypath = path + ".tmp"
-                rc = vol.unlink(mypath)
-                if rc < 0:
-                        return False, "unlink error %d" % fd
-                fd = vol.open(mypath,os.O_RDWR)
-                if isinstance(fd,File):
-                        return False, "path still usable after unlink"
-                return True, "unlink worked"
-
-        def test_mkdir (vol, path, data):
-                mypath = path + ".dir"
-                rc = vol.mkdir(mypath)
-                if rc < 0:
-                        return False, "mkdir error %d" % rc
-                return True, "mkdir worked"
-
-        def test_create_in_dir (vol, path, data):
-                mypath = path + ".dir/probe"
-                fd = vol.creat(mypath,os.O_RDWR,0644)
-                if not isinstance(fd,File):
-                        return False, "create (in dir) error"
-                return True, "create (in dir) worked"
-
-        def test_dir_listing (vol, path, data):
-                mypath = path + ".dir"
-                fd = vol.opendir(mypath)
-                if not isinstance(fd,Dir):
-                        return False, "opendir error %d" % fd
-                files = []
-                while True:
-                        ent = fd.next()
-                        if not isinstance(ent,Dirent):
-                                break
-                        name = ent.d_name[:ent.d_reclen]
-                        files.append(name)
-                if files != [".", "..", "probe"]:
-                        return False, "wrong directory contents"
-                return True, "directory listing worked"
-
-        def test_unlink_in_dir (vol, path, data):
-                mypath = path + ".dir/probe"
-                rc = vol.unlink(mypath)
-                if rc < 0:
-                        return False, "unlink (in dir) error %d" % rc
-                return True, "unlink (in dir) worked"
-
-        def test_rmdir (vol, path, data):
-                mypath = path + ".dir"
-                rc = vol.rmdir(mypath)
-                if rc < 0:
-                        return False, "rmdir error %d" % rc
-                sb = vol.lstat(mypath)
-                if not isinstance(sb,Stat):
-                        return False, "dir still there after rmdir"
-                return True, "rmdir worked"
-
-        def test_setxattr (vol, path, data):
-                mypath = path + ".xa"
-                fd = vol.creat(mypath,os.O_RDWR|os.O_EXCL,0644)
-                if not fd:
-                        return False, "creat (xattr test) error"
-                key1, key2 = "hello", "goodbye"
-                if vol.setxattr(mypath,"trusted.key1",key1,len(key1)) < 0:
-                        return False, "setxattr (key1) error"
-                if vol.setxattr(mypath,"trusted.key2",key2,len(key2)) < 0:
-                        return False, "setxattr (key2) error"
-                return True, "setxattr worked"
-
-        def test_getxattr (vol, path, data):
-                mypath = path + ".xa"
-                buf = vol.getxattr(mypath,"trusted.key1",32)
-                if type(buf) == types.IntType:
-                        return False, "getxattr error"
-                if buf != "hello":
-                        return False, "wrong getxattr value %s" % buf
-                return True, "getxattr worked"
-
-        def test_listxattr (vol, path, data):
-                mypath = path + ".xa"
-                xattrs = vol.listxattr(mypath)
-                if type(xattrs) == types.IntType:
-                        return False, "listxattr error"
-                if xattrs != ["trusted.key1","trusted.key2"]:
-                        return False, "wrong listxattr value %s" % repr(xattrs)
-                return True, "listxattr worked"
-
-        def test_fallocate (vol, path, data):
-                mypath = path + ".io"
-                fd = vol.creat(mypath,os.O_WRONLY|os.O_EXCL,0644)
-                if not fd:
-                        return False, "creat error"
-		rc = fd.fallocate(0, 0, 1024*1024)
-                if rc != 0:
-                        return False, "fallocate error"
-		rc = fd.discard(4096, 4096)
-		if rc != 0:
-			return False, "discard error"
-                return True, "fallocate/discard worked"
-
-        test_list = (
-                test_create_write,
-                test_open_read,
-                test_lstat,
-                test_rename,
-                test_unlink,
-                test_mkdir,
-                test_create_in_dir,
-                test_dir_listing,
-                test_unlink_in_dir,
-                test_rmdir,
-                test_setxattr,
-                test_getxattr,
-                test_listxattr,
-		test_fallocate,
-        )
-
-        ok_to_fail = (
-                # TBD: this fails opening the new file, even though the file
-                # did get renamed.  Looks like a gfapi bug, not ours.
-                (test_rename, "new path not working after rename"),
-                # TBD: similar, call returns error even though it worked
-                (test_rmdir, "dir still there after rmdir"),
-        )
-
-        volid, path = sys.argv[1:3]
-        data = "fubar"
-        vol = Volume("localhost",volid)
-        vol.set_logging("/dev/null",7)
-        #vol.set_logging("/dev/stderr",7)
-        vol.mount()
-
-        failures = 0
-        expected = 0
-        for t in test_list:
-                rc, msg = t(vol,path,data)
-                if rc:
-                        print "PASS: %s" % msg
-                else:
-                        print "FAIL: %s" % msg
-                        failures += 1
-                        for otf in ok_to_fail:
-                                if (t == otf[0]) and (msg == otf[1]):
-                                        print "  (skipping known failure)"
-                                        expected += 1
-                                        break # from the *inner* for loop
-                        else:
-                                break # from the *outer* for loop
-
-        print "%d failures (%d expected)" % (failures, expected)
diff --git a/api/examples/glfsxmp.c b/api/examples/glfsxmp.c
index 644793a4b1a..a55616ef739 100644
--- a/api/examples/glfsxmp.c
+++ b/api/examples/glfsxmp.c
@@ -1,247 +1,1811 @@
 #include <stdio.h>
+#include <stdlib.h>
 #include <errno.h>
-#include "api/glfs.h"
+#include <glusterfs/api/glfs.h>
+#include <glusterfs/api/glfs-handles.h>
 #include <string.h>
 #include <time.h>
 
+#define TEST_STR_LEN 2048
 
 int
-test_dirops (glfs_t *fs)
+test_dirops(glfs_t *fs)
 {
-	glfs_fd_t *fd = NULL;
-	char buf[512];
-	struct dirent *entry = NULL;
+    glfs_fd_t *fd = NULL;
+    char buf[512];
+    struct dirent *entry = NULL;
+
+    fd = glfs_opendir(fs, "/");
+    if (!fd) {
+        fprintf(stderr, "/: %s\n", strerror(errno));
+        return -1;
+    }
+
+    fprintf(stderr, "Entries:\n");
+    while (glfs_readdir_r(fd, (struct dirent *)buf, &entry), entry) {
+        fprintf(stderr, "%s: %lu\n", entry->d_name, glfs_telldir(fd));
+    }
+
+    glfs_closedir(fd);
+    return 0;
+}
 
-	fd = glfs_opendir (fs, "/");
-	if (!fd) {
-		fprintf (stderr, "/: %s\n", strerror (errno));
-		return -1;
-	}
+int
+test_xattr(glfs_t *fs)
+{
+    char *filename = "/filename2";
+    char *linkfile = "/linkfile";
+    glfs_fd_t *fd = NULL;
+    char buf[512];
+    char *ptr;
+    int ret;
+
+    ret = glfs_setxattr(fs, filename, "user.testkey", "testval", 8, 0);
+    fprintf(stderr, "setxattr(%s): %d (%s)\n", filename, ret, strerror(errno));
+
+    ret = glfs_setxattr(fs, filename, "user.testkey2", "testval", 8, 0);
+    fprintf(stderr, "setxattr(%s): %d (%s)\n", filename, ret, strerror(errno));
+
+    ret = glfs_getxattr(fs, filename, "user.testkey", buf, 512);
+    fprintf(stderr, "getxattr(%s): %d (%s)\n", filename, ret, strerror(errno));
+    if (ret < 0)
+        return -1;
+
+    ret = glfs_listxattr(fs, filename, buf, 512);
+    fprintf(stderr, "listxattr(%s): %d (%s)\n", filename, ret, strerror(errno));
+    if (ret < 0)
+        return -1;
+
+    ret = glfs_symlink(fs, "filename", linkfile);
+    fprintf(stderr, "symlink(%s %s): %s\n", filename, linkfile,
+            strerror(errno));
+    if (ret < 0)
+        return -1;
+
+    ret = glfs_readlink(fs, linkfile, buf, 512);
+    fprintf(stderr, "readlink(%s) : %d (%s)\n", filename, ret, strerror(errno));
+    if (ret < 0)
+        return -1;
+
+    ret = glfs_lsetxattr(fs, filename, "user.testkey3", "testval", 8, 0);
+    fprintf(stderr, "lsetxattr(%s) : %d (%s)\n", linkfile, ret,
+            strerror(errno));
+    if (ret < 0)
+        return -1;
+
+    ret = glfs_llistxattr(fs, linkfile, buf, 512);
+    fprintf(stderr, "llistxattr(%s): %d (%s)\n", filename, ret,
+            strerror(errno));
+    if (ret < 0)
+        return -1;
+
+    ret = glfs_lgetxattr(fs, filename, "user.testkey3", buf, 512);
+    fprintf(stderr, "lgetxattr(%s): %d (%s)\n", linkfile, ret, strerror(errno));
+    if (ret < 0)
+        return -1;
+
+    for (ptr = buf; ptr < buf + ret; ptr++) {
+        printf("key=%s\n", ptr);
+        ptr += strlen(ptr);
+    }
+
+    ret = glfs_removexattr(fs, filename, "user.testkey2");
+    fprintf(stderr, "removexattr(%s): %d (%s)\n", filename, ret,
+            strerror(errno));
+
+    fd = glfs_open(fs, filename, O_RDWR);
+    fprintf(stderr, "open(%s): (%p) %s\n", filename, fd, strerror(errno));
+
+    ret = glfs_fsetxattr(fd, "user.testkey2", "testval", 8, 0);
+    fprintf(stderr, "fsetxattr(%s): %d (%s)\n", filename, ret, strerror(errno));
+
+    ret = glfs_fgetxattr(fd, "user.testkey2", buf, 512);
+    fprintf(stderr, "fgetxattr(%s): %d (%s)\n", filename, ret, strerror(errno));
+
+    ret = glfs_flistxattr(fd, buf, 512);
+    fprintf(stderr, "flistxattr(%s): %d (%s)\n", filename, ret,
+            strerror(errno));
+    if (ret < 0)
+        return -1;
+
+    for (ptr = buf; ptr < buf + ret; ptr++) {
+        printf("key=%s\n", ptr);
+        ptr += strlen(ptr);
+    }
+
+    ret = glfs_fremovexattr(fd, "user.testkey2");
+    fprintf(stderr, "fremovexattr(%s): %d (%s)\n", filename, ret,
+            strerror(errno));
+
+    glfs_close(fd);
+
+    return 0;
+}
 
-	fprintf (stderr, "Entries:\n");
-	while (glfs_readdir_r (fd, (struct dirent *)buf, &entry), entry) {
-		fprintf (stderr, "%s: %lu\n", entry->d_name, glfs_telldir (fd));
-	}
+int
+test_chdir(glfs_t *fs)
+{
+    int ret = -1;
+    char *dir = "/dir";
+    char *topdir = "/topdir";
+    char *linkdir = "/linkdir";
+    char *linkdir2 = "/linkdir2";
+    char *subdir = "./subdir";
+    char *respath = NULL;
+    char pathbuf[4096];
+
+    ret = glfs_mkdir(fs, topdir, 0755);
+    fprintf(stderr, "mkdir(%s): %s\n", topdir, strerror(errno));
+    if (ret)
+        return -1;
+
+    ret = glfs_mkdir(fs, dir, 0755);
+    fprintf(stderr, "mkdir(%s): %s\n", dir, strerror(errno));
+    if (ret)
+        return -1;
+
+    respath = glfs_getcwd(fs, pathbuf, 4096);
+    fprintf(stdout, "getcwd() = %s\n", respath);
+
+    ret = glfs_symlink(fs, "topdir", linkdir);
+    if (ret) {
+        fprintf(stderr, "symlink(%s, %s): %s\n", topdir, linkdir,
+                strerror(errno));
+        return -1;
+    }
+
+    ret = glfs_chdir(fs, linkdir);
+    if (ret) {
+        fprintf(stderr, "chdir(%s): %s\n", linkdir, strerror(errno));
+        return -1;
+    }
+
+    respath = glfs_getcwd(fs, pathbuf, 4096);
+    fprintf(stdout, "getcwd() = %s\n", respath);
+
+    respath = glfs_realpath(fs, subdir, pathbuf);
+    if (respath) {
+        fprintf(stderr, "realpath(%s) worked unexpectedly: %s\n", subdir,
+                respath);
+        return -1;
+    }
+
+    ret = glfs_mkdir(fs, subdir, 0755);
+    if (ret) {
+        fprintf(stderr, "mkdir(%s): %s\n", subdir, strerror(errno));
+        return -1;
+    }
+
+    respath = glfs_realpath(fs, subdir, pathbuf);
+    if (!respath) {
+        fprintf(stderr, "realpath(%s): %s\n", subdir, strerror(errno));
+    } else {
+        fprintf(stdout, "realpath(%s) = %s\n", subdir, respath);
+    }
+
+    ret = glfs_chdir(fs, subdir);
+    if (ret) {
+        fprintf(stderr, "chdir(%s): %s\n", subdir, strerror(errno));
+        return -1;
+    }
+
+    respath = glfs_getcwd(fs, pathbuf, 4096);
+    fprintf(stdout, "getcwd() = %s\n", respath);
+
+    respath = glfs_realpath(fs, "/linkdir/subdir", pathbuf);
+    if (!respath) {
+        fprintf(stderr, "realpath(/linkdir/subdir): %s\n", strerror(errno));
+    } else {
+        fprintf(stdout, "realpath(/linkdir/subdir) = %s\n", respath);
+    }
+
+    return 0;
+}
 
-	glfs_closedir (fd);
-	return 0;
+#ifdef DEBUG
+static void
+peek_stat(struct stat *sb)
+{
+    printf("Dumping stat information:\n");
+    printf("File type:                ");
+
+    switch (sb->st_mode & S_IFMT) {
+        case S_IFBLK:
+            printf("block device\n");
+            break;
+        case S_IFCHR:
+            printf("character device\n");
+            break;
+        case S_IFDIR:
+            printf("directory\n");
+            break;
+        case S_IFIFO:
+            printf("FIFO/pipe\n");
+            break;
+        case S_IFLNK:
+            printf("symlink\n");
+            break;
+        case S_IFREG:
+            printf("regular file\n");
+            break;
+        case S_IFSOCK:
+            printf("socket\n");
+            break;
+        default:
+            printf("unknown?\n");
+            break;
+    }
+
+    printf("I-node number:            %ld\n", (long)sb->st_ino);
+
+    printf("Mode:                     %lo (octal)\n",
+           (unsigned long)sb->st_mode);
+
+    printf("Link count:               %ld\n", (long)sb->st_nlink);
+    printf("Ownership:                UID=%ld   GID=%ld\n", (long)sb->st_uid,
+           (long)sb->st_gid);
+
+    printf("Preferred I/O block size: %ld bytes\n", (long)sb->st_blksize);
+    printf("File size:                %lld bytes\n", (long long)sb->st_size);
+    printf("Blocks allocated:         %lld\n", (long long)sb->st_blocks);
+
+    printf("Last status change:       %s", ctime(&sb->st_ctime));
+    printf("Last file access:         %s", ctime(&sb->st_atime));
+    printf("Last file modification:   %s", ctime(&sb->st_mtime));
+
+    return;
 }
 
+static void
+peek_handle(unsigned char *glid)
+{
+    int i;
 
-int
-test_xattr (glfs_t *fs)
+    for (i = 0; i < GFAPI_HANDLE_LENGTH; i++) {
+        printf(":%02x:", glid[i]);
+    }
+    printf("\n");
+}
+#else  /* DEBUG */
+static void
+peek_stat(struct stat *sb)
 {
-	char *filename = "/filename2";
-	char buf[512];
-	char *ptr;
-	int ret;
+    return;
+}
 
-	ret = glfs_setxattr (fs, filename, "user.testkey", "testval", 8, 0);
-	fprintf (stderr, "setxattr(%s): %d (%s)\n", filename, ret,
-		 strerror (errno));
+static void
+peek_handle(unsigned char *id)
+{
+    return;
+}
+#endif /* DEBUG */
 
-	ret = glfs_setxattr (fs, filename, "user.testkey2", "testval", 8, 0);
-	fprintf (stderr, "setxattr(%s): %d (%s)\n", filename, ret,
-		 strerror (errno));
+glfs_t *fs = NULL;
+char *full_parent_name = "/testdir", *parent_name = "testdir";
 
-	ret = glfs_listxattr (fs, filename, buf, 512);
-	fprintf (stderr, "listxattr(%s): %d (%s)\n", filename, ret,
-		 strerror (errno));
-	if (ret < 0)
-		return -1;
+void
+test_h_unlink(void)
+{
+    char *my_dir = "unlinkdir";
+    char *my_file = "file.txt";
+    char *my_subdir = "dir1";
+    struct glfs_object *parent = NULL, *leaf = NULL, *dir = NULL,
+                       *subdir = NULL, *subleaf = NULL;
+    struct stat sb;
+    int ret;
+
+    printf("glfs_h_unlink tests: In Progress\n");
+
+    /* Prepare tests */
+    parent = glfs_h_lookupat(fs, NULL, full_parent_name, &sb, 0);
+    if (parent == NULL) {
+        fprintf(stderr,
+                "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                full_parent_name, NULL, strerror(errno));
+        printf("glfs_h_lookupat tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    dir = glfs_h_mkdir(fs, parent, my_dir, 0755, &sb);
+    if (dir == NULL) {
+        fprintf(stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
+                my_dir, parent, strerror(errno));
+        printf("glfs_h_unlink tests: FAILED\n");
+        goto out;
+    }
+
+    leaf = glfs_h_creat(fs, dir, my_file, O_CREAT, 0644, &sb);
+    if (leaf == NULL) {
+        fprintf(stderr, "glfs_h_creat: error creating %s: from (%p),%s\n",
+                my_file, dir, strerror(errno));
+        printf("glfs_h_unlink tests: FAILED\n");
+        goto out;
+    }
+
+    subdir = glfs_h_mkdir(fs, dir, my_subdir, 0755, &sb);
+    if (subdir == NULL) {
+        fprintf(stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
+                my_subdir, dir, strerror(errno));
+        printf("glfs_h_unlink tests: FAILED\n");
+        goto out;
+    }
+
+    subleaf = glfs_h_creat(fs, subdir, my_file, O_CREAT, 0644, &sb);
+    if (subleaf == NULL) {
+        fprintf(stderr, "glfs_h_creat: error creating %s: from (%p),%s\n",
+                my_file, subdir, strerror(errno));
+        printf("glfs_h_unlink tests: FAILED\n");
+        goto out;
+    }
+
+    /* unlink non empty directory */
+    ret = glfs_h_unlink(fs, dir, my_subdir);
+    if ((ret && errno != ENOTEMPTY) || (ret == 0)) {
+        fprintf(stderr,
+                "glfs_h_unlink: error unlinking %s: it is non empty: %s\n",
+                my_subdir, strerror(errno));
+        printf("glfs_h_unlink tests: FAILED\n");
+        goto out;
+    }
+
+    /* unlink regular file */
+    ret = glfs_h_unlink(fs, subdir, my_file);
+    if (ret) {
+        fprintf(stderr, "glfs_h_unlink: error unlinking %s: from (%p),%s\n",
+                my_file, subdir, strerror(errno));
+        printf("glfs_h_unlink tests: FAILED\n");
+        goto out;
+    }
+
+    /* unlink directory */
+    ret = glfs_h_unlink(fs, dir, my_subdir);
+    if (ret) {
+        fprintf(stderr, "glfs_h_unlink: error unlinking %s: from (%p),%s\n",
+                my_subdir, dir, strerror(errno));
+        printf("glfs_h_unlink tests: FAILED\n");
+        goto out;
+    }
+
+    /* unlink regular file */
+    ret = glfs_h_unlink(fs, dir, my_file);
+    if (ret) {
+        fprintf(stderr, "glfs_h_unlink: error unlinking %s: from (%p),%s\n",
+                my_file, dir, strerror(errno));
+        printf("glfs_h_unlink tests: FAILED\n");
+        goto out;
+    }
+
+    /* unlink non-existent regular file */
+    ret = glfs_h_unlink(fs, dir, my_file);
+    if ((ret && errno != ENOENT) || (ret == 0)) {
+        fprintf(stderr,
+                "glfs_h_unlink: error unlinking non-existent %s: invalid errno "
+                ",%d, %s\n",
+                my_file, ret, strerror(errno));
+        printf("glfs_h_unlink tests: FAILED\n");
+        goto out;
+    }
+
+    /* unlink non-existent directory */
+    ret = glfs_h_unlink(fs, dir, my_subdir);
+    if ((ret && errno != ENOENT) || (ret == 0)) {
+        fprintf(stderr,
+                "glfs_h_unlink: error unlinking non-existent %s:  invalid "
+                "errno ,%d, %s\n",
+                my_subdir, ret, strerror(errno));
+        printf("glfs_h_unlink tests: FAILED\n");
+        goto out;
+    }
+
+    /* unlink directory */
+    ret = glfs_h_unlink(fs, parent, my_dir);
+    if (ret) {
+        fprintf(stderr, "glfs_h_unlink: error unlinking %s: from (%p),%s\n",
+                my_dir, dir, strerror(errno));
+        printf("glfs_h_unlink tests: FAILED\n");
+        goto out;
+    }
+
+    printf("glfs_h_unlink tests: PASSED\n");
+
+out:
+    if (dir)
+        glfs_h_close(dir);
+    if (leaf)
+        glfs_h_close(leaf);
+    if (subdir)
+        glfs_h_close(subdir);
+    if (subleaf)
+        glfs_h_close(subleaf);
+    if (parent)
+        glfs_h_close(parent);
+
+    return;
+}
 
-	for (ptr = buf; ptr < buf + ret; ptr++) {
-		printf ("key=%s\n", ptr);
-		ptr += strlen (ptr);
-	}
+void
+test_h_getsetattrs(void)
+{
+    char *my_dir = "attrdir";
+    char *my_file = "attrfile.txt";
+    struct glfs_object *parent = NULL, *leaf = NULL, *dir = NULL;
+    struct stat sb, retsb;
+    int ret, valid;
+    struct timespec timestamp;
+
+    printf("glfs_h_getattrs and setattrs tests: In Progress\n");
+
+    /* Prepare tests */
+    parent = glfs_h_lookupat(fs, NULL, full_parent_name, &sb, 0);
+    if (parent == NULL) {
+        fprintf(stderr,
+                "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                full_parent_name, NULL, strerror(errno));
+        printf("glfs_h_lookupat tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    dir = glfs_h_mkdir(fs, parent, my_dir, 0755, &sb);
+    if (dir == NULL) {
+        fprintf(stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
+                my_dir, parent, strerror(errno));
+        printf("glfs_h_unlink tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    leaf = glfs_h_creat(fs, dir, my_file, O_CREAT, 0644, &sb);
+    if (leaf == NULL) {
+        fprintf(stderr, "glfs_h_creat: error creating %s: from (%p),%s\n",
+                my_file, dir, strerror(errno));
+        printf("glfs_h_unlink tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    ret = glfs_h_getattrs(fs, dir, &retsb);
+    if (ret != 0) {
+        fprintf(stderr, "glfs_h_getattrs: error %s: from (%p),%s\n", my_dir,
+                dir, strerror(errno));
+        printf("glfs_h_getattrs and setattrs tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&retsb);
+    /* TODO: Compare stat information */
+
+    retsb.st_mode = 00666;
+    retsb.st_uid = 1000;
+    retsb.st_gid = 1001;
+    ret = clock_gettime(CLOCK_REALTIME, &timestamp);
+    if (ret != 0) {
+        fprintf(stderr, "clock_gettime: error %s\n", strerror(errno));
+        printf("glfs_h_getattrs and setattrs tests: FAILED\n");
+        goto out;
+    }
+    retsb.st_atim = timestamp;
+    retsb.st_mtim = timestamp;
+    valid = GFAPI_SET_ATTR_MODE | GFAPI_SET_ATTR_UID | GFAPI_SET_ATTR_GID |
+            GFAPI_SET_ATTR_ATIME | GFAPI_SET_ATTR_MTIME;
+    peek_stat(&retsb);
+
+    ret = glfs_h_setattrs(fs, dir, &retsb, valid);
+    if (ret != 0) {
+        fprintf(stderr, "glfs_h_setattrs: error %s: from (%p),%s\n", my_dir,
+                dir, strerror(errno));
+        printf("glfs_h_getattrs and setattrs tests: FAILED\n");
+        goto out;
+    }
+
+    memset(&retsb, 0, sizeof(struct stat));
+    ret = glfs_h_stat(fs, dir, &retsb);
+    if (ret != 0) {
+        fprintf(stderr, "glfs_h_stat: error %s: from (%p),%s\n", my_dir, dir,
+                strerror(errno));
+        printf("glfs_h_getattrs and setattrs tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&retsb);
+
+    printf("glfs_h_getattrs and setattrs tests: PASSED\n");
+out:
+    if (parent)
+        glfs_h_close(parent);
+    if (leaf)
+        glfs_h_close(leaf);
+    if (dir)
+        glfs_h_close(dir);
+
+    return;
+}
 
-	return 0;
+void
+test_h_truncate(void)
+{
+    char *my_dir = "truncatedir";
+    char *my_file = "file.txt";
+    struct glfs_object *root = NULL, *parent = NULL, *leaf = NULL;
+    struct stat sb;
+    glfs_fd_t *fd = NULL;
+    char buf[32];
+    off_t offset = 0;
+    int ret = 0;
+
+    printf("glfs_h_truncate tests: In Progress\n");
+
+    /* Prepare tests */
+    root = glfs_h_lookupat(fs, NULL, full_parent_name, &sb, 0);
+    if (root == NULL) {
+        fprintf(stderr,
+                "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                full_parent_name, NULL, strerror(errno));
+        printf("glfs_h_truncate tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    parent = glfs_h_mkdir(fs, root, my_dir, 0755, &sb);
+    if (parent == NULL) {
+        fprintf(stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
+                my_dir, root, strerror(errno));
+        printf("glfs_h_truncate tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    leaf = glfs_h_creat(fs, parent, my_file, O_CREAT, 0644, &sb);
+    if (leaf == NULL) {
+        fprintf(stderr, "glfs_h_creat: error creating %s: from (%p),%s\n",
+                my_file, parent, strerror(errno));
+        printf("glfs_h_truncate tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    fd = glfs_h_open(fs, leaf, O_RDWR);
+    if (fd == NULL) {
+        fprintf(stderr, "glfs_h_open: error on open of %s: %s\n", my_file,
+                strerror(errno));
+        printf("glfs_h_truncate tests: FAILED\n");
+        goto out;
+    }
+
+    memcpy(buf, "abcdefghijklmnopqrstuvwxyz012345", 32);
+    ret = glfs_write(fd, buf, 32, 0);
+
+    /* run tests */
+    /* truncate lower */
+    offset = 30;
+    ret = glfs_h_truncate(fs, leaf, offset);
+    if (ret != 0) {
+        fprintf(stderr, "glfs_h_truncate: error creating %s: from (%p),%s\n",
+                my_file, parent, strerror(errno));
+        printf("glfs_h_truncate tests: FAILED\n");
+        goto out;
+    }
+    ret = glfs_h_getattrs(fs, leaf, &sb);
+    if (ret != 0) {
+        fprintf(stderr, "glfs_h_getattrs: error for %s (%p),%s\n", my_file,
+                leaf, strerror(errno));
+        printf("glfs_h_truncate tests: FAILED\n");
+        goto out;
+    }
+    if (sb.st_size != offset) {
+        fprintf(stderr, "glfs_h_truncate: post size mismatch\n");
+        printf("glfs_h_truncate tests: FAILED\n");
+        goto out;
+    }
+
+    /* truncate higher */
+    offset = 32;
+    ret = glfs_h_truncate(fs, leaf, offset);
+    if (ret != 0) {
+        fprintf(stderr, "glfs_h_truncate: error creating %s: from (%p),%s\n",
+                my_file, parent, strerror(errno));
+        printf("glfs_h_truncate tests: FAILED\n");
+        goto out;
+    }
+    ret = glfs_h_getattrs(fs, leaf, &sb);
+    if (ret != 0) {
+        fprintf(stderr, "glfs_h_getattrs: error for %s (%p),%s\n", my_file,
+                leaf, strerror(errno));
+        printf("glfs_h_truncate tests: FAILED\n");
+        goto out;
+    }
+    if (sb.st_size != offset) {
+        fprintf(stderr, "glfs_h_truncate: post size mismatch\n");
+        printf("glfs_h_truncate tests: FAILED\n");
+        goto out;
+    }
+
+    /* truncate equal */
+    offset = 30;
+    ret = glfs_h_truncate(fs, leaf, offset);
+    if (ret != 0) {
+        fprintf(stderr, "glfs_h_truncate: error creating %s: from (%p),%s\n",
+                my_file, parent, strerror(errno));
+        printf("glfs_h_truncate tests: FAILED\n");
+        goto out;
+    }
+    ret = glfs_h_getattrs(fs, leaf, &sb);
+    if (ret != 0) {
+        fprintf(stderr, "glfs_h_getattrs: error for %s (%p),%s\n", my_file,
+                leaf, strerror(errno));
+        printf("glfs_h_truncate tests: FAILED\n");
+        goto out;
+    }
+    if (sb.st_size != offset) {
+        fprintf(stderr, "glfs_h_truncate: post size mismatch\n");
+        printf("glfs_h_truncate tests: FAILED\n");
+        goto out;
+    }
+
+    printf("glfs_h_truncate tests: PASSED\n");
+out:
+    if (fd)
+        glfs_close(fd);
+    if (root)
+        glfs_h_close(root);
+    if (parent)
+        glfs_h_close(parent);
+    if (leaf)
+        glfs_h_close(leaf);
+
+    return;
 }
 
+void
+test_h_links(void)
+{
+    char *my_dir = "linkdir";
+    char *my_file = "file.txt";
+    char *my_symlnk = "slnk.txt";
+    char *my_lnk = "lnk.txt";
+    char *linksrc_dir = "dir1";
+    char *linktgt_dir = "dir2";
+    struct glfs_object *root = NULL, *parent = NULL, *leaf = NULL,
+                       *dirsrc = NULL, *dirtgt = NULL, *dleaf = NULL;
+    struct glfs_object *ln1 = NULL;
+    struct stat sb;
+    int ret;
+    char *buf = NULL;
+
+    printf("glfs_h_link(s) tests: In Progress\n");
+
+    /* Prepare tests */
+    root = glfs_h_lookupat(fs, NULL, full_parent_name, &sb, 0);
+    if (root == NULL) {
+        fprintf(stderr,
+                "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                full_parent_name, NULL, strerror(errno));
+        printf("glfs_h_link(s) tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    parent = glfs_h_mkdir(fs, root, my_dir, 0755, &sb);
+    if (parent == NULL) {
+        fprintf(stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
+                my_dir, root, strerror(errno));
+        printf("glfs_h_link(s) tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    leaf = glfs_h_creat(fs, parent, my_file, O_CREAT, 0644, &sb);
+    if (leaf == NULL) {
+        fprintf(stderr, "glfs_h_creat: error creating %s: from (%p),%s\n",
+                my_file, parent, strerror(errno));
+        printf("glfs_h_link(s) tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    dirsrc = glfs_h_mkdir(fs, parent, linksrc_dir, 0755, &sb);
+    if (dirsrc == NULL) {
+        fprintf(stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
+                linksrc_dir, parent, strerror(errno));
+        printf("glfs_h_link(s) tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    dirtgt = glfs_h_mkdir(fs, parent, linktgt_dir, 0755, &sb);
+    if (dirtgt == NULL) {
+        fprintf(stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
+                linktgt_dir, parent, strerror(errno));
+        printf("glfs_h_link(s) tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    dleaf = glfs_h_creat(fs, dirsrc, my_file, O_CREAT, 0644, &sb);
+    if (dleaf == NULL) {
+        fprintf(stderr, "glfs_h_creat: error creating %s: from (%p),%s\n",
+                my_file, dirsrc, strerror(errno));
+        printf("glfs_h_link(s) tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    /* run tests */
+    /* sym link: /testdir/linkdir/file.txt to ./slnk.txt */
+    ln1 = glfs_h_symlink(fs, parent, my_symlnk, "./file.txt", &sb);
+    if (ln1 == NULL) {
+        fprintf(stderr, "glfs_h_symlink: error creating %s: from (%p),%s\n",
+                my_symlnk, parent, strerror(errno));
+        printf("glfs_h_link(s) tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    buf = calloc(1024, sizeof(char));
+    if (buf == NULL) {
+        fprintf(stderr, "Error allocating memory\n");
+        printf("glfs_h_link(s) tests: FAILED\n");
+        goto out;
+    }
+
+    ret = glfs_h_readlink(fs, ln1, buf, 1024);
+    if (ret <= 0) {
+        fprintf(stderr, "glfs_h_readlink: error reading %s: from (%p),%s\n",
+                my_symlnk, ln1, strerror(errno));
+        printf("glfs_h_link(s) tests: FAILED\n");
+        goto out;
+    }
+    if (!(strncmp(buf, my_symlnk, strlen(my_symlnk)))) {
+        fprintf(stderr,
+                "glfs_h_readlink: error mismatch in link name: actual %s: "
+                "retrieved %s\n",
+                my_symlnk, buf);
+        printf("glfs_h_link(s) tests: FAILED\n");
+        goto out;
+    }
+
+    /* link: /testdir/linkdir/file.txt to ./lnk.txt */
+    ret = glfs_h_link(fs, leaf, parent, my_lnk);
+    if (ret != 0) {
+        fprintf(stderr, "glfs_h_link: error creating %s: from (%p),%s\n",
+                my_lnk, parent, strerror(errno));
+        printf("glfs_h_link(s) tests: FAILED\n");
+        goto out;
+    }
+    /* TODO: Should write content to a file and read from the link */
+
+    /* link: /testdir/linkdir/dir1/file.txt to ../dir2/slnk.txt */
+    ret = glfs_h_link(fs, dleaf, dirtgt, my_lnk);
+    if (ret != 0) {
+        fprintf(stderr, "glfs_h_link: error creating %s: from (%p),%s\n",
+                my_lnk, dirtgt, strerror(errno));
+        printf("glfs_h_link(s) tests: FAILED\n");
+        goto out;
+    }
+    /* TODO: Should write content to a file and read from the link */
+
+    printf("glfs_h_link(s) tests: PASSED\n");
+
+out:
+    if (root)
+        glfs_h_close(root);
+    if (parent)
+        glfs_h_close(parent);
+    if (leaf)
+        glfs_h_close(leaf);
+    if (dirsrc)
+        glfs_h_close(dirsrc);
+    if (dirtgt)
+        glfs_h_close(dirtgt);
+    if (dleaf)
+        glfs_h_close(dleaf);
+    if (ln1)
+        glfs_h_close(ln1);
+    if (buf)
+        free(buf);
+
+    return;
+}
+
+void
+test_h_rename(void)
+{
+    char *my_dir = "renamedir";
+    char *my_file = "file.txt";
+    char *src_dir = "dir1";
+    char *tgt_dir = "dir2";
+    struct glfs_object *root = NULL, *parent = NULL, *leaf = NULL,
+                       *dirsrc = NULL, *dirtgt = NULL, *dleaf = NULL;
+    struct stat sb;
+    int ret;
+
+    printf("glfs_h_rename tests: In Progress\n");
+
+    /* Prepare tests */
+    root = glfs_h_lookupat(fs, NULL, full_parent_name, &sb, 0);
+    if (root == NULL) {
+        fprintf(stderr,
+                "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                full_parent_name, NULL, strerror(errno));
+        printf("glfs_h_rename tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    parent = glfs_h_mkdir(fs, root, my_dir, 0755, &sb);
+    if (parent == NULL) {
+        fprintf(stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
+                my_dir, root, strerror(errno));
+        printf("glfs_h_rename tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    leaf = glfs_h_creat(fs, parent, my_file, O_CREAT, 0644, &sb);
+    if (leaf == NULL) {
+        fprintf(stderr, "glfs_h_creat: error creating %s: from (%p),%s\n",
+                my_file, parent, strerror(errno));
+        printf("glfs_h_rename tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    dirsrc = glfs_h_mkdir(fs, parent, src_dir, 0755, &sb);
+    if (dirsrc == NULL) {
+        fprintf(stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
+                src_dir, parent, strerror(errno));
+        printf("glfs_h_rename tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    dirtgt = glfs_h_mkdir(fs, parent, tgt_dir, 0755, &sb);
+    if (dirtgt == NULL) {
+        fprintf(stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
+                tgt_dir, parent, strerror(errno));
+        printf("glfs_h_rename tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    dleaf = glfs_h_creat(fs, dirsrc, my_file, O_CREAT, 0644, &sb);
+    if (dleaf == NULL) {
+        fprintf(stderr, "glfs_h_creat: error creating %s: from (%p),%s\n",
+                my_file, dirsrc, strerror(errno));
+        printf("glfs_h_rename tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    /* run tests */
+    /* Rename file.txt -> file1.txt */
+    ret = glfs_h_rename(fs, parent, "file.txt", parent, "file1.txt");
+    if (ret != 0) {
+        fprintf(stderr, "glfs_h_rename: error renaming %s to %s (%s)\n",
+                "file.txt", "file1.txt", strerror(errno));
+        printf("glfs_h_rename tests: FAILED\n");
+        goto out;
+    }
+
+    /* rename dir1/file.txt -> file.txt */
+    ret = glfs_h_rename(fs, dirsrc, "file.txt", parent, "file.txt");
+    if (ret != 0) {
+        fprintf(stderr, "glfs_h_rename: error renaming %s/%s to %s (%s)\n",
+                src_dir, "file.txt", "file.txt", strerror(errno));
+        printf("glfs_h_rename tests: FAILED\n");
+        goto out;
+    }
+
+    /* rename file1.txt -> file.txt (exists) */
+    ret = glfs_h_rename(fs, parent, "file1.txt", parent, "file.txt");
+    if (ret != 0) {
+        fprintf(stderr, "glfs_h_rename: error renaming %s to %s (%s)\n",
+                "file.txt", "file.txt", strerror(errno));
+        printf("glfs_h_rename tests: FAILED\n");
+        goto out;
+    }
+
+    /* rename dir1 -> dir3 */
+    ret = glfs_h_rename(fs, parent, "dir1", parent, "dir3");
+    if (ret != 0) {
+        fprintf(stderr, "glfs_h_rename: error renaming %s to %s (%s)\n", "dir1",
+                "dir3", strerror(errno));
+        printf("glfs_h_rename tests: FAILED\n");
+        goto out;
+    }
+
+    /* rename dir2 ->dir3 (exists) */
+    ret = glfs_h_rename(fs, parent, "dir2", parent, "dir3");
+    if (ret != 0) {
+        fprintf(stderr, "glfs_h_rename: error renaming %s to %s (%s)\n", "dir2",
+                "dir3", strerror(errno));
+        printf("glfs_h_rename tests: FAILED\n");
+        goto out;
+    }
+
+    /* rename file.txt -> dir3 (fail) */
+    ret = glfs_h_rename(fs, parent, "file.txt", parent, "dir3");
+    if (ret == 0) {
+        fprintf(stderr, "glfs_h_rename: NO error renaming %s to %s (%s)\n",
+                "file.txt", "dir3", strerror(errno));
+        printf("glfs_h_rename tests: FAILED\n");
+        goto out;
+    }
+
+    /* rename dir3 -> file.txt (fail) */
+    ret = glfs_h_rename(fs, parent, "dir3", parent, "file.txt");
+    if (ret == 0) {
+        fprintf(stderr, "glfs_h_rename: NO error renaming %s to %s (%s)\n",
+                "dir3", "file.txt", strerror(errno));
+        printf("glfs_h_rename tests: FAILED\n");
+        goto out;
+    }
+
+    printf("glfs_h_rename tests: PASSED\n");
+
+out:
+    if (root)
+        glfs_h_close(root);
+    if (parent)
+        glfs_h_close(parent);
+    if (leaf)
+        glfs_h_close(leaf);
+    if (dirsrc)
+        glfs_h_close(dirsrc);
+    if (dirtgt)
+        glfs_h_close(dirtgt);
+    if (dleaf)
+        glfs_h_close(dleaf);
+
+    return;
+}
+
+void
+assimilatetime(struct timespec *ts, struct timespec ts_st,
+               struct timespec ts_ed)
+{
+    if ((ts_ed.tv_nsec - ts_st.tv_nsec) < 0) {
+        ts->tv_sec += ts_ed.tv_sec - ts_st.tv_sec - 1;
+        ts->tv_nsec += 1000000000 + ts_ed.tv_nsec - ts_st.tv_nsec;
+    } else {
+        ts->tv_sec += ts_ed.tv_sec - ts_st.tv_sec;
+        ts->tv_nsec += ts_ed.tv_nsec - ts_st.tv_nsec;
+    }
+
+    if (ts->tv_nsec > 1000000000) {
+        ts->tv_nsec = ts->tv_nsec - 1000000000;
+        ts->tv_sec += 1;
+    }
+
+    return;
+}
+
+#define MAX_FILES_CREATE 10
+#define MAXPATHNAME 512
+void
+test_h_performance(void)
+{
+    char *my_dir = "perftest", *full_dir_path = "/testdir/perftest";
+    char *my_file = "file_", my_file_name[MAXPATHNAME];
+    struct glfs_object *parent = NULL, *leaf = NULL, *dir = NULL;
+    struct stat sb;
+    int ret, i;
+    struct glfs_fd *fd;
+    struct timespec c_ts = {0, 0}, c_ts_st, c_ts_ed;
+    struct timespec o_ts = {0, 0}, o_ts_st, o_ts_ed;
+
+    printf("glfs_h_performance tests: In Progress\n");
+
+    /* Prepare tests */
+    parent = glfs_h_lookupat(fs, NULL, full_parent_name, &sb, 0);
+    if (parent == NULL) {
+        fprintf(stderr,
+                "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                full_parent_name, NULL, strerror(errno));
+        printf("glfs_h_performance tests: FAILED\n");
+        goto out;
+    }
+
+    dir = glfs_h_mkdir(fs, parent, my_dir, 0755, &sb);
+    if (dir == NULL) {
+        fprintf(stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
+                my_dir, parent, strerror(errno));
+        printf("glfs_h_performance tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    /* create performance */
+    ret = clock_gettime(CLOCK_REALTIME, &o_ts_st);
+    if (ret != 0) {
+        fprintf(stderr, "clock_gettime: error %s\n", strerror(errno));
+        printf("glfs_h_getattrs and setattrs tests: FAILED\n");
+        goto out;
+    }
+
+    for (i = 0; i < MAX_FILES_CREATE; i++) {
+        sprintf(my_file_name, "%s%d", my_file, i);
+
+        ret = clock_gettime(CLOCK_REALTIME, &c_ts_st);
+        if (ret != 0) {
+            fprintf(stderr, "clock_gettime: error %s\n", strerror(errno));
+            printf("glfs_h_getattrs and setattrs tests: FAILED\n");
+            goto out;
+        }
+
+        leaf = glfs_h_lookupat(fs, dir, my_file_name, &sb, 0);
+        if (leaf != NULL) {
+            fprintf(stderr, "glfs_h_lookup: exists %s\n", my_file_name);
+            printf("glfs_h_performance tests: FAILED\n");
+            goto out;
+        }
+
+        leaf = glfs_h_creat(fs, dir, my_file_name, O_CREAT, 0644, &sb);
+        if (leaf == NULL) {
+            fprintf(stderr, "glfs_h_creat: error creating %s: from (%p),%s\n",
+                    my_file, dir, strerror(errno));
+            printf("glfs_h_performance tests: FAILED\n");
+            goto out;
+        }
+
+        ret = clock_gettime(CLOCK_REALTIME, &c_ts_ed);
+        if (ret != 0) {
+            fprintf(stderr, "clock_gettime: error %s\n", strerror(errno));
+            printf("glfs_h_getattrs and setattrs tests: FAILED\n");
+            goto out;
+        }
+
+        assimilatetime(&c_ts, c_ts_st, c_ts_ed);
+        glfs_h_close(leaf);
+        leaf = NULL;
+    }
+
+    ret = clock_gettime(CLOCK_REALTIME, &o_ts_ed);
+    if (ret != 0) {
+        fprintf(stderr, "clock_gettime: error %s\n", strerror(errno));
+        printf("glfs_h_getattrs and setattrs tests: FAILED\n");
+        goto out;
+    }
+
+    assimilatetime(&o_ts, o_ts_st, o_ts_ed);
+
+    printf("Creation performance (handle based):\n\t# empty files:%d\n",
+           MAX_FILES_CREATE);
+    printf("\tOverall time:\n\t\tSecs:%ld\n\t\tnSecs:%ld\n", o_ts.tv_sec,
+           o_ts.tv_nsec);
+    printf("\tcreate call time time:\n\t\tSecs:%ld\n\t\tnSecs:%ld\n",
+           c_ts.tv_sec, c_ts.tv_nsec);
+
+    /* create using path */
+    c_ts.tv_sec = o_ts.tv_sec = 0;
+    c_ts.tv_nsec = o_ts.tv_nsec = 0;
+
+    sprintf(my_file_name, "%s1", full_dir_path);
+    ret = glfs_mkdir(fs, my_file_name, 0755);
+    if (ret != 0) {
+        fprintf(stderr, "glfs_mkdir: error creating %s: from (%p),%s\n", my_dir,
+                parent, strerror(errno));
+        printf("glfs_h_performance tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    ret = clock_gettime(CLOCK_REALTIME, &o_ts_st);
+    if (ret != 0) {
+        fprintf(stderr, "clock_gettime: error %s\n", strerror(errno));
+        printf("glfs_h_getattrs and setattrs tests: FAILED\n");
+        goto out;
+    }
+
+    for (i = 0; i < MAX_FILES_CREATE; i++) {
+        sprintf(my_file_name, "%s1/%sn%d", full_dir_path, my_file, i);
+
+        ret = clock_gettime(CLOCK_REALTIME, &c_ts_st);
+        if (ret != 0) {
+            fprintf(stderr, "clock_gettime: error %s\n", strerror(errno));
+            printf("glfs_h_getattrs and setattrs tests: FAILED\n");
+            goto out;
+        }
+
+        ret = glfs_stat(fs, my_file_name, &sb);
+        if (ret == 0) {
+            fprintf(stderr, "glfs_stat: exists %s\n", my_file_name);
+            printf("glfs_h_performance tests: FAILED\n");
+            goto out;
+        }
+
+        fd = glfs_creat(fs, my_file_name, O_CREAT, 0644);
+        if (fd == NULL) {
+            fprintf(stderr, "glfs_creat: error creating %s: from (%p),%s\n",
+                    my_file, dir, strerror(errno));
+            printf("glfs_h_performance tests: FAILED\n");
+            goto out;
+        }
+
+        ret = clock_gettime(CLOCK_REALTIME, &c_ts_ed);
+        if (ret != 0) {
+            fprintf(stderr, "clock_gettime: error %s\n", strerror(errno));
+            printf("glfs_h_getattrs and setattrs tests: FAILED\n");
+            goto out;
+        }
+
+        assimilatetime(&c_ts, c_ts_st, c_ts_ed);
+        glfs_close(fd);
+    }
+
+    ret = clock_gettime(CLOCK_REALTIME, &o_ts_ed);
+    if (ret != 0) {
+        fprintf(stderr, "clock_gettime: error %s\n", strerror(errno));
+        printf("glfs_h_getattrs and setattrs tests: FAILED\n");
+        goto out;
+    }
+
+    assimilatetime(&o_ts, o_ts_st, o_ts_ed);
+
+    printf("Creation performance (path based):\n\t# empty files:%d\n",
+           MAX_FILES_CREATE);
+    printf("\tOverall time:\n\t\tSecs:%ld\n\t\tnSecs:%ld\n", o_ts.tv_sec,
+           o_ts.tv_nsec);
+    printf("\tcreate call time time:\n\t\tSecs:%ld\n\t\tnSecs:%ld\n",
+           c_ts.tv_sec, c_ts.tv_nsec);
+out:
+    return;
+}
 
 int
-test_chdir (glfs_t *fs)
-{
-	int ret = -1;
-	ino_t ino = 0;
-	struct stat st;
-	char *topdir = "/topdir";
-	char *linkdir = "/linkdir";
-	char *subdir = "./subdir";
-	char *respath = NULL;
-	char pathbuf[4096];
-
-	ret = glfs_mkdir (fs, topdir, 0755);
-	if (ret) {
-		fprintf (stderr, "mkdir(%s): %s\n", topdir, strerror (errno));
-		return -1;
-	}
-
-	respath = glfs_getcwd (fs, pathbuf, 4096);
-	fprintf (stdout, "getcwd() = %s\n", respath);
-
-	ret = glfs_symlink (fs, topdir, linkdir);
-	if (ret) {
-		fprintf (stderr, "symlink(%s, %s): %s\n", topdir, linkdir, strerror (errno));
-		return -1;
-	}
-
-	ret = glfs_chdir (fs, linkdir);
-	if (ret) {
-		fprintf (stderr, "chdir(%s): %s\n", linkdir, strerror (errno));
-		return -1;
-	}
-
-	respath = glfs_getcwd (fs, pathbuf, 4096);
-	fprintf (stdout, "getcwd() = %s\n", respath);
-
-	respath = glfs_realpath (fs, subdir, pathbuf);
-	if (respath) {
-		fprintf (stderr, "realpath(%s) worked unexpectedly: %s\n", subdir, respath);
-		return -1;
-	}
-
-	ret = glfs_mkdir (fs, subdir, 0755);
-	if (ret) {
-		fprintf (stderr, "mkdir(%s): %s\n", subdir, strerror (errno));
-		return -1;
-	}
-
-	respath = glfs_realpath (fs, subdir, pathbuf);
-	if (!respath) {
-		fprintf (stderr, "realpath(%s): %s\n", subdir, strerror (errno));
-	} else {
-		fprintf (stdout, "realpath(%s) = %s\n", subdir, respath);
-	}
-
-	ret = glfs_chdir (fs, subdir);
-	if (ret) {
-		fprintf (stderr, "chdir(%s): %s\n", subdir, strerror (errno));
-		return -1;
-	}
-
-	respath = glfs_getcwd (fs, pathbuf, 4096);
-	fprintf (stdout, "getcwd() = %s\n", respath);
-
-	respath = glfs_realpath (fs, "/linkdir/subdir", pathbuf);
-	if (!respath) {
-		fprintf (stderr, "realpath(/linkdir/subdir): %s\n", strerror (errno));
-	} else {
-		fprintf (stdout, "realpath(/linkdir/subdir) = %s\n", respath);
-	}
-
-	return 0;
+test_handleops(int argc, char *argv[])
+{
+    int ret = 0;
+    glfs_fd_t *fd = NULL;
+    struct stat sb = {
+        0,
+    };
+    struct glfs_object *root = NULL, *parent = NULL, *leaf = NULL, *tmp = NULL;
+    char readbuf[32], writebuf[32];
+    unsigned char leaf_handle[GFAPI_HANDLE_LENGTH];
+
+    char *full_leaf_name = "/testdir/testfile.txt", *leaf_name = "testfile.txt",
+         *relative_leaf_name = "testdir/testfile.txt";
+    char *leaf_name1 = "testfile1.txt";
+    char *full_newparent_name = "/testdir/dir1", *newparent_name = "dir1";
+    char *full_newnod_name = "/testdir/nod1", *newnod_name = "nod1";
+
+    /* Initialize test area */
+    ret = glfs_mkdir(fs, full_parent_name, 0755);
+    if (ret != 0 && errno != EEXIST) {
+        fprintf(stderr, "%s: (%p) %s\n", full_parent_name, fd, strerror(errno));
+        printf("Test initialization failed on volume %s\n", argv[1]);
+        goto out;
+    } else if (ret != 0) {
+        printf("Found test directory %s to be existing\n", full_parent_name);
+        printf("Cleanup test directory and restart tests\n");
+        goto out;
+    }
+
+    fd = glfs_creat(fs, full_leaf_name, O_CREAT, 0644);
+    if (fd == NULL) {
+        fprintf(stderr, "%s: (%p) %s\n", full_leaf_name, fd, strerror(errno));
+        printf("Test initialization failed on volume %s\n", argv[1]);
+        goto out;
+    }
+    glfs_close(fd);
+
+    printf("Initialized the test area, within volume %s\n", argv[1]);
+
+    /* Handle based APIs test area */
+
+    /* glfs_lookupat test */
+    printf("glfs_h_lookupat tests: In Progress\n");
+    /* start at root of the volume */
+    root = glfs_h_lookupat(fs, NULL, "/", &sb, 0);
+    if (root == NULL) {
+        fprintf(stderr,
+                "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n", "/",
+                NULL, strerror(errno));
+        printf("glfs_h_lookupat tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    /* lookup a parent within root */
+    parent = glfs_h_lookupat(fs, root, parent_name, &sb, 0);
+    if (parent == NULL) {
+        fprintf(stderr,
+                "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                parent_name, root, strerror(errno));
+        printf("glfs_h_lookupat tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    /* lookup a leaf/child within the parent */
+    leaf = glfs_h_lookupat(fs, parent, leaf_name, &sb, 0);
+    if (leaf == NULL) {
+        fprintf(stderr,
+                "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                leaf_name, parent, strerror(errno));
+        printf("glfs_h_lookupat tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    /* reset */
+    glfs_h_close(root);
+    root = NULL;
+    glfs_h_close(leaf);
+    leaf = NULL;
+    glfs_h_close(parent);
+    parent = NULL;
+
+    /* check absolute paths */
+    root = glfs_h_lookupat(fs, NULL, "/", &sb, 0);
+    if (root == NULL) {
+        fprintf(stderr,
+                "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n", "/",
+                NULL, strerror(errno));
+        printf("glfs_h_lookupat tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    parent = glfs_h_lookupat(fs, NULL, full_parent_name, &sb, 0);
+    if (parent == NULL) {
+        fprintf(stderr,
+                "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                full_parent_name, root, strerror(errno));
+        printf("glfs_h_lookupat tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    leaf = glfs_h_lookupat(fs, NULL, full_leaf_name, &sb, 0);
+    if (leaf == NULL) {
+        fprintf(stderr,
+                "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                full_leaf_name, parent, strerror(errno));
+        printf("glfs_h_lookupat tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    /* reset */
+    glfs_h_close(leaf);
+    leaf = NULL;
+
+    /* check multiple component paths */
+    leaf = glfs_h_lookupat(fs, root, relative_leaf_name, &sb, 0);
+    if (leaf == NULL) {
+        fprintf(stderr,
+                "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                relative_leaf_name, parent, strerror(errno));
+        goto out;
+    }
+    peek_stat(&sb);
+
+    /* reset */
+    glfs_h_close(root);
+    root = NULL;
+    glfs_h_close(parent);
+    parent = NULL;
+
+    /* check symlinks in path */
+
+    /* TODO: -ve test cases */
+    /* parent invalid
+     * path invalid
+     * path does not exist after some components
+     * no parent, but relative path
+     * parent and full path? -ve?
+     */
+
+    printf("glfs_h_lookupat tests: PASSED\n");
+
+    /* glfs_openat test */
+    printf("glfs_h_open tests: In Progress\n");
+    fd = glfs_h_open(fs, leaf, O_RDWR);
+    if (fd == NULL) {
+        fprintf(stderr, "glfs_h_open: error on open of %s: %s\n",
+                full_leaf_name, strerror(errno));
+        printf("glfs_h_open tests: FAILED\n");
+        goto out;
+    }
+
+    /* test read/write based on fd */
+    memcpy(writebuf, "abcdefghijklmnopqrstuvwxyz012345", 32);
+    ret = glfs_write(fd, writebuf, 32, 0);
+
+    glfs_lseek(fd, 0, SEEK_SET);
+
+    ret = glfs_read(fd, readbuf, 32, 0);
+    if (memcmp(readbuf, writebuf, 32)) {
+        printf("Failed to read what I wrote: %s %s\n", readbuf, writebuf);
+        glfs_close(fd);
+        printf("glfs_h_open tests: FAILED\n");
+        goto out;
+    }
+
+    glfs_h_close(leaf);
+    leaf = NULL;
+    glfs_close(fd);
+
+    printf("glfs_h_open tests: PASSED\n");
+
+    /* Create tests */
+    printf("glfs_h_creat tests: In Progress\n");
+    parent = glfs_h_lookupat(fs, NULL, full_parent_name, &sb, 0);
+    if (parent == NULL) {
+        fprintf(stderr,
+                "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                full_parent_name, root, strerror(errno));
+        printf("glfs_h_creat tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    leaf = glfs_h_creat(fs, parent, leaf_name1, O_CREAT, 0644, &sb);
+    if (leaf == NULL) {
+        fprintf(stderr, "glfs_h_creat: error on create of %s: from (%p),%s\n",
+                leaf_name1, parent, strerror(errno));
+        printf("glfs_h_creat tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    glfs_h_close(leaf);
+    leaf = NULL;
+
+    leaf = glfs_h_creat(fs, parent, leaf_name1, O_CREAT | O_EXCL, 0644, &sb);
+    if (leaf != NULL || errno != EEXIST) {
+        fprintf(stderr,
+                "glfs_h_creat: existing file, leaf = (%p), errno = %s\n", leaf,
+                strerror(errno));
+        printf("glfs_h_creat tests: FAILED\n");
+        if (leaf != NULL) {
+            glfs_h_close(leaf);
+            leaf = NULL;
+        }
+    }
+
+    tmp = glfs_h_creat(fs, root, parent_name, O_CREAT, 0644, &sb);
+    if (tmp != NULL || !(errno == EISDIR || errno == EINVAL)) {
+        fprintf(stderr, "glfs_h_creat: dir create, tmp = (%p), errno = %s\n",
+                leaf, strerror(errno));
+        printf("glfs_h_creat tests: FAILED\n");
+        if (tmp != NULL) {
+            glfs_h_close(tmp);
+            tmp = NULL;
+        }
+    }
+
+    /* TODO: Other combinations and -ve cases as applicable */
+    printf("glfs_h_creat tests: PASSED\n");
+
+    /* extract handle and create from handle test */
+    printf(
+        "glfs_h_extract_handle and glfs_h_create_from_handle tests: In "
+        "Progress\n");
+    /* TODO: Change the lookup to create below for a GIFD recovery failure,
+     * that needs to be fixed */
+    leaf = glfs_h_lookupat(fs, parent, leaf_name1, &sb, 0);
+    if (leaf == NULL) {
+        fprintf(stderr,
+                "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                leaf_name1, parent, strerror(errno));
+        printf("glfs_h_extract_handle tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    ret = glfs_h_extract_handle(leaf, leaf_handle, GFAPI_HANDLE_LENGTH);
+    if (ret < 0) {
+        fprintf(stderr,
+                "glfs_h_extract_handle: error extracting handle of %s: %s\n",
+                full_leaf_name, strerror(errno));
+        printf("glfs_h_extract_handle tests: FAILED\n");
+        goto out;
+    }
+    peek_handle(leaf_handle);
+
+    glfs_h_close(leaf);
+    leaf = NULL;
+
+    leaf = glfs_h_create_from_handle(fs, leaf_handle, GFAPI_HANDLE_LENGTH, &sb);
+    if (leaf == NULL) {
+        fprintf(
+            stderr,
+            "glfs_h_create_from_handle: error on create of %s: from (%p),%s\n",
+            leaf_name1, leaf_handle, strerror(errno));
+        printf("glfs_h_create_from_handle tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    fd = glfs_h_open(fs, leaf, O_RDWR);
+    if (fd == NULL) {
+        fprintf(stderr, "glfs_h_open: error on open of %s: %s\n",
+                full_leaf_name, strerror(errno));
+        printf("glfs_h_create_from_handle tests: FAILED\n");
+        goto out;
+    }
+
+    /* test read/write based on fd */
+    memcpy(writebuf, "abcdefghijklmnopqrstuvwxyz012345", 32);
+    ret = glfs_write(fd, writebuf, 32, 0);
+
+    glfs_lseek(fd, 0, SEEK_SET);
+
+    ret = glfs_read(fd, readbuf, 32, 0);
+    if (memcmp(readbuf, writebuf, 32)) {
+        printf("Failed to read what I wrote: %s %s\n", writebuf, writebuf);
+        printf("glfs_h_create_from_handle tests: FAILED\n");
+        glfs_close(fd);
+        goto out;
+    }
+
+    glfs_close(fd);
+    glfs_h_close(leaf);
+    leaf = NULL;
+    glfs_h_close(parent);
+    parent = NULL;
+
+    printf(
+        "glfs_h_extract_handle and glfs_h_create_from_handle tests: PASSED\n");
+
+    /* Mkdir tests */
+    printf("glfs_h_mkdir tests: In Progress\n");
+
+    ret = glfs_rmdir(fs, full_newparent_name);
+    if (ret && errno != ENOENT) {
+        fprintf(stderr, "glfs_rmdir: Failed for %s: %s\n", full_newparent_name,
+                strerror(errno));
+        printf("glfs_h_mkdir tests: FAILED\n");
+        goto out;
+    }
+
+    parent = glfs_h_lookupat(fs, NULL, full_parent_name, &sb, 0);
+    if (parent == NULL) {
+        fprintf(stderr,
+                "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                full_parent_name, root, strerror(errno));
+        printf("glfs_h_mkdir tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    leaf = glfs_h_mkdir(fs, parent, newparent_name, 0755, &sb);
+    if (leaf == NULL) {
+        fprintf(stderr, "glfs_h_mkdir: error on mkdir of %s: from (%p),%s\n",
+                newparent_name, parent, strerror(errno));
+        printf("glfs_h_mkdir tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    glfs_h_close(leaf);
+    leaf = NULL;
+
+    leaf = glfs_h_mkdir(fs, parent, newparent_name, 0755, &sb);
+    if (leaf != NULL || errno != EEXIST) {
+        fprintf(stderr,
+                "glfs_h_mkdir: existing directory, leaf = (%p), errno = %s\n",
+                leaf, strerror(errno));
+        printf("glfs_h_mkdir tests: FAILED\n");
+        if (leaf != NULL) {
+            glfs_h_close(leaf);
+            leaf = NULL;
+        }
+    }
+
+    glfs_h_close(parent);
+    parent = NULL;
+
+    printf("glfs_h_mkdir tests: PASSED\n");
+
+    /* Mknod tests */
+    printf("glfs_h_mknod tests: In Progress\n");
+    ret = glfs_unlink(fs, full_newnod_name);
+    if (ret && errno != ENOENT) {
+        fprintf(stderr, "glfs_unlink: Failed for %s: %s\n", full_newnod_name,
+                strerror(errno));
+        printf("glfs_h_mknod tests: FAILED\n");
+        goto out;
+    }
+
+    parent = glfs_h_lookupat(fs, NULL, full_parent_name, &sb, 0);
+    if (parent == NULL) {
+        fprintf(stderr,
+                "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                full_parent_name, root, strerror(errno));
+        printf("glfs_h_mknod tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    leaf = glfs_h_mknod(fs, parent, newnod_name, S_IFIFO, 0, &sb);
+    if (leaf == NULL) {
+        fprintf(stderr, "glfs_h_mkdir: error on mkdir of %s: from (%p),%s\n",
+                newnod_name, parent, strerror(errno));
+        printf("glfs_h_mknod tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    /* TODO: create op on a FIFO node hangs, need to check and fix
+    tmp = glfs_h_creat (fs, parent, newnod_name, O_CREAT, 0644, &sb);
+    if (tmp != NULL || errno != EINVAL) {
+            fprintf (stderr, "glfs_h_creat: node create, tmp = (%p), errno =
+    %s\n", tmp, strerror (errno)); printf ("glfs_h_creat/mknod tests:
+    FAILED\n"); if (tmp != NULL) { glfs_h_close(tmp); tmp = NULL;
+            }
+    } */
+
+    glfs_h_close(leaf);
+    leaf = NULL;
+
+    leaf = glfs_h_mknod(fs, parent, newnod_name, 0644, 0, &sb);
+    if (leaf != NULL || errno != EEXIST) {
+        fprintf(stderr,
+                "glfs_h_mknod: existing node, leaf = (%p), errno = %s\n", leaf,
+                strerror(errno));
+        printf("glfs_h_mknod tests: FAILED\n");
+        if (leaf != NULL) {
+            glfs_h_close(leaf);
+            leaf = NULL;
+        }
+    }
+
+    glfs_h_close(parent);
+    parent = NULL;
+
+    printf("glfs_h_mknod tests: PASSED\n");
+
+    /* unlink tests */
+    test_h_unlink();
+
+    /* TODO: opendir tests */
+
+    /* getattr tests */
+    test_h_getsetattrs();
+
+    /* TODO: setattr tests */
+
+    /* truncate tests */
+    test_h_truncate();
+
+    /* link tests */
+    test_h_links();
+
+    /* rename tests */
+    test_h_rename();
+
+    /* performance tests */
+    test_h_performance();
+
+    /* END: New APIs test area */
+
+out:
+    /* Cleanup glfs handles */
+    if (root)
+        glfs_h_close(root);
+    if (parent)
+        glfs_h_close(parent);
+    if (leaf)
+        glfs_h_close(leaf);
+
+    return ret;
 }
 
+int
+test_write_apis(glfs_t *fs)
+{
+    /* Add more content here */
+    /* Some apis we can get are */
+    /*
+      0. glfs_set_xlator_option()
+
+      Read/Write combinations:
+      . glfs_{p,}readv/{p,}writev
+      . glfs_pread/pwrite
+
+      tests/basic/gfapi/gfapi-async-calls-test.c
+      . glfs_read_async/write_async
+      . glfs_pread_async/pwrite_async
+      . glfs_readv_async/writev_async
+      . glfs_preadv_async/pwritev_async
+
+      . ftruncate/ftruncate_async
+      . fsync/fsync_async
+      . fdatasync/fdatasync_async
+
+    */
+
+    glfs_fd_t *fd = NULL;
+    char *filename = "/filename2";
+    int flags = O_RDWR;
+    char *buf = "some bytes!";
+    char writestr[TEST_STR_LEN];
+    struct iovec iov = {&writestr, TEST_STR_LEN};
+    int ret, i;
+
+    for (i = 0; i < TEST_STR_LEN; i++)
+        writestr[i] = 0x11;
+
+    fd = glfs_open(fs, filename, flags);
+    if (!fd)
+        fprintf(stderr, "open(%s): (%p) %s\n", filename, fd, strerror(errno));
+
+    ret = glfs_writev(fd, &iov, 1, flags);
+    if (ret < 0) {
+        fprintf(stderr, "writev(%s): %d (%s)\n", filename, ret,
+                strerror(errno));
+    }
+
+    ret = glfs_pwrite(fd, buf, 10, 4, flags, NULL, NULL);
+    if (ret < 0) {
+        fprintf(stderr, "pwrite(%s): %d (%s)\n", filename, ret,
+                strerror(errno));
+    }
+
+    ret = glfs_pwritev(fd, &iov, 1, 4, flags);
+    if (ret < 0) {
+        fprintf(stderr, "pwritev(%s): %d (%s)\n", filename, ret,
+                strerror(errno));
+    }
+
+    return 0;
+}
 
 int
-main (int argc, char *argv[])
+test_metadata_ops(glfs_t *fs, glfs_t *fs2)
 {
-	glfs_t    *fs = NULL;
-	glfs_t    *fs2 = NULL;
-	int        ret = 0;
-	glfs_fd_t *fd = NULL;
-	glfs_fd_t *fd2 = NULL;
-	struct stat sb = {0, };
-	char       readbuf[32];
-	char       writebuf[32];
+    glfs_fd_t *fd = NULL;
+    glfs_fd_t *fd2 = NULL;
+    struct stat sb = {
+        0,
+    };
+    struct glfs_stat gsb = {
+        0,
+    };
+    struct statvfs sfs;
+    char readbuf[32];
+    char writebuf[32];
+
+    char *filename = "/filename2";
+    int ret;
+
+    ret = glfs_lstat(fs, filename, &sb);
+    fprintf(stderr, "lstat(%s): (%d) %s\n", filename, ret, strerror(errno));
+
+    fd = glfs_creat(fs, filename, O_RDWR, 0644);
+    fprintf(stderr, "creat(%s): (%p) %s\n", filename, fd, strerror(errno));
+
+    fd2 = glfs_open(fs2, filename, O_RDWR);
+    fprintf(stderr, "open(%s): (%p) %s\n", filename, fd, strerror(errno));
 
-	char      *filename = "/filename2";
+    glfs_lseek(fd2, 0, SEEK_SET);
 
-	fs = glfs_new ("fsync");
-	if (!fs) {
-		fprintf (stderr, "glfs_new: returned NULL\n");
-		return 1;
-	}
+    ret = glfs_read(fd2, readbuf, 32, 0);
 
-//	ret = glfs_set_volfile (fs, "/tmp/posix.vol");
+    printf("read %d, %s", ret, readbuf);
 
-	ret = glfs_set_volfile_server (fs, "tcp", "localhost", 24007);
+    /* get stat */
+    ret = glfs_fstat(fd2, &sb);
 
-//	ret = glfs_set_volfile_server (fs, "unix", "/tmp/gluster.sock", 0);
+    ret = glfs_access(fs, filename, R_OK);
 
-	ret = glfs_set_logging (fs, "/dev/stderr", 7);
+    /* set stat */
+    /* TODO: got some errors, need to fix */
+    /* ret = glfs_fsetattr(fd2, &gsb); */
 
-	ret = glfs_init (fs);
+    glfs_close(fd);
+    glfs_close(fd2);
 
-	fprintf (stderr, "glfs_init: returned %d\n", ret);
+    filename = "/filename3";
+    ret = glfs_mknod(fs, filename, S_IFIFO, 0);
+    fprintf(stderr, "%s: (%d) %s\n", filename, ret, strerror(errno));
 
-	sleep (2);
+    ret = glfs_lstat(fs, filename, &sb);
+    fprintf(stderr, "%s: (%d) %s\n", filename, ret, strerror(errno));
 
-	fs2 = glfs_new ("fsync");
-	if (!fs2) {
-		fprintf (stderr, "glfs_new: returned NULL\n");
-		return 1;
-	}
+    ret = glfs_rename(fs, filename, "/filename4");
+    fprintf(stderr, "rename(%s): (%d) %s\n", filename, ret, strerror(errno));
 
+    ret = glfs_unlink(fs, "/filename4");
+    fprintf(stderr, "unlink(%s): (%d) %s\n", "/filename4", ret,
+            strerror(errno));
 
-//	ret = glfs_set_volfile (fs2, "/tmp/posix.vol");
+    filename = "/dirname2";
+    ret = glfs_mkdir(fs, filename, 0);
+    fprintf(stderr, "%s: (%d) %s\n", filename, ret, strerror(errno));
+
+    ret = glfs_lstat(fs, filename, &sb);
+    fprintf(stderr, "lstat(%s): (%d) %s\n", filename, ret, strerror(errno));
+
+    ret = glfs_rmdir(fs, filename);
+    fprintf(stderr, "rmdir(%s): (%d) %s\n", filename, ret, strerror(errno));
+}
+int
+main(int argc, char *argv[])
+{
+    glfs_t *fs2 = NULL;
+    int ret = 0;
+    glfs_fd_t *fd = NULL;
+    glfs_fd_t *fd2 = NULL;
+    struct stat sb = {
+        0,
+    };
+    struct glfs_stat gsb = {
+        0,
+    };
+    struct statvfs sfs;
+    char readbuf[32];
+    char writebuf[32];
 
-	ret = glfs_set_volfile_server (fs2, "tcp", "localhost", 24007);
+    char *filename = "/filename2";
 
-	ret = glfs_set_logging (fs2, "/dev/stderr", 7);
+    if (argc != 3) {
+        printf("Expect following args\n\t%s <volname> <hostname>\n", argv[0]);
+        return -1;
+    }
 
-	ret = glfs_init (fs2);
+    fs = glfs_new(argv[1]);
+    if (!fs) {
+        fprintf(stderr, "glfs_new: returned NULL\n");
+        return 1;
+    }
 
-	fprintf (stderr, "glfs_init: returned %d\n", ret);
+    //      ret = glfs_set_volfile (fs, "/tmp/posix.vol");
 
-	ret = glfs_lstat (fs, filename, &sb);
-	fprintf (stderr, "%s: (%d) %s\n", filename, ret, strerror (errno));
+    ret = glfs_set_volfile_server(fs, "tcp", argv[2], 24007);
 
-	fd = glfs_creat (fs, filename, O_RDWR, 0644);
-	fprintf (stderr, "%s: (%p) %s\n", filename, fd, strerror (errno));
+    //      ret = glfs_set_volfile_server (fs, "unix", "/tmp/gluster.sock", 0);
 
-	fd2 = glfs_open (fs2, filename, O_RDWR);
-	fprintf (stderr, "%s: (%p) %s\n", filename, fd, strerror (errno));
+    ret = glfs_set_logging(fs, "/dev/stderr", 7);
 
-	sprintf (writebuf, "hi there\n");
-	ret = glfs_write (fd, writebuf, 32, 0);
+    ret = glfs_init(fs);
 
-	glfs_lseek (fd2, 0, SEEK_SET);
+    fprintf(stderr, "glfs_init: returned %d\n", ret);
 
-	ret = glfs_read (fd2, readbuf, 32, 0);
+    if (ret)
+        goto out;
 
-	printf ("read %d, %s", ret, readbuf);
+    sleep(2);
 
-	glfs_close (fd);
-	glfs_close (fd2);
+    fs2 = glfs_new(argv[1]);
+    if (!fs2) {
+        fprintf(stderr, "glfs_new: returned NULL\n");
+        return 1;
+    }
 
-	filename = "/filename3";
-	ret = glfs_mknod (fs, filename, S_IFIFO, 0);
-	fprintf (stderr, "%s: (%d) %s\n", filename, ret, strerror (errno));
+    //      ret = glfs_set_volfile (fs2, "/tmp/posix.vol");
 
-	ret = glfs_lstat (fs, filename, &sb);
-	fprintf (stderr, "%s: (%d) %s\n", filename, ret, strerror (errno));
+    ret = glfs_set_volfile_server(fs2, "tcp", argv[2], 24007);
 
+    ret = glfs_set_logging(fs2, "/dev/stderr", 7);
 
-	ret = glfs_rename (fs, filename, "/filename4");
-	fprintf (stderr, "rename(%s): (%d) %s\n", filename, ret,
-		 strerror (errno));
+    ret = glfs_init(fs2);
 
-	ret = glfs_unlink (fs, "/filename4");
-	fprintf (stderr, "unlink(%s): (%d) %s\n", "/filename4", ret,
-		 strerror (errno));
+    fprintf(stderr, "glfs_init: returned %d\n", ret);
 
-	filename = "/dirname2";
-	ret = glfs_mkdir (fs, filename, 0);
-	fprintf (stderr, "%s: (%d) %s\n", filename, ret, strerror (errno));
+    test_metadata_ops(fs, fs2);
 
-	ret = glfs_lstat (fs, filename, &sb);
-	fprintf (stderr, "lstat(%s): (%d) %s\n", filename, ret, strerror (errno));
+    test_dirops(fs);
 
-	ret = glfs_rmdir (fs, filename);
-	fprintf (stderr, "rmdir(%s): (%d) %s\n", filename, ret, strerror (errno));
+    test_xattr(fs);
 
-	test_dirops (fs);
+    test_chdir(fs);
 
-	test_xattr (fs);
+    test_handleops(argc, argv);
+    // done
 
-	test_chdir (fs);
+    /* Test some extra apis */
+    test_write_apis(fs);
 
-	// done
+    glfs_statvfs(fs, "/", &sfs);
 
-	glfs_fini (fs);
-	glfs_fini (fs2);
+    glfs_fini(fs);
+    glfs_fini(fs2);
 
-	return ret;
+    ret = 0;
+out:
+    return ret;
 }
diff --git a/api/src/Makefile.am b/api/src/Makefile.am
index 0782435e065..7f9a7d17b35 100644
--- a/api/src/Makefile.am
+++ b/api/src/Makefile.am
@@ -1,19 +1,27 @@
 lib_LTLIBRARIES = libgfapi.la
-noinst_HEADERS = glfs-mem-types.h glfs-internal.h
-libgfapi_HEADERS = glfs.h
+noinst_HEADERS = glfs-mem-types.h glfs-internal.h gfapi-messages.h
+libgfapi_HEADERS = glfs.h glfs-handles.h
 libgfapidir = $(includedir)/glusterfs/api
 
-libgfapi_la_SOURCES = glfs.c glfs-mgmt.c glfs-fops.c glfs-resolve.c
+EXTRA_DIST = gfapi.map gfapi.aliases
+
+libgfapi_la_SOURCES = glfs.c glfs-mgmt.c glfs-fops.c glfs-resolve.c \
+	glfs-handleops.c
 libgfapi_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la \
 	$(top_builddir)/rpc/rpc-lib/src/libgfrpc.la \
-	$(top_builddir)/rpc/xdr/src/libgfxdr.la \
-	$(GF_LDADD)
+	$(top_builddir)/rpc/xdr/src/libgfxdr.la
 
-libgfapi_la_CPPFLAGS = $(GF_CPPFLAGS) -D__USE_FILE_OFFSET64 \
-	-I$(top_srcdir)/libglusterfs/src \
-	-I$(top_srcdir)/rpc/rpc-lib/src \
-	-I$(top_srcdir)/rpc/xdr/src
+libgfapi_la_LDFLAGS = -version-info $(GFAPI_LT_VERSION) $(GF_LDFLAGS) \
+	$(GFAPI_EXTRA_LDFLAGS) $(ACL_LIBS)
 
+libgfapi_la_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
+        -I$(top_srcdir)/rpc/rpc-lib/src \
+        -I$(top_srcdir)/rpc/xdr/src \
+        -I$(top_builddir)/rpc/xdr/src \
+        -DDATADIR=\"$(localstatedir)\" \
+        -D__USE_FILE_OFFSET64 -D__USE_LARGEFILE64
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
 
 xlator_LTLIBRARIES = api.la
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/mount
@@ -24,12 +32,12 @@ $(install_xlatorLTLIBRARIES): install-libLTLIBRARIES
 
 api_la_SOURCES = glfs-master.c
 api_la_DEPENDENCIES = libgfapi.la
-api_la_LDFLAGS = -module -avoid-version
+api_la_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
+	-I$(top_srcdir)/rpc/xdr/src \
+	-I$(top_builddir)/rpc/xdr/src
+api_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
+#api_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS) $(GF_LDFLAGS)
 api_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la \
 	$(top_builddir)/rpc/rpc-lib/src/libgfrpc.la \
 	$(top_builddir)/rpc/xdr/src/libgfxdr.la \
 	$(top_builddir)/api/src/libgfapi.la
-
-AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
-
-AM_CFLAGS = -Wall $(GF_CFLAGS)
diff --git a/api/src/README.Symbol_Versions b/api/src/README.Symbol_Versions
new file mode 100644
index 00000000000..b6ec95f9311
--- /dev/null
+++ b/api/src/README.Symbol_Versions
@@ -0,0 +1,3 @@
+
+See ../../doc/developer-guide/gfapi-symbol-versions.md
+
diff --git a/api/src/gfapi-messages.h b/api/src/gfapi-messages.h
new file mode 100644
index 00000000000..b9223940416
--- /dev/null
+++ b/api/src/gfapi-messages.h
@@ -0,0 +1,147 @@
+/*
+ *   Copyright (c) 2015-2018 Red Hat, Inc. <http://www.redhat.com>
+ *   This file is part of GlusterFS.
+ *
+ *   This file is licensed to you under your choice of the GNU Lesser
+ *   General Public License, version 3 or any later version (LGPLv3 or
+ *   later), or the GNU General Public License, version 2 (GPLv2), in all
+ *   cases as published by the Free Software Foundation.
+ *   */
+
+#ifndef _GFAPI_MESSAGES_H__
+#define _GFAPI_MESSAGES_H__
+
+#include <glusterfs/glfs-message-id.h>
+
+/* To add new message IDs, append new identifiers at the end of the list.
+ *
+ * Never remove a message ID. If it's not used anymore, you can rename it or
+ * leave it as it is, but not delete it. This is to prevent reutilization of
+ * IDs by other messages.
+ *
+ * The component name must match one of the entries defined in
+ * glfs-message-id.h.
+ */
+
+GLFS_MSGID(API, API_MSG_MEM_ACCT_INIT_FAILED, API_MSG_MASTER_XLATOR_INIT_FAILED,
+           API_MSG_GFAPI_XLATOR_INIT_FAILED, API_MSG_VOLFILE_OPEN_FAILED,
+           API_MSG_VOL_SPEC_FILE_ERROR, API_MSG_GLFS_FSOBJ_NULL,
+           API_MSG_INVALID_ENTRY, API_MSG_FSMUTEX_LOCK_FAILED,
+           API_MSG_COND_WAIT_FAILED, API_MSG_FSMUTEX_UNLOCK_FAILED,
+           API_MSG_INODE_REFRESH_FAILED, API_MSG_GRAPH_CONSTRUCT_FAILED,
+           API_MSG_API_XLATOR_ERROR, API_MSG_XDR_PAYLOAD_FAILED,
+           API_MSG_GET_VOLINFO_CBK_FAILED, API_MSG_FETCH_VOLUUID_FAILED,
+           API_MSG_INSUFF_SIZE, API_MSG_FRAME_CREAT_FAILED,
+           API_MSG_DICT_SET_FAILED, API_MSG_XDR_DECODE_FAILED,
+           API_MSG_GET_VOLFILE_FAILED, API_MSG_WRONG_OPVERSION,
+           API_MSG_DICT_SERIALIZE_FAILED, API_MSG_REMOTE_HOST_CONN_FAILED,
+           API_MSG_VOLFILE_SERVER_EXHAUST, API_MSG_CREATE_RPC_CLIENT_FAILED,
+           API_MSG_REG_NOTIFY_FUNC_FAILED, API_MSG_REG_CBK_FUNC_FAILED,
+           API_MSG_GET_CWD_FAILED, API_MSG_FGETXATTR_FAILED,
+           API_MSG_LOCKINFO_KEY_MISSING, API_MSG_FSETXATTR_FAILED,
+           API_MSG_FSYNC_FAILED, API_MSG_FDCREATE_FAILED,
+           API_MSG_INODE_PATH_FAILED, API_MSG_SYNCOP_OPEN_FAILED,
+           API_MSG_LOCK_MIGRATE_FAILED, API_MSG_OPENFD_SKIPPED,
+           API_MSG_FIRST_LOOKUP_GRAPH_FAILED, API_MSG_CWD_GRAPH_REF_FAILED,
+           API_MSG_SWITCHED_GRAPH, API_MSG_XDR_RESPONSE_DECODE_FAILED,
+           API_MSG_VOLFILE_INFO, API_MSG_VOLFILE_CONNECTING, API_MSG_NEW_GRAPH,
+           API_MSG_ALLOC_FAILED, API_MSG_CREATE_HANDLE_FAILED,
+           API_MSG_INODE_LINK_FAILED, API_MSG_STATEDUMP_FAILED,
+           API_MSG_XREADDIRP_R_FAILED, API_MSG_LOCK_INSERT_MERGE_FAILED,
+           API_MSG_SETTING_LOCK_TYPE_FAILED, API_MSG_INODE_FIND_FAILED,
+           API_MSG_FDCTX_SET_FAILED, API_MSG_UPCALL_SYNCOP_FAILED,
+           API_MSG_INVALID_ARG, API_MSG_UPCALL_EVENT_NULL_RECEIVED,
+           API_MSG_FLAGS_HANDLE, API_MSG_FDCREATE_FAILED_ON_GRAPH,
+           API_MSG_TRANS_RDMA_DEP, API_MSG_TRANS_NOT_SUPPORTED,
+           API_MSG_FS_NOT_INIT, API_MSG_INVALID_SYSRQ,
+           API_MSG_DECODE_XDR_FAILED, API_MSG_NULL, API_MSG_CALL_NOT_SUCCESSFUL,
+           API_MSG_CALL_NOT_VALID, API_MSG_UNABLE_TO_DEL,
+           API_MSG_REMOTE_HOST_DISCONN, API_MSG_HANDLE_NOT_SET);
+
+#define API_MSG_ALLOC_FAILED_STR "Upcall allocation failed"
+#define API_MSG_LOCK_INSERT_MERGE_FAILED_STR                                   \
+    "Lock insertion and splitting/merging failed"
+#define API_MSG_SETTING_LOCK_TYPE_FAILED_STR "Setting lock type failed"
+
+#define API_MSG_INVALID_ARG_STR "Invalid"
+#define API_MSG_INVALID_ENTRY_STR "Upcall entry validation failed"
+#define API_MSG_INODE_FIND_FAILED_STR "Unable to find inode entry"
+#define API_MSG_CREATE_HANDLE_FAILED_STR "handle creation failed"
+#define API_MSG_UPCALL_EVENT_NULL_RECEIVED_STR                                 \
+    "Upcall_EVENT_NULL received. Skipping it"
+#define API_MSG_UPCALL_SYNCOP_FAILED_STR "Synctask for upcall failed"
+#define API_MSG_FDCREATE_FAILED_STR "Allocating anonymous fd failed"
+#define API_MSG_XREADDIRP_R_FAILED_STR "glfs_x_readdirp_r failed"
+#define API_MSG_FDCTX_SET_FAILED_STR "Setting fd ctx failed"
+#define API_MSG_FLAGS_HANDLE_STR "arg not set. Flags handled are"
+#define API_MSG_INODE_REFRESH_FAILED_STR "inode refresh failed"
+#define API_MSG_INODE_LINK_FAILED_STR "inode linking failed"
+#define API_MSG_GET_CWD_FAILED_STR "Failed to get cwd"
+#define API_MSG_FGETXATTR_FAILED_STR "fgetxattr failed"
+#define API_MSG_LOCKINFO_KEY_MISSING_STR "missing lockinfo key"
+#define API_MSG_FSYNC_FAILED_STR "fsync() failed"
+#define API_MSG_FDCREATE_FAILED_ON_GRAPH_STR "fd_create failed on graph"
+#define API_MSG_INODE_PATH_FAILED_STR "inode_path failed"
+#define API_MSG_SYNCOP_OPEN_FAILED_STR "syncop_open failed"
+#define API_MSG_LOCK_MIGRATE_FAILED_STR "lock migration failed on graph"
+#define API_MSG_OPENFD_SKIPPED_STR "skipping openfd in graph"
+#define API_MSG_FIRST_LOOKUP_GRAPH_FAILED_STR "first lookup on graph failed"
+#define API_MSG_CWD_GRAPH_REF_FAILED_STR "cwd refresh of graph failed"
+#define API_MSG_SWITCHED_GRAPH_STR "switched to graph"
+#define API_MSG_FSETXATTR_FAILED_STR "fsetxattr failed"
+#define API_MSG_MEM_ACCT_INIT_FAILED_STR "Memory accounting init failed"
+#define API_MSG_MASTER_XLATOR_INIT_FAILED_STR                                  \
+    "master xlator for initialization failed"
+#define API_MSG_GFAPI_XLATOR_INIT_FAILED_STR                                   \
+    "failed to initialize gfapi translator"
+#define API_MSG_VOLFILE_OPEN_FAILED_STR "volume file open failed"
+#define API_MSG_VOL_SPEC_FILE_ERROR_STR "Cannot reach volume specification file"
+#define API_MSG_TRANS_RDMA_DEP_STR                                             \
+    "transport RDMA is deprecated, falling back to tcp"
+#define API_MSG_TRANS_NOT_SUPPORTED_STR                                        \
+    "transport is not supported, possible values tcp|unix"
+#define API_MSG_GLFS_FSOBJ_NULL_STR "fs is NULL"
+#define API_MSG_FS_NOT_INIT_STR "fs is not properly initialized"
+#define API_MSG_FSMUTEX_LOCK_FAILED_STR                                        \
+    "pthread lock on glfs mutex, returned error"
+#define API_MSG_FSMUTEX_UNLOCK_FAILED_STR                                      \
+    "pthread unlock on glfs mutex, returned error"
+#define API_MSG_COND_WAIT_FAILED_STR "cond wait failed"
+#define API_MSG_INVALID_SYSRQ_STR "not a valid sysrq"
+#define API_MSG_GRAPH_CONSTRUCT_FAILED_STR "failed to construct the graph"
+#define API_MSG_API_XLATOR_ERROR_STR                                           \
+    "api master xlator cannot be specified in volume file"
+#define API_MSG_STATEDUMP_FAILED_STR "statedump failed"
+#define API_MSG_DECODE_XDR_FAILED_STR                                          \
+    "Failed to decode xdr response for GF_CBK_STATEDUMP"
+#define API_MSG_NULL_STR "NULL"
+#define API_MSG_XDR_PAYLOAD_FAILED_STR "failed to create XDR payload"
+#define API_MSG_CALL_NOT_SUCCESSFUL_STR                                        \
+    "GET_VOLUME_INFO RPC call is not successful"
+#define API_MSG_XDR_RESPONSE_DECODE_FAILED_STR                                 \
+    "Failed to decode xdr response for GET_VOLUME_INFO"
+#define API_MSG_CALL_NOT_VALID_STR                                             \
+    "Response received for GET_VOLUME_INFO RPC is not valid"
+#define API_MSG_GET_VOLINFO_CBK_FAILED_STR                                     \
+    "In GET_VOLUME_INFO cbk, received error"
+#define API_MSG_FETCH_VOLUUID_FAILED_STR "Unable to fetch volume UUID"
+#define API_MSG_INSUFF_SIZE_STR "Insufficient size passed"
+#define API_MSG_FRAME_CREAT_FAILED_STR "failed to create the frame"
+#define API_MSG_DICT_SET_FAILED_STR "failed to set"
+#define API_MSG_XDR_DECODE_FAILED_STR "XDR decoding error"
+#define API_MSG_GET_VOLFILE_FAILED_STR "failed to get the volume file"
+#define API_MSG_VOLFILE_INFO_STR "No change in volfile, continuing"
+#define API_MSG_UNABLE_TO_DEL_STR "unable to delete file"
+#define API_MSG_WRONG_OPVERSION_STR                                            \
+    "Server is operating at an op-version which is not supported"
+#define API_MSG_DICT_SERIALIZE_FAILED_STR "Failed to serialize dictionary"
+#define API_MSG_REMOTE_HOST_CONN_FAILED_STR "Failed to connect to remote-host"
+#define API_MSG_REMOTE_HOST_DISCONN_STR "disconnected from remote-host"
+#define API_MSG_VOLFILE_SERVER_EXHAUST_STR "Exhausted all volfile servers"
+#define API_MSG_VOLFILE_CONNECTING_STR "connecting to next volfile server"
+#define API_MSG_CREATE_RPC_CLIENT_FAILED_STR "failed to create rpc clnt"
+#define API_MSG_REG_NOTIFY_FUNC_FAILED_STR "failed to register notify function"
+#define API_MSG_REG_CBK_FUNC_FAILED_STR "failed to register callback function"
+#define API_MSG_NEW_GRAPH_STR "New graph coming up"
+#define API_MSG_HANDLE_NOT_SET_STR "handle not set. Flags handled for xstat are"
+#endif /* !_GFAPI_MESSAGES_H__ */
diff --git a/api/src/gfapi.aliases b/api/src/gfapi.aliases
new file mode 100644
index 00000000000..bc639e6b99f
--- /dev/null
+++ b/api/src/gfapi.aliases
@@ -0,0 +1,201 @@
+
+_priv_glfs_loc_touchup _glfs_loc_touchup$GFAPI_PRIVATE_3.4.0
+_priv_glfs_active_subvol _glfs_active_subvol$GFAPI_PRIVATE_3.4.0
+_priv_glfs_subvol_done _glfs_subvol_done$GFAPI_PRIVATE_3.4.0
+_priv_glfs_init_done _glfs_init_done$GFAPI_PRIVATE_3.4.0
+_priv_glfs_resolve_at _glfs_resolve_at$GFAPI_PRIVATE_3.4.0
+
+_pub_glfs_new _glfs_new$GFAPI_3.4.0
+_pub_glfs_set_volfile _glfs_set_volfile$GFAPI_3.4.0
+_pub_glfs_set_volfile_server _glfs_set_volfile_server$GFAPI_3.4.0
+_pub_glfs_set_logging _glfs_set_logging$GFAPI_3.4.0
+_pub_glfs_init _glfs_init$GFAPI_3.4.0
+_pub_glfs_fini _glfs_fini$GFAPI_3.4.0
+_pub_glfs_open _glfs_open$GFAPI_3.4.0
+_pub_glfs_creat _glfs_creat$GFAPI_3.4.0
+_pub_glfs_close _glfs_close$GFAPI_3.4.0
+_pub_glfs_from_glfd _glfs_from_glfd$GFAPI_3.4.0
+_pub_glfs_set_xlator_option _glfs_set_xlator_option$GFAPI_3.4.0
+_pub_glfs_read _glfs_read$GFAPI_3.4.0
+_pub_glfs_write _glfs_write$GFAPI_3.4.0
+_pub_glfs_readv _glfs_readv$GFAPI_3.4.0
+_pub_glfs_writev _glfs_writev$GFAPI_3.4.0
+_pub_glfs_pread34 _glfs_pread$GFAPI_3.4.0
+_pub_glfs_pwrite34 _glfs_pwrite$GFAPI_3.4.0
+_pub_glfs_pread_async34 _glfs_pread_async$GFAPI_3.4.0
+_pub_glfs_pwrite_async34 _glfs_pwrite_async$GFAPI_3.4.0
+_pub_glfs_preadv _glfs_preadv$GFAPI_3.4.0
+_pub_glfs_pwritev _glfs_pwritev$GFAPI_3.4.0
+_pub_glfs_lseek _glfs_lseek$GFAPI_3.4.0
+_pub_glfs_ftruncate34 _glfs_ftruncate$GFAPI_3.4.0
+_pub_glfs_ftruncate_async34 _glfs_ftruncate_async$GFAPI_3.4.0
+_pub_glfs_lstat _glfs_lstat$GFAPI_3.4.0
+_pub_glfs_stat _glfs_stat$GFAPI_3.4.0
+_pub_glfs_fstat _glfs_fstat$GFAPI_3.4.0
+_pub_glfs_fsync34 _glfs_fsync$GFAPI_3.4.0
+_pub_glfs_fsync_async34 _glfs_fsync_async$GFAPI_3.4.0
+_pub_glfs_fdatasync34 _glfs_fdatasync$GFAPI_3.4.0
+_pub_glfs_fdatasync_async34 _glfs_fdatasync_async$GFAPI_3.4.0
+_pub_glfs_access _glfs_access$GFAPI_3.4.0
+_pub_glfs_symlink _glfs_symlink$GFAPI_3.4.0
+_pub_glfs_readlink _glfs_readlink$GFAPI_3.4.0
+_pub_glfs_mknod _glfs_mknod$GFAPI_3.4.0
+_pub_glfs_mkdir _glfs_mkdir$GFAPI_3.4.0
+_pub_glfs_unlink _glfs_unlink$GFAPI_3.4.0
+_pub_glfs_rmdir _glfs_rmdir$GFAPI_3.4.0
+_pub_glfs_rename _glfs_rename$GFAPI_3.4.0
+_pub_glfs_link _glfs_link$GFAPI_3.4.0
+_pub_glfs_opendir _glfs_opendir$GFAPI_3.4.0
+_pub_glfs_readdir_r _glfs_readdir_r$GFAPI_3.4.0
+_pub_glfs_readdirplus_r _glfs_readdirplus_r$GFAPI_3.4.0
+_pub_glfs_telldir _glfs_telldir$GFAPI_3.4.0
+_pub_glfs_seekdir _glfs_seekdir$GFAPI_3.4.0
+_pub_glfs_closedir _glfs_closedir$GFAPI_3.4.0
+_pub_glfs_statvfs _glfs_statvfs$GFAPI_3.4.0
+_pub_glfs_chmod _glfs_chmod$GFAPI_3.4.0
+_pub_glfs_fchmod _glfs_fchmod$GFAPI_3.4.0
+_pub_glfs_chown _glfs_chown$GFAPI_3.4.0
+_pub_glfs_lchown _glfs_lchown$GFAPI_3.4.0
+_pub_glfs_fchown _glfs_fchown$GFAPI_3.4.0
+_pub_glfs_utimens _glfs_utimens$GFAPI_3.4.0
+_pub_glfs_lutimens _glfs_lutimens$GFAPI_3.4.0
+_pub_glfs_futimens _glfs_futimens$GFAPI_3.4.0
+_pub_glfs_getxattr _glfs_getxattr$GFAPI_3.4.0
+_pub_glfs_lgetxattr _glfs_lgetxattr$GFAPI_3.4.0
+_pub_glfs_fgetxattr _glfs_fgetxattr$GFAPI_3.4.0
+_pub_glfs_listxattr _glfs_listxattr$GFAPI_3.4.0
+_pub_glfs_llistxattr _glfs_llistxattr$GFAPI_3.4.0
+_pub_glfs_flistxattr _glfs_flistxattr$GFAPI_3.4.0
+_pub_glfs_setxattr _glfs_setxattr$GFAPI_3.4.0
+_pub_glfs_lsetxattr _glfs_lsetxattr$GFAPI_3.4.0
+_pub_glfs_fsetxattr _glfs_fsetxattr$GFAPI_3.4.0
+_pub_glfs_removexattr _glfs_removexattr$GFAPI_3.4.0
+_pub_glfs_lremovexattr _glfs_lremovexattr$GFAPI_3.4.0
+_pub_glfs_fremovexattr _glfs_fremovexattr$GFAPI_3.4.0
+_pub_glfs_getcwd _glfs_getcwd$GFAPI_3.4.0
+_pub_glfs_chdir _glfs_chdir$GFAPI_3.4.0
+_pub_glfs_fchdir _glfs_fchdir$GFAPI_3.4.0
+_pub_glfs_realpath34 _glfs_realpath$GFAPI_3.4.0
+_pub_glfs_posix_lock _glfs_posix_lock$GFAPI_3.4.0
+_pub_glfs_dup _glfs_dup$GFAPI_3.4.0
+
+_pub_glfs_setfsuid _glfs_setfsuid$GFAPI_3.4.2
+_pub_glfs_setfsgid _glfs_setfsgid$GFAPI_3.4.2
+_pub_glfs_setfsgroups _glfs_setfsgroups$GFAPI_3.4.2
+_pub_glfs_h_lookupat34 _glfs_h_lookupat$GFAPI_3.4.2
+_pub_glfs_h_creat _glfs_h_creat$GFAPI_3.4.2
+_pub_glfs_h_mkdir _glfs_h_mkdir$GFAPI_3.4.2
+_pub_glfs_h_mknod _glfs_h_mknod$GFAPI_3.4.2
+_pub_glfs_h_symlink _glfs_h_symlink$GFAPI_3.4.2
+_pub_glfs_h_unlink _glfs_h_unlink$GFAPI_3.4.2
+_pub_glfs_h_close _glfs_h_close$GFAPI_3.4.2
+_pub_glfs_h_truncate _glfs_h_truncate$GFAPI_3.4.2
+_pub_glfs_h_stat _glfs_h_stat$GFAPI_3.4.2
+_pub_glfs_h_getattrs _glfs_h_getattrs$GFAPI_3.4.2
+_pub_glfs_h_setattrs _glfs_h_setattrs$GFAPI_3.4.2
+_pub_glfs_h_readlink _glfs_h_readlink$GFAPI_3.4.2
+_pub_glfs_h_link _glfs_h_link$GFAPI_3.4.2
+_pub_glfs_h_rename _glfs_h_rename$GFAPI_3.4.2
+_pub_glfs_h_extract_handle _glfs_h_extract_handle$GFAPI_3.4.2
+_pub_glfs_h_create_from_handle _glfs_h_create_from_handle$GFAPI_3.4.2
+_pub_glfs_h_opendir _glfs_h_opendir$GFAPI_3.4.2
+_pub_glfs_h_open _glfs_h_open$GFAPI_3.4.2
+
+_pub_glfs_get_volumeid _glfs_get_volumeid$GFAPI_3.5.0
+_pub_glfs_readdir _glfs_readdir$GFAPI_3.5.0
+_pub_glfs_readdirplus _glfs_readdirplus$GFAPI_3.5.0
+_pub_glfs_fallocate _glfs_fallocate$GFAPI_3.5.0
+_pub_glfs_discard _glfs_discard$GFAPI_3.5.0
+_pub_glfs_zerofill _glfs_zerofill$GFAPI_3.5.0
+_pub_glfs_caller_specific_init _glfs_caller_specific_init$GFAPI_3.5.0
+_pub_glfs_h_setxattrs _glfs_h_setxattrs$GFAPI_3.5.0
+
+_pub_glfs_unset_volfile_server _glfs_unset_volfile_server$GFAPI_3.5.1
+_pub_glfs_h_getxattrs _glfs_h_getxattrs$GFAPI_3.5.1
+_pub_glfs_h_removexattrs _glfs_h_removexattrs$GFAPI_3.5.1
+
+_pub_glfs_get_volfile _glfs_get_volfile$GFAPI_3.6.0
+_pub_glfs_h_access _glfs_h_access$GFAPI_3.6.0
+
+_pub_glfs_h_poll_upcall370 _glfs_h_poll_upcall$GFAPI_3.7.0
+_pub_glfs_h_acl_set _glfs_h_acl_set$GFAPI_3.7.0
+_pub_glfs_h_acl_get _glfs_h_acl_get$GFAPI_3.7.0
+_pub_glfs_h_statfs _glfs_h_statfs$GFAPI_3.7.0
+_pub_glfs_h_anonymous_read _glfs_h_anonymous_read$GFAPI_3.7.0
+_pub_glfs_h_anonymous_write _glfs_h_anonymous_write$GFAPI_3.7.0
+
+_priv_glfs_free_from_ctx _glfs_free_from_ctx$GFAPI_PRIVATE_3.7.0
+_priv_glfs_new_from_ctx _glfs_new_from_ctx$GFAPI_PRIVATE_3.7.0
+_priv_glfs_resolve _glfs_resolve$GFAPI_PRIVATE_3.7.0
+_priv_glfs_process_upcall_event _glfs_process_upcall_event$GFAPI_PRIVATE_3.7.0
+
+_pub_glfs_h_lookupat _glfs_h_lookupat$GFAPI_3.7.4
+
+_pub_glfs_truncate _glfs_truncate$GFAPI_3.7.15
+
+_pub_glfs_free _glfs_free$GFAPI_3.7.16
+_pub_glfs_h_poll_upcall _glfs_h_poll_upcall$GFAPI_3.7.16
+_pub_glfs_upcall_get_fs _glfs_upcall_get_fs$GFAPI_3.7.16
+_pub_glfs_upcall_get_reason _glfs_upcall_get_reason$GFAPI_3.7.16
+_pub_glfs_upcall_get_event _glfs_upcall_get_event$GFAPI_3.7.16
+_pub_glfs_upcall_inode_get_object _glfs_upcall_inode_get_object$GFAPI_3.7.16
+_pub_glfs_upcall_inode_get_flags _glfs_upcall_inode_get_flags$GFAPI_3.7.16
+_pub_glfs_upcall_inode_get_stat _glfs_upcall_inode_get_stat$GFAPI_3.7.16
+_pub_glfs_upcall_inode_get_expire _glfs_upcall_inode_get_expire$GFAPI_3.7.16
+_pub_glfs_upcall_inode_get_pobject _glfs_upcall_inode_get_pobject$GFAPI_3.7.16
+_pub_glfs_upcall_inode_get_pstat _glfs_upcall_inode_get_pstat$GFAPI_3.7.16
+_pub_glfs_upcall_inode_get_oldpobject _glfs_upcall_inode_get_oldpobject$GFAPI_3.7.16
+_pub_glfs_upcall_inode_get_oldpstat _glfs_upcall_inode_get_oldpstat$GFAPI_3.7.16
+
+_pub_glfs_realpath _glfs_realpath$GFAPI_3.7.17
+
+_pub_glfs_sysrq _glfs_sysrq$GFAPI_3.10.0
+
+_pub_glfs_fd_set_lkowner _glfs_fd_set_lkowner$GFAPI_3.10.7
+
+_pub_glfs_xreaddirplus_r _glfs_xreaddirplus_r$GFAPI_3.11.0
+_pub_glfs_xreaddirplus_r_get_stat _glfs_xreaddirplus_r_get_stat$GFAPI_3.11.0
+_pub_glfs_xreaddirplus_r_get_object _glfs_xreaddirplus_r_get_object$GFAPI_3.11.0
+_pub_glfs_object_copy _glfs_object_copy$GFAPI_3.11.0
+
+_priv_glfs_ipc _glfs_ipc$GFAPI_3.12.0
+
+_pub_glfs_upcall_register _glfs_upcall_register$GFAPI_3.13.0
+_pub_glfs_upcall_unregister _glfs_upcall_unregister$GFAPI_3.13.0
+
+_pub_glfs_setfsleaseid _glfs_setfsleaseid$GFAPI_4.0.0
+_pub_glfs_file_lock _glfs_file_lock$GFAPI_4.0.0
+_pub_glfs_lease _glfs_lease$GFAPI_4.0.0
+_pub_glfs_h_lease _glfs_h_lease$GFAPI_4.0.0
+_pub_glfs_upcall_lease_get_object _glfs_upcall_lease_get_object$GFAPI_4.1.6
+_pub_glfs_upcall_lease_get_lease_type _glfs_upcall_lease_get_lease_type$GFAPI_4.1.6
+
+_priv_glfs_statx _glfs_statx$GFAPI_6.0
+_priv_glfs_iatt_from_statx _glfs_iatt_from_statx$GFAPI_6.0
+_priv_glfs_setfspid _glfs_setfspid$GFAPI_6.1
+
+_pub_glfs_read_async _glfs_read_async$GFAPI_6.0
+_pub_glfs_write_async _glfs_write_async$GFAPI_6.0
+_pub_glfs_readv_async _glfs_readv_async$GFAPI_6.0
+_pub_glfs_writev_async _glfs_writev_async$GFAPI_6.0
+_pub_glfs_pread _glfs_pread$GFAPI_6.0
+_pub_glfs_pwrite _glfs_pwrite$GFAPI_6.0
+_pub_glfs_pread_async _glfs_pread_async$GFAPI_6.0
+_pub_glfs_pwrite_async _glfs_pwrite_async$GFAPI_6.0
+_pub_glfs_preadv_async _glfs_preadv_async$GFAPI_6.0
+_pub_glfs_pwritev_async _glfs_pwritev_async$GFAPI_6.0
+_pub_glfs_fsync _glfs_fsync$GFAPI_6.0
+_pub_glfs_fsync_async _glfs_fsync_async$GFAPI_6.0
+_pub_glfs_fdatasync _glfs_fdatasync$GFAPI_6.0
+_pub_glfs_fdatasync_async _glfs_fdatasync_async$GFAPI_6.0
+_pub_glfs_ftruncate _glfs_ftruncate$GFAPI_6.0
+_pub_glfs_ftruncate_async _glfs_ftruncate_async$GFAPI_6.0
+_pub_glfs_discard_async _glfs_discard_async$GFAPI_6.0
+_pub_glfs_zerofill_async _glfs_zerofill_async$GFAPI_6.0
+_pub_glfs_copy_file_range _glfs_copy_file_range$GFAPI_6.0
+_pub_glfs_fsetattr _glfs_fsetattr$GFAPI_6.0
+_pub_glfs_setattr _glfs_setattr$GFAPI_6.0
+
+_pub_glfs_set_statedump_path _glfs_set_statedump_path@GFAPI_7.0
+
+_pub_glfs_h_creat_open _glfs_h_creat_open@GFAPI_6.6
diff --git a/api/src/gfapi.map b/api/src/gfapi.map
new file mode 100644
index 00000000000..228ac47c084
--- /dev/null
+++ b/api/src/gfapi.map
@@ -0,0 +1,283 @@
+
+GFAPI_PRIVATE_3.4.0 {
+	global:
+		glfs_loc_touchup;
+		glfs_active_subvol;
+		glfs_subvol_done;
+		glfs_init_done;
+		glfs_resolve_at;
+	local: *;
+};
+
+GFAPI_3.4.0 {
+	global:
+		glfs_new;
+		glfs_set_volfile;
+		glfs_set_volfile_server;
+		glfs_set_logging;
+		glfs_init;
+		glfs_fini;
+		glfs_open;
+		glfs_creat;
+		glfs_close;
+		glfs_from_glfd;
+		glfs_set_xlator_option;
+		glfs_read;
+		glfs_write;
+		glfs_read_async;
+		glfs_write_async;
+		glfs_readv;
+		glfs_writev;
+		glfs_readv_async;
+		glfs_writev_async;
+		glfs_pread;
+		glfs_pwrite;
+		glfs_pread_async;
+		glfs_pwrite_async;
+		glfs_preadv;
+		glfs_pwritev;
+		glfs_preadv_async;
+		glfs_pwritev_async;
+		glfs_lseek;
+		glfs_ftruncate;
+		glfs_ftruncate_async;
+		glfs_lstat;
+		glfs_stat;
+		glfs_fstat;
+		glfs_fsync;
+		glfs_fsync_async;
+		glfs_fdatasync;
+		glfs_fdatasync_async;
+		glfs_access;
+		glfs_symlink;
+		glfs_readlink;
+		glfs_mknod;
+		glfs_mkdir;
+		glfs_unlink;
+		glfs_rmdir;
+		glfs_rename;
+		glfs_link;
+		glfs_opendir;
+		glfs_readdir_r;
+		glfs_readdirplus_r;
+		glfs_telldir;
+		glfs_seekdir;
+		glfs_closedir;
+		glfs_statvfs;
+		glfs_chmod;
+		glfs_fchmod;
+		glfs_chown;
+		glfs_lchown;
+		glfs_fchown;
+		glfs_utimens;
+		glfs_lutimens;
+		glfs_futimens;
+		glfs_getxattr;
+		glfs_lgetxattr;
+		glfs_fgetxattr;
+		glfs_listxattr;
+		glfs_llistxattr;
+		glfs_flistxattr;
+		glfs_setxattr;
+		glfs_lsetxattr;
+		glfs_fsetxattr;
+		glfs_removexattr;
+		glfs_lremovexattr;
+		glfs_fremovexattr;
+		glfs_getcwd;
+		glfs_chdir;
+		glfs_fchdir;
+		glfs_realpath;
+		glfs_posix_lock;
+		glfs_dup;
+} GFAPI_PRIVATE_3.4.0;
+
+GFAPI_3.4.2 {
+	global:
+		glfs_setfsuid;
+		glfs_setfsgid;
+		glfs_setfsgroups;
+		glfs_h_creat;
+		glfs_h_mkdir;
+		glfs_h_mknod;
+		glfs_h_symlink;
+		glfs_h_unlink;
+		glfs_h_close;
+		glfs_h_truncate;
+		glfs_h_stat;
+		glfs_h_getattrs;
+		glfs_h_setattrs;
+		glfs_h_readlink;
+		glfs_h_link;
+		glfs_h_rename;
+		glfs_h_extract_handle;
+		glfs_h_create_from_handle;
+		glfs_h_opendir;
+		glfs_h_open;
+		glfs_h_lookupat;
+} GFAPI_3.4.0;
+
+GFAPI_3.5.0 {
+	global:
+		glfs_get_volumeid;
+		glfs_readdir;
+		glfs_readdirplus;
+		glfs_fallocate;
+		glfs_discard;
+		glfs_discard_async;
+		glfs_zerofill;
+		glfs_zerofill_async;
+		glfs_caller_specific_init;
+		glfs_h_setxattrs;
+} GFAPI_3.4.2;
+
+GFAPI_3.5.1 {
+	global:
+		glfs_unset_volfile_server;
+		glfs_h_getxattrs;
+		glfs_h_removexattrs;
+} GFAPI_3.5.0;
+
+GFAPI_3.6.0 {
+	global:
+		glfs_get_volfile;
+		glfs_h_access;
+} GFAPI_3.5.1;
+
+GFAPI_3.7.0 {
+	global:
+		glfs_h_poll_upcall;
+		glfs_h_acl_set;
+		glfs_h_acl_get;
+		glfs_h_statfs;
+		glfs_h_anonymous_read;
+		glfs_h_anonymous_write;
+} GFAPI_3.6.0;
+
+GFAPI_PRIVATE_3.7.0 {
+	global:
+		glfs_free_from_ctx;
+		glfs_new_from_ctx;
+		glfs_resolve;
+		glfs_process_upcall_event;
+} GFAPI_3.7.0;
+
+GFAPI_3.7.4 {
+	global:
+		glfs_h_lookupat;
+} GFAPI_PRIVATE_3.7.0;
+
+GFAPI_3.7.15 {
+	global:
+		glfs_truncate;
+} GFAPI_3.7.4;
+
+GFAPI_3.7.16 {
+	global:
+		glfs_free;
+		glfs_upcall_get_fs;
+		glfs_upcall_get_reason;
+		glfs_upcall_get_event;
+		glfs_upcall_inode_get_object;
+		glfs_upcall_inode_get_flags;
+		glfs_upcall_inode_get_stat;
+		glfs_upcall_inode_get_expire;
+		glfs_upcall_inode_get_pobject;
+		glfs_upcall_inode_get_pstat;
+		glfs_upcall_inode_get_oldpobject;
+		glfs_upcall_inode_get_oldpstat;
+		glfs_h_poll_upcall;
+} GFAPI_3.7.15;
+
+GFAPI_3.7.17 {
+	global:
+		glfs_realpath;
+} GFAPI_3.7.16;
+
+GFAPI_3.10.0 {
+	global:
+		glfs_sysrq;
+} GFAPI_3.7.17;
+
+GFAPI_3.10.7 {
+	global:
+		glfs_fd_set_lkowner;
+} GFAPI_3.10.0;
+
+GFAPI_3.11.0 {
+		glfs_xreaddirplus_r;
+		glfs_xreaddirplus_r_get_stat;
+		glfs_xreaddirplus_r_get_object;
+		glfs_object_copy;
+} GFAPI_3.10.7;
+
+GFAPI_PRIVATE_3.12.0 {
+	global:
+		glfs_ipc;
+} GFAPI_3.11.0;
+
+GFAPI_3.13.0 {
+	global:
+		glfs_upcall_register;
+		glfs_upcall_unregister;
+} GFAPI_PRIVATE_3.12.0;
+
+GFAPI_4.0.0 {
+	global:
+		glfs_setfsleaseid;
+		glfs_file_lock;
+		glfs_lease;
+		glfs_h_lease;
+} GFAPI_3.13.0;
+
+GFAPI_4.1.6 {
+	global:
+		glfs_upcall_lease_get_object;
+		glfs_upcall_lease_get_lease_type;
+} GFAPI_4.0.0;
+
+GFAPI_PRIVATE_6.0 {
+	global:
+		glfs_statx;
+		glfs_iatt_from_statx;
+} GFAPI_4.1.6;
+
+GFAPI_6.0 {
+	global:
+		glfs_read_async;
+		glfs_write_async;
+		glfs_readv_async;
+		glfs_writev_async;
+		glfs_pread;
+		glfs_pwrite;
+		glfs_pread_async;
+		glfs_pwrite_async;
+		glfs_preadv_async;
+		glfs_pwritev_async;
+		glfs_fsync;
+		glfs_fsync_async;
+		glfs_fdatasync;
+		glfs_fdatasync_async;
+		glfs_ftruncate;
+		glfs_ftruncate_async;
+		glfs_discard_async;
+		glfs_zerofill_async;
+		glfs_copy_file_range;
+		glfs_setattr;
+		glfs_fsetattr;
+} GFAPI_PRIVATE_6.0;
+
+GFAPI_PRIVATE_6.1 {
+	global:
+		glfs_setfspid;
+} GFAPI_6.0;
+
+GFAPI_6.6 {
+	global:
+		glfs_h_creat_open;
+} GFAPI_PRIVATE_6.1;
+
+GFAPI_7.0 {
+	global:
+		glfs_set_statedump_path;
+} GFAPI_6.6;
diff --git a/api/src/glfs-fops.c b/api/src/glfs-fops.c
index e411f5a8304..6aa3c5602d1 100644
--- a/api/src/glfs-fops.c
+++ b/api/src/glfs-fops.c
@@ -1,5 +1,6 @@
+
 /*
-  Copyright (c) 2012 Red Hat, Inc. <http://www.redhat.com>
+  Copyright (c) 2012-2018 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
   This file is licensed to you under your choice of the GNU Lesser
@@ -8,3122 +9,6437 @@
   cases as published by the Free Software Foundation.
 */
 
+/* for SEEK_HOLE and SEEK_DATA */
+#ifndef _GNU_SOURCE
+#define _GNU_SOURCE
+#endif
+
+#include <unistd.h>
 
 #include "glfs-internal.h"
 #include "glfs-mem-types.h"
-#include "syncop.h"
+#include <glusterfs/syncop.h>
 #include "glfs.h"
+#include "gfapi-messages.h"
+#include <glusterfs/compat-errno.h>
+#include <limits.h>
+#include "glusterfs3.h"
+#include <glusterfs/iatt.h>
+
+#ifdef NAME_MAX
+#define GF_NAME_MAX NAME_MAX
+#else
+#define GF_NAME_MAX 255
+#endif
 
-#define DEFAULT_REVAL_COUNT 1
+struct upcall_syncop_args {
+    struct glfs *fs;
+    struct gf_upcall upcall_data;
+};
 
-#define ESTALE_RETRY(ret,errno,reval,loc,label) do {	\
-	if (ret == -1 && errno == ESTALE) {	        \
-		if (reval < DEFAULT_REVAL_COUNT) {	\
-			reval++;			\
-			loc_wipe (loc);			\
-			goto label;			\
-		}					\
-	}						\
-	} while (0)
+#define READDIRBUF_SIZE (sizeof(struct dirent) + GF_NAME_MAX + 1)
 
+typedef void (*glfs_io_cbk34)(glfs_fd_t *fd, ssize_t ret, void *data);
 
-static int
-glfs_loc_link (loc_t *loc, struct iatt *iatt)
+/*
+ * This function will mark glfd for deletion and decrement its refcount.
+ */
+int
+glfs_mark_glfd_for_deletion(struct glfs_fd *glfd)
 {
-	int ret = -1;
-	inode_t *linked_inode = NULL;
-
-	if (!loc->inode) {
-		errno = EINVAL;
-		return -1;
-	}
+    LOCK(&glfd->lock);
+    {
+        glfd->state = GLFD_CLOSE;
+    }
+    UNLOCK(&glfd->lock);
 
-	linked_inode = inode_link (loc->inode, loc->parent, loc->name, iatt);
-	if (linked_inode) {
-		inode_lookup (linked_inode);
-		inode_unref (linked_inode);
-		ret = 0;
-	} else {
-		ret = -1;
-		errno = ENOMEM;
-	}
+    GF_REF_PUT(glfd);
 
-	return ret;
+    return 0;
 }
 
-
-static void
-glfs_iatt_to_stat (struct glfs *fs, struct iatt *iatt, struct stat *stat)
+/* This function is useful for all async fops. There is chance that glfd is
+ * closed before async fop is completed. When glfd is closed we change the
+ * state to GLFD_CLOSE.
+ *
+ * This function will return _gf_true if the glfd is still valid else return
+ * _gf_false.
+ */
+gf_boolean_t
+glfs_is_glfd_still_valid(struct glfs_fd *glfd)
 {
-	iatt_to_stat (iatt, stat);
-	stat->st_dev = fs->dev_id;
-}
+    gf_boolean_t ret = _gf_false;
 
+    LOCK(&glfd->lock);
+    {
+        if (glfd->state != GLFD_CLOSE)
+            ret = _gf_true;
+    }
+    UNLOCK(&glfd->lock);
 
-static int
-glfs_loc_unlink (loc_t *loc)
+    return ret;
+}
+
+void
+glfd_set_state_bind(struct glfs_fd *glfd)
 {
-	inode_unlink (loc->inode, loc->parent, loc->name);
+    LOCK(&glfd->lock);
+    {
+        glfd->state = GLFD_OPEN;
+    }
+    UNLOCK(&glfd->lock);
 
-	return 0;
-}
+    fd_bind(glfd->fd);
+    glfs_fd_bind(glfd);
 
+    return;
+}
 
-struct glfs_fd *
-glfs_open (struct glfs *fs, const char *path, int flags)
+/*
+ * This routine is called when an upcall event of type
+ * 'GF_UPCALL_CACHE_INVALIDATION' is received.
+ * It makes a copy of the contents of the upcall cache-invalidation
+ * data received into an entry which is stored in the upcall list
+ * maintained by gfapi.
+ */
+int
+glfs_get_upcall_cache_invalidation(struct gf_upcall *to_up_data,
+                                   struct gf_upcall *from_up_data)
 {
-	int              ret = -1;
-	struct glfs_fd  *glfd = NULL;
-	xlator_t        *subvol = NULL;
-	loc_t            loc = {0, };
-	struct iatt      iatt = {0, };
-	int              reval = 0;
+    struct gf_upcall_cache_invalidation *ca_data = NULL;
+    struct gf_upcall_cache_invalidation *f_ca_data = NULL;
+    int ret = -1;
 
-	__glfs_entry_fs (fs);
+    GF_VALIDATE_OR_GOTO(THIS->name, to_up_data, out);
+    GF_VALIDATE_OR_GOTO(THIS->name, from_up_data, out);
 
-	subvol = glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+    f_ca_data = from_up_data->data;
+    GF_VALIDATE_OR_GOTO(THIS->name, f_ca_data, out);
 
-	glfd = glfs_fd_new (fs);
-	if (!glfd)
-		goto out;
+    ca_data = GF_CALLOC(1, sizeof(*ca_data), glfs_mt_upcall_entry_t);
 
-retry:
-	ret = glfs_resolve (fs, subvol, path, &loc, &iatt, reval);
+    if (!ca_data) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, errno, API_MSG_ALLOC_FAILED, "entry",
+                NULL);
+        goto out;
+    }
+
+    to_up_data->data = ca_data;
+
+    ca_data->flags = f_ca_data->flags;
+    ca_data->expire_time_attr = f_ca_data->expire_time_attr;
+    ca_data->stat = f_ca_data->stat;
+    ca_data->p_stat = f_ca_data->p_stat;
+    ca_data->oldp_stat = f_ca_data->oldp_stat;
+
+    ret = 0;
+out:
+    return ret;
+}
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+int
+glfs_get_upcall_lease(struct gf_upcall *to_up_data,
+                      struct gf_upcall *from_up_data)
+{
+    struct gf_upcall_recall_lease *ca_data = NULL;
+    struct gf_upcall_recall_lease *f_ca_data = NULL;
+    int ret = -1;
 
-	if (ret)
-		goto out;
+    GF_VALIDATE_OR_GOTO(THIS->name, to_up_data, out);
+    GF_VALIDATE_OR_GOTO(THIS->name, from_up_data, out);
 
-	if (IA_ISDIR (iatt.ia_type)) {
-		ret = -1;
-		errno = EISDIR;
-		goto out;
-	}
+    f_ca_data = from_up_data->data;
+    GF_VALIDATE_OR_GOTO(THIS->name, f_ca_data, out);
 
-	if (!IA_ISREG (iatt.ia_type)) {
-		ret = -1;
-		errno = EINVAL;
-		goto out;
-	}
+    ca_data = GF_CALLOC(1, sizeof(*ca_data), glfs_mt_upcall_entry_t);
 
-	if (glfd->fd) {
-		/* Retry. Safe to touch glfd->fd as we
-		   still have not glfs_fd_bind() yet.
-		*/
-		fd_unref (glfd->fd);
-		glfd->fd = NULL;
-	}
+    if (!ca_data) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, errno, API_MSG_ALLOC_FAILED, "entry",
+                NULL);
+        goto out;
+    }
 
-	glfd->fd = fd_create (loc.inode, getpid());
-	if (!glfd->fd) {
-		ret = -1;
-		errno = ENOMEM;
-		goto out;
-	}
+    to_up_data->data = ca_data;
 
-	ret = syncop_open (subvol, &loc, flags, glfd->fd);
+    ca_data->lease_type = f_ca_data->lease_type;
+    gf_uuid_copy(ca_data->tid, f_ca_data->tid);
+    ca_data->dict = f_ca_data->dict;
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    ret = 0;
 out:
-	loc_wipe (&loc);
+    return ret;
+}
+int
+glfs_loc_link(loc_t *loc, struct iatt *iatt)
+{
+    int ret = -1;
+    inode_t *old_inode = NULL;
+    uint64_t ctx_value = LOOKUP_NOT_NEEDED;
+
+    if (!loc->inode) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    old_inode = loc->inode;
+
+    /* If the inode already exists in the cache, the inode
+     * returned here points to the existing one. We need
+     * to update loc.inode accordingly.
+     */
+    loc->inode = inode_link(loc->inode, loc->parent, loc->name, iatt);
+    if (loc->inode) {
+        inode_ctx_set(loc->inode, THIS, &ctx_value);
+        inode_lookup(loc->inode);
+        inode_unref(old_inode);
+        ret = 0;
+    } else {
+        ret = -1;
+    }
+
+    return ret;
+}
 
-	if (ret && glfd) {
-		glfs_fd_destroy (glfd);
-		glfd = NULL;
-	} else if (glfd) {
-                glfd->fd->flags = flags;
-		fd_bind (glfd->fd);
-		glfs_fd_bind (glfd);
-	}
+void
+glfs_iatt_to_stat(struct glfs *fs, struct iatt *iatt, struct stat *stat)
+{
+    iatt_to_stat(iatt, stat);
+    stat->st_dev = fs->dev_id;
+}
 
-	glfs_subvol_done (fs, subvol);
+void
+glfs_iatt_to_statx(struct glfs *fs, const struct iatt *iatt,
+                   struct glfs_stat *statx)
+{
+    statx->glfs_st_mask = 0;
+
+    statx->glfs_st_mode = 0;
+    if (IATT_TYPE_VALID(iatt->ia_flags)) {
+        statx->glfs_st_mode |= st_mode_type_from_ia(iatt->ia_type);
+        statx->glfs_st_mask |= GLFS_STAT_TYPE;
+    }
+
+    if (IATT_MODE_VALID(iatt->ia_flags)) {
+        statx->glfs_st_mode |= st_mode_prot_from_ia(iatt->ia_prot);
+        statx->glfs_st_mask |= GLFS_STAT_MODE;
+    }
+
+    if (IATT_NLINK_VALID(iatt->ia_flags)) {
+        statx->glfs_st_nlink = iatt->ia_nlink;
+        statx->glfs_st_mask |= GLFS_STAT_NLINK;
+    }
+
+    if (IATT_UID_VALID(iatt->ia_flags)) {
+        statx->glfs_st_uid = iatt->ia_uid;
+        statx->glfs_st_mask |= GLFS_STAT_UID;
+    }
+
+    if (IATT_GID_VALID(iatt->ia_flags)) {
+        statx->glfs_st_gid = iatt->ia_gid;
+        statx->glfs_st_mask |= GLFS_STAT_GID;
+    }
+
+    if (IATT_ATIME_VALID(iatt->ia_flags)) {
+        statx->glfs_st_atime.tv_sec = iatt->ia_atime;
+        statx->glfs_st_atime.tv_nsec = iatt->ia_atime_nsec;
+        statx->glfs_st_mask |= GLFS_STAT_ATIME;
+    }
+
+    if (IATT_MTIME_VALID(iatt->ia_flags)) {
+        statx->glfs_st_mtime.tv_sec = iatt->ia_mtime;
+        statx->glfs_st_mtime.tv_nsec = iatt->ia_mtime_nsec;
+        statx->glfs_st_mask |= GLFS_STAT_MTIME;
+    }
+
+    if (IATT_CTIME_VALID(iatt->ia_flags)) {
+        statx->glfs_st_ctime.tv_sec = iatt->ia_ctime;
+        statx->glfs_st_ctime.tv_nsec = iatt->ia_ctime_nsec;
+        statx->glfs_st_mask |= GLFS_STAT_CTIME;
+    }
+
+    if (IATT_BTIME_VALID(iatt->ia_flags)) {
+        statx->glfs_st_btime.tv_sec = iatt->ia_btime;
+        statx->glfs_st_btime.tv_nsec = iatt->ia_btime_nsec;
+        statx->glfs_st_mask |= GLFS_STAT_BTIME;
+    }
+
+    if (IATT_INO_VALID(iatt->ia_flags)) {
+        statx->glfs_st_ino = iatt->ia_ino;
+        statx->glfs_st_mask |= GLFS_STAT_INO;
+    }
+
+    if (IATT_SIZE_VALID(iatt->ia_flags)) {
+        statx->glfs_st_size = iatt->ia_size;
+        statx->glfs_st_mask |= GLFS_STAT_SIZE;
+    }
+
+    if (IATT_BLOCKS_VALID(iatt->ia_flags)) {
+        statx->glfs_st_blocks = iatt->ia_blocks;
+        statx->glfs_st_mask |= GLFS_STAT_BLOCKS;
+    }
+
+    /* unconditionally present, encode as is */
+    statx->glfs_st_blksize = iatt->ia_blksize;
+    statx->glfs_st_rdev_major = ia_major(iatt->ia_rdev);
+    statx->glfs_st_rdev_minor = ia_minor(iatt->ia_rdev);
+    statx->glfs_st_dev_major = ia_major(fs->dev_id);
+    statx->glfs_st_dev_minor = ia_minor(fs->dev_id);
+
+    /* At present we do not read any localFS attributes and pass them along,
+     * so setting this to 0. As we start supporting file attributes we can
+     * populate the same here as well */
+    statx->glfs_st_attributes = 0;
+    statx->glfs_st_attributes_mask = 0;
+}
 
-	return glfd;
+GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_iatt_from_statx, 6.0)
+void
+priv_glfs_iatt_from_statx(struct iatt *iatt, const struct glfs_stat *statx)
+{
+    /* Most code in xlators are not checking validity flags before accessing
+    the items. Hence zero everything before setting valid items */
+    memset(iatt, 0, sizeof(struct iatt));
+
+    if (GLFS_STAT_TYPE_VALID(statx->glfs_st_mask)) {
+        iatt->ia_type = ia_type_from_st_mode(statx->glfs_st_mode);
+        iatt->ia_flags |= IATT_TYPE;
+    }
+
+    if (GLFS_STAT_MODE_VALID(statx->glfs_st_mask)) {
+        iatt->ia_prot = ia_prot_from_st_mode(statx->glfs_st_mode);
+        iatt->ia_flags |= IATT_MODE;
+    }
+
+    if (GLFS_STAT_NLINK_VALID(statx->glfs_st_mask)) {
+        iatt->ia_nlink = statx->glfs_st_nlink;
+        iatt->ia_flags |= IATT_NLINK;
+    }
+
+    if (GLFS_STAT_UID_VALID(statx->glfs_st_mask)) {
+        iatt->ia_uid = statx->glfs_st_uid;
+        iatt->ia_flags |= IATT_UID;
+    }
+
+    if (GLFS_STAT_GID_VALID(statx->glfs_st_mask)) {
+        iatt->ia_gid = statx->glfs_st_gid;
+        iatt->ia_flags |= IATT_GID;
+    }
+
+    if (GLFS_STAT_ATIME_VALID(statx->glfs_st_mask)) {
+        iatt->ia_atime = statx->glfs_st_atime.tv_sec;
+        iatt->ia_atime_nsec = statx->glfs_st_atime.tv_nsec;
+        iatt->ia_flags |= IATT_ATIME;
+    }
+
+    if (GLFS_STAT_MTIME_VALID(statx->glfs_st_mask)) {
+        iatt->ia_mtime = statx->glfs_st_mtime.tv_sec;
+        iatt->ia_mtime_nsec = statx->glfs_st_mtime.tv_nsec;
+        iatt->ia_flags |= IATT_MTIME;
+    }
+
+    if (GLFS_STAT_CTIME_VALID(statx->glfs_st_mask)) {
+        iatt->ia_ctime = statx->glfs_st_ctime.tv_sec;
+        iatt->ia_ctime_nsec = statx->glfs_st_ctime.tv_nsec;
+        iatt->ia_flags |= IATT_CTIME;
+    }
+
+    if (GLFS_STAT_BTIME_VALID(statx->glfs_st_mask)) {
+        iatt->ia_btime = statx->glfs_st_btime.tv_sec;
+        iatt->ia_btime_nsec = statx->glfs_st_btime.tv_nsec;
+        iatt->ia_flags |= IATT_BTIME;
+    }
+
+    if (GLFS_STAT_INO_VALID(statx->glfs_st_mask)) {
+        iatt->ia_ino = statx->glfs_st_ino;
+        iatt->ia_flags |= IATT_INO;
+    }
+
+    if (GLFS_STAT_SIZE_VALID(statx->glfs_st_mask)) {
+        iatt->ia_size = statx->glfs_st_size;
+        iatt->ia_flags |= IATT_SIZE;
+    }
+
+    if (GLFS_STAT_BLOCKS_VALID(statx->glfs_st_mask)) {
+        iatt->ia_blocks = statx->glfs_st_blocks;
+        iatt->ia_flags |= IATT_BLOCKS;
+    }
+
+    /* unconditionally present, encode as is */
+    iatt->ia_blksize = statx->glfs_st_blksize;
+    iatt->ia_rdev = makedev(statx->glfs_st_rdev_major,
+                            statx->glfs_st_rdev_minor);
+    iatt->ia_dev = makedev(statx->glfs_st_dev_major, statx->glfs_st_dev_minor);
+    iatt->ia_attributes = statx->glfs_st_attributes;
+    iatt->ia_attributes_mask = statx->glfs_st_attributes_mask;
 }
 
+void
+glfsflags_from_gfapiflags(struct glfs_stat *stat, int *glvalid)
+{
+    *glvalid = 0;
+    if (stat->glfs_st_mask & GLFS_STAT_MODE) {
+        *glvalid |= GF_SET_ATTR_MODE;
+    }
+
+    if (stat->glfs_st_mask & GLFS_STAT_SIZE) {
+        *glvalid |= GF_SET_ATTR_SIZE;
+    }
+
+    if (stat->glfs_st_mask & GLFS_STAT_UID) {
+        *glvalid |= GF_SET_ATTR_UID;
+    }
+
+    if (stat->glfs_st_mask & GLFS_STAT_GID) {
+        *glvalid |= GF_SET_ATTR_GID;
+    }
+
+    if (stat->glfs_st_mask & GLFS_STAT_ATIME) {
+        *glvalid |= GF_SET_ATTR_ATIME;
+    }
+
+    if (stat->glfs_st_mask & GLFS_STAT_MTIME) {
+        *glvalid |= GF_SET_ATTR_MTIME;
+    }
+}
 
 int
-glfs_close (struct glfs_fd *glfd)
+glfs_loc_unlink(loc_t *loc)
 {
-	xlator_t  *subvol = NULL;
-	int        ret = -1;
-	fd_t      *fd = NULL;
-	struct glfs *fs = NULL;
+    inode_unlink(loc->inode, loc->parent, loc->name);
 
-	__glfs_entry_fd (glfd);
+    /* since glfs_h_* objects hold a reference to inode
+     * it is safe to keep lookup count to '0' */
+    if (!inode_has_dentry(loc->inode))
+        inode_forget(loc->inode, 0);
 
-	subvol = glfs_active_subvol (glfd->fs);
-        if (!subvol) {
-                ret = -1;
-                errno = EIO;
-                goto out;
-        }
+    return 0;
+}
 
-	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
-	if (!fd) {
-		ret = -1;
-		errno = EBADFD;
-		goto out;
-	}
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_open, 3.4.0)
+struct glfs_fd *
+pub_glfs_open(struct glfs *fs, const char *path, int flags)
+{
+    int ret = -1;
+    struct glfs_fd *glfd = NULL;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    int reval = 0;
+    dict_t *fop_attr = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    glfd = glfs_fd_new(fs);
+    if (!glfd)
+        goto out;
+
+retry:
+    ret = glfs_resolve(fs, subvol, path, &loc, &iatt, reval);
 
-	ret = syncop_flush (subvol, fd);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
+
+    if (ret)
+        goto out;
+
+    if (IA_ISDIR(iatt.ia_type)) {
+        ret = -1;
+        errno = EISDIR;
+        goto out;
+    }
+
+    if (!IA_ISREG(iatt.ia_type)) {
+        ret = -1;
+        errno = EINVAL;
+        goto out;
+    }
+
+    if (glfd->fd) {
+        /* Retry. Safe to touch glfd->fd as we
+           still have not glfs_fd_bind() yet.
+        */
+        fd_unref(glfd->fd);
+        glfd->fd = NULL;
+    }
+
+    glfd->fd = fd_create(loc.inode, getpid());
+    if (!glfd->fd) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+    glfd->fd->flags = flags;
+
+    ret = get_fop_attr_thrd_key(&fop_attr);
+    if (ret)
+        gf_msg_debug("gfapi", 0, "Getting leaseid from thread failed");
+
+    ret = syncop_open(subvol, &loc, flags, glfd->fd, fop_attr, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 out:
-	fs = glfd->fs;
-	glfs_fd_destroy (glfd);
+    loc_wipe(&loc);
 
-	if (fd)
-		fd_unref (fd);
+    if (fop_attr)
+        dict_unref(fop_attr);
 
-	glfs_subvol_done (fs, subvol);
+    if (ret && glfd) {
+        GF_REF_PUT(glfd);
+        glfd = NULL;
+    } else if (glfd) {
+        glfd_set_state_bind(glfd);
+    }
 
-	return ret;
-}
+    glfs_subvol_done(fs, subvol);
+
+    __GLFS_EXIT_FS;
 
+invalid_fs:
+    return glfd;
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_close, 3.4.0)
 int
-glfs_lstat (struct glfs *fs, const char *path, struct stat *stat)
+pub_glfs_close(struct glfs_fd *glfd)
 {
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	loc_t            loc = {0, };
-	struct iatt      iatt = {0, };
-	int              reval = 0;
+    xlator_t *subvol = NULL;
+    int ret = -1;
+    fd_t *fd = NULL;
+    struct glfs *fs = NULL;
+    dict_t *fop_attr = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    if (glfd->lk_owner.len != 0) {
+        ret = syncopctx_setfslkowner(&glfd->lk_owner);
+        if (ret)
+            goto out;
+    }
+    ret = get_fop_attr_thrd_key(&fop_attr);
+    if (ret)
+        gf_msg_debug("gfapi", 0, "Getting leaseid from thread failed");
+
+    ret = syncop_flush(subvol, fd, fop_attr, NULL);
+    DECODE_SYNCOP_ERR(ret);
+out:
+    fs = glfd->fs;
+
+    if (fd)
+        fd_unref(fd);
+    if (fop_attr)
+        dict_unref(fop_attr);
 
-	__glfs_entry_fs (fs);
+    glfs_mark_glfd_for_deletion(glfd);
+    glfs_subvol_done(fs, subvol);
 
-	subvol = glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_lstat, 3.4.0)
+int
+pub_glfs_lstat(struct glfs *fs, const char *path, struct stat *stat)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
 retry:
-	ret = glfs_lresolve (fs, subvol, path, &loc, &iatt, reval);
+    ret = glfs_lresolve(fs, subvol, path, &loc, &iatt, reval);
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	if (ret == 0 && stat)
-		glfs_iatt_to_stat (fs, &iatt, stat);
+    if (ret == 0 && stat)
+        glfs_iatt_to_stat(fs, &iatt, stat);
 out:
-	loc_wipe (&loc);
+    loc_wipe(&loc);
 
-	glfs_subvol_done (fs, subvol);
+    glfs_subvol_done(fs, subvol);
 
-	return ret;
-}
+    __GLFS_EXIT_FS;
 
+invalid_fs:
+    return ret;
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_stat, 3.4.0)
 int
-glfs_stat (struct glfs *fs, const char *path, struct stat *stat)
+pub_glfs_stat(struct glfs *fs, const char *path, struct stat *stat)
 {
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	loc_t            loc = {0, };
-	struct iatt      iatt = {0, };
-	int              reval = 0;
-
-	__glfs_entry_fs (fs);
-
-	subvol = glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
 retry:
-	ret = glfs_resolve (fs, subvol, path, &loc, &iatt, reval);
+    ret = glfs_resolve(fs, subvol, path, &loc, &iatt, reval);
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	if (ret == 0 && stat)
-		glfs_iatt_to_stat (fs, &iatt, stat);
+    if (ret == 0 && stat)
+        glfs_iatt_to_stat(fs, &iatt, stat);
 out:
-	loc_wipe (&loc);
+    loc_wipe(&loc);
 
-	glfs_subvol_done (fs, subvol);
+    glfs_subvol_done(fs, subvol);
 
-	return ret;
-}
+    __GLFS_EXIT_FS;
 
+invalid_fs:
+    return ret;
+}
 
+GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_statx, 6.0)
 int
-glfs_fstat (struct glfs_fd *glfd, struct stat *stat)
+priv_glfs_statx(struct glfs *fs, const char *path, const unsigned int mask,
+                struct glfs_stat *statxbuf)
 {
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	struct iatt      iatt = {0, };
-	fd_t            *fd = NULL;
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    if (path == NULL) {
+        ret = -1;
+        errno = EINVAL;
+        goto out;
+    }
+
+    if (mask & ~GLFS_STAT_ALL) {
+        ret = -1;
+        errno = EINVAL;
+        goto out;
+    }
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
 
-	__glfs_entry_fd (glfd);
+retry:
+    ret = glfs_resolve(fs, subvol, path, &loc, &iatt, reval);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	subvol = glfs_active_subvol (glfd->fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+    if (ret == 0 && statxbuf)
+        glfs_iatt_to_statx(fs, &iatt, statxbuf);
+out:
+    loc_wipe(&loc);
 
-	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
-	if (!fd) {
-		ret = -1;
-		errno = EBADFD;
-		goto out;
-	}
+    glfs_subvol_done(fs, subvol);
 
-	ret = syncop_fstat (subvol, fd, &iatt);
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
 
-	if (ret == 0 && stat)
-		glfs_iatt_to_stat (glfd->fs, &iatt, stat);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fstat, 3.4.0)
+int
+pub_glfs_fstat(struct glfs_fd *glfd, struct stat *stat)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    struct iatt iatt = {
+        0,
+    };
+    fd_t *fd = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    GF_REF_GET(glfd);
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    ret = syncop_fstat(subvol, fd, &iatt, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    if (ret == 0 && stat)
+        glfs_iatt_to_stat(glfd->fs, &iatt, stat);
 out:
-	if (fd)
-		fd_unref (fd);
+    if (fd)
+        fd_unref(fd);
+    if (glfd)
+        GF_REF_PUT(glfd);
 
-	glfs_subvol_done (glfd->fs, subvol);
+    glfs_subvol_done(glfd->fs, subvol);
 
-	return ret;
-}
+    __GLFS_EXIT_FS;
 
+invalid_fs:
+    return ret;
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_creat, 3.4.0)
 struct glfs_fd *
-glfs_creat (struct glfs *fs, const char *path, int flags, mode_t mode)
-{
-	int              ret = -1;
-	struct glfs_fd  *glfd = NULL;
-	xlator_t        *subvol = NULL;
-	loc_t            loc = {0, };
-	struct iatt      iatt = {0, };
-	uuid_t           gfid;
-	dict_t          *xattr_req = NULL;
-	int              reval = 0;
-
-	__glfs_entry_fs (fs);
-
-	subvol = glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
-
-	xattr_req = dict_new ();
-	if (!xattr_req) {
-		ret = -1;
-		errno = ENOMEM;
-		goto out;
-	}
-
-	uuid_generate (gfid);
-	ret = dict_set_static_bin (xattr_req, "gfid-req", gfid, 16);
-	if (ret) {
-		ret = -1;
-		errno = ENOMEM;
-		goto out;
-	}
-
-	glfd = glfs_fd_new (fs);
-	if (!glfd)
-		goto out;
-
-	/* This must be glfs_resolve() and NOT glfs_lresolve().
-	   That is because open("name", O_CREAT) where "name"
-	   is a danging symlink must create the dangling
-	   destinataion.
-	*/
+pub_glfs_creat(struct glfs *fs, const char *path, int flags, mode_t mode)
+{
+    int ret = -1;
+    struct glfs_fd *glfd = NULL;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    uuid_t gfid;
+    dict_t *xattr_req = NULL;
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    xattr_req = dict_new();
+    if (!xattr_req) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gf_uuid_generate(gfid);
+    ret = dict_set_gfuuid(xattr_req, "gfid-req", gfid, true);
+    if (ret) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    glfd = glfs_fd_new(fs);
+    if (!glfd)
+        goto out;
+
+    /* This must be glfs_resolve() and NOT glfs_lresolve().
+       That is because open("name", O_CREAT) where "name"
+       is a danging symlink must create the dangling
+       destination.
+    */
 retry:
-	ret = glfs_resolve (fs, subvol, path, &loc, &iatt, reval);
-
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
-
-	if (ret == -1 && errno != ENOENT)
-		/* Any other type of error is fatal */
-		goto out;
-
-	if (ret == -1 && errno == ENOENT && !loc.parent)
-		/* The parent directory or an ancestor even
-		   higher does not exist
-		*/
-		goto out;
-
-	if (loc.inode) {
-		if (flags & O_EXCL) {
-			ret = -1;
-			errno = EEXIST;
-			goto out;
-		}
-
-		if (IA_ISDIR (iatt.ia_type)) {
-			ret = -1;
-			errno = EISDIR;
-			goto out;
-		}
-
-		if (!IA_ISREG (iatt.ia_type)) {
-			ret = -1;
-			errno = EINVAL;
-			goto out;
-		}
-	}
-
-	if (ret == -1 && errno == ENOENT) {
-		loc.inode = inode_new (loc.parent->table);
-		if (!loc.inode) {
-			ret = -1;
-			errno = ENOMEM;
-			goto out;
-		}
-	}
-
-	if (glfd->fd) {
-		/* Retry. Safe to touch glfd->fd as we
-		   still have not glfs_fd_bind() yet.
-		*/
-		fd_unref (glfd->fd);
-		glfd->fd = NULL;
-	}
-
-	glfd->fd = fd_create (loc.inode, getpid());
-	if (!glfd->fd) {
-		ret = -1;
-		errno = ENOMEM;
-		goto out;
-	}
-
-	if (ret == 0) {
-		ret = syncop_open (subvol, &loc, flags, glfd->fd);
-	} else {
-		ret = syncop_create (subvol, &loc, flags, mode, glfd->fd,
-				     xattr_req, &iatt);
-	}
-
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
-
-	if (ret == 0)
-		ret = glfs_loc_link (&loc, &iatt);
-out:
-	loc_wipe (&loc);
-
-	if (xattr_req)
-		dict_unref (xattr_req);
-
-	if (ret && glfd) {
-		glfs_fd_destroy (glfd);
-		glfd = NULL;
-	} else if (glfd) {
-                glfd->fd->flags = flags;
-		fd_bind (glfd->fd);
-		glfs_fd_bind (glfd);
-	}
-
-	glfs_subvol_done (fs, subvol);
-
-	return glfd;
-}
+    ret = glfs_resolve(fs, subvol, path, &loc, &iatt, reval);
 
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-off_t
-glfs_lseek (struct glfs_fd *glfd, off_t offset, int whence)
-{
-	struct stat sb = {0, };
-	int         ret = -1;
+    if (ret == -1 && errno != ENOENT)
+        /* Any other type of error is fatal */
+        goto out;
 
-	__glfs_entry_fd (glfd);
+    if (ret == -1 && errno == ENOENT && !loc.parent)
+        /* The parent directory or an ancestor even
+           higher does not exist
+        */
+        goto out;
 
-	switch (whence) {
-	case SEEK_SET:
-		glfd->offset = offset;
-		break;
-	case SEEK_CUR:
-		glfd->offset += offset;
-		break;
-	case SEEK_END:
-		ret = glfs_fstat (glfd, &sb);
-		if (ret) {
-			/* seek cannot fail :O */
-			break;
-		}
-		glfd->offset = sb.st_size + offset;
-		break;
-	}
+    if (loc.inode) {
+        if (flags & O_EXCL) {
+            ret = -1;
+            errno = EEXIST;
+            goto out;
+        }
 
-	return glfd->offset;
-}
+        if (IA_ISDIR(iatt.ia_type)) {
+            ret = -1;
+            errno = EISDIR;
+            goto out;
+        }
 
+        if (!IA_ISREG(iatt.ia_type)) {
+            ret = -1;
+            errno = EINVAL;
+            goto out;
+        }
+    }
+
+    if (ret == -1 && errno == ENOENT) {
+        loc.inode = inode_new(loc.parent->table);
+        if (!loc.inode) {
+            ret = -1;
+            errno = ENOMEM;
+            goto out;
+        }
+    }
 
-//////////////
+    if (glfd->fd) {
+        /* Retry. Safe to touch glfd->fd as we
+           still have not glfs_fd_bind() yet.
+        */
+        fd_unref(glfd->fd);
+        glfd->fd = NULL;
+    }
+
+    glfd->fd = fd_create(loc.inode, getpid());
+    if (!glfd->fd) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+    glfd->fd->flags = flags;
+
+    if (get_fop_attr_thrd_key(&xattr_req))
+        gf_msg_debug("gfapi", 0, "Getting leaseid from thread failed");
+    if (ret == 0) {
+        ret = syncop_open(subvol, &loc, flags, glfd->fd, xattr_req, NULL);
+        DECODE_SYNCOP_ERR(ret);
+    } else {
+        ret = syncop_create(subvol, &loc, flags, mode, glfd->fd, &iatt,
+                            xattr_req, NULL);
+        DECODE_SYNCOP_ERR(ret);
+    }
+
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
+
+    if (ret == 0)
+        ret = glfs_loc_link(&loc, &iatt);
+out:
+    loc_wipe(&loc);
 
-ssize_t
-glfs_preadv (struct glfs_fd *glfd, const struct iovec *iovec, int iovcnt,
-	     off_t offset, int flags)
-{
-	xlator_t       *subvol = NULL;
-	ssize_t         ret = -1;
-	ssize_t         size = -1;
-	struct iovec   *iov = NULL;
-	int             cnt = 0;
-	struct iobref  *iobref = NULL;
-	fd_t           *fd = NULL;
+    if (xattr_req)
+        dict_unref(xattr_req);
 
-	__glfs_entry_fd (glfd);
+    if (ret && glfd) {
+        GF_REF_PUT(glfd);
+        glfd = NULL;
+    } else if (glfd) {
+        glfd_set_state_bind(glfd);
+    }
 
-	subvol = glfs_active_subvol (glfd->fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+    glfs_subvol_done(fs, subvol);
 
-	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
-	if (!fd) {
-		ret = -1;
-		errno = EBADFD;
-		goto out;
-	}
+    __GLFS_EXIT_FS;
 
-	size = iov_length (iovec, iovcnt);
+invalid_fs:
+    return glfd;
+}
 
-	ret = syncop_readv (subvol, fd, size, offset, 0, &iov, &cnt, &iobref);
-	if (ret <= 0)
-		goto out;
+#ifdef HAVE_SEEK_HOLE
+static int
+glfs_seek(struct glfs_fd *glfd, off_t offset, int whence)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    fd_t *fd = NULL;
+    gf_seek_what_t what = 0;
+    off_t off = -1;
+
+    switch (whence) {
+        case SEEK_DATA:
+            what = GF_SEEK_DATA;
+            break;
+        case SEEK_HOLE:
+            what = GF_SEEK_HOLE;
+            break;
+        default:
+            /* other SEEK_* do not make sense, all operations get an offset
+             * and the position in the fd is not tracked */
+            errno = EINVAL;
+            goto out;
+    }
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        errno = EBADFD;
+        goto done;
+    }
+
+    ret = syncop_seek(subvol, fd, offset, what, NULL, &off);
+    DECODE_SYNCOP_ERR(ret);
+
+    if (ret != -1)
+        glfd->offset = off;
+
+done:
+    if (fd)
+        fd_unref(fd);
+
+    glfs_subvol_done(glfd->fs, subvol);
 
-	size = iov_copy (iovec, iovcnt, iov, cnt); /* FIXME!!! */
+out:
+    return ret;
+}
+#endif
 
-	glfd->offset = (offset + size);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_lseek, 3.4.0)
+off_t
+pub_glfs_lseek(struct glfs_fd *glfd, off_t offset, int whence)
+{
+    struct stat sb = {
+        0,
+    };
+    int ret = -1;
+    off_t off = -1;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    GF_REF_GET(glfd);
+
+    switch (whence) {
+        case SEEK_SET:
+            glfd->offset = offset;
+            ret = 0;
+            break;
+        case SEEK_CUR:
+            glfd->offset += offset;
+            ret = 0;
+            break;
+        case SEEK_END:
+            ret = pub_glfs_fstat(glfd, &sb);
+            if (ret) {
+                /* seek cannot fail :O */
+                break;
+            }
+            glfd->offset = sb.st_size + offset;
+            break;
+#ifdef HAVE_SEEK_HOLE
+        case SEEK_DATA:
+        case SEEK_HOLE:
+            ret = glfs_seek(glfd, offset, whence);
+            break;
+#endif
+        default:
+            errno = EINVAL;
+    }
+
+    if (glfd)
+        GF_REF_PUT(glfd);
+
+    __GLFS_EXIT_FS;
+
+    if (ret != -1)
+        off = glfd->offset;
+
+    return off;
 
-	if (iov)
-		GF_FREE (iov);
-	if (iobref)
-		iobref_unref (iobref);
+invalid_fs:
+    return -1;
+}
 
-	ret = size;
+static ssize_t
+glfs_preadv_common(struct glfs_fd *glfd, const struct iovec *iovec, int iovcnt,
+                   off_t offset, int flags, struct glfs_stat *poststat)
+{
+    xlator_t *subvol = NULL;
+    ssize_t ret = -1;
+    ssize_t size = -1;
+    struct iovec *iov = NULL;
+    int cnt = 0;
+    struct iobref *iobref = NULL;
+    fd_t *fd = NULL;
+    struct iatt iatt = {
+        0,
+    };
+    dict_t *fop_attr = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    GF_REF_GET(glfd);
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    size = iov_length(iovec, iovcnt);
+
+    ret = get_fop_attr_thrd_key(&fop_attr);
+    if (ret)
+        gf_msg_debug("gfapi", 0, "Getting leaseid from thread failed");
+
+    ret = syncop_readv(subvol, fd, size, offset, 0, &iov, &cnt, &iobref, &iatt,
+                       fop_attr, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    if (ret >= 0 && poststat)
+        glfs_iatt_to_statx(glfd->fs, &iatt, poststat);
+
+    if (ret <= 0)
+        goto out;
+
+    size = iov_copy(iovec, iovcnt, iov, cnt); /* FIXME!!! */
+
+    glfd->offset = (offset + size);
+
+    ret = size;
 out:
-	if (fd)
-		fd_unref (fd);
+    if (iov)
+        GF_FREE(iov);
+    if (iobref)
+        iobref_unref(iobref);
+
+    if (fd)
+        fd_unref(fd);
+    if (glfd)
+        GF_REF_PUT(glfd);
+    if (fop_attr)
+        dict_unref(fop_attr);
 
-	glfs_subvol_done (glfd->fs, subvol);
+    glfs_subvol_done(glfd->fs, subvol);
 
-	return ret;
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
 }
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_preadv, 3.4.0)
+ssize_t
+pub_glfs_preadv(struct glfs_fd *glfd, const struct iovec *iovec, int iovcnt,
+                off_t offset, int flags)
+{
+    return glfs_preadv_common(glfd, iovec, iovcnt, offset, flags, NULL);
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_read, 3.4.0)
 ssize_t
-glfs_read (struct glfs_fd *glfd, void *buf, size_t count, int flags)
+pub_glfs_read(struct glfs_fd *glfd, void *buf, size_t count, int flags)
 {
-	struct iovec iov = {0, };
-	ssize_t      ret = 0;
+    struct iovec iov = {
+        0,
+    };
+    ssize_t ret = 0;
 
-	iov.iov_base = buf;
-	iov.iov_len = count;
+    if (glfd == NULL) {
+        errno = EBADF;
+        return -1;
+    }
 
-	ret = glfs_preadv (glfd, &iov, 1, glfd->offset, flags);
+    iov.iov_base = buf;
+    iov.iov_len = count;
 
-	return ret;
-}
+    ret = pub_glfs_preadv(glfd, &iov, 1, glfd->offset, flags);
 
+    return ret;
+}
 
+GFAPI_SYMVER_PUBLIC(glfs_pread34, glfs_pread, 3.4.0)
 ssize_t
-glfs_pread (struct glfs_fd *glfd, void *buf, size_t count, off_t offset,
-	    int flags)
+pub_glfs_pread34(struct glfs_fd *glfd, void *buf, size_t count, off_t offset,
+                 int flags)
 {
-	struct iovec iov = {0, };
-	ssize_t      ret = 0;
+    struct iovec iov = {
+        0,
+    };
+    ssize_t ret = 0;
 
-	iov.iov_base = buf;
-	iov.iov_len = count;
+    iov.iov_base = buf;
+    iov.iov_len = count;
 
-	ret = glfs_preadv (glfd, &iov, 1, offset, flags);
+    ret = pub_glfs_preadv(glfd, &iov, 1, offset, flags);
 
-	return ret;
+    return ret;
 }
 
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_pread, 6.0)
 ssize_t
-glfs_readv (struct glfs_fd *glfd, const struct iovec *iov, int count,
-	    int flags)
+pub_glfs_pread(struct glfs_fd *glfd, void *buf, size_t count, off_t offset,
+               int flags, struct glfs_stat *poststat)
 {
-	ssize_t      ret = 0;
+    struct iovec iov = {
+        0,
+    };
+    ssize_t ret = 0;
 
-	ret = glfs_preadv (glfd, iov, count, glfd->offset, flags);
+    iov.iov_base = buf;
+    iov.iov_len = count;
 
-	return ret;
+    ret = glfs_preadv_common(glfd, &iov, 1, offset, flags, poststat);
+
+    return ret;
 }
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_readv, 3.4.0)
+ssize_t
+pub_glfs_readv(struct glfs_fd *glfd, const struct iovec *iov, int count,
+               int flags)
+{
+    ssize_t ret = 0;
+
+    if (glfd == NULL) {
+        errno = EBADF;
+        return -1;
+    }
+
+    ret = pub_glfs_preadv(glfd, iov, count, glfd->offset, flags);
+
+    return ret;
+}
 
 struct glfs_io {
-	struct glfs_fd      *glfd;
-	int                  op;
-	off_t                offset;
-	struct iovec        *iov;
-	int                  count;
-	int                  flags;
-	glfs_io_cbk          fn;
-	void                *data;
+    struct glfs_fd *glfd;
+    int op;
+    off_t offset;
+    struct iovec *iov;
+    int count;
+    int flags;
+    gf_boolean_t oldcb;
+    union {
+        glfs_io_cbk34 fn34;
+        glfs_io_cbk fn;
+    };
+    void *data;
 };
 
-
 static int
-glfs_io_async_cbk (int ret, call_frame_t *frame, void *data)
+glfs_io_async_cbk(int op_ret, int op_errno, call_frame_t *frame, void *cookie,
+                  struct iovec *iovec, int count, struct iatt *prebuf,
+                  struct iatt *postbuf)
 {
-	struct glfs_io  *gio = data;
+    struct glfs_io *gio = NULL;
+    xlator_t *subvol = NULL;
+    struct glfs *fs = NULL;
+    struct glfs_fd *glfd = NULL;
+    int ret = -1;
+    struct glfs_stat prestat = {}, *prestatp = NULL;
+    struct glfs_stat poststat = {}, *poststatp = NULL;
+
+    GF_VALIDATE_OR_GOTO("gfapi", frame, inval);
+    GF_VALIDATE_OR_GOTO("gfapi", cookie, inval);
+
+    gio = frame->local;
+    frame->local = NULL;
+    subvol = cookie;
+    glfd = gio->glfd;
+    fs = glfd->fs;
+
+    if (!glfs_is_glfd_still_valid(glfd))
+        goto err;
+
+    if (op_ret <= 0) {
+        goto out;
+    } else if (gio->op == GF_FOP_READ) {
+        if (!iovec) {
+            op_ret = -1;
+            op_errno = EINVAL;
+            goto out;
+        }
 
-	gio->fn (gio->glfd, ret, gio->data);
+        op_ret = iov_copy(gio->iov, gio->count, iovec, count);
+        glfd->offset = gio->offset + op_ret;
+    } else if (gio->op == GF_FOP_WRITE) {
+        glfd->offset = gio->offset + gio->iov->iov_len;
+    }
 
-	GF_FREE (gio->iov);
-	GF_FREE (gio);
+out:
+    errno = op_errno;
+    if (gio->oldcb) {
+        gio->fn34(gio->glfd, op_ret, gio->data);
+    } else {
+        if (prebuf) {
+            prestatp = &prestat;
+            glfs_iatt_to_statx(fs, prebuf, prestatp);
+        }
 
-	return 0;
-}
+        if (postbuf) {
+            poststatp = &poststat;
+            glfs_iatt_to_statx(fs, postbuf, poststatp);
+        }
 
+        gio->fn(gio->glfd, op_ret, prestatp, poststatp, gio->data);
+    }
+err:
+    fd_unref(glfd->fd);
+    /* Since the async operation is complete
+     * release the ref taken during the start
+     * of async operation
+     */
+    GF_REF_PUT(glfd);
+
+    GF_FREE(gio->iov);
+    GF_FREE(gio);
+    STACK_DESTROY(frame->root);
+    glfs_subvol_done(fs, subvol);
+
+    ret = 0;
+inval:
+    return ret;
+}
 
 static int
-glfs_io_async_task (void *data)
+glfs_preadv_async_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int op_ret, int op_errno, struct iovec *iovec, int count,
+                      struct iatt *stbuf, struct iobref *iobref, dict_t *xdata)
 {
-	struct glfs_io *gio = data;
-	ssize_t         ret = 0;
+    glfs_io_async_cbk(op_ret, op_errno, frame, cookie, iovec, count, NULL,
+                      stbuf);
 
-	switch (gio->op) {
-	case GF_FOP_WRITE:
-		ret = glfs_pwritev (gio->glfd, gio->iov, gio->count,
-				    gio->offset, gio->flags);
-		break;
-	case GF_FOP_FTRUNCATE:
-		ret = glfs_ftruncate (gio->glfd, gio->offset);
-		break;
-	case GF_FOP_FSYNC:
-		if (gio->flags)
-			ret = glfs_fdatasync (gio->glfd);
-		else
-			ret = glfs_fsync (gio->glfd);
-		break;
-	case GF_FOP_DISCARD:
-		ret = glfs_discard (gio->glfd, gio->offset, gio->count);
-		break;
-	}
-
-	return (int) ret;
+    return 0;
 }
 
-
-int
-glfs_preadv_async_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		       int op_ret, int op_errno, struct iovec *iovec,
-		       int count, struct iatt *stbuf, struct iobref *iobref,
-		       dict_t *xdata)
+static int
+glfs_preadv_async_common(struct glfs_fd *glfd, const struct iovec *iovec,
+                         int count, off_t offset, int flags, gf_boolean_t oldcb,
+                         glfs_io_cbk fn, void *data)
 {
-	struct glfs_io *gio = NULL;
-	xlator_t       *subvol = NULL;
-	struct glfs    *fs = NULL;
-	struct glfs_fd *glfd = NULL;
+    struct glfs_io *gio = NULL;
+    int ret = 0;
+    call_frame_t *frame = NULL;
+    xlator_t *subvol = NULL;
+    struct glfs *fs = NULL;
+    fd_t *fd = NULL;
+    dict_t *fop_attr = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    GF_REF_GET(glfd);
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    fs = glfd->fs;
+
+    frame = syncop_create_frame(THIS);
+    if (!frame) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gio = GF_CALLOC(1, sizeof(*gio), glfs_mt_glfs_io_t);
+    if (!gio) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gio->iov = iov_dup(iovec, count);
+    if (!gio->iov) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gio->op = GF_FOP_READ;
+    gio->glfd = glfd;
+    gio->count = count;
+    gio->offset = offset;
+    gio->flags = flags;
+    gio->oldcb = oldcb;
+    gio->fn = fn;
+    gio->data = data;
+
+    frame->local = gio;
+
+    ret = get_fop_attr_thrd_key(&fop_attr);
+    if (ret)
+        gf_msg_debug("gfapi", 0, "Getting leaseid from thread failed");
+
+    STACK_WIND_COOKIE(frame, glfs_preadv_async_cbk, subvol, subvol,
+                      subvol->fops->readv, fd, iov_length(iovec, count), offset,
+                      flags, fop_attr);
 
+out:
+    if (ret) {
+        if (fd)
+            fd_unref(fd);
+        if (glfd)
+            GF_REF_PUT(glfd);
+        if (gio) {
+            GF_FREE(gio->iov);
+            GF_FREE(gio);
+        }
+        if (frame) {
+            STACK_DESTROY(frame->root);
+        }
+        glfs_subvol_done(fs, subvol);
+    }
+    if (fop_attr)
+        dict_unref(fop_attr);
 
-	gio = frame->local;
-	frame->local = NULL;
-	subvol = cookie;
-	glfd = gio->glfd;
-	fs = glfd->fs;
-
-	if (op_ret <= 0)
-		goto out;
-
-	op_ret = iov_copy (gio->iov, gio->count, iovec, count);
+    __GLFS_EXIT_FS;
 
-	glfd->offset = gio->offset + op_ret;
-out:
-	errno = op_errno;
-	gio->fn (gio->glfd, op_ret, gio->data);
+    return ret;
 
-	GF_FREE (gio->iov);
-	GF_FREE (gio);
-	STACK_DESTROY (frame->root);
-	glfs_subvol_done (fs, subvol);
+invalid_fs:
+    return -1;
+}
 
-	return 0;
+GFAPI_SYMVER_PUBLIC(glfs_preadv_async34, glfs_preadv_async, 3.4.0)
+int
+pub_glfs_preadv_async34(struct glfs_fd *glfd, const struct iovec *iovec,
+                        int count, off_t offset, int flags, glfs_io_cbk34 fn,
+                        void *data)
+{
+    return glfs_preadv_async_common(glfd, iovec, count, offset, flags, _gf_true,
+                                    (void *)fn, data);
 }
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_preadv_async, 6.0)
+int
+pub_glfs_preadv_async(struct glfs_fd *glfd, const struct iovec *iovec,
+                      int count, off_t offset, int flags, glfs_io_cbk fn,
+                      void *data)
+{
+    return glfs_preadv_async_common(glfd, iovec, count, offset, flags,
+                                    _gf_false, fn, data);
+}
 
+GFAPI_SYMVER_PUBLIC(glfs_read_async34, glfs_read_async, 3.4.0)
 int
-glfs_preadv_async (struct glfs_fd *glfd, const struct iovec *iovec, int count,
-		   off_t offset, int flags, glfs_io_cbk fn, void *data)
+pub_glfs_read_async34(struct glfs_fd *glfd, void *buf, size_t count, int flags,
+                      glfs_io_cbk34 fn, void *data)
 {
-	struct glfs_io *gio = NULL;
-	int             ret = 0;
-	call_frame_t   *frame = NULL;
-	xlator_t       *subvol = NULL;
-	glfs_t         *fs = NULL;
-	fd_t           *fd = NULL;
+    struct iovec iov = {
+        0,
+    };
+    ssize_t ret = 0;
 
-	__glfs_entry_fd (glfd);
+    if (glfd == NULL) {
+        errno = EBADF;
+        return -1;
+    }
 
-	subvol = glfs_active_subvol (glfd->fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+    iov.iov_base = buf;
+    iov.iov_len = count;
 
-	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
-	if (!fd) {
-		ret = -1;
-		errno = EBADFD;
-		goto out;
-	}
+    ret = glfs_preadv_async_common(glfd, &iov, 1, glfd->offset, flags, _gf_true,
+                                   (void *)fn, data);
 
-	fs = glfd->fs;
+    return ret;
+}
 
-	frame = syncop_create_frame (THIS);
-	if (!frame) {
-		ret = -1;
-		errno = ENOMEM;
-		goto out;
-	}
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_read_async, 6.0)
+int
+pub_glfs_read_async(struct glfs_fd *glfd, void *buf, size_t count, int flags,
+                    glfs_io_cbk fn, void *data)
+{
+    struct iovec iov = {
+        0,
+    };
+    ssize_t ret = 0;
 
-	gio = GF_CALLOC (1, sizeof (*gio), glfs_mt_glfs_io_t);
-	if (!gio) {
-		ret = -1;
-		errno = ENOMEM;
-		goto out;
-	}
+    if (glfd == NULL) {
+        errno = EBADF;
+        return -1;
+    }
 
-	gio->iov = iov_dup (iovec, count);
-	if (!gio->iov) {
-		ret = -1;
-		errno = ENOMEM;
-		goto out;
-	}
+    iov.iov_base = buf;
+    iov.iov_len = count;
 
-	gio->op     = GF_FOP_READ;
-	gio->glfd   = glfd;
-	gio->count  = count;
-	gio->offset = offset;
-	gio->flags  = flags;
-	gio->fn     = fn;
-	gio->data   = data;
+    ret = glfs_preadv_async_common(glfd, &iov, 1, glfd->offset, flags,
+                                   _gf_false, fn, data);
 
-	frame->local = gio;
+    return ret;
+}
 
-	STACK_WIND_COOKIE (frame, glfs_preadv_async_cbk, subvol, subvol,
-			   subvol->fops->readv, fd, iov_length (iovec, count),
-			   offset, flags, NULL);
+GFAPI_SYMVER_PUBLIC(glfs_pread_async34, glfs_pread_async, 3.4.0)
+int
+pub_glfs_pread_async34(struct glfs_fd *glfd, void *buf, size_t count,
+                       off_t offset, int flags, glfs_io_cbk34 fn, void *data)
+{
+    struct iovec iov = {
+        0,
+    };
+    ssize_t ret = 0;
 
-out:
-	if (ret) {
-		GF_FREE (gio->iov);
-		GF_FREE (gio);
-		STACK_DESTROY (frame->root);
-		glfs_subvol_done (fs, subvol);
-	}
+    iov.iov_base = buf;
+    iov.iov_len = count;
 
-	if (fd)
-		fd_unref (fd);
+    ret = glfs_preadv_async_common(glfd, &iov, 1, offset, flags, _gf_true,
+                                   (void *)fn, data);
 
-	return ret;
+    return ret;
 }
 
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_pread_async, 6.0)
 int
-glfs_read_async (struct glfs_fd *glfd, void *buf, size_t count, int flags,
-		 glfs_io_cbk fn, void *data)
+pub_glfs_pread_async(struct glfs_fd *glfd, void *buf, size_t count,
+                     off_t offset, int flags, glfs_io_cbk fn, void *data)
 {
-	struct iovec iov = {0, };
-	ssize_t      ret = 0;
+    struct iovec iov = {
+        0,
+    };
+    ssize_t ret = 0;
 
-	iov.iov_base = buf;
-	iov.iov_len = count;
+    iov.iov_base = buf;
+    iov.iov_len = count;
 
-	ret = glfs_preadv_async (glfd, &iov, 1, glfd->offset, flags, fn, data);
+    ret = glfs_preadv_async_common(glfd, &iov, 1, offset, flags, _gf_false, fn,
+                                   data);
 
-	return ret;
+    return ret;
 }
 
-
+GFAPI_SYMVER_PUBLIC(glfs_readv_async34, glfs_readv_async, 3.4.0)
 int
-glfs_pread_async (struct glfs_fd *glfd, void *buf, size_t count, off_t offset,
-		  int flags, glfs_io_cbk fn, void *data)
+pub_glfs_readv_async34(struct glfs_fd *glfd, const struct iovec *iov, int count,
+                       int flags, glfs_io_cbk34 fn, void *data)
 {
-	struct iovec iov = {0, };
-	ssize_t      ret = 0;
-
-	iov.iov_base = buf;
-	iov.iov_len = count;
+    ssize_t ret = 0;
 
-	ret = glfs_preadv_async (glfd, &iov, 1, offset, flags, fn, data);
+    if (glfd == NULL) {
+        errno = EBADF;
+        return -1;
+    }
 
-	return ret;
+    ret = glfs_preadv_async_common(glfd, iov, count, glfd->offset, flags,
+                                   _gf_true, (void *)fn, data);
+    return ret;
 }
 
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_readv_async, 6.0)
 int
-glfs_readv_async (struct glfs_fd *glfd, const struct iovec *iov, int count,
-		  int flags, glfs_io_cbk fn, void *data)
+pub_glfs_readv_async(struct glfs_fd *glfd, const struct iovec *iov, int count,
+                     int flags, glfs_io_cbk fn, void *data)
 {
-	ssize_t      ret = 0;
+    ssize_t ret = 0;
 
-	ret = glfs_preadv_async (glfd, iov, count, glfd->offset, flags,
-				 fn, data);
-	return ret;
+    if (glfd == NULL) {
+        errno = EBADF;
+        return -1;
+    }
+
+    ret = glfs_preadv_async_common(glfd, iov, count, glfd->offset, flags,
+                                   _gf_false, fn, data);
+    return ret;
 }
 
-///// writev /////
+static ssize_t
+glfs_pwritev_common(struct glfs_fd *glfd, const struct iovec *iovec, int iovcnt,
+                    off_t offset, int flags, struct glfs_stat *prestat,
+                    struct glfs_stat *poststat)
+{
+    xlator_t *subvol = NULL;
+    int ret = -1;
+    struct iobref *iobref = NULL;
+    struct iobuf *iobuf = NULL;
+    struct iovec iov = {
+        0,
+    };
+    fd_t *fd = NULL;
+    struct iatt preiatt =
+                    {
+                        0,
+                    },
+                postiatt = {
+                    0,
+                };
+    dict_t *fop_attr = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    GF_REF_GET(glfd);
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    ret = iobuf_copy(subvol->ctx->iobuf_pool, iovec, iovcnt, &iobref, &iobuf,
+                     &iov);
+    if (ret)
+        goto out;
+
+    ret = get_fop_attr_thrd_key(&fop_attr);
+    if (ret)
+        gf_msg_debug("gfapi", 0, "Getting leaseid from thread failed");
+
+    ret = syncop_writev(subvol, fd, &iov, 1, offset, iobref, flags, &preiatt,
+                        &postiatt, fop_attr, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    if (ret >= 0) {
+        if (prestat)
+            glfs_iatt_to_statx(glfd->fs, &preiatt, prestat);
+        if (poststat)
+            glfs_iatt_to_statx(glfd->fs, &postiatt, poststat);
+    }
+
+    if (ret <= 0)
+        goto out;
+
+    glfd->offset = (offset + iov.iov_len);
+out:
+    if (iobuf)
+        iobuf_unref(iobuf);
+    if (iobref)
+        iobref_unref(iobref);
+    if (fd)
+        fd_unref(fd);
+    if (glfd)
+        GF_REF_PUT(glfd);
+    if (fop_attr)
+        dict_unref(fop_attr);
+
+    glfs_subvol_done(glfd->fs, subvol);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_copy_file_range, 6.0)
 ssize_t
-glfs_pwritev (struct glfs_fd *glfd, const struct iovec *iovec, int iovcnt,
-	      off_t offset, int flags)
+pub_glfs_copy_file_range(struct glfs_fd *glfd_in, off64_t *off_in,
+                         struct glfs_fd *glfd_out, off64_t *off_out, size_t len,
+                         unsigned int flags, struct glfs_stat *statbuf,
+                         struct glfs_stat *prestat, struct glfs_stat *poststat)
 {
-	xlator_t       *subvol = NULL;
-	int             ret = -1;
-	size_t          size = -1;
-	struct iobref  *iobref = NULL;
-	struct iobuf   *iobuf = NULL;
-	struct iovec    iov = {0, };
-	fd_t           *fd = NULL;
-
-	__glfs_entry_fd (glfd);
-
-	subvol = glfs_active_subvol (glfd->fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
-
-	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
-	if (!fd) {
-		ret = -1;
-		errno = EBADFD;
-		goto out;
-	}
+    xlator_t *subvol = NULL;
+    int ret = -1;
+    fd_t *fd_in = NULL;
+    fd_t *fd_out = NULL;
+    struct iatt preiatt =
+                    {
+                        0,
+                    },
+                iattbuf =
+                    {
+                        0,
+                    },
+                postiatt = {
+                    0,
+                };
+    dict_t *fop_attr = NULL;
+    off64_t pos_in;
+    off64_t pos_out;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd_in, invalid_fs);
+    __GLFS_ENTRY_VALIDATE_FD(glfd_out, invalid_fs);
+
+    GF_REF_GET(glfd_in);
+    GF_REF_GET(glfd_out);
+
+    if (glfd_in->fs != glfd_out->fs) {
+        ret = -1;
+        errno = EXDEV;
+        goto out;
+    }
+
+    subvol = glfs_active_subvol(glfd_in->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd_in = glfs_resolve_fd(glfd_in->fs, subvol, glfd_in);
+    if (!fd_in) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    fd_out = glfs_resolve_fd(glfd_out->fs, subvol, glfd_out);
+    if (!fd_out) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    /*
+     * This is based on how the vfs layer in the kernel handles
+     * copy_file_range call. Upon receiving it follows the
+     * below method to consider the offset.
+     * if (off_in != NULL)
+     *    use the value off_in to perform the op
+     * else if off_in == NULL
+     *    use the current file offset position to perform the op
+     *
+     * For gfapi, glfd->offset is used. For a freshly opened
+     * fd, the offset is set to 0.
+     */
+    if (off_in)
+        pos_in = *off_in;
+    else
+        pos_in = glfd_in->offset;
+
+    if (off_out)
+        pos_out = *off_out;
+    else
+        pos_out = glfd_out->offset;
+
+    ret = get_fop_attr_thrd_key(&fop_attr);
+    if (ret)
+        gf_msg_debug("gfapi", 0, "Getting leaseid from thread failed");
+
+    ret = syncop_copy_file_range(subvol, fd_in, pos_in, fd_out, pos_out, len,
+                                 flags, &iattbuf, &preiatt, &postiatt, fop_attr,
+                                 NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    if (ret >= 0) {
+        pos_in += ret;
+        pos_out += ret;
+
+        if (off_in)
+            *off_in = pos_in;
+        if (off_out)
+            *off_out = pos_out;
+
+        if (statbuf)
+            glfs_iatt_to_statx(glfd_in->fs, &iattbuf, statbuf);
+        if (prestat)
+            glfs_iatt_to_statx(glfd_in->fs, &preiatt, prestat);
+        if (poststat)
+            glfs_iatt_to_statx(glfd_in->fs, &postiatt, poststat);
+    }
+
+    if (ret <= 0)
+        goto out;
+
+    /*
+     * If *off_in is NULL, then there is no offset info that can
+     * obtained from the input argument. Hence follow below method.
+     *  If *off_in is NULL, then
+     *     glfd->offset = offset + ret;
+     * else
+     *     do nothing.
+     *
+     * According to the man page of copy_file_range, if off_in is
+     * NULL, then the offset of the source file is advanced by
+     * the return value of the fop. The same applies to off_out as
+     * well. Otherwise, if *off_in is not NULL, then the offset
+     * is not advanced by the filesystem. The entity which sends
+     * the copy_file_range call is supposed to advance the offset
+     * value in its buffer (pointed to by *off_in or *off_out)
+     * by the return value of copy_file_range.
+     */
+    if (!off_in)
+        glfd_in->offset += ret;
+
+    if (!off_out)
+        glfd_out->offset += ret;
 
-	size = iov_length (iovec, iovcnt);
-
-	iobuf = iobuf_get2 (subvol->ctx->iobuf_pool, size);
-	if (!iobuf) {
-		ret = -1;
-		errno = ENOMEM;
-		goto out;
-	}
-
-	iobref = iobref_new ();
-	if (!iobref) {
-		iobuf_unref (iobuf);
-		errno = ENOMEM;
-		ret = -1;
-		goto out;
-	}
+out:
+    if (fd_in)
+        fd_unref(fd_in);
+    if (fd_out)
+        fd_unref(fd_out);
+    if (glfd_in)
+        GF_REF_PUT(glfd_in);
+    if (glfd_out)
+        GF_REF_PUT(glfd_out);
+    if (fop_attr)
+        dict_unref(fop_attr);
+
+    glfs_subvol_done(glfd_in->fs, subvol);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
 
-	ret = iobref_add (iobref, iobuf);
-	if (ret) {
-		iobuf_unref (iobuf);
-		iobref_unref (iobref);
-		errno = ENOMEM;
-		ret = -1;
-		goto out;
-	}
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_pwritev, 3.4.0)
+ssize_t
+pub_glfs_pwritev(struct glfs_fd *glfd, const struct iovec *iovec, int iovcnt,
+                 off_t offset, int flags)
+{
+    return glfs_pwritev_common(glfd, iovec, iovcnt, offset, flags, NULL, NULL);
+}
 
-	iov_unload (iobuf_ptr (iobuf), iovec, iovcnt);  /* FIXME!!! */
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_write, 3.4.0)
+ssize_t
+pub_glfs_write(struct glfs_fd *glfd, const void *buf, size_t count, int flags)
+{
+    struct iovec iov = {
+        0,
+    };
+    ssize_t ret = 0;
 
-	iov.iov_base = iobuf_ptr (iobuf);
-	iov.iov_len = size;
+    if (glfd == NULL) {
+        errno = EBADF;
+        return -1;
+    }
 
-	ret = syncop_writev (subvol, fd, &iov, 1, offset, iobref, flags);
+    iov.iov_base = (void *)buf;
+    iov.iov_len = count;
 
-	iobuf_unref (iobuf);
-	iobref_unref (iobref);
+    ret = pub_glfs_pwritev(glfd, &iov, 1, glfd->offset, flags);
 
-	if (ret <= 0)
-		goto out;
+    return ret;
+}
 
-	glfd->offset = (offset + size);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_writev, 3.4.0)
+ssize_t
+pub_glfs_writev(struct glfs_fd *glfd, const struct iovec *iov, int count,
+                int flags)
+{
+    ssize_t ret = 0;
 
-out:
-	if (fd)
-		fd_unref (fd);
+    if (glfd == NULL) {
+        errno = EBADF;
+        return -1;
+    }
 
-	glfs_subvol_done (glfd->fs, subvol);
+    ret = pub_glfs_pwritev(glfd, iov, count, glfd->offset, flags);
 
-	return ret;
+    return ret;
 }
 
-
+GFAPI_SYMVER_PUBLIC(glfs_pwrite34, glfs_pwrite, 3.4.0)
 ssize_t
-glfs_write (struct glfs_fd *glfd, const void *buf, size_t count, int flags)
+pub_glfs_pwrite34(struct glfs_fd *glfd, const void *buf, size_t count,
+                  off_t offset, int flags)
 {
-	struct iovec iov = {0, };
-	ssize_t      ret = 0;
+    struct iovec iov = {
+        0,
+    };
+    ssize_t ret = 0;
 
-	iov.iov_base = (void *) buf;
-	iov.iov_len = count;
+    iov.iov_base = (void *)buf;
+    iov.iov_len = count;
 
-	ret = glfs_pwritev (glfd, &iov, 1, glfd->offset, flags);
+    ret = pub_glfs_pwritev(glfd, &iov, 1, offset, flags);
 
-	return ret;
+    return ret;
 }
 
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_pwrite, 6.0)
 ssize_t
-glfs_writev (struct glfs_fd *glfd, const struct iovec *iov, int count,
-	     int flags)
+pub_glfs_pwrite(struct glfs_fd *glfd, const void *buf, size_t count,
+                off_t offset, int flags, struct glfs_stat *prestat,
+                struct glfs_stat *poststat)
 {
-	ssize_t      ret = 0;
+    struct iovec iov = {
+        0,
+    };
+    ssize_t ret = 0;
 
-	ret = glfs_pwritev (glfd, iov, count, glfd->offset, flags);
+    iov.iov_base = (void *)buf;
+    iov.iov_len = count;
 
-	return ret;
+    ret = glfs_pwritev_common(glfd, &iov, 1, offset, flags, prestat, poststat);
+
+    return ret;
 }
 
+extern glfs_t *
+pub_glfs_from_glfd(glfs_fd_t *);
 
-ssize_t
-glfs_pwrite (struct glfs_fd *glfd, const void *buf, size_t count, off_t offset,
-	     int flags)
+static int
+glfs_pwritev_async_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int op_ret, int op_errno, struct iatt *prebuf,
+                       struct iatt *postbuf, dict_t *xdata)
 {
-	struct iovec iov = {0, };
-	ssize_t      ret = 0;
-
-	iov.iov_base = (void *) buf;
-	iov.iov_len = count;
-
-	ret = glfs_pwritev (glfd, &iov, 1, offset, flags);
+    glfs_io_async_cbk(op_ret, op_errno, frame, cookie, NULL, 0, prebuf,
+                      postbuf);
 
-	return ret;
+    return 0;
 }
 
+static int
+glfs_pwritev_async_common(struct glfs_fd *glfd, const struct iovec *iovec,
+                          int count, off_t offset, int flags,
+                          gf_boolean_t oldcb, glfs_io_cbk fn, void *data)
+{
+    struct glfs_io *gio = NULL;
+    int ret = -1;
+    call_frame_t *frame = NULL;
+    xlator_t *subvol = NULL;
+    fd_t *fd = NULL;
+    struct iobref *iobref = NULL;
+    struct iobuf *iobuf = NULL;
+    dict_t *fop_attr = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    /* Need to take explicit ref so that the fd
+     * is not destroyed before the fop is complete
+     */
+    GF_REF_GET(glfd);
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        errno = EBADFD;
+        goto out;
+    }
+
+    gio = GF_CALLOC(1, sizeof(*gio), glfs_mt_glfs_io_t);
+    if (!gio) {
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gio->op = GF_FOP_WRITE;
+    gio->glfd = glfd;
+    gio->offset = offset;
+    gio->flags = flags;
+    gio->oldcb = oldcb;
+    gio->fn = fn;
+    gio->data = data;
+    gio->count = 1;
+    gio->iov = GF_CALLOC(gio->count, sizeof(*(gio->iov)), gf_common_mt_iovec);
+    if (!gio->iov) {
+        errno = ENOMEM;
+        goto out;
+    }
+
+    ret = iobuf_copy(subvol->ctx->iobuf_pool, iovec, count, &iobref, &iobuf,
+                     gio->iov);
+    if (ret)
+        goto out;
+
+    frame = syncop_create_frame(THIS);
+    if (!frame) {
+        errno = ENOMEM;
+        ret = -1;
+        goto out;
+    }
+
+    frame->local = gio;
+
+    ret = get_fop_attr_thrd_key(&fop_attr);
+    if (ret)
+        gf_msg_debug("gfapi", 0, "Getting leaseid from thread failed");
+
+    STACK_WIND_COOKIE(frame, glfs_pwritev_async_cbk, subvol, subvol,
+                      subvol->fops->writev, fd, gio->iov, gio->count, offset,
+                      flags, iobref, fop_attr);
+
+    ret = 0;
+out:
+    if (ret) {
+        if (fd)
+            fd_unref(fd);
+        if (glfd)
+            GF_REF_PUT(glfd);
+        GF_FREE(gio);
+        /*
+         * If there is any error condition check after the frame
+         * creation, we have to destroy the frame root.
+         */
+        glfs_subvol_done(glfd->fs, subvol);
+    }
+    if (fop_attr)
+        dict_unref(fop_attr);
+
+    if (iobuf)
+        iobuf_unref(iobuf);
+    if (iobref)
+        iobref_unref(iobref);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
 
+GFAPI_SYMVER_PUBLIC(glfs_pwritev_async34, glfs_pwritev_async, 3.4.0)
 int
-glfs_pwritev_async (struct glfs_fd *glfd, const struct iovec *iovec, int count,
-		    off_t offset, int flags, glfs_io_cbk fn, void *data)
+pub_glfs_pwritev_async34(struct glfs_fd *glfd, const struct iovec *iovec,
+                         int count, off_t offset, int flags, glfs_io_cbk34 fn,
+                         void *data)
 {
-	struct glfs_io *gio = NULL;
-	int             ret = 0;
+    return glfs_pwritev_async_common(glfd, iovec, count, offset, flags,
+                                     _gf_true, (void *)fn, data);
+}
 
-	gio = GF_CALLOC (1, sizeof (*gio), glfs_mt_glfs_io_t);
-	if (!gio) {
-		errno = ENOMEM;
-		return -1;
-	}
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_pwritev_async, 6.0)
+int
+pub_glfs_pwritev_async(struct glfs_fd *glfd, const struct iovec *iovec,
+                       int count, off_t offset, int flags, glfs_io_cbk fn,
+                       void *data)
+{
+    return glfs_pwritev_async_common(glfd, iovec, count, offset, flags,
+                                     _gf_false, fn, data);
+}
 
-	gio->iov = iov_dup (iovec, count);
-	if (!gio->iov) {
-		GF_FREE (gio);
-		errno = ENOMEM;
-		return -1;
-	}
+GFAPI_SYMVER_PUBLIC(glfs_write_async34, glfs_write_async, 3.4.0)
+int
+pub_glfs_write_async34(struct glfs_fd *glfd, const void *buf, size_t count,
+                       int flags, glfs_io_cbk34 fn, void *data)
+{
+    struct iovec iov = {
+        0,
+    };
+    ssize_t ret = 0;
 
-	gio->op     = GF_FOP_WRITE;
-	gio->glfd   = glfd;
-	gio->count  = count;
-	gio->offset = offset;
-	gio->flags  = flags;
-	gio->fn     = fn;
-	gio->data   = data;
+    if (glfd == NULL) {
+        errno = EBADF;
+        return -1;
+    }
 
-	ret = synctask_new (glfs_from_glfd (glfd)->ctx->env,
-			    glfs_io_async_task, glfs_io_async_cbk,
-			    NULL, gio);
+    iov.iov_base = (void *)buf;
+    iov.iov_len = count;
 
-	if (ret) {
-		GF_FREE (gio->iov);
-		GF_FREE (gio);
-	}
+    ret = glfs_pwritev_async_common(glfd, &iov, 1, glfd->offset, flags,
+                                    _gf_true, (void *)fn, data);
 
-	return ret;
+    return ret;
 }
 
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_write_async, 6.0)
 int
-glfs_write_async (struct glfs_fd *glfd, const void *buf, size_t count, int flags,
-		  glfs_io_cbk fn, void *data)
+pub_glfs_write_async(struct glfs_fd *glfd, const void *buf, size_t count,
+                     int flags, glfs_io_cbk fn, void *data)
 {
-	struct iovec iov = {0, };
-	ssize_t      ret = 0;
+    struct iovec iov = {
+        0,
+    };
+    ssize_t ret = 0;
 
-	iov.iov_base = (void *) buf;
-	iov.iov_len = count;
+    if (glfd == NULL) {
+        errno = EBADF;
+        return -1;
+    }
 
-	ret = glfs_pwritev_async (glfd, &iov, 1, glfd->offset, flags, fn, data);
+    iov.iov_base = (void *)buf;
+    iov.iov_len = count;
 
-	return ret;
-}
+    ret = glfs_pwritev_async_common(glfd, &iov, 1, glfd->offset, flags,
+                                    _gf_false, fn, data);
 
+    return ret;
+}
 
+GFAPI_SYMVER_PUBLIC(glfs_pwrite_async34, glfs_pwrite_async, 3.4.0)
 int
-glfs_pwrite_async (struct glfs_fd *glfd, const void *buf, int count,
-		   off_t offset, int flags, glfs_io_cbk fn, void *data)
+pub_glfs_pwrite_async34(struct glfs_fd *glfd, const void *buf, int count,
+                        off_t offset, int flags, glfs_io_cbk34 fn, void *data)
 {
-	struct iovec iov = {0, };
-	ssize_t      ret = 0;
+    struct iovec iov = {
+        0,
+    };
+    ssize_t ret = 0;
 
-	iov.iov_base = (void *) buf;
-	iov.iov_len = count;
+    iov.iov_base = (void *)buf;
+    iov.iov_len = count;
 
-	ret = glfs_pwritev_async (glfd, &iov, 1, offset, flags, fn, data);
+    ret = glfs_pwritev_async_common(glfd, &iov, 1, offset, flags, _gf_true,
+                                    (void *)fn, data);
 
-	return ret;
+    return ret;
 }
 
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_pwrite_async, 6.0)
 int
-glfs_writev_async (struct glfs_fd *glfd, const struct iovec *iov, int count,
-		   int flags, glfs_io_cbk fn, void *data)
+pub_glfs_pwrite_async(struct glfs_fd *glfd, const void *buf, int count,
+                      off_t offset, int flags, glfs_io_cbk fn, void *data)
 {
-	ssize_t      ret = 0;
+    struct iovec iov = {
+        0,
+    };
+    ssize_t ret = 0;
 
-	ret = glfs_pwritev_async (glfd, iov, count, glfd->offset, flags,
-				  fn, data);
-	return ret;
-}
+    iov.iov_base = (void *)buf;
+    iov.iov_len = count;
 
+    ret = glfs_pwritev_async_common(glfd, &iov, 1, offset, flags, _gf_false, fn,
+                                    data);
 
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC(glfs_writev_async34, glfs_writev_async, 3.4.0)
 int
-glfs_fsync (struct glfs_fd *glfd)
+pub_glfs_writev_async34(struct glfs_fd *glfd, const struct iovec *iov,
+                        int count, int flags, glfs_io_cbk34 fn, void *data)
 {
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	fd_t            *fd = NULL;
-
-	__glfs_entry_fd (glfd);
+    ssize_t ret = 0;
 
-	subvol = glfs_active_subvol (glfd->fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+    if (glfd == NULL) {
+        errno = EBADF;
+        return -1;
+    }
 
-	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
-	if (!fd) {
-		ret = -1;
-		errno = EBADFD;
-		goto out;
-	}
+    ret = glfs_pwritev_async_common(glfd, iov, count, glfd->offset, flags,
+                                    _gf_true, (void *)fn, data);
+    return ret;
+}
 
-	ret = syncop_fsync (subvol, fd, 0);
-out:
-	if (fd)
-		fd_unref (fd);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_writev_async, 6.0)
+int
+pub_glfs_writev_async(struct glfs_fd *glfd, const struct iovec *iov, int count,
+                      int flags, glfs_io_cbk fn, void *data)
+{
+    ssize_t ret = 0;
 
-	glfs_subvol_done (glfd->fs, subvol);
+    if (glfd == NULL) {
+        errno = EBADF;
+        return -1;
+    }
 
-	return ret;
+    ret = glfs_pwritev_async_common(glfd, iov, count, glfd->offset, flags,
+                                    _gf_false, fn, data);
+    return ret;
 }
 
-
 static int
-glfs_fsync_async_common (struct glfs_fd *glfd, glfs_io_cbk fn, void *data,
-			 int dataonly)
+glfs_fsync_common(struct glfs_fd *glfd, struct glfs_stat *prestat,
+                  struct glfs_stat *poststat)
 {
-	struct glfs_io *gio = NULL;
-	int             ret = 0;
-
-	gio = GF_CALLOC (1, sizeof (*gio), glfs_mt_glfs_io_t);
-	if (!gio) {
-		errno = ENOMEM;
-		return -1;
-	}
-
-	gio->op     = GF_FOP_FSYNC;
-	gio->glfd   = glfd;
-	gio->flags  = dataonly;
-	gio->fn     = fn;
-	gio->data   = data;
-
-	ret = synctask_new (glfs_from_glfd (glfd)->ctx->env,
-			    glfs_io_async_task, glfs_io_async_cbk,
-			    NULL, gio);
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    fd_t *fd = NULL;
+    struct iatt preiatt =
+                    {
+                        0,
+                    },
+                postiatt = {
+                    0,
+                };
+    dict_t *fop_attr = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    GF_REF_GET(glfd);
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    ret = get_fop_attr_thrd_key(&fop_attr);
+    if (ret)
+        gf_msg_debug("gfapi", 0, "Getting leaseid from thread failed");
+
+    ret = syncop_fsync(subvol, fd, 0, &preiatt, &postiatt, fop_attr, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    if (ret >= 0) {
+        if (prestat)
+            glfs_iatt_to_statx(glfd->fs, &preiatt, prestat);
+        if (poststat)
+            glfs_iatt_to_statx(glfd->fs, &postiatt, poststat);
+    }
+out:
+    if (fd)
+        fd_unref(fd);
+    if (glfd)
+        GF_REF_PUT(glfd);
+    if (fop_attr)
+        dict_unref(fop_attr);
 
-	if (ret) {
-		GF_FREE (gio->iov);
-		GF_FREE (gio);
-	}
+    glfs_subvol_done(glfd->fs, subvol);
 
-	return ret;
+    __GLFS_EXIT_FS;
 
+invalid_fs:
+    return ret;
 }
 
-
+GFAPI_SYMVER_PUBLIC(glfs_fsync34, glfs_fsync, 3.4.0)
 int
-glfs_fsync_async (struct glfs_fd *glfd, glfs_io_cbk fn, void *data)
+pub_glfs_fsync34(struct glfs_fd *glfd)
 {
-	return glfs_fsync_async_common (glfd, fn, data, 0);
+    return glfs_fsync_common(glfd, NULL, NULL);
 }
 
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fsync, 6.0)
 int
-glfs_fdatasync (struct glfs_fd *glfd)
+pub_glfs_fsync(struct glfs_fd *glfd, struct glfs_stat *prestat,
+               struct glfs_stat *poststat)
 {
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	fd_t            *fd = NULL;
+    return glfs_fsync_common(glfd, prestat, poststat);
+}
 
-	__glfs_entry_fd (glfd);
+static int
+glfs_fsync_async_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                     struct iatt *postbuf, dict_t *xdata)
+{
+    glfs_io_async_cbk(op_ret, op_errno, frame, cookie, NULL, 0, prebuf,
+                      postbuf);
 
-	subvol = glfs_active_subvol (glfd->fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+    return 0;
+}
 
-	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
-	if (!fd) {
-		ret = -1;
-		errno = EBADFD;
-		goto out;
-	}
+static int
+glfs_fsync_async_common(struct glfs_fd *glfd, gf_boolean_t oldcb,
+                        glfs_io_cbk fn, void *data, int dataonly)
+{
+    struct glfs_io *gio = NULL;
+    int ret = 0;
+    call_frame_t *frame = NULL;
+    xlator_t *subvol = NULL;
+    fd_t *fd = NULL;
+
+    /* Need to take explicit ref so that the fd
+     * is not destroyed before the fop is complete
+     */
+    GF_REF_GET(glfd);
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    frame = syncop_create_frame(THIS);
+    if (!frame) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gio = GF_CALLOC(1, sizeof(*gio), glfs_mt_glfs_io_t);
+    if (!gio) {
+        errno = ENOMEM;
+        ret = -1;
+        goto out;
+    }
+
+    gio->op = GF_FOP_FSYNC;
+    gio->glfd = glfd;
+    gio->flags = dataonly;
+    gio->oldcb = oldcb;
+    gio->fn = fn;
+    gio->data = data;
+
+    frame->local = gio;
+
+    STACK_WIND_COOKIE(frame, glfs_fsync_async_cbk, subvol, subvol,
+                      subvol->fops->fsync, fd, dataonly, NULL);
 
-	ret = syncop_fsync (subvol, fd, 1);
 out:
-	if (fd)
-		fd_unref (fd);
-
-	glfs_subvol_done (glfd->fs, subvol);
-
-	return ret;
+    if (ret) {
+        if (fd)
+            fd_unref(fd);
+        GF_REF_PUT(glfd);
+        GF_FREE(gio);
+        if (frame)
+            STACK_DESTROY(frame->root);
+        glfs_subvol_done(glfd->fs, subvol);
+    }
+
+    return ret;
 }
 
-
+GFAPI_SYMVER_PUBLIC(glfs_fsync_async34, glfs_fsync_async, 3.4.0)
 int
-glfs_fdatasync_async (struct glfs_fd *glfd, glfs_io_cbk fn, void *data)
+pub_glfs_fsync_async34(struct glfs_fd *glfd, glfs_io_cbk34 fn, void *data)
 {
-	return glfs_fsync_async_common (glfd, fn, data, 1);
-}
+    int ret = -1;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    ret = glfs_fsync_async_common(glfd, _gf_true, (void *)fn, data, 0);
 
+    __GLFS_EXIT_FS;
 
+invalid_fs:
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fsync_async, 6.0)
 int
-glfs_ftruncate (struct glfs_fd *glfd, off_t offset)
+pub_glfs_fsync_async(struct glfs_fd *glfd, glfs_io_cbk fn, void *data)
 {
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	fd_t            *fd = NULL;
+    int ret = -1;
 
-	__glfs_entry_fd (glfd);
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
 
-	subvol = glfs_active_subvol (glfd->fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+    ret = glfs_fsync_async_common(glfd, _gf_false, fn, data, 0);
 
-	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
-	if (!fd) {
-		ret = -1;
-		errno = EBADFD;
-		goto out;
-	}
+    __GLFS_EXIT_FS;
 
-	ret = syncop_ftruncate (subvol, fd, offset);
+invalid_fs:
+    return ret;
+}
+
+static int
+glfs_fdatasync_common(struct glfs_fd *glfd, struct glfs_stat *prestat,
+                      struct glfs_stat *poststat)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    fd_t *fd = NULL;
+    struct iatt preiatt =
+                    {
+                        0,
+                    },
+                postiatt = {
+                    0,
+                };
+    dict_t *fop_attr = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    GF_REF_GET(glfd);
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    ret = get_fop_attr_thrd_key(&fop_attr);
+    if (ret)
+        gf_msg_debug("gfapi", 0, "Getting leaseid from thread failed");
+
+    ret = syncop_fsync(subvol, fd, 1, &preiatt, &postiatt, fop_attr, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    if (ret >= 0) {
+        if (prestat)
+            glfs_iatt_to_statx(glfd->fs, &preiatt, prestat);
+        if (poststat)
+            glfs_iatt_to_statx(glfd->fs, &postiatt, poststat);
+    }
 out:
-	if (fd)
-		fd_unref (fd);
+    if (fd)
+        fd_unref(fd);
+    if (glfd)
+        GF_REF_PUT(glfd);
+    if (fop_attr)
+        dict_unref(fop_attr);
+
+    glfs_subvol_done(glfd->fs, subvol);
 
-	glfs_subvol_done (glfd->fs, subvol);
+    __GLFS_EXIT_FS;
 
-	return ret;
+invalid_fs:
+    return ret;
 }
 
+GFAPI_SYMVER_PUBLIC(glfs_fdatasync34, glfs_fdatasync, 3.4.0)
+int
+pub_glfs_fdatasync34(struct glfs_fd *glfd)
+{
+    return glfs_fdatasync_common(glfd, NULL, NULL);
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fdatasync, 6.0)
 int
-glfs_ftruncate_async (struct glfs_fd *glfd, off_t offset,
-		      glfs_io_cbk fn, void *data)
+pub_glfs_fdatasync(struct glfs_fd *glfd, struct glfs_stat *prestat,
+                   struct glfs_stat *poststat)
 {
-	struct glfs_io *gio = NULL;
-	int             ret = 0;
+    return glfs_fdatasync_common(glfd, prestat, poststat);
+}
 
-	gio = GF_CALLOC (1, sizeof (*gio), glfs_mt_glfs_io_t);
-	if (!gio) {
-		errno = ENOMEM;
-		return -1;
-	}
+GFAPI_SYMVER_PUBLIC(glfs_fdatasync_async34, glfs_fdatasync_async, 3.4.0)
+int
+pub_glfs_fdatasync_async34(struct glfs_fd *glfd, glfs_io_cbk34 fn, void *data)
+{
+    int ret = -1;
 
-	gio->op     = GF_FOP_FTRUNCATE;
-	gio->glfd   = glfd;
-	gio->offset = offset;
-	gio->fn     = fn;
-	gio->data   = data;
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
 
-	ret = synctask_new (glfs_from_glfd (glfd)->ctx->env,
-			    glfs_io_async_task, glfs_io_async_cbk,
-			    NULL, gio);
+    ret = glfs_fsync_async_common(glfd, _gf_true, (void *)fn, data, 1);
 
-	if (ret) {
-		GF_FREE (gio->iov);
-		GF_FREE (gio);
-	}
+    __GLFS_EXIT_FS;
 
-	return ret;
+invalid_fs:
+    return ret;
 }
 
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fdatasync_async, 6.0)
 int
-glfs_access (struct glfs *fs, const char *path, int mode)
+pub_glfs_fdatasync_async(struct glfs_fd *glfd, glfs_io_cbk fn, void *data)
 {
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	loc_t            loc = {0, };
-	struct iatt      iatt = {0, };
-	int              reval = 0;
-
-	__glfs_entry_fs (fs);
+    int ret = -1;
 
-	subvol = glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
-retry:
-	ret = glfs_resolve (fs, subvol, path, &loc, &iatt, reval);
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    ret = glfs_fsync_async_common(glfd, _gf_false, fn, data, 1);
 
-	if (ret)
-		goto out;
+    __GLFS_EXIT_FS;
 
-	ret = syncop_access (subvol, &loc, mode);
+invalid_fs:
+    return ret;
+}
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+static int
+glfs_ftruncate_common(struct glfs_fd *glfd, off_t offset,
+                      struct glfs_stat *prestat, struct glfs_stat *poststat)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    fd_t *fd = NULL;
+    struct iatt preiatt =
+                    {
+                        0,
+                    },
+                postiatt = {
+                    0,
+                };
+    dict_t *fop_attr = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    GF_REF_GET(glfd);
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    ret = get_fop_attr_thrd_key(&fop_attr);
+    if (ret)
+        gf_msg_debug("gfapi", 0, "Getting leaseid from thread failed");
+
+    ret = syncop_ftruncate(subvol, fd, offset, &preiatt, &postiatt, fop_attr,
+                           NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    if (ret >= 0) {
+        if (prestat)
+            glfs_iatt_to_statx(glfd->fs, &preiatt, prestat);
+        if (poststat)
+            glfs_iatt_to_statx(glfd->fs, &postiatt, poststat);
+    }
 out:
-	loc_wipe (&loc);
+    if (fd)
+        fd_unref(fd);
+    if (glfd)
+        GF_REF_PUT(glfd);
+    if (fop_attr)
+        dict_unref(fop_attr);
 
-	glfs_subvol_done (fs, subvol);
+    glfs_subvol_done(glfd->fs, subvol);
 
-	return ret;
-}
+    __GLFS_EXIT_FS;
 
+invalid_fs:
+    return ret;
+}
 
+GFAPI_SYMVER_PUBLIC(glfs_ftruncate34, glfs_ftruncate, 3.4.0)
 int
-glfs_symlink (struct glfs *fs, const char *data, const char *path)
+pub_glfs_ftruncate34(struct glfs_fd *glfd, off_t offset)
 {
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	loc_t            loc = {0, };
-	struct iatt      iatt = {0, };
-	uuid_t           gfid;
-	dict_t          *xattr_req = NULL;
-	int              reval = 0;
-
-	__glfs_entry_fs (fs);
-
-	subvol = glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+    return glfs_ftruncate_common(glfd, offset, NULL, NULL);
+}
 
-	xattr_req = dict_new ();
-	if (!xattr_req) {
-		ret = -1;
-		errno = ENOMEM;
-		goto out;
-	}
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_ftruncate, 6.0)
+int
+pub_glfs_ftruncate(struct glfs_fd *glfd, off_t offset,
+                   struct glfs_stat *prestat, struct glfs_stat *poststat)
+{
+    return glfs_ftruncate_common(glfd, offset, prestat, poststat);
+}
 
-	uuid_generate (gfid);
-	ret = dict_set_static_bin (xattr_req, "gfid-req", gfid, 16);
-	if (ret) {
-		ret = -1;
-		errno = ENOMEM;
-		goto out;
-	}
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_truncate, 3.7.15)
+int
+pub_glfs_truncate(struct glfs *fs, const char *path, off_t length)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
 retry:
-	ret = glfs_lresolve (fs, subvol, path, &loc, &iatt, reval);
+    ret = glfs_resolve(fs, subvol, path, &loc, &iatt, reval);
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	if (loc.inode) {
-		errno = EEXIST;
-		ret = -1;
-		goto out;
-	}
+    if (ret)
+        goto out;
 
-	if (ret == -1 && errno != ENOENT)
-		/* Any other type of error is fatal */
-		goto out;
+    ret = syncop_truncate(subvol, &loc, length, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
 
-	if (ret == -1 && errno == ENOENT && !loc.parent)
-		/* The parent directory or an ancestor even
-		   higher does not exist
-		*/
-		goto out;
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
+out:
+    loc_wipe(&loc);
 
-	/* ret == -1 && errno == ENOENT */
-	loc.inode = inode_new (loc.parent->table);
-	if (!loc.inode) {
-		ret = -1;
-		errno = ENOMEM;
-		goto out;
-	}
+    glfs_subvol_done(fs, subvol);
 
-	ret = syncop_symlink (subvol, &loc, data, xattr_req, &iatt);
+    __GLFS_EXIT_FS;
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+invalid_fs:
+    return ret;
+}
 
-	if (ret == 0)
-		ret = glfs_loc_link (&loc, &iatt);
-out:
-	loc_wipe (&loc);
+static int
+glfs_ftruncate_async_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                         int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                         struct iatt *postbuf, dict_t *xdata)
+{
+    glfs_io_async_cbk(op_ret, op_errno, frame, cookie, NULL, 0, prebuf,
+                      postbuf);
 
-	if (xattr_req)
-		dict_unref (xattr_req);
+    return 0;
+}
 
-	glfs_subvol_done (fs, subvol);
+static int
+glfs_ftruncate_async_common(struct glfs_fd *glfd, off_t offset,
+                            gf_boolean_t oldcb, glfs_io_cbk fn, void *data)
+{
+    struct glfs_io *gio = NULL;
+    int ret = -1;
+    call_frame_t *frame = NULL;
+    xlator_t *subvol = NULL;
+    fd_t *fd = NULL;
+    dict_t *fop_attr = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    /* Need to take explicit ref so that the fd
+     * is not destroyed before the fop is complete
+     */
+    GF_REF_GET(glfd);
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        errno = EBADFD;
+        goto out;
+    }
+
+    frame = syncop_create_frame(THIS);
+    if (!frame) {
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gio = GF_CALLOC(1, sizeof(*gio), glfs_mt_glfs_io_t);
+    if (!gio) {
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gio->op = GF_FOP_FTRUNCATE;
+    gio->glfd = glfd;
+    gio->offset = offset;
+    gio->oldcb = oldcb;
+    gio->fn = fn;
+    gio->data = data;
+
+    frame->local = gio;
+
+    ret = get_fop_attr_thrd_key(&fop_attr);
+    if (ret)
+        gf_msg_debug("gfapi", 0, "Getting leaseid from thread failed");
+
+    STACK_WIND_COOKIE(frame, glfs_ftruncate_async_cbk, subvol, subvol,
+                      subvol->fops->ftruncate, fd, offset, fop_attr);
+
+    ret = 0;
 
-	return ret;
+out:
+    if (ret) {
+        if (fd)
+            fd_unref(fd);
+        if (glfd)
+            GF_REF_PUT(glfd);
+        GF_FREE(gio);
+        if (frame)
+            STACK_DESTROY(frame->root);
+        glfs_subvol_done(glfd->fs, subvol);
+    }
+    if (fop_attr)
+        dict_unref(fop_attr);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
 }
 
-
+GFAPI_SYMVER_PUBLIC(glfs_ftruncate_async34, glfs_ftruncate_async, 3.4.0)
 int
-glfs_readlink (struct glfs *fs, const char *path, char *buf, size_t bufsiz)
+pub_glfs_ftruncate_async34(struct glfs_fd *glfd, off_t offset, glfs_io_cbk34 fn,
+                           void *data)
 {
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	loc_t            loc = {0, };
-	struct iatt      iatt = {0, };
-	int              reval = 0;
-	char            *linkval = NULL;
+    return glfs_ftruncate_async_common(glfd, offset, _gf_true, (void *)fn,
+                                       data);
+}
 
-	__glfs_entry_fs (fs);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_ftruncate_async, 6.0)
+int
+pub_glfs_ftruncate_async(struct glfs_fd *glfd, off_t offset, glfs_io_cbk fn,
+                         void *data)
+{
+    return glfs_ftruncate_async_common(glfd, offset, _gf_false, fn, data);
+}
 
-	subvol = glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_access, 3.4.0)
+int
+pub_glfs_access(struct glfs *fs, const char *path, int mode)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
 retry:
-	ret = glfs_lresolve (fs, subvol, path, &loc, &iatt, reval);
-
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    ret = glfs_resolve(fs, subvol, path, &loc, &iatt, reval);
 
-	if (ret)
-		goto out;
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	if (iatt.ia_type != IA_IFLNK) {
-		ret = -1;
-		errno = EINVAL;
-		goto out;
-	}
+    if (ret)
+        goto out;
 
-	ret = syncop_readlink (subvol, &loc, &linkval, bufsiz);
-	if (ret > 0) {
-		memcpy (buf, linkval, ret);
-		GF_FREE (linkval);
-	}
+    ret = syncop_access(subvol, &loc, mode, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 out:
-	loc_wipe (&loc);
+    loc_wipe(&loc);
 
-	glfs_subvol_done (fs, subvol);
+    glfs_subvol_done(fs, subvol);
 
-	return ret;
-}
+    __GLFS_EXIT_FS;
 
+invalid_fs:
+    return ret;
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_symlink, 3.4.0)
 int
-glfs_mknod (struct glfs *fs, const char *path, mode_t mode, dev_t dev)
+pub_glfs_symlink(struct glfs *fs, const char *data, const char *path)
 {
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	loc_t            loc = {0, };
-	struct iatt      iatt = {0, };
-	uuid_t           gfid;
-	dict_t          *xattr_req = NULL;
-	int              reval = 0;
-
-	__glfs_entry_fs (fs);
-
-	subvol = glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
-
-	xattr_req = dict_new ();
-	if (!xattr_req) {
-		ret = -1;
-		errno = ENOMEM;
-		goto out;
-	}
-
-	uuid_generate (gfid);
-	ret = dict_set_static_bin (xattr_req, "gfid-req", gfid, 16);
-	if (ret) {
-		ret = -1;
-		errno = ENOMEM;
-		goto out;
-	}
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    uuid_t gfid;
+    dict_t *xattr_req = NULL;
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    xattr_req = dict_new();
+    if (!xattr_req) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gf_uuid_generate(gfid);
+    ret = dict_set_gfuuid(xattr_req, "gfid-req", gfid, true);
+    if (ret) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
 retry:
-	ret = glfs_lresolve (fs, subvol, path, &loc, &iatt, reval);
+    ret = glfs_lresolve(fs, subvol, path, &loc, &iatt, reval);
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	if (loc.inode) {
-		errno = EEXIST;
-		ret = -1;
-		goto out;
-	}
+    if (loc.inode) {
+        errno = EEXIST;
+        ret = -1;
+        goto out;
+    }
 
-	if (ret == -1 && errno != ENOENT)
-		/* Any other type of error is fatal */
-		goto out;
+    if (ret == -1 && errno != ENOENT)
+        /* Any other type of error is fatal */
+        goto out;
 
-	if (ret == -1 && errno == ENOENT && !loc.parent)
-		/* The parent directory or an ancestor even
-		   higher does not exist
-		*/
-		goto out;
+    if (ret == -1 && errno == ENOENT && !loc.parent)
+        /* The parent directory or an ancestor even
+           higher does not exist
+        */
+        goto out;
 
-	/* ret == -1 && errno == ENOENT */
-	loc.inode = inode_new (loc.parent->table);
-	if (!loc.inode) {
-		ret = -1;
-		errno = ENOMEM;
-		goto out;
-	}
+    /* ret == -1 && errno == ENOENT */
+    loc.inode = inode_new(loc.parent->table);
+    if (!loc.inode) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
 
-	ret = syncop_mknod (subvol, &loc, mode, dev, xattr_req, &iatt);
+    ret = syncop_symlink(subvol, &loc, data, &iatt, xattr_req, NULL);
+    DECODE_SYNCOP_ERR(ret);
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	if (ret == 0)
-		ret = glfs_loc_link (&loc, &iatt);
+    if (ret == 0)
+        ret = glfs_loc_link(&loc, &iatt);
 out:
-	loc_wipe (&loc);
+    loc_wipe(&loc);
 
-	if (xattr_req)
-		dict_unref (xattr_req);
+    if (xattr_req)
+        dict_unref(xattr_req);
 
-	glfs_subvol_done (fs, subvol);
+    glfs_subvol_done(fs, subvol);
 
-	return ret;
-}
+    __GLFS_EXIT_FS;
 
+invalid_fs:
+    return ret;
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_readlink, 3.4.0)
 int
-glfs_mkdir (struct glfs *fs, const char *path, mode_t mode)
+pub_glfs_readlink(struct glfs *fs, const char *path, char *buf, size_t bufsiz)
 {
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	loc_t            loc = {0, };
-	struct iatt      iatt = {0, };
-	uuid_t           gfid;
-	dict_t          *xattr_req = NULL;
-	int              reval = 0;
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    int reval = 0;
+    char *linkval = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+retry:
+    ret = glfs_lresolve(fs, subvol, path, &loc, &iatt, reval);
+
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
+
+    if (ret)
+        goto out;
+
+    if (iatt.ia_type != IA_IFLNK) {
+        ret = -1;
+        errno = EINVAL;
+        goto out;
+    }
+
+    ret = syncop_readlink(subvol, &loc, &linkval, bufsiz, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+    if (ret > 0) {
+        memcpy(buf, linkval, ret);
+        GF_FREE(linkval);
+    }
+
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
+out:
+    loc_wipe(&loc);
 
-	__glfs_entry_fs (fs);
+    glfs_subvol_done(fs, subvol);
 
-	subvol = glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+    __GLFS_EXIT_FS;
 
-	xattr_req = dict_new ();
-	if (!xattr_req) {
-		ret = -1;
-		errno = ENOMEM;
-		goto out;
-	}
+invalid_fs:
+    return ret;
+}
 
-	uuid_generate (gfid);
-	ret = dict_set_static_bin (xattr_req, "gfid-req", gfid, 16);
-	if (ret) {
-		ret = -1;
-		errno = ENOMEM;
-		goto out;
-	}
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_mknod, 3.4.0)
+int
+pub_glfs_mknod(struct glfs *fs, const char *path, mode_t mode, dev_t dev)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    uuid_t gfid;
+    dict_t *xattr_req = NULL;
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    xattr_req = dict_new();
+    if (!xattr_req) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gf_uuid_generate(gfid);
+    ret = dict_set_gfuuid(xattr_req, "gfid-req", gfid, true);
+    if (ret) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
 retry:
-	ret = glfs_lresolve (fs, subvol, path, &loc, &iatt, reval);
+    ret = glfs_lresolve(fs, subvol, path, &loc, &iatt, reval);
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	if (loc.inode) {
-		errno = EEXIST;
-		ret = -1;
-		goto out;
-	}
+    if (loc.inode) {
+        errno = EEXIST;
+        ret = -1;
+        goto out;
+    }
 
-	if (ret == -1 && errno != ENOENT)
-		/* Any other type of error is fatal */
-		goto out;
+    if (ret == -1 && errno != ENOENT)
+        /* Any other type of error is fatal */
+        goto out;
 
-	if (ret == -1 && errno == ENOENT && !loc.parent)
-		/* The parent directory or an ancestor even
-		   higher does not exist
-		*/
-		goto out;
+    if (ret == -1 && errno == ENOENT && !loc.parent)
+        /* The parent directory or an ancestor even
+           higher does not exist
+        */
+        goto out;
 
-	/* ret == -1 && errno == ENOENT */
-	loc.inode = inode_new (loc.parent->table);
-	if (!loc.inode) {
-		ret = -1;
-		errno = ENOMEM;
-		goto out;
-	}
+    /* ret == -1 && errno == ENOENT */
+    loc.inode = inode_new(loc.parent->table);
+    if (!loc.inode) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
 
-	ret = syncop_mkdir (subvol, &loc, mode, xattr_req, &iatt);
+    ret = syncop_mknod(subvol, &loc, mode, dev, &iatt, xattr_req, NULL);
+    DECODE_SYNCOP_ERR(ret);
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	if (ret == 0)
-		ret = glfs_loc_link (&loc, &iatt);
+    if (ret == 0)
+        ret = glfs_loc_link(&loc, &iatt);
 out:
-	loc_wipe (&loc);
+    loc_wipe(&loc);
 
-	if (xattr_req)
-		dict_unref (xattr_req);
+    if (xattr_req)
+        dict_unref(xattr_req);
 
-	glfs_subvol_done (fs, subvol);
+    glfs_subvol_done(fs, subvol);
 
-	return ret;
-}
+    __GLFS_EXIT_FS;
 
+invalid_fs:
+    return ret;
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_mkdir, 3.4.0)
 int
-glfs_unlink (struct glfs *fs, const char *path)
+pub_glfs_mkdir(struct glfs *fs, const char *path, mode_t mode)
 {
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	loc_t            loc = {0, };
-	struct iatt      iatt = {0, };
-	int              reval = 0;
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    uuid_t gfid;
+    dict_t *xattr_req = NULL;
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    xattr_req = dict_new();
+    if (!xattr_req) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gf_uuid_generate(gfid);
+    ret = dict_set_gfuuid(xattr_req, "gfid-req", gfid, true);
+    if (ret) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+retry:
+    ret = glfs_lresolve(fs, subvol, path, &loc, &iatt, reval);
 
-	__glfs_entry_fs (fs);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	subvol = glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
-retry:
-	ret = glfs_lresolve (fs, subvol, path, &loc, &iatt, reval);
+    if (loc.inode) {
+        errno = EEXIST;
+        ret = -1;
+        goto out;
+    }
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    if (ret == -1 && errno != ENOENT)
+        /* Any other type of error is fatal */
+        goto out;
 
-	if (ret)
-		goto out;
+    if (ret == -1 && errno == ENOENT && !loc.parent)
+        /* The parent directory or an ancestor even
+           higher does not exist
+        */
+        goto out;
 
-	if (iatt.ia_type == IA_IFDIR) {
-		ret = -1;
-		errno = EISDIR;
-		goto out;
-	}
+    /* ret == -1 && errno == ENOENT */
+    loc.inode = inode_new(loc.parent->table);
+    if (!loc.inode) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
 
-	ret = syncop_unlink (subvol, &loc);
+    ret = syncop_mkdir(subvol, &loc, mode, &iatt, xattr_req, NULL);
+    DECODE_SYNCOP_ERR(ret);
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	if (ret == 0)
-		ret = glfs_loc_unlink (&loc);
+    if (ret == 0)
+        ret = glfs_loc_link(&loc, &iatt);
 out:
-	loc_wipe (&loc);
+    loc_wipe(&loc);
 
-	glfs_subvol_done (fs, subvol);
+    if (xattr_req)
+        dict_unref(xattr_req);
 
-	return ret;
-}
+    glfs_subvol_done(fs, subvol);
 
+    __GLFS_EXIT_FS;
 
+invalid_fs:
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_unlink, 3.4.0)
 int
-glfs_rmdir (struct glfs *fs, const char *path)
+pub_glfs_unlink(struct glfs *fs, const char *path)
 {
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	loc_t            loc = {0, };
-	struct iatt      iatt = {0, };
-	int              reval = 0;
-
-	__glfs_entry_fs (fs);
-
-	subvol = glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
 retry:
-	ret = glfs_lresolve (fs, subvol, path, &loc, &iatt, reval);
+    ret = glfs_lresolve(fs, subvol, path, &loc, &iatt, reval);
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	if (ret)
-		goto out;
+    if (ret)
+        goto out;
 
-	if (iatt.ia_type != IA_IFDIR) {
-		ret = -1;
-		errno = ENOTDIR;
-		goto out;
-	}
+    if (iatt.ia_type == IA_IFDIR) {
+        ret = -1;
+        errno = EISDIR;
+        goto out;
+    }
 
-	ret = syncop_rmdir (subvol, &loc);
+    /* TODO: Add leaseid */
+    ret = syncop_unlink(subvol, &loc, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	if (ret == 0)
-		ret = glfs_loc_unlink (&loc);
+    if (ret == 0)
+        ret = glfs_loc_unlink(&loc);
 out:
-	loc_wipe (&loc);
+    loc_wipe(&loc);
 
-	glfs_subvol_done (fs, subvol);
+    glfs_subvol_done(fs, subvol);
 
-	return ret;
-}
+    __GLFS_EXIT_FS;
 
+invalid_fs:
+    return ret;
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_rmdir, 3.4.0)
 int
-glfs_rename (struct glfs *fs, const char *oldpath, const char *newpath)
+pub_glfs_rmdir(struct glfs *fs, const char *path)
 {
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	loc_t            oldloc = {0, };
-	loc_t            newloc = {0, };
-	struct iatt      oldiatt = {0, };
-	struct iatt      newiatt = {0, };
-	int              reval = 0;
-
-	__glfs_entry_fs (fs);
-
-	subvol = glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
 retry:
-	ret = glfs_lresolve (fs, subvol, oldpath, &oldloc, &oldiatt, reval);
+    ret = glfs_lresolve(fs, subvol, path, &loc, &iatt, reval);
 
-	ESTALE_RETRY (ret, errno, reval, &oldloc, retry);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	if (ret)
-		goto out;
-retrynew:
-	ret = glfs_lresolve (fs, subvol, newpath, &newloc, &newiatt, reval);
-
-	ESTALE_RETRY (ret, errno, reval, &newloc, retrynew);
-
-	if (ret && errno != ENOENT && newloc.parent)
-		goto out;
-
-	if (newiatt.ia_type != IA_INVAL) {
-                if ((oldiatt.ia_type == IA_IFDIR) !=
-                    (newiatt.ia_type == IA_IFDIR)) {
-                       /* Either both old and new must be dirs,
-                        * or both must be non-dirs. Else, fail.
-                        */
-                       ret = -1;
-                       errno = EISDIR;
-                       goto out;
-                }
-        }
+    if (ret)
+        goto out;
 
-	/* TODO: check if new or old is a prefix of the other, and fail EINVAL */
+    if (iatt.ia_type != IA_IFDIR) {
+        ret = -1;
+        errno = ENOTDIR;
+        goto out;
+    }
 
-	ret = syncop_rename (subvol, &oldloc, &newloc);
+    ret = syncop_rmdir(subvol, &loc, 0, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
 
-	if (ret == -1 && errno == ESTALE) {
-		if (reval < DEFAULT_REVAL_COUNT) {
-			reval++;
-			loc_wipe (&oldloc);
-			loc_wipe (&newloc);
-			goto retry;
-		}
-	}
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	if (ret == 0)
-		inode_rename (oldloc.parent->table, oldloc.parent, oldloc.name,
-			      newloc.parent, newloc.name, oldloc.inode,
-			      &oldiatt);
+    if (ret == 0)
+        ret = glfs_loc_unlink(&loc);
 out:
-	loc_wipe (&oldloc);
-	loc_wipe (&newloc);
+    loc_wipe(&loc);
 
-	glfs_subvol_done (fs, subvol);
+    glfs_subvol_done(fs, subvol);
 
-	return ret;
-}
+    __GLFS_EXIT_FS;
 
+invalid_fs:
+    return ret;
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_rename, 3.4.0)
 int
-glfs_link (struct glfs *fs, const char *oldpath, const char *newpath)
+pub_glfs_rename(struct glfs *fs, const char *oldpath, const char *newpath)
 {
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	loc_t            oldloc = {0, };
-	loc_t            newloc = {0, };
-	struct iatt      oldiatt = {0, };
-	struct iatt      newiatt = {0, };
-	int              reval = 0;
-
-	__glfs_entry_fs (fs);
-
-	subvol = glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    loc_t oldloc = {
+        0,
+    };
+    loc_t newloc = {
+        0,
+    };
+    struct iatt oldiatt = {
+        0,
+    };
+    struct iatt newiatt = {
+        0,
+    };
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
 retry:
-	ret = glfs_lresolve (fs, subvol, oldpath, &oldloc, &oldiatt, reval);
+    ret = glfs_lresolve(fs, subvol, oldpath, &oldloc, &oldiatt, reval);
 
-	ESTALE_RETRY (ret, errno, reval, &oldloc, retry);
+    ESTALE_RETRY(ret, errno, reval, &oldloc, retry);
 
-	if (ret)
-		goto out;
+    if (ret)
+        goto out;
 retrynew:
-	ret = glfs_lresolve (fs, subvol, newpath, &newloc, &newiatt, reval);
+    ret = glfs_lresolve(fs, subvol, newpath, &newloc, &newiatt, reval);
 
-	ESTALE_RETRY (ret, errno, reval, &newloc, retrynew);
+    ESTALE_RETRY(ret, errno, reval, &newloc, retrynew);
 
-	if (ret == 0) {
-		ret = -1;
-		errno = EEXIST;
-		goto out;
-	}
+    if (ret && errno != ENOENT && newloc.parent)
+        goto out;
 
-	if (oldiatt.ia_type == IA_IFDIR) {
-		ret = -1;
-		errno = EISDIR;
-		goto out;
-	}
+    if (newiatt.ia_type != IA_INVAL) {
+        if ((oldiatt.ia_type == IA_IFDIR) != (newiatt.ia_type == IA_IFDIR)) {
+            /* Either both old and new must be dirs,
+             * or both must be non-dirs. Else, fail.
+             */
+            ret = -1;
+            errno = EISDIR;
+            goto out;
+        }
+    }
 
-        /* Filling the inode of the hard link to be same as that of the
-           original file
-        */
-	if (newloc.inode) {
-		inode_unref (newloc.inode);
-		newloc.inode = NULL;
-	}
-        newloc.inode = inode_ref (oldloc.inode);
+    /* TODO: - check if new or old is a prefix of the other, and fail EINVAL
+     *       - Add leaseid */
 
-	ret = syncop_link (subvol, &oldloc, &newloc);
+    ret = syncop_rename(subvol, &oldloc, &newloc, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
 
-	if (ret == -1 && errno == ESTALE) {
-		loc_wipe (&oldloc);
-		loc_wipe (&newloc);
-		if (reval--)
-			goto retry;
-	}
+    if (ret == -1 && errno == ESTALE) {
+        if (reval < DEFAULT_REVAL_COUNT) {
+            reval++;
+            loc_wipe(&oldloc);
+            loc_wipe(&newloc);
+            goto retry;
+        }
+    }
 
-	if (ret == 0)
-		ret = glfs_loc_link (&newloc, &oldiatt);
+    if (ret == 0) {
+        inode_rename(oldloc.parent->table, oldloc.parent, oldloc.name,
+                     newloc.parent, newloc.name, oldloc.inode, &oldiatt);
+
+        if (newloc.inode && !inode_has_dentry(newloc.inode))
+            inode_forget(newloc.inode, 0);
+    }
 out:
-	loc_wipe (&oldloc);
-	loc_wipe (&newloc);
+    loc_wipe(&oldloc);
+    loc_wipe(&newloc);
 
-	glfs_subvol_done (fs, subvol);
+    glfs_subvol_done(fs, subvol);
 
-	return ret;
-}
+    __GLFS_EXIT_FS;
 
+invalid_fs:
+    return ret;
+}
 
-struct glfs_fd *
-glfs_opendir (struct glfs *fs, const char *path)
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_link, 3.4.0)
+int
+pub_glfs_link(struct glfs *fs, const char *oldpath, const char *newpath)
 {
-	int              ret = -1;
-	struct glfs_fd  *glfd = NULL;
-	xlator_t        *subvol = NULL;
-	loc_t            loc = {0, };
-	struct iatt      iatt = {0, };
-	int              reval = 0;
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    loc_t oldloc = {
+        0,
+    };
+    loc_t newloc = {
+        0,
+    };
+    struct iatt oldiatt = {
+        0,
+    };
+    struct iatt newiatt = {
+        0,
+    };
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+retry:
+    ret = glfs_lresolve(fs, subvol, oldpath, &oldloc, &oldiatt, reval);
+
+    ESTALE_RETRY(ret, errno, reval, &oldloc, retry);
 
-	__glfs_entry_fs (fs);
+    if (ret)
+        goto out;
+retrynew:
+    ret = glfs_lresolve(fs, subvol, newpath, &newloc, &newiatt, reval);
+
+    ESTALE_RETRY(ret, errno, reval, &newloc, retrynew);
+
+    if (ret == 0) {
+        ret = -1;
+        errno = EEXIST;
+        goto out;
+    }
+
+    if (oldiatt.ia_type == IA_IFDIR) {
+        ret = -1;
+        errno = EISDIR;
+        goto out;
+    }
+
+    /* Filling the inode of the hard link to be same as that of the
+       original file
+    */
+    if (newloc.inode) {
+        inode_unref(newloc.inode);
+        newloc.inode = NULL;
+    }
+    newloc.inode = inode_ref(oldloc.inode);
+
+    ret = syncop_link(subvol, &oldloc, &newloc, &newiatt, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    if (ret == -1 && errno == ESTALE) {
+        loc_wipe(&oldloc);
+        loc_wipe(&newloc);
+        if (reval--)
+            goto retry;
+    }
+
+    if (ret == 0)
+        ret = glfs_loc_link(&newloc, &newiatt);
+out:
+    loc_wipe(&oldloc);
+    loc_wipe(&newloc);
 
-	subvol = glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+    glfs_subvol_done(fs, subvol);
 
-	glfd = glfs_fd_new (fs);
-	if (!glfd)
-		goto out;
+    __GLFS_EXIT_FS;
 
-	INIT_LIST_HEAD (&glfd->entries);
+invalid_fs:
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_opendir, 3.4.0)
+struct glfs_fd *
+pub_glfs_opendir(struct glfs *fs, const char *path)
+{
+    int ret = -1;
+    struct glfs_fd *glfd = NULL;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    glfd = glfs_fd_new(fs);
+    if (!glfd)
+        goto out;
+
+    INIT_LIST_HEAD(&glfd->entries);
 retry:
-	ret = glfs_resolve (fs, subvol, path, &loc, &iatt, reval);
+    ret = glfs_resolve(fs, subvol, path, &loc, &iatt, reval);
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	if (ret)
-		goto out;
+    if (ret)
+        goto out;
 
-	if (!IA_ISDIR (iatt.ia_type)) {
-		ret = -1;
-		errno = ENOTDIR;
-		goto out;
-	}
+    if (!IA_ISDIR(iatt.ia_type)) {
+        ret = -1;
+        errno = ENOTDIR;
+        goto out;
+    }
 
-	if (glfd->fd) {
-		/* Retry. Safe to touch glfd->fd as we
-		   still have not glfs_fd_bind() yet.
-		*/
-		fd_unref (glfd->fd);
-		glfd->fd = NULL;
-	}
+    if (glfd->fd) {
+        /* Retry. Safe to touch glfd->fd as we
+           still have not glfs_fd_bind() yet.
+        */
+        fd_unref(glfd->fd);
+        glfd->fd = NULL;
+    }
 
-	glfd->fd = fd_create (loc.inode, getpid());
-	if (!glfd->fd) {
-		ret = -1;
-		errno = ENOMEM;
-		goto out;
-	}
+    glfd->fd = fd_create(loc.inode, getpid());
+    if (!glfd->fd) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
 
-	ret = syncop_opendir (subvol, &loc, glfd->fd);
+    ret = syncop_opendir(subvol, &loc, glfd->fd, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 out:
-	loc_wipe (&loc);
+    loc_wipe(&loc);
 
-	if (ret && glfd) {
-		glfs_fd_destroy (glfd);
-		glfd = NULL;
-	} else {
-		fd_bind (glfd->fd);
-		glfs_fd_bind (glfd);
-	}
+    if (ret && glfd) {
+        GF_REF_PUT(glfd);
+        glfd = NULL;
+    } else if (glfd) {
+        glfd_set_state_bind(glfd);
+    }
 
-	glfs_subvol_done (fs, subvol);
+    glfs_subvol_done(fs, subvol);
 
-	return glfd;
-}
+    __GLFS_EXIT_FS;
 
+invalid_fs:
+    return glfd;
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_closedir, 3.4.0)
 int
-glfs_closedir (struct glfs_fd *glfd)
+pub_glfs_closedir(struct glfs_fd *glfd)
 {
-	__glfs_entry_fd (glfd);
+    int ret = -1;
 
-	gf_dirent_free (list_entry (&glfd->entries, gf_dirent_t, list));
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
 
-	glfs_fd_destroy (glfd);
+    gf_dirent_free(list_entry(&glfd->entries, gf_dirent_t, list));
 
-	return 0;
-}
+    glfs_mark_glfd_for_deletion(glfd);
+
+    __GLFS_EXIT_FS;
 
+    ret = 0;
 
+invalid_fs:
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_telldir, 3.4.0)
 long
-glfs_telldir (struct glfs_fd *fd)
+pub_glfs_telldir(struct glfs_fd *fd)
 {
-	return fd->offset;
-}
+    if (fd == NULL) {
+        errno = EBADF;
+        return -1;
+    }
 
+    return fd->offset;
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_seekdir, 3.4.0)
 void
-glfs_seekdir (struct glfs_fd *fd, long offset)
+pub_glfs_seekdir(struct glfs_fd *fd, long offset)
 {
-	gf_dirent_t *entry = NULL;
-	gf_dirent_t *tmp = NULL;
+    gf_dirent_t *entry = NULL;
+    gf_dirent_t *tmp = NULL;
+
+    if (fd == NULL) {
+        errno = EBADF;
+        return;
+    }
 
-	if (fd->offset == offset)
-		return;
+    if (fd->offset == offset)
+        return;
 
-	fd->offset = offset;
-	fd->next = NULL;
+    fd->offset = offset;
+    fd->next = NULL;
 
-	list_for_each_entry_safe (entry, tmp, &fd->entries, list) {
-		if (entry->d_off != offset)
-			continue;
+    list_for_each_entry_safe(entry, tmp, &fd->entries, list)
+    {
+        if (entry->d_off != offset)
+            continue;
 
-		if (&tmp->list != &fd->entries) {
-			/* found! */
-			fd->next = tmp;
-			return;
-		}
-	}
-	/* could not find entry at requested offset in the cache.
-	   next readdir_r() will result in glfd_entry_refresh()
-	*/
+        if (&tmp->list != &fd->entries) {
+            /* found! */
+            fd->next = tmp;
+            return;
+        }
+    }
+    /* could not find entry at requested offset in the cache.
+       next readdir_r() will result in glfd_entry_refresh()
+    */
+}
+
+static int
+glfs_discard_async_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno,
+                       struct iatt *preop_stbuf, struct iatt *postop_stbuf,
+                       dict_t *xdata)
+{
+    glfs_io_async_cbk(op_ret, op_errno, frame, cookie, NULL, 0, preop_stbuf,
+                      postop_stbuf);
+
+    return 0;
 }
 
+static int
+glfs_discard_async_common(struct glfs_fd *glfd, off_t offset, size_t len,
+                          gf_boolean_t oldcb, glfs_io_cbk fn, void *data)
+{
+    struct glfs_io *gio = NULL;
+    int ret = -1;
+    call_frame_t *frame = NULL;
+    xlator_t *subvol = NULL;
+    fd_t *fd = NULL;
+    dict_t *fop_attr = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    /* Need to take explicit ref so that the fd
+     * is not destroyed before the fop is complete
+     */
+    GF_REF_GET(glfd);
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        errno = EBADFD;
+        goto out;
+    }
+
+    frame = syncop_create_frame(THIS);
+    if (!frame) {
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gio = GF_CALLOC(1, sizeof(*gio), glfs_mt_glfs_io_t);
+    if (!gio) {
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gio->op = GF_FOP_DISCARD;
+    gio->glfd = glfd;
+    gio->offset = offset;
+    gio->count = len;
+    gio->oldcb = oldcb;
+    gio->fn = fn;
+    gio->data = data;
+
+    frame->local = gio;
+    ret = get_fop_attr_thrd_key(&fop_attr);
+    if (ret)
+        gf_msg_debug("gfapi", 0, "Getting leaseid from thread failed");
+
+    STACK_WIND_COOKIE(frame, glfs_discard_async_cbk, subvol, subvol,
+                      subvol->fops->discard, fd, offset, len, fop_attr);
+
+    ret = 0;
+out:
+    if (ret) {
+        if (fd)
+            fd_unref(fd);
+        if (glfd)
+            GF_REF_PUT(glfd);
+        GF_FREE(gio);
+        if (frame)
+            STACK_DESTROY(frame->root);
+        glfs_subvol_done(glfd->fs, subvol);
+    }
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC(glfs_discard_async35, glfs_discard_async, 3.5.0)
 int
-glfs_discard_async (struct glfs_fd *glfd, off_t offset, size_t len,
-		      glfs_io_cbk fn, void *data)
+pub_glfs_discard_async35(struct glfs_fd *glfd, off_t offset, size_t len,
+                         glfs_io_cbk34 fn, void *data)
 {
-	struct glfs_io *gio = NULL;
-	int             ret = 0;
+    return glfs_discard_async_common(glfd, offset, len, _gf_true, (void *)fn,
+                                     data);
+}
 
-	gio = GF_CALLOC (1, sizeof (*gio), glfs_mt_glfs_io_t);
-	if (!gio) {
-		errno = ENOMEM;
-		return -1;
-	}
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_discard_async, 6.0)
+int
+pub_glfs_discard_async(struct glfs_fd *glfd, off_t offset, size_t len,
+                       glfs_io_cbk fn, void *data)
+{
+    return glfs_discard_async_common(glfd, offset, len, _gf_false, fn, data);
+}
 
-	gio->op     = GF_FOP_DISCARD;
-	gio->glfd   = glfd;
-	gio->offset = offset;
-	gio->count  = len;
-	gio->fn     = fn;
-	gio->data   = data;
+static int
+glfs_zerofill_async_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno,
+                        struct iatt *preop_stbuf, struct iatt *postop_stbuf,
+                        dict_t *xdata)
+{
+    glfs_io_async_cbk(op_ret, op_errno, frame, cookie, NULL, 0, preop_stbuf,
+                      postop_stbuf);
 
-	ret = synctask_new (glfs_from_glfd (glfd)->ctx->env,
-			    glfs_io_async_task, glfs_io_async_cbk,
-			    NULL, gio);
+    return 0;
+}
 
-	if (ret) {
-		GF_FREE (gio->iov);
-		GF_FREE (gio);
-	}
+static int
+glfs_zerofill_async_common(struct glfs_fd *glfd, off_t offset, off_t len,
+                           gf_boolean_t oldcb, glfs_io_cbk fn, void *data)
+{
+    struct glfs_io *gio = NULL;
+    int ret = -1;
+    call_frame_t *frame = NULL;
+    xlator_t *subvol = NULL;
+    fd_t *fd = NULL;
+    dict_t *fop_attr = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    /* Need to take explicit ref so that the fd
+     * is not destroyed before the fop is complete
+     */
+    GF_REF_GET(glfd);
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        errno = EBADFD;
+        goto out;
+    }
+
+    frame = syncop_create_frame(THIS);
+    if (!frame) {
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gio = GF_CALLOC(1, sizeof(*gio), glfs_mt_glfs_io_t);
+    if (!gio) {
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gio->op = GF_FOP_ZEROFILL;
+    gio->glfd = glfd;
+    gio->offset = offset;
+    gio->count = len;
+    gio->oldcb = oldcb;
+    gio->fn = fn;
+    gio->data = data;
+
+    frame->local = gio;
+
+    ret = get_fop_attr_thrd_key(&fop_attr);
+    if (ret)
+        gf_msg_debug("gfapi", 0, "Getting leaseid from thread failed");
+
+    STACK_WIND_COOKIE(frame, glfs_zerofill_async_cbk, subvol, subvol,
+                      subvol->fops->zerofill, fd, offset, len, fop_attr);
+    ret = 0;
+out:
+    if (ret) {
+        if (fd)
+            fd_unref(fd);
+        if (glfd)
+            GF_REF_PUT(glfd);
+        GF_FREE(gio);
+        if (frame)
+            STACK_DESTROY(frame->root);
+        glfs_subvol_done(glfd->fs, subvol);
+    }
+    if (fop_attr)
+        dict_unref(fop_attr);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
 
-	return ret;
+GFAPI_SYMVER_PUBLIC(glfs_zerofill_async35, glfs_zerofill_async, 3.5.0)
+int
+pub_glfs_zerofill_async35(struct glfs_fd *glfd, off_t offset, off_t len,
+                          glfs_io_cbk34 fn, void *data)
+{
+    return glfs_zerofill_async_common(glfd, offset, len, _gf_true, (void *)fn,
+                                      data);
 }
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_zerofill_async, 6.0)
+int
+pub_glfs_zerofill_async(struct glfs_fd *glfd, off_t offset, off_t len,
+                        glfs_io_cbk fn, void *data)
+{
+    return glfs_zerofill_async_common(glfd, offset, len, _gf_false, fn, data);
+}
 
 void
-gf_dirent_to_dirent (gf_dirent_t *gf_dirent, struct dirent *dirent)
+gf_dirent_to_dirent(gf_dirent_t *gf_dirent, struct dirent *dirent)
 {
-	dirent->d_ino = gf_dirent->d_ino;
+    dirent->d_ino = gf_dirent->d_ino;
 
 #ifdef _DIRENT_HAVE_D_OFF
-	dirent->d_off = gf_dirent->d_off;
+    dirent->d_off = gf_dirent->d_off;
 #endif
 
 #ifdef _DIRENT_HAVE_D_TYPE
-	dirent->d_type = gf_dirent->d_type;
+    dirent->d_type = gf_dirent->d_type;
 #endif
 
 #ifdef _DIRENT_HAVE_D_NAMLEN
-	dirent->d_namlen = strlen (gf_dirent->d_name);
+    dirent->d_namlen = strlen(gf_dirent->d_name);
 #endif
 
-	strncpy (dirent->d_name, gf_dirent->d_name, 256);
+    snprintf(dirent->d_name, NAME_MAX + 1, "%s", gf_dirent->d_name);
 }
 
-
 int
-glfd_entry_refresh (struct glfs_fd *glfd, int plus)
+glfd_entry_refresh(struct glfs_fd *glfd, int plus)
 {
-	xlator_t        *subvol = NULL;
-	gf_dirent_t      entries;
-	gf_dirent_t      old;
-	int              ret = -1;
-	fd_t            *fd = NULL;
-
-	subvol = glfs_active_subvol (glfd->fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
-
-	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
-	if (!fd) {
-		ret = -1;
-		errno = EBADFD;
-		goto out;
-	}
-
-	if (fd->inode->ia_type != IA_IFDIR) {
-		ret = -1;
-		errno = EBADF;
-		goto out;
-	}
-
-	INIT_LIST_HEAD (&entries.list);
-	INIT_LIST_HEAD (&old.list);
+    xlator_t *subvol = NULL;
+    gf_dirent_t entries;
+    gf_dirent_t old;
+    gf_dirent_t *entry = NULL;
+    int ret = -1;
+    fd_t *fd = NULL;
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    if (fd->inode->ia_type != IA_IFDIR) {
+        ret = -1;
+        errno = EBADF;
+        goto out;
+    }
+
+    INIT_LIST_HEAD(&entries.list);
+    INIT_LIST_HEAD(&old.list);
+
+    if (plus)
+        ret = syncop_readdirp(subvol, fd, 131072, glfd->offset, &entries, NULL,
+                              NULL);
+    else
+        ret = syncop_readdir(subvol, fd, 131072, glfd->offset, &entries, NULL,
+                             NULL);
+    DECODE_SYNCOP_ERR(ret);
+    if (ret >= 0) {
+        if (plus) {
+            list_for_each_entry(entry, &entries.list, list)
+            {
+                if ((!entry->inode && (!IA_ISDIR(entry->d_stat.ia_type))) ||
+                    ((entry->d_stat.ia_ctime == 0) &&
+                     strcmp(entry->d_name, ".") &&
+                     strcmp(entry->d_name, ".."))) {
+                    /* entry->inode for directories will be
+                     * always set to null to force a lookup
+                     * on the dentry. Hence to not degrade
+                     * readdir performance, we skip lookups
+                     * for directory entries. Also we will have
+                     * proper stat if directory present on
+                     * hashed subvolume.
+                     *
+                     * In addition, if the stat is invalid, force
+                     * lookup to fetch proper stat.
+                     */
+                    gf_fill_iatt_for_dirent(entry, fd->inode, subvol);
+                }
+            }
 
-	if (plus)
-		ret = syncop_readdirp (subvol, fd, 131072, glfd->offset,
-				       NULL, &entries);
-	else
-		ret = syncop_readdir (subvol, fd, 131072, glfd->offset,
-				      &entries);
-	if (ret >= 0) {
-		if (plus)
-			gf_link_inodes_from_dirent (THIS, fd->inode, &entries);
+            gf_link_inodes_from_dirent(THIS, fd->inode, &entries);
+        }
 
-		list_splice_init (&glfd->entries, &old.list);
-		list_splice_init (&entries.list, &glfd->entries);
+        list_splice_init(&glfd->entries, &old.list);
+        list_splice_init(&entries.list, &glfd->entries);
 
-		/* spurious errno is dangerous for glfd_entry_next() */
-		errno = 0;
-	}
+        /* spurious errno is dangerous for glfd_entry_next() */
+        errno = 0;
+    }
 
-	if (ret > 0)
-		glfd->next = list_entry (glfd->entries.next, gf_dirent_t, list);
+    if (ret > 0)
+        glfd->next = list_entry(glfd->entries.next, gf_dirent_t, list);
 
-	gf_dirent_free (&old);
+    gf_dirent_free(&old);
 out:
-	if (fd)
-		fd_unref (fd);
+    if (fd)
+        fd_unref(fd);
 
-	glfs_subvol_done (glfd->fs, subvol);
+    glfs_subvol_done(glfd->fs, subvol);
 
-	return ret;
+    return ret;
 }
 
-
 gf_dirent_t *
-glfd_entry_next (struct glfs_fd *glfd, int plus)
+glfd_entry_next(struct glfs_fd *glfd, int plus)
 {
-	gf_dirent_t     *entry = NULL;
-	int              ret = -1;
+    gf_dirent_t *entry = NULL;
+    int ret = -1;
 
-	if (!glfd->offset || !glfd->next) {
-		ret = glfd_entry_refresh (glfd, plus);
-		if (ret < 0)
-			return NULL;
-	}
+    if (!glfd->offset || !glfd->next) {
+        ret = glfd_entry_refresh(glfd, plus);
+        if (ret < 0)
+            return NULL;
+    }
 
-	entry = glfd->next;
-	if (!entry)
-		return NULL;
+    entry = glfd->next;
+    if (!entry)
+        return NULL;
 
-	if (&entry->next->list == &glfd->entries)
-		glfd->next = NULL;
-	else
-		glfd->next = entry->next;
+    if (&entry->next->list == &glfd->entries)
+        glfd->next = NULL;
+    else
+        glfd->next = entry->next;
 
-	glfd->offset = entry->d_off;
+    glfd->offset = entry->d_off;
 
-	return entry;
+    return entry;
 }
 
-
-int
-glfs_readdirplus_r (struct glfs_fd *glfd, struct stat *stat, struct dirent *buf,
-		    struct dirent **res)
+struct dirent *
+glfs_readdirbuf_get(struct glfs_fd *glfd)
 {
-	int              ret = 0;
-	gf_dirent_t     *entry = NULL;
-
-	__glfs_entry_fd (glfd);
-
-	errno = 0;
-	entry = glfd_entry_next (glfd, !!stat);
-	if (errno)
-		ret = -1;
+    struct dirent *buf = NULL;
+
+    LOCK(&glfd->fd->lock);
+    {
+        buf = glfd->readdirbuf;
+        if (buf) {
+            memset(buf, 0, READDIRBUF_SIZE);
+            goto unlock;
+        }
 
-	if (res) {
-		if (entry)
-			*res = buf;
-		else
-			*res = NULL;
-	}
+        buf = GF_CALLOC(1, READDIRBUF_SIZE, glfs_mt_readdirbuf_t);
+        if (!buf) {
+            errno = ENOMEM;
+            goto unlock;
+        }
 
-	if (entry) {
-		gf_dirent_to_dirent (entry, buf);
-		if (stat)
-			glfs_iatt_to_stat (glfd->fs, &entry->d_stat, stat);
-	}
+        glfd->readdirbuf = buf;
+    }
+unlock:
+    UNLOCK(&glfd->fd->lock);
 
-	return ret;
+    return buf;
 }
 
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_readdirplus_r, 3.4.0)
 int
-glfs_readdir_r (struct glfs_fd *glfd, struct dirent *buf, struct dirent **res)
+pub_glfs_readdirplus_r(struct glfs_fd *glfd, struct stat *stat,
+                       struct dirent *ext, struct dirent **res)
 {
-	return glfs_readdirplus_r (glfd, 0, buf, res);
-}
+    int ret = 0;
+    gf_dirent_t *entry = NULL;
+    struct dirent *buf = NULL;
 
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
 
-int
-glfs_statvfs (struct glfs *fs, const char *path, struct statvfs *buf)
-{
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	loc_t            loc = {0, };
-	struct iatt      iatt = {0, };
-	int              reval = 0;
+    GF_REF_GET(glfd);
 
-	__glfs_entry_fs (fs);
+    errno = 0;
 
-	subvol = glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
-retry:
-	ret = glfs_resolve (fs, subvol, path, &loc, &iatt, reval);
+    if (ext)
+        buf = ext;
+    else
+        buf = glfs_readdirbuf_get(glfd);
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    if (!buf) {
+        errno = ENOMEM;
+        ret = -1;
+        goto out;
+    }
 
-	if (ret)
-		goto out;
+    entry = glfd_entry_next(glfd, !!stat);
+    if (errno)
+        ret = -1;
 
-	ret = syncop_statfs (subvol, &loc, buf);
+    if (res) {
+        if (entry)
+            *res = buf;
+        else
+            *res = NULL;
+    }
+
+    if (entry) {
+        gf_dirent_to_dirent(entry, buf);
+        if (stat)
+            glfs_iatt_to_stat(glfd->fs, &entry->d_stat, stat);
+    }
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
 out:
-	loc_wipe (&loc);
+    if (glfd)
+        GF_REF_PUT(glfd);
 
-	glfs_subvol_done (fs, subvol);
+    __GLFS_EXIT_FS;
 
-	return ret;
-}
+    return ret;
 
+invalid_fs:
+    return -1;
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_readdir_r, 3.4.0)
 int
-glfs_setattr (struct glfs *fs, const char *path, struct iatt *iatt,
-	      int valid, int follow)
+pub_glfs_readdir_r(struct glfs_fd *glfd, struct dirent *buf,
+                   struct dirent **res)
 {
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	loc_t            loc = {0, };
-	struct iatt      riatt = {0, };
-	int              reval = 0;
+    return pub_glfs_readdirplus_r(glfd, 0, buf, res);
+}
 
-	__glfs_entry_fs (fs);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_readdirplus, 3.5.0)
+struct dirent *
+pub_glfs_readdirplus(struct glfs_fd *glfd, struct stat *stat)
+{
+    struct dirent *res = NULL;
+    int ret = -1;
 
-	subvol = glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+    ret = pub_glfs_readdirplus_r(glfd, stat, NULL, &res);
+    if (ret)
+        return NULL;
+
+    return res;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_readdir, 3.5.0)
+struct dirent *
+pub_glfs_readdir(struct glfs_fd *glfd)
+{
+    return pub_glfs_readdirplus(glfd, NULL);
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_statvfs, 3.4.0)
+int
+pub_glfs_statvfs(struct glfs *fs, const char *path, struct statvfs *buf)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
 retry:
-	if (follow)
-		ret = glfs_resolve (fs, subvol, path, &loc, &riatt, reval);
-	else
-		ret = glfs_lresolve (fs, subvol, path, &loc, &riatt, reval);
+    ret = glfs_resolve(fs, subvol, path, &loc, &iatt, reval);
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	if (ret)
-		goto out;
+    if (ret)
+        goto out;
 
-	ret = syncop_setattr (subvol, &loc, iatt, valid, 0, 0);
+    ret = syncop_statfs(subvol, &loc, buf, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 out:
-	loc_wipe (&loc);
+    loc_wipe(&loc);
 
-	glfs_subvol_done (fs, subvol);
+    glfs_subvol_done(fs, subvol);
 
-	return ret;
-}
+    __GLFS_EXIT_FS;
 
+invalid_fs:
+    return ret;
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_setattr, 6.0)
 int
-glfs_fsetattr (struct glfs_fd *glfd, struct iatt *iatt, int valid)
+pub_glfs_setattr(struct glfs *fs, const char *path, struct glfs_stat *stat,
+                 int follow)
 {
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	fd_t            *fd = NULL;
+    int ret = -1;
+    int glvalid;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt riatt = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    GF_VALIDATE_OR_GOTO("glfs_setattr", stat, out);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+retry:
+    if (follow)
+        ret = glfs_resolve(fs, subvol, path, &loc, &riatt, reval);
+    else
+        ret = glfs_lresolve(fs, subvol, path, &loc, &riatt, reval);
+
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	__glfs_entry_fd (glfd);
+    if (ret)
+        goto out;
 
-	subvol = glfs_active_subvol (glfd->fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+    glfs_iatt_from_statx(&iatt, stat);
+    glfsflags_from_gfapiflags(stat, &glvalid);
 
-	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
-	if (!fd) {
-		ret = -1;
-		errno = EBADFD;
-		goto out;
-	}
+    /* TODO : Add leaseid */
+    ret = syncop_setattr(subvol, &loc, &iatt, glvalid, 0, 0, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
 
-	ret = syncop_fsetattr (subvol, fd, iatt, valid, 0, 0);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 out:
-	if (fd)
-		fd_unref (fd);
+    loc_wipe(&loc);
 
-	glfs_subvol_done (glfd->fs, subvol);
+    glfs_subvol_done(fs, subvol);
 
-	return ret;
-}
+    __GLFS_EXIT_FS;
 
+invalid_fs:
+    return ret;
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fsetattr, 6.0)
 int
-glfs_chmod (struct glfs *fs, const char *path, mode_t mode)
+pub_glfs_fsetattr(struct glfs_fd *glfd, struct glfs_stat *stat)
 {
-	int              ret = -1;
-	struct iatt      iatt = {0, };
-	int              valid = 0;
+    int ret = -1;
+    int glvalid;
+    struct iatt iatt = {
+        0,
+    };
+    xlator_t *subvol = NULL;
+    fd_t *fd = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    GF_REF_GET(glfd);
+
+    GF_VALIDATE_OR_GOTO("glfs_fsetattr", stat, out);
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    glfs_iatt_from_statx(&iatt, stat);
+    glfsflags_from_gfapiflags(stat, &glvalid);
+
+    /* TODO : Add leaseid */
+    ret = syncop_fsetattr(subvol, fd, &iatt, glvalid, 0, 0, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+out:
+    if (fd)
+        fd_unref(fd);
+    if (glfd)
+        GF_REF_PUT(glfd);
 
-	iatt.ia_prot = ia_prot_from_st_mode (mode);
-	valid = GF_SET_ATTR_MODE;
+    glfs_subvol_done(glfd->fs, subvol);
 
-	ret = glfs_setattr (fs, path, &iatt, valid, 1);
+    __GLFS_EXIT_FS;
 
-	return ret;
+invalid_fs:
+    return ret;
 }
 
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_chmod, 3.4.0)
 int
-glfs_fchmod (struct glfs_fd *glfd, mode_t mode)
+pub_glfs_chmod(struct glfs *fs, const char *path, mode_t mode)
 {
-	int              ret = -1;
-	struct iatt      iatt = {0, };
-	int              valid = 0;
+    int ret = -1;
+    struct glfs_stat stat = {
+        0,
+    };
 
-	iatt.ia_prot = ia_prot_from_st_mode (mode);
-	valid = GF_SET_ATTR_MODE;
+    stat.glfs_st_mode = mode;
+    stat.glfs_st_mask = GLFS_STAT_MODE;
 
-	ret = glfs_fsetattr (glfd, &iatt, valid);
+    ret = glfs_setattr(fs, path, &stat, 1);
 
-	return ret;
+    return ret;
 }
 
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fchmod, 3.4.0)
 int
-glfs_chown (struct glfs *fs, const char *path, uid_t uid, gid_t gid)
+pub_glfs_fchmod(struct glfs_fd *glfd, mode_t mode)
 {
-	int              ret = -1;
-	int              valid = 0;
-	struct iatt      iatt = {0, };
+    int ret = -1;
+    struct glfs_stat stat = {
+        0,
+    };
 
-	iatt.ia_uid = uid;
-	iatt.ia_gid = gid;
-	valid = GF_SET_ATTR_UID|GF_SET_ATTR_GID;
+    stat.glfs_st_mode = mode;
+    stat.glfs_st_mask = GLFS_STAT_MODE;
 
-	ret = glfs_setattr (fs, path, &iatt, valid, 1);
+    ret = glfs_fsetattr(glfd, &stat);
 
-	return ret;
+    return ret;
 }
 
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_chown, 3.4.0)
 int
-glfs_lchown (struct glfs *fs, const char *path, uid_t uid, gid_t gid)
+pub_glfs_chown(struct glfs *fs, const char *path, uid_t uid, gid_t gid)
 {
-	int              ret = -1;
-	int              valid = 0;
-	struct iatt      iatt = {0, };
+    int ret = 0;
+    struct glfs_stat stat = {
+        0,
+    };
 
-	iatt.ia_uid = uid;
-	iatt.ia_gid = gid;
-	valid = GF_SET_ATTR_UID|GF_SET_ATTR_GID;
+    if (uid != (uid_t)-1) {
+        stat.glfs_st_uid = uid;
+        stat.glfs_st_mask = GLFS_STAT_UID;
+    }
 
-	ret = glfs_setattr (fs, path, &iatt, valid, 0);
+    if (gid != (uid_t)-1) {
+        stat.glfs_st_gid = gid;
+        stat.glfs_st_mask = stat.glfs_st_mask | GLFS_STAT_GID;
+    }
 
-	return ret;
-}
+    if (stat.glfs_st_mask)
+        ret = glfs_setattr(fs, path, &stat, 1);
 
+    return ret;
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_lchown, 3.4.0)
 int
-glfs_fchown (struct glfs_fd *glfd, uid_t uid, gid_t gid)
+pub_glfs_lchown(struct glfs *fs, const char *path, uid_t uid, gid_t gid)
 {
-	int              ret = -1;
-	int              valid = 0;
-	struct iatt      iatt = {0, };
+    int ret = 0;
+    struct glfs_stat stat = {
+        0,
+    };
 
-	iatt.ia_uid = uid;
-	iatt.ia_gid = gid;
-	valid = GF_SET_ATTR_UID|GF_SET_ATTR_GID;
+    if (uid != (uid_t)-1) {
+        stat.glfs_st_uid = uid;
+        stat.glfs_st_mask = GLFS_STAT_UID;
+    }
 
-	ret = glfs_fsetattr (glfd, &iatt, valid);
+    if (gid != (uid_t)-1) {
+        stat.glfs_st_gid = gid;
+        stat.glfs_st_mask = stat.glfs_st_mask | GLFS_STAT_GID;
+    }
 
-	return ret;
-}
+    if (stat.glfs_st_mask)
+        ret = glfs_setattr(fs, path, &stat, 0);
 
+    return ret;
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fchown, 3.4.0)
 int
-glfs_utimens (struct glfs *fs, const char *path, struct timespec times[2])
+pub_glfs_fchown(struct glfs_fd *glfd, uid_t uid, gid_t gid)
 {
-	int              ret = -1;
-	int              valid = 0;
-	struct iatt      iatt = {0, };
+    int ret = 0;
+    struct glfs_stat stat = {
+        0,
+    };
 
-	iatt.ia_atime = times[0].tv_sec;
-	iatt.ia_atime_nsec = times[0].tv_nsec;
-	iatt.ia_mtime = times[1].tv_sec;
-	iatt.ia_mtime_nsec = times[1].tv_nsec;
+    if (uid != (uid_t)-1) {
+        stat.glfs_st_uid = uid;
+        stat.glfs_st_mask = GLFS_STAT_UID;
+    }
 
-	valid = GF_SET_ATTR_ATIME|GF_SET_ATTR_MTIME;
+    if (gid != (uid_t)-1) {
+        stat.glfs_st_gid = gid;
+        stat.glfs_st_mask = stat.glfs_st_mask | GLFS_STAT_GID;
+    }
 
-	ret = glfs_setattr (fs, path, &iatt, valid, 1);
+    if (stat.glfs_st_mask)
+        ret = glfs_fsetattr(glfd, &stat);
 
-	return ret;
+    return ret;
 }
 
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_utimens, 3.4.0)
 int
-glfs_lutimens (struct glfs *fs, const char *path, struct timespec times[2])
+pub_glfs_utimens(struct glfs *fs, const char *path,
+                 const struct timespec times[2])
 {
-	int              ret = -1;
-	int              valid = 0;
-	struct iatt      iatt = {0, };
+    int ret = -1;
+    struct glfs_stat stat = {
+        0,
+    };
 
-	iatt.ia_atime = times[0].tv_sec;
-	iatt.ia_atime_nsec = times[0].tv_nsec;
-	iatt.ia_mtime = times[1].tv_sec;
-	iatt.ia_mtime_nsec = times[1].tv_nsec;
+    stat.glfs_st_atime = times[0];
+    stat.glfs_st_mtime = times[1];
 
-	valid = GF_SET_ATTR_ATIME|GF_SET_ATTR_MTIME;
+    stat.glfs_st_mask = GLFS_STAT_ATIME | GLFS_STAT_MTIME;
 
-	ret = glfs_setattr (fs, path, &iatt, valid, 0);
+    ret = glfs_setattr(fs, path, &stat, 1);
 
-	return ret;
+    return ret;
 }
 
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_lutimens, 3.4.0)
 int
-glfs_futimens (struct glfs_fd *glfd, struct timespec times[2])
+pub_glfs_lutimens(struct glfs *fs, const char *path,
+                  const struct timespec times[2])
 {
-	int              ret = -1;
-	int              valid = 0;
-	struct iatt      iatt = {0, };
+    int ret = -1;
+    struct glfs_stat stat = {
+        0,
+    };
 
-	iatt.ia_atime = times[0].tv_sec;
-	iatt.ia_atime_nsec = times[0].tv_nsec;
-	iatt.ia_mtime = times[1].tv_sec;
-	iatt.ia_mtime_nsec = times[1].tv_nsec;
+    stat.glfs_st_atime = times[0];
+    stat.glfs_st_mtime = times[1];
 
-	valid = GF_SET_ATTR_ATIME|GF_SET_ATTR_MTIME;
+    stat.glfs_st_mask = GLFS_STAT_ATIME | GLFS_STAT_MTIME;
 
-	ret = glfs_fsetattr (glfd, &iatt, valid);
+    ret = glfs_setattr(fs, path, &stat, 0);
 
-	return ret;
+    return ret;
 }
 
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_futimens, 3.4.0)
 int
-glfs_getxattr_process (void *value, size_t size, dict_t *xattr,
-		       const char *name)
+pub_glfs_futimens(struct glfs_fd *glfd, const struct timespec times[2])
 {
-	data_t *data = NULL;
-	int     ret = -1;
+    int ret = -1;
+    struct glfs_stat stat = {
+        0,
+    };
 
-	data = dict_get (xattr, (char *)name);
-	if (!data) {
-		errno = ENODATA;
-		ret = -1;
-		goto out;
-	}
+    stat.glfs_st_atime = times[0];
+    stat.glfs_st_mtime = times[1];
 
-	ret = data->len;
-	if (!value || !size)
-		goto out;
+    stat.glfs_st_mask = GLFS_STAT_ATIME | GLFS_STAT_MTIME;
 
-	if (size < ret) {
-		ret = -1;
-		errno = ERANGE;
-		goto out;
-	}
+    ret = glfs_fsetattr(glfd, &stat);
 
-	memcpy (value, data->data, ret);
-out:
-	if (xattr)
-		dict_unref (xattr);
-	return ret;
+    return ret;
 }
 
+int
+glfs_getxattr_process(void *value, size_t size, dict_t *xattr, const char *name)
+{
+    data_t *data = NULL;
+    int ret = -1;
+
+    data = dict_get(xattr, (char *)name);
+    if (!data) {
+        errno = ENODATA;
+        ret = -1;
+        goto out;
+    }
+
+    ret = data->len;
+    if (!value || !size)
+        goto out;
+
+    if (size < ret) {
+        ret = -1;
+        errno = ERANGE;
+        goto out;
+    }
+
+    memcpy(value, data->data, ret);
+out:
+    return ret;
+}
 
 ssize_t
-glfs_getxattr_common (struct glfs *fs, const char *path, const char *name,
-		      void *value, size_t size, int follow)
-{
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	loc_t            loc = {0, };
-	struct iatt      iatt = {0, };
-	dict_t          *xattr = NULL;
-	int              reval = 0;
-
-	__glfs_entry_fs (fs);
-
-	subvol = glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+glfs_getxattr_common(struct glfs *fs, const char *path, const char *name,
+                     void *value, size_t size, int follow)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    dict_t *xattr = NULL;
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    if (!name || *name == '\0') {
+        ret = -1;
+        errno = EINVAL;
+        goto out;
+    }
+
+    if (strlen(name) > GF_XATTR_NAME_MAX) {
+        ret = -1;
+        errno = ENAMETOOLONG;
+        goto out;
+    }
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
 retry:
-	if (follow)
-		ret = glfs_resolve (fs, subvol, path, &loc, &iatt, reval);
-	else
-		ret = glfs_lresolve (fs, subvol, path, &loc, &iatt, reval);
+    if (follow)
+        ret = glfs_resolve(fs, subvol, path, &loc, &iatt, reval);
+    else
+        ret = glfs_lresolve(fs, subvol, path, &loc, &iatt, reval);
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	if (ret)
-		goto out;
+    if (ret)
+        goto out;
 
-	ret = syncop_getxattr (subvol, &loc, &xattr, name);
+    ret = syncop_getxattr(subvol, &loc, &xattr, name, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	if (ret)
-		goto out;
+    if (ret)
+        goto out;
 
-	ret = glfs_getxattr_process (value, size, xattr, name);
+    ret = glfs_getxattr_process(value, size, xattr, name);
 out:
-	loc_wipe (&loc);
+    loc_wipe(&loc);
 
-	glfs_subvol_done (fs, subvol);
+    if (xattr)
+        dict_unref(xattr);
 
-	return ret;
-}
+    glfs_subvol_done(fs, subvol);
 
+    __GLFS_EXIT_FS;
 
+invalid_fs:
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_getxattr, 3.4.0)
 ssize_t
-glfs_getxattr (struct glfs *fs, const char *path, const char *name,
-	       void *value, size_t size)
+pub_glfs_getxattr(struct glfs *fs, const char *path, const char *name,
+                  void *value, size_t size)
 {
-	return glfs_getxattr_common (fs, path, name, value, size, 1);
+    return glfs_getxattr_common(fs, path, name, value, size, 1);
 }
 
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_lgetxattr, 3.4.0)
 ssize_t
-glfs_lgetxattr (struct glfs *fs, const char *path, const char *name,
-		void *value, size_t size)
+pub_glfs_lgetxattr(struct glfs *fs, const char *path, const char *name,
+                   void *value, size_t size)
 {
-	return glfs_getxattr_common (fs, path, name, value, size, 0);
+    return glfs_getxattr_common(fs, path, name, value, size, 0);
 }
 
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fgetxattr, 3.4.0)
 ssize_t
-glfs_fgetxattr (struct glfs_fd *glfd, const char *name, void *value,
-		size_t size)
+pub_glfs_fgetxattr(struct glfs_fd *glfd, const char *name, void *value,
+                   size_t size)
 {
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	dict_t          *xattr = NULL;
-	fd_t            *fd = NULL;
-
-	__glfs_entry_fd (glfd);
-
-	subvol = glfs_active_subvol (glfd->fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
-
-	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
-	if (!fd) {
-		ret = -1;
-		errno = EBADFD;
-		goto out;
-	}
-
-	ret = syncop_fgetxattr (subvol, fd, &xattr, name);
-	if (ret)
-		goto out;
-
-	ret = glfs_getxattr_process (value, size, xattr, name);
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    dict_t *xattr = NULL;
+    fd_t *fd = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    GF_REF_GET(glfd);
+
+    if (!name || *name == '\0') {
+        ret = -1;
+        errno = EINVAL;
+        goto out;
+    }
+
+    if (strlen(name) > GF_XATTR_NAME_MAX) {
+        ret = -1;
+        errno = ENAMETOOLONG;
+        goto out;
+    }
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    ret = syncop_fgetxattr(subvol, fd, &xattr, name, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+    if (ret)
+        goto out;
+
+    ret = glfs_getxattr_process(value, size, xattr, name);
 out:
-	if (fd)
-		fd_unref (fd);
+    if (fd)
+        fd_unref(fd);
+    if (glfd)
+        GF_REF_PUT(glfd);
+    if (xattr)
+        dict_unref(xattr);
 
-	glfs_subvol_done (glfd->fs, subvol);
+    glfs_subvol_done(glfd->fs, subvol);
 
-	return ret;
-}
+    __GLFS_EXIT_FS;
 
+invalid_fs:
+    return ret;
+}
 
 int
-glfs_listxattr_process (void *value, size_t size, dict_t *xattr)
+glfs_listxattr_process(void *value, size_t size, dict_t *xattr)
 {
-	int     ret = -1;
+    int ret = -1;
 
-	ret = dict_keys_join (NULL, 0, xattr, NULL);
+    if (!xattr)
+        goto out;
 
-	if (!value || !size)
-		goto out;
+    ret = dict_keys_join(NULL, 0, xattr, NULL);
 
-	if (size < ret) {
-		ret = -1;
-		errno = ERANGE;
-		goto out;
-	}
+    if (!value || !size)
+        goto out;
+
+    if (size < ret) {
+        ret = -1;
+        errno = ERANGE;
+    } else {
+        dict_keys_join(value, size, xattr, NULL);
+    }
 
-	dict_keys_join (value, size, xattr, NULL);
 out:
-	if (xattr)
-		dict_unref (xattr);
-	return ret;
+    return ret;
 }
 
-
 ssize_t
-glfs_listxattr_common (struct glfs *fs, const char *path, void *value,
-		       size_t size, int follow)
+glfs_listxattr_common(struct glfs *fs, const char *path, void *value,
+                      size_t size, int follow)
 {
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	loc_t            loc = {0, };
-	struct iatt      iatt = {0, };
-	dict_t          *xattr = NULL;
-	int              reval = 0;
-
-	__glfs_entry_fs (fs);
-
-	subvol = glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    dict_t *xattr = NULL;
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
 
 retry:
-	if (follow)
-		ret = glfs_resolve (fs, subvol, path, &loc, &iatt, reval);
-	else
-		ret = glfs_lresolve (fs, subvol, path, &loc, &iatt, reval);
+    if (follow)
+        ret = glfs_resolve(fs, subvol, path, &loc, &iatt, reval);
+    else
+        ret = glfs_lresolve(fs, subvol, path, &loc, &iatt, reval);
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	if (ret)
-		goto out;
+    if (ret)
+        goto out;
 
-	ret = syncop_getxattr (subvol, &loc, &xattr, NULL);
+    ret = syncop_getxattr(subvol, &loc, &xattr, NULL, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	if (ret)
-		goto out;
+    if (ret)
+        goto out;
 
-	ret = glfs_listxattr_process (value, size, xattr);
+    ret = glfs_listxattr_process(value, size, xattr);
 out:
-	loc_wipe (&loc);
+    loc_wipe(&loc);
 
-	glfs_subvol_done (fs, subvol);
+    if (xattr)
+        dict_unref(xattr);
 
-	return ret;
-}
+    glfs_subvol_done(fs, subvol);
 
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_listxattr, 3.4.0)
 ssize_t
-glfs_listxattr (struct glfs *fs, const char *path, void *value, size_t size)
+pub_glfs_listxattr(struct glfs *fs, const char *path, void *value, size_t size)
 {
-	return glfs_listxattr_common (fs, path, value, size, 1);
+    return glfs_listxattr_common(fs, path, value, size, 1);
 }
 
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_llistxattr, 3.4.0)
 ssize_t
-glfs_llistxattr (struct glfs *fs, const char *path, void *value, size_t size)
+pub_glfs_llistxattr(struct glfs *fs, const char *path, void *value, size_t size)
 {
-	return glfs_listxattr_common (fs, path, value, size, 0);
+    return glfs_listxattr_common(fs, path, value, size, 0);
 }
 
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_flistxattr, 3.4.0)
 ssize_t
-glfs_flistxattr (struct glfs_fd *glfd, void *value, size_t size)
+pub_glfs_flistxattr(struct glfs_fd *glfd, void *value, size_t size)
 {
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	dict_t          *xattr = NULL;
-	fd_t            *fd = NULL;
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    dict_t *xattr = NULL;
+    fd_t *fd = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    GF_REF_GET(glfd);
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    ret = syncop_fgetxattr(subvol, fd, &xattr, NULL, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+    if (ret)
+        goto out;
+
+    ret = glfs_listxattr_process(value, size, xattr);
+out:
+    if (fd)
+        fd_unref(fd);
+    if (glfd)
+        GF_REF_PUT(glfd);
+    if (xattr)
+        dict_unref(xattr);
 
-	__glfs_entry_fd (glfd);
+    glfs_subvol_done(glfd->fs, subvol);
 
-	subvol = glfs_active_subvol (glfd->fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+    __GLFS_EXIT_FS;
 
-	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
-	if (!fd) {
-		ret = -1;
-		errno = EBADFD;
-		goto out;
-	}
+invalid_fs:
+    return ret;
+}
 
-	ret = syncop_fgetxattr (subvol, fd, &xattr, NULL);
-	if (ret)
-		goto out;
+int
+glfs_setxattr_common(struct glfs *fs, const char *path, const char *name,
+                     const void *value, size_t size, int flags, int follow)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    dict_t *xattr = NULL;
+    int reval = 0;
+    void *value_cp = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    if (!name || *name == '\0') {
+        ret = -1;
+        errno = EINVAL;
+        goto out;
+    }
+
+    if (strlen(name) > GF_XATTR_NAME_MAX) {
+        ret = -1;
+        errno = ENAMETOOLONG;
+        goto out;
+    }
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
 
-	ret = glfs_listxattr_process (value, size, xattr);
-out:
-	if (fd)
-		fd_unref (fd);
+retry:
+    if (follow)
+        ret = glfs_resolve(fs, subvol, path, &loc, &iatt, reval);
+    else
+        ret = glfs_lresolve(fs, subvol, path, &loc, &iatt, reval);
 
-	glfs_subvol_done (glfd->fs, subvol);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	return ret;
-}
+    if (ret)
+        goto out;
 
+    value_cp = gf_memdup(value, size);
+    GF_CHECK_ALLOC_AND_LOG(subvol->name, value_cp, ret,
+                           "Failed to"
+                           " duplicate setxattr value",
+                           out);
 
-dict_t *
-dict_for_key_value (const char *name, const char *value, size_t size)
-{
-	dict_t *xattr = NULL;
-	int     ret = 0;
+    xattr = dict_for_key_value(name, value_cp, size, _gf_false);
+    if (!xattr) {
+        GF_FREE(value_cp);
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
 
-	xattr = dict_new ();
-	if (!xattr)
-		return NULL;
+    ret = syncop_setxattr(subvol, &loc, xattr, flags, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
 
-	ret = dict_set_static_bin (xattr, (char *)name, (void *)value, size);
-	if (ret) {
-		dict_destroy (xattr);
-		xattr = NULL;
-	}
+out:
+    loc_wipe(&loc);
+    if (xattr)
+        dict_unref(xattr);
+
+    glfs_subvol_done(fs, subvol);
+
+    __GLFS_EXIT_FS;
 
-	return xattr;
+invalid_fs:
+    return ret;
 }
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_setxattr, 3.4.0)
+int
+pub_glfs_setxattr(struct glfs *fs, const char *path, const char *name,
+                  const void *value, size_t size, int flags)
+{
+    return glfs_setxattr_common(fs, path, name, value, size, flags, 1);
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_lsetxattr, 3.4.0)
 int
-glfs_setxattr_common (struct glfs *fs, const char *path, const char *name,
-		      const void *value, size_t size, int flags, int follow)
+pub_glfs_lsetxattr(struct glfs *fs, const char *path, const char *name,
+                   const void *value, size_t size, int flags)
 {
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	loc_t            loc = {0, };
-	struct iatt      iatt = {0, };
-	dict_t          *xattr = NULL;
-	int              reval = 0;
+    return glfs_setxattr_common(fs, path, name, value, size, flags, 0);
+}
 
-	__glfs_entry_fs (fs);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fsetxattr, 3.4.0)
+int
+pub_glfs_fsetxattr(struct glfs_fd *glfd, const char *name, const void *value,
+                   size_t size, int flags)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    dict_t *xattr = NULL;
+    fd_t *fd = NULL;
+    void *value_cp = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    GF_REF_GET(glfd);
+
+    if (!name || *name == '\0') {
+        ret = -1;
+        errno = EINVAL;
+        goto out;
+    }
+
+    if (strlen(name) > GF_XATTR_NAME_MAX) {
+        ret = -1;
+        errno = ENAMETOOLONG;
+        goto out;
+    }
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    value_cp = gf_memdup(value, size);
+    GF_CHECK_ALLOC_AND_LOG(subvol->name, value_cp, ret,
+                           "Failed to"
+                           " duplicate setxattr value",
+                           out);
+
+    xattr = dict_for_key_value(name, value_cp, size, _gf_false);
+    if (!xattr) {
+        GF_FREE(value_cp);
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    ret = syncop_fsetxattr(subvol, fd, xattr, flags, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+out:
+    if (xattr)
+        dict_unref(xattr);
 
-	subvol = glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
-retry:
-	if (follow)
-		ret = glfs_resolve (fs, subvol, path, &loc, &iatt, reval);
-	else
-		ret = glfs_lresolve (fs, subvol, path, &loc, &iatt, reval);
+    if (fd)
+        fd_unref(fd);
+    if (glfd)
+        GF_REF_PUT(glfd);
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    glfs_subvol_done(glfd->fs, subvol);
 
-	if (ret)
-		goto out;
+    __GLFS_EXIT_FS;
 
-	xattr = dict_for_key_value (name, value, size);
-	if (!xattr) {
-		ret = -1;
-		errno = ENOMEM;
-		goto out;
-	}
+invalid_fs:
+    return ret;
+}
+
+int
+glfs_removexattr_common(struct glfs *fs, const char *path, const char *name,
+                        int follow)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+retry:
+    if (follow)
+        ret = glfs_resolve(fs, subvol, path, &loc, &iatt, reval);
+    else
+        ret = glfs_lresolve(fs, subvol, path, &loc, &iatt, reval);
 
-	ret = syncop_setxattr (subvol, &loc, xattr, flags);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    if (ret)
+        goto out;
+
+    ret = syncop_removexattr(subvol, &loc, name, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
 out:
-	loc_wipe (&loc);
-	if (xattr)
-		dict_unref (xattr);
+    loc_wipe(&loc);
 
-	glfs_subvol_done (fs, subvol);
+    glfs_subvol_done(fs, subvol);
 
-	return ret;
-}
+    __GLFS_EXIT_FS;
 
+invalid_fs:
+    return ret;
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_removexattr, 3.4.0)
 int
-glfs_setxattr (struct glfs *fs, const char *path, const char *name,
-	       const void *value, size_t size, int flags)
+pub_glfs_removexattr(struct glfs *fs, const char *path, const char *name)
 {
-	return glfs_setxattr_common (fs, path, name, value, size, flags, 1);
+    return glfs_removexattr_common(fs, path, name, 1);
 }
 
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_lremovexattr, 3.4.0)
 int
-glfs_lsetxattr (struct glfs *fs, const char *path, const char *name,
-		const void *value, size_t size, int flags)
+pub_glfs_lremovexattr(struct glfs *fs, const char *path, const char *name)
 {
-	return glfs_setxattr_common (fs, path, name, value, size, flags, 0);
+    return glfs_removexattr_common(fs, path, name, 0);
 }
 
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fremovexattr, 3.4.0)
 int
-glfs_fsetxattr (struct glfs_fd *glfd, const char *name, const void *value,
-		size_t size, int flags)
+pub_glfs_fremovexattr(struct glfs_fd *glfd, const char *name)
 {
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	dict_t          *xattr = NULL;
-	fd_t            *fd = NULL;
-
-	__glfs_entry_fd (glfd);
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    fd_t *fd = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    GF_REF_GET(glfd);
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    ret = syncop_fremovexattr(subvol, fd, name, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+out:
+    if (fd)
+        fd_unref(fd);
+    if (glfd)
+        GF_REF_PUT(glfd);
 
-	subvol = glfs_active_subvol (glfd->fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+    glfs_subvol_done(glfd->fs, subvol);
 
-	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
-	if (!fd) {
-		ret = -1;
-		errno = EBADFD;
-		goto out;
-	}
+    __GLFS_EXIT_FS;
 
-	xattr = dict_for_key_value (name, value, size);
-	if (!xattr) {
-		ret = -1;
-		errno = ENOMEM;
-		goto out;
-	}
+invalid_fs:
+    return ret;
+}
 
-	ret = syncop_fsetxattr (subvol, fd, xattr, flags);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fallocate, 3.5.0)
+int
+pub_glfs_fallocate(struct glfs_fd *glfd, int keep_size, off_t offset,
+                   size_t len)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    fd_t *fd = NULL;
+    dict_t *fop_attr = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    GF_REF_GET(glfd);
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    ret = get_fop_attr_thrd_key(&fop_attr);
+    if (ret)
+        gf_msg_debug("gfapi", 0, "Getting leaseid from thread failed");
+
+    ret = syncop_fallocate(subvol, fd, keep_size, offset, len, fop_attr, NULL);
+    DECODE_SYNCOP_ERR(ret);
 out:
-	if (xattr)
-		dict_unref (xattr);
+    if (fd)
+        fd_unref(fd);
+    if (glfd)
+        GF_REF_PUT(glfd);
+    if (fop_attr)
+        dict_unref(fop_attr);
 
-	if (fd)
-		fd_unref (fd);
+    glfs_subvol_done(glfd->fs, subvol);
 
-	glfs_subvol_done (glfd->fs, subvol);
+    __GLFS_EXIT_FS;
 
-	return ret;
+invalid_fs:
+    return ret;
 }
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_discard, 3.5.0)
+int
+pub_glfs_discard(struct glfs_fd *glfd, off_t offset, size_t len)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    fd_t *fd = NULL;
+    dict_t *fop_attr = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    GF_REF_GET(glfd);
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    ret = get_fop_attr_thrd_key(&fop_attr);
+    if (ret)
+        gf_msg_debug("gfapi", 0, "Getting leaseid from thread failed");
+
+    ret = syncop_discard(subvol, fd, offset, len, fop_attr, NULL);
+    DECODE_SYNCOP_ERR(ret);
+out:
+    if (fd)
+        fd_unref(fd);
+    if (glfd)
+        GF_REF_PUT(glfd);
+    if (fop_attr)
+        dict_unref(fop_attr);
+
+    glfs_subvol_done(glfd->fs, subvol);
 
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_zerofill, 3.5.0)
 int
-glfs_removexattr_common (struct glfs *fs, const char *path, const char *name,
-			 int follow)
+pub_glfs_zerofill(struct glfs_fd *glfd, off_t offset, off_t len)
 {
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	loc_t            loc = {0, };
-	struct iatt      iatt = {0, };
-	int              reval = 0;
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    fd_t *fd = NULL;
+    dict_t *fop_attr = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    GF_REF_GET(glfd);
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        errno = EBADFD;
+        goto out;
+    }
+
+    ret = get_fop_attr_thrd_key(&fop_attr);
+    if (ret)
+        gf_msg_debug("gfapi", 0, "Getting leaseid from thread failed");
+
+    ret = syncop_zerofill(subvol, fd, offset, len, fop_attr, NULL);
+    DECODE_SYNCOP_ERR(ret);
+out:
+    if (fd)
+        fd_unref(fd);
+    if (glfd)
+        GF_REF_PUT(glfd);
+    if (fop_attr)
+        dict_unref(fop_attr);
+
+    glfs_subvol_done(glfd->fs, subvol);
 
-	__glfs_entry_fs (fs);
+    __GLFS_EXIT_FS;
 
-	subvol = glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+invalid_fs:
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_chdir, 3.4.0)
+int
+pub_glfs_chdir(struct glfs *fs, const char *path)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
 retry:
-	if (follow)
-		ret = glfs_resolve (fs, subvol, path, &loc, &iatt, reval);
-	else
-		ret = glfs_lresolve (fs, subvol, path, &loc, &iatt, reval);
+    ret = glfs_resolve(fs, subvol, path, &loc, &iatt, reval);
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	if (ret)
-		goto out;
+    if (ret)
+        goto out;
 
-	ret = syncop_removexattr (subvol, &loc, name);
+    if (!IA_ISDIR(iatt.ia_type)) {
+        ret = -1;
+        errno = ENOTDIR;
+        goto out;
+    }
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    glfs_cwd_set(fs, loc.inode);
 
 out:
-	loc_wipe (&loc);
+    loc_wipe(&loc);
 
-	glfs_subvol_done (fs, subvol);
+    glfs_subvol_done(fs, subvol);
 
-	return ret;
-}
+    __GLFS_EXIT_FS;
 
+invalid_fs:
+    return ret;
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fchdir, 3.4.0)
 int
-glfs_removexattr (struct glfs *fs, const char *path, const char *name)
+pub_glfs_fchdir(struct glfs_fd *glfd)
 {
-	return glfs_removexattr_common (fs, path, name, 1);
-}
+    int ret = -1;
+    inode_t *inode = NULL;
+    xlator_t *subvol = NULL;
+    fd_t *fd = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    GF_REF_GET(glfd);
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    inode = fd->inode;
+
+    if (!IA_ISDIR(inode->ia_type)) {
+        ret = -1;
+        errno = ENOTDIR;
+        goto out;
+    }
+
+    glfs_cwd_set(glfd->fs, inode);
+    ret = 0;
+out:
+    if (fd)
+        fd_unref(fd);
+    if (glfd)
+        GF_REF_PUT(glfd);
 
+    glfs_subvol_done(glfd->fs, subvol);
 
-int
-glfs_lremovexattr (struct glfs *fs, const char *path, const char *name)
-{
-	return glfs_removexattr_common (fs, path, name, 0);
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
 }
 
+static gf_boolean_t warn_realpath = _gf_true; /* log once */
 
-int
-glfs_fremovexattr (struct glfs_fd *glfd, const char *name)
+static char *
+glfs_realpath_common(struct glfs *fs, const char *path, char *resolved_path,
+                     gf_boolean_t warn_deprecated)
 {
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	fd_t            *fd = NULL;
+    int ret = -1;
+    char *retpath = NULL;
+    char *allocpath = NULL;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    if (resolved_path)
+        retpath = resolved_path;
+    else if (warn_deprecated) {
+        retpath = allocpath = malloc(PATH_MAX + 1);
+        if (warn_realpath) {
+            warn_realpath = _gf_false;
+            gf_log(THIS->name, GF_LOG_WARNING,
+                   "this application "
+                   "is compiled against an old version of "
+                   "libgfapi, it should use glfs_free() to "
+                   "release the path returned by "
+                   "glfs_realpath()");
+        }
+    } else {
+        retpath = allocpath = GLFS_CALLOC(1, PATH_MAX + 1, NULL,
+                                          glfs_mt_realpath_t);
+    }
+
+    if (!retpath) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+retry:
+    ret = glfs_resolve(fs, subvol, path, &loc, &iatt, reval);
 
-	__glfs_entry_fd (glfd);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	subvol = glfs_active_subvol (glfd->fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+    if (ret)
+        goto out;
 
-	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
-	if (!fd) {
-		ret = -1;
-		errno = EBADFD;
-		goto out;
-	}
+    if (loc.path) {
+        snprintf(retpath, PATH_MAX + 1, "%s", loc.path);
+    }
 
-	ret = syncop_fremovexattr (subvol, fd, name);
 out:
-	if (fd)
-		fd_unref (fd);
+    loc_wipe(&loc);
+
+    if (ret == -1) {
+        if (warn_deprecated && allocpath)
+            free(allocpath);
+        else if (allocpath)
+            GLFS_FREE(allocpath);
+        retpath = NULL;
+    }
+
+    glfs_subvol_done(fs, subvol);
 
-	glfs_subvol_done (glfd->fs, subvol);
+    __GLFS_EXIT_FS;
 
-	return ret;
+invalid_fs:
+    return retpath;
 }
 
+GFAPI_SYMVER_PUBLIC(glfs_realpath34, glfs_realpath, 3.4.0)
+char *
+pub_glfs_realpath34(struct glfs *fs, const char *path, char *resolved_path)
+{
+    return glfs_realpath_common(fs, path, resolved_path, _gf_true);
+}
 
-int
-glfs_fallocate (struct glfs_fd *glfd, int keep_size, off_t offset, size_t len)
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_realpath, 3.7.17)
+char *
+pub_glfs_realpath(struct glfs *fs, const char *path, char *resolved_path)
 {
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	fd_t		*fd = NULL;
+    return glfs_realpath_common(fs, path, resolved_path, _gf_false);
+}
 
-	__glfs_entry_fd (glfd);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_getcwd, 3.4.0)
+char *
+pub_glfs_getcwd(struct glfs *fs, char *buf, size_t n)
+{
+    int ret = -1;
+    inode_t *inode = NULL;
+    char *path = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    if (!buf || n < 2) {
+        ret = -1;
+        errno = EINVAL;
+        goto out;
+    }
+
+    inode = glfs_cwd_get(fs);
+
+    if (!inode) {
+        strncpy(buf, "/", n);
+        ret = 0;
+        goto out;
+    }
+
+    ret = inode_path(inode, 0, &path);
+    if (n <= ret) {
+        ret = -1;
+        errno = ERANGE;
+        goto out;
+    }
+
+    strncpy(buf, path, n);
+    ret = 0;
+out:
+    GF_FREE(path);
 
-	subvol = glfs_active_subvol (glfd->fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+    if (inode)
+        inode_unref(inode);
 
-	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
-	if (!fd) {
-		ret = -1;
-		errno = EBADFD;
-		goto out;
-	}
+    __GLFS_EXIT_FS;
 
-	ret = syncop_fallocate (subvol, fd, keep_size, offset, len);
-out:
-	if (fd)
-		fd_unref(fd);
+invalid_fs:
+    if (ret < 0)
+        return NULL;
 
-	glfs_subvol_done (glfd->fs, subvol);
+    return buf;
+}
 
-	return ret;
+static void
+gf_flock_to_flock(struct gf_flock *gf_flock, struct flock *flock)
+{
+    flock->l_type = gf_flock->l_type;
+    flock->l_whence = gf_flock->l_whence;
+    flock->l_start = gf_flock->l_start;
+    flock->l_len = gf_flock->l_len;
+    flock->l_pid = gf_flock->l_pid;
 }
 
+static void
+gf_flock_from_flock(struct gf_flock *gf_flock, struct flock *flock)
+{
+    gf_flock->l_type = flock->l_type;
+    gf_flock->l_whence = flock->l_whence;
+    gf_flock->l_start = flock->l_start;
+    gf_flock->l_len = flock->l_len;
+    gf_flock->l_pid = flock->l_pid;
+}
 
-int
-glfs_discard (struct glfs_fd *glfd, off_t offset, size_t len)
+static int
+glfs_lock_common(struct glfs_fd *glfd, int cmd, struct flock *flock,
+                 dict_t *xdata)
 {
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	fd_t		*fd = NULL;
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    struct gf_flock gf_flock = {
+        0,
+    };
+    struct gf_flock saved_flock = {
+        0,
+    };
+    fd_t *fd = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    if (!flock) {
+        errno = EINVAL;
+        goto out;
+    }
+
+    GF_REF_GET(glfd);
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    /* Generate glusterfs flock structure from client flock
+     * structure to be processed by server */
+    gf_flock_from_flock(&gf_flock, flock);
+
+    /* Keep another copy of flock for split/merge of locks
+     * at client side */
+    gf_flock_from_flock(&saved_flock, flock);
+
+    if (glfd->lk_owner.len != 0) {
+        ret = syncopctx_setfslkowner(&glfd->lk_owner);
+
+        if (ret)
+            goto out;
+    }
+
+    ret = get_fop_attr_thrd_key(&xdata);
+    if (ret)
+        gf_msg_debug("gfapi", 0, "Getting leaseid from thread failed");
+
+    ret = syncop_lk(subvol, fd, cmd, &gf_flock, xdata, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    /* Convert back from gf_flock to flock as expected by application */
+    gf_flock_to_flock(&gf_flock, flock);
+
+    if (ret == 0 && (cmd == F_SETLK || cmd == F_SETLKW)) {
+        ret = fd_lk_insert_and_merge(fd, cmd, &saved_flock);
+        if (ret) {
+            gf_smsg(THIS->name, GF_LOG_ERROR, 0,
+                    API_MSG_LOCK_INSERT_MERGE_FAILED, "gfid=%s",
+                    uuid_utoa(fd->inode->gfid), NULL);
+            ret = 0;
+        }
+    }
+
+out:
+    if (fd)
+        fd_unref(fd);
+    if (glfd)
+        GF_REF_PUT(glfd);
 
-	__glfs_entry_fd (glfd);
+    glfs_subvol_done(glfd->fs, subvol);
 
-	subvol = glfs_active_subvol (glfd->fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+    __GLFS_EXIT_FS;
 
-	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
-	if (!fd) {
-		ret = -1;
-		errno = EBADFD;
-		goto out;
-	}
+invalid_fs:
+    return ret;
+}
 
-	ret = syncop_discard (subvol, fd, offset, len);
-out:
-	if (fd)
-		fd_unref(fd);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_file_lock, 4.0.0)
+int
+pub_glfs_file_lock(struct glfs_fd *glfd, int cmd, struct flock *flock,
+                   glfs_lock_mode_t lk_mode)
+{
+    int ret = -1;
+    dict_t *xdata_in = NULL;
+
+    if (lk_mode == GLFS_LK_MANDATORY) {
+        /* Create a new dictionary */
+        xdata_in = dict_new();
+        if (xdata_in == NULL) {
+            ret = -1;
+            errno = ENOMEM;
+            goto out;
+        }
 
-	glfs_subvol_done (glfd->fs, subvol);
+        /* Set GF_LK_MANDATORY internally within dictionary to map
+         * GLFS_LK_MANDATORY */
+        ret = dict_set_uint32(xdata_in, GF_LOCK_MODE, GF_LK_MANDATORY);
+        if (ret) {
+            gf_smsg(THIS->name, GF_LOG_ERROR, 0,
+                    API_MSG_SETTING_LOCK_TYPE_FAILED, NULL);
+            ret = -1;
+            errno = ENOMEM;
+            goto out;
+        }
+    }
 
-	return ret;
+    ret = glfs_lock_common(glfd, cmd, flock, xdata_in);
+out:
+    if (xdata_in)
+        dict_unref(xdata_in);
+
+    return ret;
 }
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_posix_lock, 3.4.0)
+int
+pub_glfs_posix_lock(struct glfs_fd *glfd, int cmd, struct flock *flock)
+{
+    return glfs_lock_common(glfd, cmd, flock, NULL);
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fd_set_lkowner, 3.10.7)
 int
-glfs_chdir (struct glfs *fs, const char *path)
+pub_glfs_fd_set_lkowner(struct glfs_fd *glfd, void *data, int len)
 {
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	loc_t            loc = {0, };
-	struct iatt      iatt = {0, };
-	int              reval = 0;
+    int ret = -1;
 
-	__glfs_entry_fs (fs);
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
 
-	subvol = glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
-retry:
-	ret = glfs_resolve (fs, subvol, path, &loc, &iatt, reval);
+    if (!GF_REF_GET(glfd)) {
+        goto invalid_fs;
+    }
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    GF_VALIDATE_OR_GOTO(THIS->name, data, out);
 
-	if (ret)
-		goto out;
+    if ((len <= 0) || (len > GFAPI_MAX_LOCK_OWNER_LEN)) {
+        errno = EINVAL;
+        gf_smsg(THIS->name, GF_LOG_ERROR, errno, API_MSG_INVALID_ARG,
+                "lk_owner len=%d", len, NULL);
+        goto out;
+    }
 
-	if (!IA_ISDIR (iatt.ia_type)) {
-		ret = -1;
-		errno = ENOTDIR;
-		goto out;
-	}
+    glfd->lk_owner.len = len;
 
-	glfs_cwd_set (fs, loc.inode);
+    memcpy(glfd->lk_owner.data, data, len);
 
+    ret = 0;
 out:
-	loc_wipe (&loc);
+    if (glfd)
+        GF_REF_PUT(glfd);
 
-	glfs_subvol_done (fs, subvol);
+    __GLFS_EXIT_FS;
 
-	return ret;
+invalid_fs:
+    return ret;
 }
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_dup, 3.4.0)
+struct glfs_fd *
+pub_glfs_dup(struct glfs_fd *glfd)
+{
+    xlator_t *subvol = NULL;
+    fd_t *fd = NULL;
+    struct glfs_fd *dupfd = NULL;
+    struct glfs *fs = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    GF_REF_GET(glfd);
+
+    fs = glfd->fs;
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(fs, subvol, glfd);
+    if (!fd) {
+        errno = EBADFD;
+        goto out;
+    }
+
+    dupfd = glfs_fd_new(fs);
+    if (!dupfd) {
+        errno = ENOMEM;
+        goto out;
+    }
+
+    dupfd->fd = fd_ref(fd);
+    dupfd->state = glfd->state;
+out:
+    if (fd)
+        fd_unref(fd);
+    if (dupfd)
+        glfs_fd_bind(dupfd);
+    if (glfd)
+        GF_REF_PUT(glfd);
 
-int
-glfs_fchdir (struct glfs_fd *glfd)
+    glfs_subvol_done(fs, subvol);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return dupfd;
+}
+
+static void
+glfs_enqueue_upcall_data(struct glfs *fs, struct gf_upcall *upcall_data)
 {
-	int       ret = -1;
-	inode_t  *inode = NULL;
-	xlator_t *subvol = NULL;
-	fd_t     *fd = NULL;
+    int ret = -1;
+    upcall_entry *u_list = NULL;
+
+    if (!fs || !upcall_data)
+        goto out;
+
+    u_list = GF_CALLOC(1, sizeof(*u_list), glfs_mt_upcall_entry_t);
+
+    if (!u_list) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, ENOMEM, API_MSG_ALLOC_FAILED, "entry",
+                NULL);
+        goto out;
+    }
+
+    INIT_LIST_HEAD(&u_list->upcall_list);
+
+    gf_uuid_copy(u_list->upcall_data.gfid, upcall_data->gfid);
+    u_list->upcall_data.event_type = upcall_data->event_type;
+
+    switch (upcall_data->event_type) {
+        case GF_UPCALL_CACHE_INVALIDATION:
+            ret = glfs_get_upcall_cache_invalidation(&u_list->upcall_data,
+                                                     upcall_data);
+            break;
+        case GF_UPCALL_RECALL_LEASE:
+            ret = glfs_get_upcall_lease(&u_list->upcall_data, upcall_data);
+            break;
+        default:
+            break;
+    }
+
+    if (ret) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, errno, API_MSG_INVALID_ENTRY, NULL);
+        goto out;
+    }
+
+    pthread_mutex_lock(&fs->upcall_list_mutex);
+    {
+        list_add_tail(&u_list->upcall_list, &fs->upcall_list);
+    }
+    pthread_mutex_unlock(&fs->upcall_list_mutex);
+
+    ret = 0;
 
-	__glfs_entry_fd (glfd);
+out:
+    if (ret && u_list) {
+        GF_FREE(u_list->upcall_data.data);
+        GF_FREE(u_list);
+    }
+}
 
-	subvol = glfs_active_subvol (glfd->fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+static void
+glfs_free_upcall_lease(void *to_free)
+{
+    struct glfs_upcall_lease *arg = to_free;
 
-	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
-	if (!fd) {
-		ret = -1;
-		errno = EBADFD;
-		goto out;
-	}
+    if (!arg)
+        return;
 
-	inode = fd->inode;
+    if (arg->object)
+        glfs_h_close(arg->object);
 
-	if (!IA_ISDIR (inode->ia_type)) {
-		ret = -1;
-		errno = ENOTDIR;
-		goto out;
-	}
+    GF_FREE(arg);
+}
+
+int
+glfs_recall_lease_fd(struct glfs *fs, struct gf_upcall *up_data)
+{
+    struct gf_upcall_recall_lease *recall_lease = NULL;
+    xlator_t *subvol = NULL;
+    int ret = 0;
+    inode_t *inode = NULL;
+    struct glfs_fd *glfd = NULL;
+    struct glfs_fd *tmp = NULL;
+    struct list_head glfd_list;
+    fd_t *fd = NULL;
+    uint64_t value = 0;
+    struct glfs_lease lease = {
+        0,
+    };
+
+    GF_VALIDATE_OR_GOTO("gfapi", up_data, out);
+    GF_VALIDATE_OR_GOTO("gfapi", fs, out);
+
+    recall_lease = up_data->data;
+    GF_VALIDATE_OR_GOTO("gfapi", recall_lease, out);
+
+    INIT_LIST_HEAD(&glfd_list);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    gf_msg_debug(THIS->name, 0, "Recall lease received for gfid:%s",
+                 uuid_utoa(up_data->gfid));
+
+    inode = inode_find(subvol->itable, up_data->gfid);
+    if (!inode) {
+        ret = -1;
+        gf_smsg(THIS->name, GF_LOG_ERROR, errno, API_MSG_INODE_FIND_FAILED,
+                "gfid=%s", uuid_utoa(up_data->gfid), "graph_id=%d",
+                subvol->graph->id, NULL);
+        goto out;
+    }
+
+    LOCK(&inode->lock);
+    {
+        list_for_each_entry(fd, &inode->fd_list, inode_list)
+        {
+            ret = fd_ctx_get(fd, subvol, &value);
+            glfd = (struct glfs_fd *)(uintptr_t)value;
+            if (glfd) {
+                gf_msg_trace(THIS->name, 0, "glfd (%p) has held lease", glfd);
+                GF_REF_GET(glfd);
+                list_add_tail(&glfd->list, &glfd_list);
+            }
+        }
+    }
+    UNLOCK(&inode->lock);
+
+    if (!list_empty(&glfd_list)) {
+        list_for_each_entry_safe(glfd, tmp, &glfd_list, list)
+        {
+            LOCK(&glfd->lock);
+            {
+                if (glfd->state != GLFD_CLOSE) {
+                    gf_msg_trace(THIS->name, 0,
+                                 "glfd (%p) has held lease, "
+                                 "calling recall cbk",
+                                 glfd);
+                    glfd->cbk(lease, glfd->cookie);
+                }
+            }
+            UNLOCK(&glfd->lock);
+
+            list_del_init(&glfd->list);
+            GF_REF_PUT(glfd);
+        }
+    }
 
-	glfs_cwd_set (glfd->fs, inode);
-	ret = 0;
 out:
-	if (fd)
-		fd_unref (fd);
+    return ret;
+}
 
-	glfs_subvol_done (glfd->fs, subvol);
+static int
+glfs_recall_lease_upcall(struct glfs *fs, struct glfs_upcall *up_arg,
+                         struct gf_upcall *up_data)
+{
+    struct gf_upcall_recall_lease *recall_lease = NULL;
+    struct glfs_object *object = NULL;
+    xlator_t *subvol = NULL;
+    int ret = -1;
+    struct glfs_upcall_lease *up_lease_arg = NULL;
+
+    GF_VALIDATE_OR_GOTO("gfapi", up_data, out);
+    GF_VALIDATE_OR_GOTO("gfapi", fs, out);
+
+    recall_lease = up_data->data;
+    GF_VALIDATE_OR_GOTO("gfapi", recall_lease, out);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        errno = EIO;
+        goto out;
+    }
+
+    gf_msg_debug(THIS->name, 0, "Recall lease received for gfid:%s",
+                 uuid_utoa(up_data->gfid));
+
+    object = glfs_h_find_handle(fs, up_data->gfid, GFAPI_HANDLE_LENGTH);
+    if (!object) {
+        /* The reason handle creation will fail is because we
+         * couldn't find the inode in the gfapi inode table.
+         *
+         * But since application would have taken inode_ref, the
+         * only case when this can happen is when it has closed
+         * the handle and hence will no more be interested in
+         * the upcall for this particular gfid.
+         */
+        gf_smsg(THIS->name, GF_LOG_DEBUG, errno, API_MSG_CREATE_HANDLE_FAILED,
+                "gfid=%s", uuid_utoa(up_data->gfid), NULL);
+        errno = ESTALE;
+        goto out;
+    }
+
+    up_lease_arg = GF_CALLOC(1, sizeof(struct glfs_upcall_lease),
+                             glfs_mt_upcall_inode_t);
+    up_lease_arg->object = object;
+
+    GF_VALIDATE_OR_GOTO("glfs_recall_lease", up_lease_arg, out);
+
+    up_lease_arg->lease_type = recall_lease->lease_type;
+
+    up_arg->reason = GLFS_UPCALL_RECALL_LEASE;
+    up_arg->event = up_lease_arg;
+    up_arg->free_event = glfs_free_upcall_lease;
+
+    ret = 0;
 
-	return ret;
+out:
+    if (ret) {
+        /* Close p_object and oldp_object as well if being referenced.*/
+        if (object)
+            glfs_h_close(object);
+
+        /* Set reason to prevent applications from using ->event */
+        up_arg->reason = GF_UPCALL_EVENT_NULL;
+    }
+    return ret;
 }
 
+static int
+upcall_syncop_args_free(struct upcall_syncop_args *args)
+{
+    dict_t *dict = NULL;
+    struct gf_upcall *upcall_data = NULL;
+
+    if (args) {
+        upcall_data = &args->upcall_data;
+        switch (upcall_data->event_type) {
+            case GF_UPCALL_CACHE_INVALIDATION:
+                dict = ((struct gf_upcall_cache_invalidation *)(upcall_data
+                                                                    ->data))
+                           ->dict;
+                break;
+            case GF_UPCALL_RECALL_LEASE:
+                dict = ((struct gf_upcall_recall_lease *)(upcall_data->data))
+                           ->dict;
+                break;
+        }
+        if (dict)
+            dict_unref(dict);
+
+        GF_FREE(upcall_data->client_uid);
+        GF_FREE(upcall_data->data);
+    }
+    GF_FREE(args);
+    return 0;
+}
 
-char *
-glfs_realpath (struct glfs *fs, const char *path, char *resolved_path)
-{
-	int              ret = -1;
-	char            *retpath = NULL;
-	char            *allocpath = NULL;
-	xlator_t        *subvol = NULL;
-	loc_t            loc = {0, };
-	struct iatt      iatt = {0, };
-	int              reval = 0;
-
-	__glfs_entry_fs (fs);
-
-	if (resolved_path)
-		retpath = resolved_path;
-	else
-		retpath = allocpath = malloc (PATH_MAX + 1);
-
-	if (!retpath) {
-		ret = -1;
-		errno = ENOMEM;
-		goto out;
-	}
-
-	subvol = glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
-retry:
-	ret = glfs_resolve (fs, subvol, path, &loc, &iatt, reval);
+static int
+glfs_upcall_syncop_cbk(int ret, call_frame_t *frame, void *opaque)
+{
+    struct upcall_syncop_args *args = opaque;
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    (void)upcall_syncop_args_free(args);
 
-	if (ret)
-		goto out;
+    return 0;
+}
 
-	if (loc.path) {
-		strncpy (retpath, loc.path, PATH_MAX);
-		retpath[PATH_MAX] = 0;
-	}
+static int
+glfs_cbk_upcall_syncop(void *opaque)
+{
+    struct upcall_syncop_args *args = opaque;
+    struct gf_upcall *upcall_data = NULL;
+    struct glfs_upcall *up_arg = NULL;
+    struct glfs *fs;
+    int ret = -1;
+
+    fs = args->fs;
+    upcall_data = &args->upcall_data;
+
+    if (!upcall_data) {
+        goto out;
+    }
+
+    up_arg = GLFS_CALLOC(1, sizeof(struct gf_upcall), glfs_release_upcall,
+                         glfs_mt_upcall_entry_t);
+    if (!up_arg) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, ENOMEM, API_MSG_ALLOC_FAILED, "entry",
+                NULL);
+        goto out;
+    }
+
+    switch (upcall_data->event_type) {
+        case GF_UPCALL_CACHE_INVALIDATION:
+            ret = glfs_h_poll_cache_invalidation(fs, up_arg, upcall_data);
+            break;
+        case GF_UPCALL_RECALL_LEASE:
+            ret = glfs_recall_lease_upcall(fs, up_arg, upcall_data);
+            break;
+        default:
+            errno = EINVAL;
+    }
+
+    /* It could so happen that the file which got
+     * upcall notification may have got deleted by
+     * the same client. In such cases up_arg->reason
+     * is set to GLFS_UPCALL_EVENT_NULL. No need to
+     * send upcall then
+     */
+    if (up_arg->reason == GLFS_UPCALL_EVENT_NULL) {
+        gf_smsg(THIS->name, GF_LOG_DEBUG, errno,
+                API_MSG_UPCALL_EVENT_NULL_RECEIVED, NULL);
+        ret = 0;
+        GLFS_FREE(up_arg);
+        goto out;
+    } else if (ret) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, errno, API_MSG_INVALID_ENTRY, NULL);
+        GLFS_FREE(up_arg);
+        goto out;
+    }
+
+    if (fs->up_cbk && up_arg)
+        (fs->up_cbk)(up_arg, fs->up_data);
+
+    /* application takes care of calling glfs_free on up_arg post
+     * their processing */
 
 out:
-	loc_wipe (&loc);
+    return ret;
+}
 
-	if (ret == -1) {
-		if (allocpath)
-			free (allocpath);
-		retpath = NULL;
-	}
+static struct gf_upcall_cache_invalidation *
+gf_copy_cache_invalidation(struct gf_upcall_cache_invalidation *src)
+{
+    struct gf_upcall_cache_invalidation *dst = NULL;
 
-	glfs_subvol_done (fs, subvol);
+    if (!src)
+        goto out;
 
-	return retpath;
-}
+    dst = GF_CALLOC(1, sizeof(struct gf_upcall_cache_invalidation),
+                    glfs_mt_upcall_entry_t);
 
+    if (!dst) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, ENOMEM, API_MSG_ALLOC_FAILED, "entry",
+                NULL);
+        goto out;
+    }
 
-char *
-glfs_getcwd (struct glfs *fs, char *buf, size_t n)
+    dst->flags = src->flags;
+    dst->expire_time_attr = src->expire_time_attr;
+    dst->stat = src->stat;
+    dst->p_stat = src->p_stat;
+    dst->oldp_stat = src->oldp_stat;
+
+    if (src->dict)
+        dst->dict = dict_copy_with_ref(src->dict, NULL);
+
+    return dst;
+out:
+    return NULL;
+}
+
+static struct gf_upcall_recall_lease *
+gf_copy_recall_lease(struct gf_upcall_recall_lease *src)
 {
-	int              ret = -1;
-	inode_t         *inode = NULL;
-	char            *path = NULL;
+    struct gf_upcall_recall_lease *dst = NULL;
 
-	__glfs_entry_fs (fs);
+    if (!src)
+        goto out;
 
-	if (!buf || n < 2) {
-		ret = -1;
-		errno = EINVAL;
-		goto out;
-	}
+    dst = GF_CALLOC(1, sizeof(struct gf_upcall_recall_lease),
+                    glfs_mt_upcall_entry_t);
 
-	inode = glfs_cwd_get (fs);
+    if (!dst) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, ENOMEM, API_MSG_ALLOC_FAILED, "entry",
+                NULL);
+        goto out;
+    }
 
-	if (!inode) {
-		strncpy (buf, "/", n);
-		ret = 0;
-		goto out;
-	}
+    dst->lease_type = src->lease_type;
+    memcpy(dst->tid, src->tid, 16);
 
-	ret = inode_path (inode, 0, &path);
-	if (n <= ret) {
-		ret = -1;
-		errno = ERANGE;
-		goto out;
-	}
+    if (src->dict)
+        dst->dict = dict_copy_with_ref(src->dict, NULL);
 
-	strncpy (buf, path, n);
-	ret = 0;
+    return dst;
 out:
-	GF_FREE (path);
+    return NULL;
+}
 
-	if (inode)
-		inode_unref (inode);
+static struct upcall_syncop_args *
+upcall_syncop_args_init(struct glfs *fs, struct gf_upcall *upcall_data)
+{
+    struct upcall_syncop_args *args = NULL;
+    int ret = -1;
+    struct gf_upcall *t_data = NULL;
+
+    if (!fs || !upcall_data)
+        goto out;
+
+    args = GF_CALLOC(1, sizeof(struct upcall_syncop_args),
+                     glfs_mt_upcall_entry_t);
+    if (!args) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, ENOMEM, API_MSG_ALLOC_FAILED,
+                "syncop args", NULL);
+        goto out;
+    }
+
+    /* Note: we are not taking any ref on fs here.
+     * Ideally applications have to unregister for upcall events
+     * or stop polling for upcall events before performing
+     * glfs_fini. And as for outstanding synctasks created, we wait
+     * for all syncenv threads to finish tasks before cleaning up the
+     * fs->ctx. Hence it seems safe to process these callback
+     * notification without taking any lock/ref.
+     */
+    args->fs = fs;
+    t_data = &(args->upcall_data);
+    t_data->client_uid = gf_strdup(upcall_data->client_uid);
+
+    gf_uuid_copy(t_data->gfid, upcall_data->gfid);
+    t_data->event_type = upcall_data->event_type;
+
+    switch (t_data->event_type) {
+        case GF_UPCALL_CACHE_INVALIDATION:
+            t_data->data = gf_copy_cache_invalidation(
+                (struct gf_upcall_cache_invalidation *)upcall_data->data);
+            break;
+        case GF_UPCALL_RECALL_LEASE:
+            t_data->data = gf_copy_recall_lease(
+                (struct gf_upcall_recall_lease *)upcall_data->data);
+            break;
+    }
+
+    if (!t_data->data)
+        goto out;
+
+    return args;
+out:
+    if (ret) {
+        if (args) {
+            GF_FREE(args->upcall_data.client_uid);
+            GF_FREE(args);
+        }
+    }
 
-	if (ret < 0)
-		return NULL;
+    return NULL;
+}
+
+static void
+glfs_cbk_upcall_data(struct glfs *fs, struct gf_upcall *upcall_data)
+{
+    struct upcall_syncop_args *args = NULL;
+    int ret = -1;
+
+    if (!fs || !upcall_data)
+        goto out;
 
-	return buf;
+    if (!(fs->upcall_events & upcall_data->event_type)) {
+        /* ignore events which application hasn't registered*/
+        goto out;
+    }
+
+    args = upcall_syncop_args_init(fs, upcall_data);
+
+    if (!args)
+        goto out;
+
+    ret = synctask_new(THIS->ctx->env, glfs_cbk_upcall_syncop,
+                       glfs_upcall_syncop_cbk, NULL, args);
+    /* should we retry incase of failure? */
+    if (ret) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, errno, API_MSG_UPCALL_SYNCOP_FAILED,
+                "event_type=%d", upcall_data->event_type, "gfid=%s",
+                (char *)(upcall_data->gfid), NULL);
+        upcall_syncop_args_free(args);
+    }
+
+out:
+    return;
 }
 
+/*
+ * This routine is called in case of any notification received
+ * from the server. All the upcall events are queued up in a list
+ * to be read by the applications.
+ *
+ * In case if the application registers a cbk function, that shall
+ * be called by this routine in case of any event received.
+ * The cbk fn is responsible for notifying the
+ * applications the way it desires for each event queued (for eg.,
+ * can raise a signal or broadcast a cond variable etc.)
+ *
+ * Otherwise all the upcall events are queued up in a list
+ * to be read/polled by the applications.
+ */
+GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_process_upcall_event, 3.7.0)
+void
+priv_glfs_process_upcall_event(struct glfs *fs, void *data)
+{
+    glusterfs_ctx_t *ctx = NULL;
+    struct gf_upcall *upcall_data = NULL;
 
-static void
-gf_flock_to_flock (struct gf_flock *gf_flock, struct flock *flock)
+    DECLARE_OLD_THIS;
+
+    gf_msg_debug(THIS->name, 0, "Upcall gfapi callback is called");
+
+    __GLFS_ENTRY_VALIDATE_FS(fs, err);
+
+    if (!data)
+        goto out;
+
+    /* Unlike in I/O path, "glfs_fini" would not have freed
+     * 'fs' by the time we take lock as it waits for all epoll
+     * threads to exit including this
+     */
+    pthread_mutex_lock(&fs->mutex);
+    {
+        ctx = fs->ctx;
+
+        /* if we're not interested in upcalls (anymore), skip them */
+        if (ctx->cleanup_started || !fs->cache_upcalls) {
+            pthread_mutex_unlock(&fs->mutex);
+            goto out;
+        }
+
+        fs->pin_refcnt++;
+    }
+    pthread_mutex_unlock(&fs->mutex);
+
+    upcall_data = (struct gf_upcall *)data;
+
+    gf_msg_trace(THIS->name, 0, "Upcall gfapi gfid = %s",
+                 (char *)(upcall_data->gfid));
+
+    /* *
+     * TODO: RECALL LEASE for each glfd
+     *
+     * In case of RECALL_LEASE, we could associate separate
+     * cbk function for each glfd either by
+     * - extending pub_glfs_lease to accept new args (recall_cbk_fn, cookie)
+     * - or by defining new API "glfs_register_recall_cbk_fn (glfd,
+     * recall_cbk_fn, cookie) . In such cases, flag it and instead of calling
+     * below upcall functions, define a new one to go through the glfd list and
+     * invoke each of theirs recall_cbk_fn.
+     * */
+
+    if (fs->up_cbk) { /* upcall cbk registered */
+        (void)glfs_cbk_upcall_data(fs, upcall_data);
+    } else {
+        (void)glfs_enqueue_upcall_data(fs, upcall_data);
+    }
+
+    pthread_mutex_lock(&fs->mutex);
+    {
+        fs->pin_refcnt--;
+    }
+    pthread_mutex_unlock(&fs->mutex);
+
+out:
+    __GLFS_EXIT_FS;
+err:
+    return;
+}
+
+ssize_t
+glfs_anonymous_pwritev(struct glfs *fs, struct glfs_object *object,
+                       const struct iovec *iovec, int iovcnt, off_t offset,
+                       int flags)
 {
-	flock->l_type   = gf_flock->l_type;
-	flock->l_whence = gf_flock->l_whence;
-	flock->l_start  = gf_flock->l_start;
-	flock->l_len    = gf_flock->l_len;
-	flock->l_pid    = gf_flock->l_pid;
+    xlator_t *subvol = NULL;
+    struct iobref *iobref = NULL;
+    struct iobuf *iobuf = NULL;
+    struct iovec iov = {
+        0,
+    };
+    inode_t *inode = NULL;
+    fd_t *fd = NULL;
+    int ret = -1;
+    size_t size = -1;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, object);
+    if (!inode) {
+        ret = -1;
+        errno = ESTALE;
+        goto out;
+    }
+
+    fd = fd_anonymous(inode);
+    if (!fd) {
+        ret = -1;
+        gf_smsg("gfapi", GF_LOG_ERROR, ENOMEM, API_MSG_FDCREATE_FAILED, NULL);
+        errno = ENOMEM;
+        goto out;
+    }
+
+    size = iov_length(iovec, iovcnt);
+
+    iobuf = iobuf_get2(subvol->ctx->iobuf_pool, size);
+    if (!iobuf) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    iobref = iobref_new();
+    if (!iobref) {
+        iobuf_unref(iobuf);
+        errno = ENOMEM;
+        ret = -1;
+        goto out;
+    }
+
+    ret = iobref_add(iobref, iobuf);
+    if (ret) {
+        iobuf_unref(iobuf);
+        iobref_unref(iobref);
+        errno = ENOMEM;
+        ret = -1;
+        goto out;
+    }
+
+    iov_unload(iobuf_ptr(iobuf), iovec, iovcnt);
+
+    iov.iov_base = iobuf_ptr(iobuf);
+    iov.iov_len = size;
+
+    /* TODO : set leaseid */
+    ret = syncop_writev(subvol, fd, &iov, 1, offset, iobref, flags, NULL, NULL,
+                        NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    iobuf_unref(iobuf);
+    iobref_unref(iobref);
+
+    if (ret <= 0)
+        goto out;
+
+out:
+
+    if (fd)
+        fd_unref(fd);
+
+    if (inode)
+        inode_unref(inode);
+
+    glfs_subvol_done(fs, subvol);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
 }
 
+ssize_t
+glfs_anonymous_preadv(struct glfs *fs, struct glfs_object *object,
+                      const struct iovec *iovec, int iovcnt, off_t offset,
+                      int flags)
+{
+    xlator_t *subvol = NULL;
+    struct iovec *iov = NULL;
+    struct iobref *iobref = NULL;
+    inode_t *inode = NULL;
+    fd_t *fd = NULL;
+    int cnt = 0;
+    ssize_t ret = -1;
+    ssize_t size = -1;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, object);
+    if (!inode) {
+        ret = -1;
+        errno = ESTALE;
+        goto out;
+    }
+
+    fd = fd_anonymous(inode);
+    if (!fd) {
+        ret = -1;
+        gf_smsg("gfapi", GF_LOG_ERROR, ENOMEM, API_MSG_FDCREATE_FAILED, NULL);
+        errno = ENOMEM;
+        goto out;
+    }
+
+    size = iov_length(iovec, iovcnt);
+
+    /* TODO : set leaseid */
+    ret = syncop_readv(subvol, fd, size, offset, flags, &iov, &cnt, &iobref,
+                       NULL, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+    if (ret <= 0)
+        goto out;
+
+    size = iov_copy(iovec, iovcnt, iov, cnt);
+
+    ret = size;
+out:
+    if (iov)
+        GF_FREE(iov);
+    if (iobref)
+        iobref_unref(iobref);
+    if (fd)
+        fd_unref(fd);
+
+    if (inode)
+        inode_unref(inode);
+
+    glfs_subvol_done(fs, subvol);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
 
 static void
-gf_flock_from_flock (struct gf_flock *gf_flock, struct flock *flock)
+glfs_release_xreaddirp_stat(void *ptr)
 {
-	gf_flock->l_type   = flock->l_type;
-	gf_flock->l_whence = flock->l_whence;
-	gf_flock->l_start  = flock->l_start;
-	gf_flock->l_len    = flock->l_len;
-	gf_flock->l_pid    = flock->l_pid;
-}
+    struct glfs_xreaddirp_stat *to_free = ptr;
 
+    if (to_free->object)
+        glfs_h_close(to_free->object);
+}
 
+/*
+ * Given glfd of a directory, this function does readdirp and returns
+ * xstat along with dirents.
+ */
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_xreaddirplus_r, 3.11.0)
 int
-glfs_posix_lock (struct glfs_fd *glfd, int cmd, struct flock *flock)
+pub_glfs_xreaddirplus_r(struct glfs_fd *glfd, uint32_t flags,
+                        struct glfs_xreaddirp_stat **xstat_p,
+                        struct dirent *ext, struct dirent **res)
 {
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	struct gf_flock  gf_flock = {0, };
-	struct gf_flock  saved_flock = {0, };
-	fd_t            *fd = NULL;
+    int ret = -1;
+    gf_dirent_t *entry = NULL;
+    struct dirent *buf = NULL;
+    struct glfs_xreaddirp_stat *xstat = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    GF_REF_GET(glfd);
+
+    GF_VALIDATE_OR_GOTO(THIS->name, xstat_p, out);
+    GF_VALIDATE_OR_GOTO(THIS->name, res, out);
+
+    errno = 0;
+
+    if (ext)
+        buf = ext;
+    else
+        buf = glfs_readdirbuf_get(glfd);
+
+    if (!buf)
+        goto out;
+
+    xstat = GLFS_CALLOC(1, sizeof(struct glfs_xreaddirp_stat),
+                        glfs_release_xreaddirp_stat, glfs_mt_xreaddirp_stat_t);
+
+    if (!xstat)
+        goto out;
+
+    /* this is readdirplus operation */
+    entry = glfd_entry_next(glfd, 1);
+
+    /* XXX: Ideally when we reach EOD, errno should have been
+     * set to ENOENT. But that doesn't seem to be the case.
+     *
+     * The only way to confirm if its EOD at this point is that
+     * errno == 0 and entry == NULL
+     */
+    if (errno)
+        goto out;
+
+    if (!entry) {
+        /* reached EOD, ret = 0  */
+        ret = 0;
+        *res = NULL;
+        *xstat_p = NULL;
+
+        /* free xstat as applications shall not be using it */
+        GLFS_FREE(xstat);
+
+        goto out;
+    }
+
+    *res = buf;
+    gf_dirent_to_dirent(entry, buf);
+
+    if (flags & GFAPI_XREADDIRP_STAT) {
+        glfs_iatt_to_stat(glfd->fs, &entry->d_stat, &xstat->st);
+        xstat->flags_handled |= GFAPI_XREADDIRP_STAT;
+    }
+
+    if ((flags & GFAPI_XREADDIRP_HANDLE) &&
+        /* skip . and .. */
+        strcmp(buf->d_name, ".") && strcmp(buf->d_name, "..")) {
+        /* Now create object.
+         * We can use "glfs_h_find_handle" as well as inodes would have
+         * already got linked as part of 'gf_link_inodes_from_dirent' */
+        xstat->object = glfs_h_create_from_handle(
+            glfd->fs, entry->d_stat.ia_gfid, GFAPI_HANDLE_LENGTH, NULL);
+
+        if (xstat->object) { /* success */
+            /* note: xstat->object->inode->ref is taken
+             * This shall be unref'ed when application does
+             * glfs_free(xstat) */
+            xstat->flags_handled |= GFAPI_XREADDIRP_HANDLE;
+        }
+    }
 
-	__glfs_entry_fd (glfd);
+    ret = xstat->flags_handled;
+    *xstat_p = xstat;
 
-	subvol = glfs_active_subvol (glfd->fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+    gf_msg_debug(THIS->name, 0,
+                 "xreaddirp- requested_flags (%x) , processed_flags (%x)",
+                 flags, xstat->flags_handled);
 
-	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
-	if (!fd) {
-		ret = -1;
-		errno = EBADFD;
-		goto out;
-	}
+out:
+    GF_REF_PUT(glfd);
 
-	gf_flock_from_flock (&gf_flock, flock);
-	gf_flock_from_flock (&saved_flock, flock);
-	ret = syncop_lk (subvol, fd, cmd, &gf_flock);
-	gf_flock_to_flock (&gf_flock, flock);
+    if (ret < 0) {
+        gf_smsg(THIS->name, GF_LOG_WARNING, errno, API_MSG_XREADDIRP_R_FAILED,
+                "reason=%s", strerror(errno), NULL);
 
-	if (ret == 0 && (cmd == F_SETLK || cmd == F_SETLKW))
-		fd_lk_insert_and_merge (fd, cmd, &saved_flock);
-out:
-	if (fd)
-		fd_unref (fd);
+        if (xstat)
+            GLFS_FREE(xstat);
+    }
 
-	glfs_subvol_done (glfd->fs, subvol);
+    __GLFS_EXIT_FS;
 
-	return ret;
-}
+    return ret;
 
+invalid_fs:
+    return -1;
+}
 
-struct glfs_fd *
-glfs_dup (struct glfs_fd *glfd)
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_xreaddirplus_get_stat, 3.11.0)
+struct stat *
+pub_glfs_xreaddirplus_get_stat(struct glfs_xreaddirp_stat *xstat)
 {
-	xlator_t  *subvol = NULL;
-	fd_t      *fd = NULL;
-	glfs_fd_t *dupfd = NULL;
-	struct glfs *fs = NULL;
+    GF_VALIDATE_OR_GOTO("glfs_xreaddirplus_get_stat", xstat, out);
 
-	__glfs_entry_fd (glfd);
+    if (!xstat->flags_handled & GFAPI_XREADDIRP_STAT)
+        gf_smsg(THIS->name, GF_LOG_ERROR, errno, API_MSG_FLAGS_HANDLE,
+                "GFAPI_XREADDIRP_STAT"
+                "xstat=%p",
+                xstat, "handles=%x", xstat->flags_handled, NULL);
+    return &xstat->st;
 
-	fs = glfd->fs;
-	subvol = glfs_active_subvol (fs);
-	if (!subvol) {
-		errno = EIO;
-		goto out;
-	}
+out:
+    return NULL;
+}
+
+void
+gf_lease_to_glfs_lease(struct gf_lease *gf_lease, struct glfs_lease *lease)
+{
+    u_int lease_type = gf_lease->lease_type;
+    lease->cmd = gf_lease->cmd;
+    lease->lease_type = lease_type;
+    memcpy(lease->lease_id, gf_lease->lease_id, LEASE_ID_SIZE);
+}
 
-	fd = glfs_resolve_fd (fs, subvol, glfd);
-	if (!fd) {
-		errno = EBADFD;
-		goto out;
-	}
+void
+glfs_lease_to_gf_lease(struct glfs_lease *lease, struct gf_lease *gf_lease)
+{
+    u_int lease_type = lease->lease_type;
+    gf_lease->cmd = lease->cmd;
+    gf_lease->lease_type = lease_type;
+    memcpy(gf_lease->lease_id, lease->lease_id, LEASE_ID_SIZE);
+}
 
-	dupfd = glfs_fd_new (fs);
-	if (!dupfd) {
-		errno = ENOMEM;
-		goto out;
-	}
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_lease, 4.0.0)
+int
+pub_glfs_lease(struct glfs_fd *glfd, struct glfs_lease *lease,
+               glfs_recall_cbk fn, void *data)
+{
+    int ret = -1;
+    loc_t loc = {
+        0,
+    };
+    xlator_t *subvol = NULL;
+    fd_t *fd = NULL;
+    struct gf_lease gf_lease = {
+        0,
+    };
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    GF_REF_GET(glfd);
+
+    if (!is_valid_lease_id(lease->lease_id)) {
+        ret = -1;
+        errno = EINVAL;
+        goto out;
+    }
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    switch (lease->lease_type) {
+        case GLFS_RD_LEASE:
+            if ((fd->flags != O_RDONLY) && !(fd->flags & O_RDWR)) {
+                ret = -1;
+                errno = EINVAL;
+                goto out;
+            }
+            break;
+        case GLFS_RW_LEASE:
+            if (!((fd->flags & O_WRONLY) || (fd->flags & O_RDWR))) {
+                ret = -1;
+                errno = EINVAL;
+                goto out;
+            }
+            break;
+        default:
+            if (lease->cmd != GLFS_GET_LEASE) {
+                ret = -1;
+                errno = EINVAL;
+                goto out;
+            }
+            break;
+    }
+
+    /* populate loc */
+    GLFS_LOC_FILL_INODE(fd->inode, loc, out);
+
+    glfs_lease_to_gf_lease(lease, &gf_lease);
+
+    ret = syncop_lease(subvol, &loc, &gf_lease, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    gf_lease_to_glfs_lease(&gf_lease, lease);
+
+    /* TODO: Add leases for client replay
+    if (ret == 0 && (cmd == F_SETLK || cmd == F_SETLKW))
+            fd_lk_insert_and_merge (fd, cmd, &saved_flock);
+    */
+    if (ret == 0) {
+        ret = fd_ctx_set(glfd->fd, subvol, (uint64_t)(long)glfd);
+        if (ret) {
+            gf_smsg(subvol->name, GF_LOG_ERROR, ENOMEM,
+                    API_MSG_FDCTX_SET_FAILED, "fd=%p", glfd->fd, NULL);
+            goto out;
+        }
+        glfd->cbk = fn;
+        glfd->cookie = data;
+    }
 
-	dupfd->fd = fd_ref (fd);
 out:
-	if (fd)
-		fd_unref (fd);
-	if (dupfd)
-		glfs_fd_bind (dupfd);
 
-	glfs_subvol_done (fs, subvol);
+    if (glfd)
+        GF_REF_PUT(glfd);
+
+    if (subvol)
+        glfs_subvol_done(glfd->fs, subvol);
+
+    __GLFS_EXIT_FS;
 
-	return dupfd;
+invalid_fs:
+    return ret;
 }
diff --git a/api/src/glfs-handleops.c b/api/src/glfs-handleops.c
new file mode 100644
index 00000000000..53c2ee896f9
--- /dev/null
+++ b/api/src/glfs-handleops.c
@@ -0,0 +1,2655 @@
+/*
+ *  Copyright (c) 2013-2018 Red Hat, Inc. <http://www.redhat.com>
+ *  This file is part of GlusterFS.
+ *
+ *  This file is licensed to you under your choice of the GNU Lesser
+ *  General Public License, version 3 or any later version (LGPLv3 or
+ *  later), or the GNU General Public License, version 2 (GPLv2), in all
+ *  cases as published by the Free Software Foundation.
+ */
+
+#include "glfs-internal.h"
+#include "glfs-mem-types.h"
+#include <glusterfs/syncop.h>
+#include "glfs.h"
+#include "glfs-handles.h"
+#include "gfapi-messages.h"
+
+int
+glfs_listxattr_process(void *value, size_t size, dict_t *xattr);
+
+void
+glfs_iatt_from_stat(struct stat *stat, int valid, struct iatt *iatt,
+                    int *glvalid)
+{
+    /* validate in args */
+    if ((stat == NULL) || (iatt == NULL) || (glvalid == NULL)) {
+        errno = EINVAL;
+        return;
+    }
+
+    *glvalid = 0;
+
+    if (valid & GFAPI_SET_ATTR_MODE) {
+        iatt->ia_prot = ia_prot_from_st_mode(stat->st_mode);
+        *glvalid |= GF_SET_ATTR_MODE;
+    }
+
+    if (valid & GFAPI_SET_ATTR_UID) {
+        iatt->ia_uid = stat->st_uid;
+        *glvalid |= GF_SET_ATTR_UID;
+    }
+
+    if (valid & GFAPI_SET_ATTR_GID) {
+        iatt->ia_gid = stat->st_gid;
+        *glvalid |= GF_SET_ATTR_GID;
+    }
+
+    if (valid & GFAPI_SET_ATTR_ATIME) {
+        iatt->ia_atime = stat->st_atime;
+        iatt->ia_atime_nsec = ST_ATIM_NSEC(stat);
+        *glvalid |= GF_SET_ATTR_ATIME;
+    }
+
+    if (valid & GFAPI_SET_ATTR_MTIME) {
+        iatt->ia_mtime = stat->st_mtime;
+        iatt->ia_mtime_nsec = ST_MTIM_NSEC(stat);
+        *glvalid |= GF_SET_ATTR_MTIME;
+    }
+
+    return;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_lookupat, 3.7.4)
+struct glfs_object *
+pub_glfs_h_lookupat(struct glfs *fs, struct glfs_object *parent,
+                    const char *path, struct stat *stat, int follow)
+{
+    int ret = 0;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    struct iatt iatt = {
+        0,
+    };
+    struct glfs_object *object = NULL;
+    loc_t loc = {
+        0,
+    };
+
+    DECLARE_OLD_THIS;
+
+    /* validate in args */
+    if (path == NULL) {
+        errno = EINVAL;
+        return NULL;
+    }
+
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    if (parent) {
+        inode = glfs_resolve_inode(fs, subvol, parent);
+        if (!inode) {
+            errno = ESTALE;
+            goto out;
+        }
+    }
+
+    /* fop/op */
+    ret = glfs_resolve_at(fs, subvol, inode, path, &loc, &iatt, follow, 0);
+
+    /* populate out args */
+    if (!ret) {
+        if (stat)
+            glfs_iatt_to_stat(fs, &iatt, stat);
+
+        ret = glfs_create_object(&loc, &object);
+    }
+
+out:
+    loc_wipe(&loc);
+
+    if (inode)
+        inode_unref(inode);
+
+    glfs_subvol_done(fs, subvol);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return object;
+}
+
+GFAPI_SYMVER_PUBLIC(glfs_h_lookupat34, glfs_h_lookupat, 3.4.2)
+struct glfs_object *
+pub_glfs_h_lookupat34(struct glfs *fs, struct glfs_object *parent,
+                      const char *path, struct stat *stat)
+{
+    return pub_glfs_h_lookupat(fs, parent, path, stat, 0);
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_statfs, 3.7.0)
+int
+pub_glfs_h_statfs(struct glfs *fs, struct glfs_object *object,
+                  struct statvfs *statvfs)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    loc_t loc = {
+        0,
+    };
+
+    DECLARE_OLD_THIS;
+
+    /* validate in args */
+    if ((fs == NULL) || (object == NULL || statvfs == NULL)) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, object);
+    if (!inode) {
+        errno = ESTALE;
+        goto out;
+    }
+
+    /* populate loc */
+    GLFS_LOC_FILL_INODE(inode, loc, out);
+
+    /* fop/op */
+    ret = syncop_statfs(subvol, &loc, statvfs, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    loc_wipe(&loc);
+
+out:
+    if (inode)
+        inode_unref(inode);
+
+    glfs_subvol_done(fs, subvol);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_stat, 3.4.2)
+int
+pub_glfs_h_stat(struct glfs *fs, struct glfs_object *object, struct stat *stat)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+
+    DECLARE_OLD_THIS;
+
+    /* validate in args */
+    if ((fs == NULL) || (object == NULL)) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, object);
+    if (!inode) {
+        errno = ESTALE;
+        goto out;
+    }
+
+    /* populate loc */
+    GLFS_LOC_FILL_INODE(inode, loc, out);
+
+    /* fop/op */
+    ret = syncop_stat(subvol, &loc, &iatt, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    /* populate out args */
+    if (!ret && stat) {
+        glfs_iatt_to_stat(fs, &iatt, stat);
+    }
+out:
+    loc_wipe(&loc);
+
+    if (inode)
+        inode_unref(inode);
+
+    glfs_subvol_done(fs, subvol);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_getattrs, 3.4.2)
+int
+pub_glfs_h_getattrs(struct glfs *fs, struct glfs_object *object,
+                    struct stat *stat)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    struct iatt iatt = {
+        0,
+    };
+
+    /* validate in args */
+    if ((fs == NULL) || (object == NULL)) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, object);
+    if (!inode) {
+        ret = 0;
+        errno = ESTALE;
+        goto out;
+    }
+
+    /* fop/op */
+    ret = glfs_resolve_base(fs, subvol, inode, &iatt);
+
+    /* populate out args */
+    if (!ret && stat) {
+        glfs_iatt_to_stat(fs, &iatt, stat);
+    }
+
+out:
+    if (inode)
+        inode_unref(inode);
+
+    glfs_subvol_done(fs, subvol);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
+
+int
+glfs_h_getxattrs_common(struct glfs *fs, struct glfs_object *object,
+                        dict_t **xattr, const char *name,
+                        gf_boolean_t is_listxattr)
+{
+    int ret = 0;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    loc_t loc = {
+        0,
+    };
+
+    /* validate in args */
+    if ((fs == NULL) || (object == NULL)) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    if (!is_listxattr) {
+        if (!name || *name == '\0') {
+            errno = EINVAL;
+            return -1;
+        }
+
+        if (strlen(name) > GF_XATTR_NAME_MAX) {
+            errno = ENAMETOOLONG;
+            return -1;
+        }
+    }
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, object);
+    if (!inode) {
+        errno = ESTALE;
+        goto out;
+    }
+
+    /* populate loc */
+    GLFS_LOC_FILL_INODE(inode, loc, out);
+
+    ret = syncop_getxattr(subvol, &loc, xattr, name, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+out:
+    loc_wipe(&loc);
+
+    if (inode)
+        inode_unref(inode);
+
+    glfs_subvol_done(fs, subvol);
+
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_getxattrs, 3.5.1)
+int
+pub_glfs_h_getxattrs(struct glfs *fs, struct glfs_object *object,
+                     const char *name, void *value, size_t size)
+{
+    int ret = -1;
+    dict_t *xattr = NULL;
+
+    /* validate in args */
+    if ((fs == NULL) || (object == NULL)) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    ret = glfs_h_getxattrs_common(fs, object, &xattr, name, (name == NULL));
+    if (ret)
+        goto out;
+
+    /* If @name is NULL, means get all the xattrs (i.e listxattr). */
+    if (name)
+        ret = glfs_getxattr_process(value, size, xattr, name);
+    else
+        ret = glfs_listxattr_process(value, size, xattr);
+
+out:
+    if (xattr)
+        dict_unref(xattr);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_setattrs, 3.4.2)
+int
+pub_glfs_h_setattrs(struct glfs *fs, struct glfs_object *object,
+                    struct stat *stat, int valid)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    int glvalid = 0;
+
+    /* validate in args */
+    if ((fs == NULL) || (object == NULL) || (stat == NULL)) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, object);
+    if (!inode) {
+        errno = ESTALE;
+        goto out;
+    }
+
+    /* map valid masks from in args */
+    glfs_iatt_from_stat(stat, valid, &iatt, &glvalid);
+
+    /* populate loc */
+    GLFS_LOC_FILL_INODE(inode, loc, out);
+
+    /* fop/op */
+    ret = syncop_setattr(subvol, &loc, &iatt, glvalid, 0, 0, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+out:
+    loc_wipe(&loc);
+
+    if (inode)
+        inode_unref(inode);
+
+    glfs_subvol_done(fs, subvol);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_setxattrs, 3.5.0)
+int
+pub_glfs_h_setxattrs(struct glfs *fs, struct glfs_object *object,
+                     const char *name, const void *value, size_t size,
+                     int flags)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    loc_t loc = {
+        0,
+    };
+    dict_t *xattr = NULL;
+    void *value_cp = NULL;
+
+    /* validate in args */
+    if ((fs == NULL) || (object == NULL) || (name == NULL) || (value == NULL)) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    if (!name || *name == '\0') {
+        errno = EINVAL;
+        return -1;
+    }
+
+    if (strlen(name) > GF_XATTR_NAME_MAX) {
+        errno = ENAMETOOLONG;
+        return -1;
+    }
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, object);
+    if (!inode) {
+        errno = ESTALE;
+        goto out;
+    }
+
+    value_cp = gf_memdup(value, size);
+    GF_CHECK_ALLOC_AND_LOG(subvol->name, value_cp, ret,
+                           "Failed to"
+                           " duplicate setxattr value",
+                           out);
+
+    xattr = dict_for_key_value(name, value_cp, size, _gf_false);
+    if (!xattr) {
+        GF_FREE(value_cp);
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    /* populate loc */
+    GLFS_LOC_FILL_INODE(inode, loc, out);
+
+    /* fop/op */
+    ret = syncop_setxattr(subvol, &loc, xattr, flags, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+out:
+    loc_wipe(&loc);
+
+    if (inode)
+        inode_unref(inode);
+
+    if (xattr)
+        dict_unref(xattr);
+
+    glfs_subvol_done(fs, subvol);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_removexattrs, 3.5.1)
+int
+pub_glfs_h_removexattrs(struct glfs *fs, struct glfs_object *object,
+                        const char *name)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    loc_t loc = {
+        0,
+    };
+
+    /* validate in args */
+    if ((fs == NULL) || (object == NULL) || (name == NULL)) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, object);
+    if (!inode) {
+        errno = ESTALE;
+        goto out;
+    }
+
+    /* populate loc */
+    GLFS_LOC_FILL_INODE(inode, loc, out);
+
+    /* fop/op */
+    ret = syncop_removexattr(subvol, &loc, name, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+out:
+    loc_wipe(&loc);
+
+    if (inode)
+        inode_unref(inode);
+
+    glfs_subvol_done(fs, subvol);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_open, 3.4.2)
+struct glfs_fd *
+pub_glfs_h_open(struct glfs *fs, struct glfs_object *object, int flags)
+{
+    int ret = -1;
+    struct glfs_fd *glfd = NULL;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    loc_t loc = {
+        0,
+    };
+    dict_t *fop_attr = NULL;
+
+    /* validate in args */
+    if ((fs == NULL) || (object == NULL)) {
+        errno = EINVAL;
+        return NULL;
+    }
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, object);
+    if (!inode) {
+        errno = ESTALE;
+        goto out;
+    }
+
+    /* check types to open */
+    if (IA_ISDIR(inode->ia_type)) {
+        ret = -1;
+        errno = EISDIR;
+        goto out;
+    }
+
+    if (!IA_ISREG(inode->ia_type)) {
+        ret = -1;
+        errno = EINVAL;
+        goto out;
+    }
+
+    glfd = glfs_fd_new(fs);
+    if (!glfd) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    glfd->fd = fd_create(inode, getpid());
+    if (!glfd->fd) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+    glfd->fd->flags = flags;
+
+    /* populate loc */
+    GLFS_LOC_FILL_INODE(inode, loc, out);
+
+    /* fop/op */
+    ret = get_fop_attr_thrd_key(&fop_attr);
+    if (ret)
+        gf_msg_debug("gfapi", 0, "Getting leaseid from thread failed");
+
+    ret = syncop_open(subvol, &loc, flags, glfd->fd, fop_attr, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    glfd->fd->flags = flags;
+
+out:
+    loc_wipe(&loc);
+
+    if (inode)
+        inode_unref(inode);
+    if (fop_attr)
+        dict_unref(fop_attr);
+
+    if (ret && glfd) {
+        GF_REF_PUT(glfd);
+        glfd = NULL;
+    } else if (glfd) {
+        glfd_set_state_bind(glfd);
+    }
+
+    glfs_subvol_done(fs, subvol);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return glfd;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_creat, 3.4.2)
+struct glfs_object *
+pub_glfs_h_creat(struct glfs *fs, struct glfs_object *parent, const char *path,
+                 int flags, mode_t mode, struct stat *stat)
+{
+    int ret = -1;
+    fd_t *fd = NULL;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    uuid_t gfid;
+    dict_t *xattr_req = NULL;
+    struct glfs_object *object = NULL;
+
+    /* validate in args */
+    if ((fs == NULL) || (parent == NULL) || (path == NULL)) {
+        errno = EINVAL;
+        return NULL;
+    }
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, parent);
+    if (!inode) {
+        ret = -1;
+        errno = ESTALE;
+        goto out;
+    }
+
+    xattr_req = dict_new();
+    if (!xattr_req) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gf_uuid_generate(gfid);
+    ret = dict_set_gfuuid(xattr_req, "gfid-req", gfid, true);
+    if (ret) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    GLFS_LOC_FILL_PINODE(inode, loc, ret, errno, out, path);
+
+    fd = fd_create(loc.inode, getpid());
+    if (!fd) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+    fd->flags = flags;
+
+    /* fop/op */
+    ret = syncop_create(subvol, &loc, flags, mode, fd, &iatt, xattr_req, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    /* populate out args */
+    if (ret == 0) {
+        ret = glfs_loc_link(&loc, &iatt);
+        if (ret != 0) {
+            goto out;
+        }
+
+        if (stat)
+            glfs_iatt_to_stat(fs, &iatt, stat);
+
+        ret = glfs_create_object(&loc, &object);
+    }
+
+out:
+    if (ret && object != NULL) {
+        /* Release the held reference */
+        glfs_h_close(object);
+        object = NULL;
+    }
+
+    loc_wipe(&loc);
+
+    if (inode)
+        inode_unref(inode);
+
+    if (xattr_req)
+        dict_unref(xattr_req);
+
+    if (fd)
+        fd_unref(fd);
+
+    glfs_subvol_done(fs, subvol);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return object;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_creat_open, 6.6)
+struct glfs_object *
+pub_glfs_h_creat_open(struct glfs *fs, struct glfs_object *parent,
+                      const char *path, int flags, mode_t mode,
+                      struct stat *stat, struct glfs_fd **out_fd)
+{
+    int ret = -1;
+    struct glfs_fd *glfd = NULL;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    uuid_t gfid;
+    dict_t *xattr_req = NULL;
+    struct glfs_object *object = NULL;
+    dict_t *fop_attr = NULL;
+
+    /* validate in args */
+    if ((fs == NULL) || (parent == NULL) || (path == NULL) ||
+        (out_fd == NULL)) {
+        errno = EINVAL;
+        return NULL;
+    }
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, parent);
+    if (!inode) {
+        ret = -1;
+        goto out;
+    }
+
+    xattr_req = dict_new();
+    if (!xattr_req) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gf_uuid_generate(gfid);
+    ret = dict_set_gfuuid(xattr_req, "gfid-req", gfid, true);
+    if (ret) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    GLFS_LOC_FILL_PINODE(inode, loc, ret, errno, out, path);
+
+    glfd = glfs_fd_new(fs);
+    if (!glfd) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    glfd->fd = fd_create(loc.inode, getpid());
+    if (!glfd->fd) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+    glfd->fd->flags = flags;
+
+    ret = get_fop_attr_thrd_key(&fop_attr);
+    if (ret)
+        gf_msg_debug("gfapi", 0, "Getting leaseid from thread failed");
+
+    /* fop/op */
+    ret = syncop_create(subvol, &loc, flags, mode, glfd->fd, &iatt, xattr_req,
+                        NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    /* populate out args */
+    if (ret == 0) {
+        glfd->fd->flags = flags;
+
+        ret = glfs_loc_link(&loc, &iatt);
+        if (ret != 0) {
+            goto out;
+        }
+
+        if (stat)
+            glfs_iatt_to_stat(fs, &iatt, stat);
+
+        ret = glfs_create_object(&loc, &object);
+    }
+
+out:
+    if (ret && object != NULL) {
+        /* Release the held reference */
+        glfs_h_close(object);
+        object = NULL;
+    }
+
+    loc_wipe(&loc);
+
+    if (inode)
+        inode_unref(inode);
+
+    if (fop_attr)
+        dict_unref(fop_attr);
+
+    if (xattr_req)
+        dict_unref(xattr_req);
+
+    if (ret && glfd) {
+        GF_REF_PUT(glfd);
+    } else if (glfd) {
+        glfd_set_state_bind(glfd);
+        *out_fd = glfd;
+    }
+
+    glfs_subvol_done(fs, subvol);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return object;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_mkdir, 3.4.2)
+struct glfs_object *
+pub_glfs_h_mkdir(struct glfs *fs, struct glfs_object *parent, const char *path,
+                 mode_t mode, struct stat *stat)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    uuid_t gfid;
+    dict_t *xattr_req = NULL;
+    struct glfs_object *object = NULL;
+
+    /* validate in args */
+    if ((fs == NULL) || (parent == NULL) || (path == NULL)) {
+        errno = EINVAL;
+        return NULL;
+    }
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, parent);
+    if (!inode) {
+        errno = ESTALE;
+        goto out;
+    }
+
+    xattr_req = dict_new();
+    if (!xattr_req) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gf_uuid_generate(gfid);
+    ret = dict_set_gfuuid(xattr_req, "gfid-req", gfid, true);
+    if (ret) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    GLFS_LOC_FILL_PINODE(inode, loc, ret, errno, out, path);
+
+    /* fop/op */
+    ret = syncop_mkdir(subvol, &loc, mode, &iatt, xattr_req, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    /* populate out args */
+    if (ret == 0) {
+        ret = glfs_loc_link(&loc, &iatt);
+        if (ret != 0) {
+            goto out;
+        }
+
+        if (stat)
+            glfs_iatt_to_stat(fs, &iatt, stat);
+
+        ret = glfs_create_object(&loc, &object);
+    }
+
+out:
+    if (ret && object != NULL) {
+        glfs_h_close(object);
+        object = NULL;
+    }
+
+    loc_wipe(&loc);
+
+    if (inode)
+        inode_unref(inode);
+
+    if (xattr_req)
+        dict_unref(xattr_req);
+
+    glfs_subvol_done(fs, subvol);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return object;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_mknod, 3.4.2)
+struct glfs_object *
+pub_glfs_h_mknod(struct glfs *fs, struct glfs_object *parent, const char *path,
+                 mode_t mode, dev_t dev, struct stat *stat)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    uuid_t gfid;
+    dict_t *xattr_req = NULL;
+    struct glfs_object *object = NULL;
+
+    /* validate in args */
+    if ((fs == NULL) || (parent == NULL) || (path == NULL)) {
+        errno = EINVAL;
+        return NULL;
+    }
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, parent);
+    if (!inode) {
+        errno = ESTALE;
+        goto out;
+    }
+
+    xattr_req = dict_new();
+    if (!xattr_req) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gf_uuid_generate(gfid);
+    ret = dict_set_gfuuid(xattr_req, "gfid-req", gfid, true);
+    if (ret) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    GLFS_LOC_FILL_PINODE(inode, loc, ret, errno, out, path);
+
+    /* fop/op */
+    ret = syncop_mknod(subvol, &loc, mode, dev, &iatt, xattr_req, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    /* populate out args */
+    if (ret == 0) {
+        ret = glfs_loc_link(&loc, &iatt);
+        if (ret != 0) {
+            goto out;
+        }
+
+        if (stat)
+            glfs_iatt_to_stat(fs, &iatt, stat);
+
+        ret = glfs_create_object(&loc, &object);
+    }
+out:
+    if (ret && object != NULL) {
+        glfs_h_close(object);
+        object = NULL;
+    }
+
+    loc_wipe(&loc);
+
+    if (inode)
+        inode_unref(inode);
+
+    if (xattr_req)
+        dict_unref(xattr_req);
+
+    glfs_subvol_done(fs, subvol);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return object;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_unlink, 3.4.2)
+int
+pub_glfs_h_unlink(struct glfs *fs, struct glfs_object *parent, const char *path)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    loc_t loc = {
+        0,
+    };
+
+    /* validate in args */
+    if ((fs == NULL) || (parent == NULL) || (path == NULL)) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, parent);
+    if (!inode) {
+        errno = ESTALE;
+        goto out;
+    }
+
+    ret = glfs_resolve_at(fs, subvol, inode, path, &loc, NULL, 0, 0);
+    if (ret != 0) {
+        goto out;
+    }
+
+    if (!IA_ISDIR(loc.inode->ia_type)) {
+        ret = syncop_unlink(subvol, &loc, NULL, NULL);
+        DECODE_SYNCOP_ERR(ret);
+        if (ret != 0) {
+            goto out;
+        }
+    } else {
+        ret = syncop_rmdir(subvol, &loc, 0, NULL, NULL);
+        DECODE_SYNCOP_ERR(ret);
+        if (ret != 0) {
+            goto out;
+        }
+    }
+
+    if (ret == 0)
+        ret = glfs_loc_unlink(&loc);
+
+out:
+    loc_wipe(&loc);
+
+    if (inode)
+        inode_unref(inode);
+
+    glfs_subvol_done(fs, subvol);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_opendir, 3.4.2)
+struct glfs_fd *
+pub_glfs_h_opendir(struct glfs *fs, struct glfs_object *object)
+{
+    int ret = -1;
+    struct glfs_fd *glfd = NULL;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    loc_t loc = {
+        0,
+    };
+
+    /* validate in args */
+    if ((fs == NULL) || (object == NULL)) {
+        errno = EINVAL;
+        return NULL;
+    }
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, object);
+    if (!inode) {
+        errno = ESTALE;
+        goto out;
+    }
+
+    if (!IA_ISDIR(inode->ia_type)) {
+        ret = -1;
+        errno = ENOTDIR;
+        goto out;
+    }
+
+    glfd = glfs_fd_new(fs);
+    if (!glfd)
+        goto out;
+
+    INIT_LIST_HEAD(&glfd->entries);
+
+    glfd->fd = fd_create(inode, getpid());
+    if (!glfd->fd) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    GLFS_LOC_FILL_INODE(inode, loc, out);
+
+    /* fop/op */
+    ret = syncop_opendir(subvol, &loc, glfd->fd, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+out:
+    loc_wipe(&loc);
+
+    if (inode)
+        inode_unref(inode);
+
+    if (ret && glfd) {
+        GF_REF_PUT(glfd);
+        glfd = NULL;
+    } else if (glfd) {
+        glfd_set_state_bind(glfd);
+    }
+
+    glfs_subvol_done(fs, subvol);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return glfd;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_access, 3.6.0)
+int
+pub_glfs_h_access(struct glfs *fs, struct glfs_object *object, int mask)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    loc_t loc = {
+        0,
+    };
+
+    DECLARE_OLD_THIS;
+
+    /* validate in args */
+    if ((fs == NULL) || (object == NULL)) {
+        errno = EINVAL;
+        return ret;
+    }
+
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, object);
+    if (!inode) {
+        errno = ESTALE;
+        goto out;
+    }
+
+    GLFS_LOC_FILL_INODE(inode, loc, out);
+
+    /* fop/op */
+
+    ret = syncop_access(subvol, &loc, mask, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+out:
+    loc_wipe(&loc);
+
+    if (inode)
+        inode_unref(inode);
+
+    glfs_subvol_done(fs, subvol);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_extract_handle, 3.4.2)
+ssize_t
+pub_glfs_h_extract_handle(struct glfs_object *object, unsigned char *handle,
+                          int len)
+{
+    ssize_t ret = -1;
+
+    /* validate in args */
+    if (object == NULL) {
+        errno = EINVAL;
+        goto out;
+    }
+
+    if (!handle || !len) {
+        ret = GFAPI_HANDLE_LENGTH;
+        goto out;
+    }
+
+    if (len < GFAPI_HANDLE_LENGTH) {
+        errno = ERANGE;
+        goto out;
+    }
+
+    memcpy(handle, object->gfid, GFAPI_HANDLE_LENGTH);
+
+    ret = GFAPI_HANDLE_LENGTH;
+
+out:
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_create_from_handle, 3.4.2)
+struct glfs_object *
+pub_glfs_h_create_from_handle(struct glfs *fs, unsigned char *handle, int len,
+                              struct stat *stat)
+{
+    loc_t loc = {
+        0,
+    };
+    int ret = -1;
+    struct iatt iatt = {
+        0,
+    };
+    inode_t *newinode = NULL;
+    xlator_t *subvol = NULL;
+    struct glfs_object *object = NULL;
+    uint64_t ctx_value = LOOKUP_NOT_NEEDED;
+    gf_boolean_t lookup_needed = _gf_false;
+
+    /* validate in args */
+    if ((fs == NULL) || (handle == NULL) || (len != GFAPI_HANDLE_LENGTH)) {
+        errno = EINVAL;
+        return NULL;
+    }
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        errno = EIO;
+        goto out;
+    }
+
+    memcpy(loc.gfid, handle, GFAPI_HANDLE_LENGTH);
+
+    /* make sure the gfid received is valid */
+    GF_VALIDATE_OR_GOTO("glfs_h_create_from_handle",
+                        !(gf_uuid_is_null(loc.gfid)), out);
+
+    newinode = inode_find(subvol->itable, loc.gfid);
+    if (newinode) {
+        if (!stat) /* No need of lookup */
+            goto found;
+
+        lookup_needed = inode_needs_lookup(newinode, THIS);
+        if (lookup_needed) {
+            loc.inode = newinode;
+        } else {
+            /* populate loc */
+            GLFS_LOC_FILL_INODE(newinode, loc, fill_out);
+
+            /* fop/op */
+            ret = syncop_stat(subvol, &loc, &iatt, NULL, NULL);
+            DECODE_SYNCOP_ERR(ret);
+
+            if (ret) {
+            fill_out:
+                /* Drop the reference hold in inode_find */
+                inode_unref(newinode);
+                goto out;
+            }
+
+            glfs_iatt_to_stat(fs, &iatt, stat);
+            goto found;
+        }
+    } else {
+        loc.inode = inode_new(subvol->itable);
+        if (!loc.inode) {
+            errno = ENOMEM;
+            goto out;
+        }
+    }
+
+    ret = syncop_lookup(subvol, &loc, &iatt, 0, 0, 0);
+    DECODE_SYNCOP_ERR(ret);
+    if (ret) {
+        gf_smsg(subvol->name, GF_LOG_WARNING, errno,
+                API_MSG_INODE_REFRESH_FAILED, "gfid=%s", uuid_utoa(loc.gfid),
+                "error=%s", strerror(errno), NULL);
+        goto out;
+    }
+
+    newinode = inode_link(loc.inode, 0, 0, &iatt);
+    if (newinode) {
+        if (newinode == loc.inode) {
+            inode_ctx_set(newinode, THIS, &ctx_value);
+        }
+        inode_lookup(newinode);
+    } else {
+        gf_smsg(subvol->name, GF_LOG_WARNING, errno, API_MSG_INODE_LINK_FAILED,
+                "gfid=%s", uuid_utoa(loc.gfid), NULL);
+        goto out;
+    }
+
+    /* populate stat */
+    if (stat)
+        glfs_iatt_to_stat(fs, &iatt, stat);
+
+found:
+    object = GF_CALLOC(1, sizeof(struct glfs_object), glfs_mt_glfs_object_t);
+    if (object == NULL) {
+        errno = ENOMEM;
+        ret = -1;
+        goto out;
+    }
+
+    /* populate the return object */
+    object->inode = newinode;
+    gf_uuid_copy(object->gfid, object->inode->gfid);
+
+out:
+    /* TODO: Check where the inode ref is being held? */
+    loc_wipe(&loc);
+
+    glfs_subvol_done(fs, subvol);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return object;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_close, 3.4.2)
+int
+pub_glfs_h_close(struct glfs_object *object)
+{
+    /* since glfs_h_* objects hold a reference to inode
+     * it is safe to keep lookup count to '0' */
+    inode_forget(object->inode, 0);
+    inode_unref(object->inode);
+    GF_FREE(object);
+
+    return 0;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_truncate, 3.4.2)
+int
+pub_glfs_h_truncate(struct glfs *fs, struct glfs_object *object, off_t offset)
+{
+    loc_t loc = {
+        0,
+    };
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+
+    DECLARE_OLD_THIS;
+
+    /* validate in args */
+    if (object == NULL) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, object);
+    if (!inode) {
+        errno = ESTALE;
+        goto out;
+    }
+
+    GLFS_LOC_FILL_INODE(inode, loc, out);
+
+    /* fop/op */
+    ret = syncop_truncate(subvol, &loc, (off_t)offset, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    /* populate out args */
+    if (ret == 0)
+        ret = glfs_loc_unlink(&loc);
+
+out:
+    loc_wipe(&loc);
+
+    if (inode)
+        inode_unref(inode);
+
+    glfs_subvol_done(fs, subvol);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_symlink, 3.4.2)
+struct glfs_object *
+pub_glfs_h_symlink(struct glfs *fs, struct glfs_object *parent,
+                   const char *name, const char *data, struct stat *stat)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    uuid_t gfid;
+    dict_t *xattr_req = NULL;
+    struct glfs_object *object = NULL;
+
+    DECLARE_OLD_THIS;
+
+    /* validate in args */
+    if ((parent == NULL) || (name == NULL) || (data == NULL)) {
+        errno = EINVAL;
+        return NULL;
+    }
+
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, parent);
+    if (!inode) {
+        errno = ESTALE;
+        goto out;
+    }
+
+    xattr_req = dict_new();
+    if (!xattr_req) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gf_uuid_generate(gfid);
+    ret = dict_set_gfuuid(xattr_req, "gfid-req", gfid, true);
+    if (ret) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    GLFS_LOC_FILL_PINODE(inode, loc, ret, errno, out, name);
+
+    /* fop/op */
+    ret = syncop_symlink(subvol, &loc, data, &iatt, xattr_req, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    /* populate out args */
+    if (ret == 0) {
+        ret = glfs_loc_link(&loc, &iatt);
+        if (ret != 0) {
+            goto out;
+        }
+
+        if (stat)
+            glfs_iatt_to_stat(fs, &iatt, stat);
+
+        ret = glfs_create_object(&loc, &object);
+    }
+
+out:
+    if (ret && object != NULL) {
+        pub_glfs_h_close(object);
+        object = NULL;
+    }
+
+    loc_wipe(&loc);
+
+    if (inode)
+        inode_unref(inode);
+
+    if (xattr_req)
+        dict_unref(xattr_req);
+
+    glfs_subvol_done(fs, subvol);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return object;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_readlink, 3.4.2)
+int
+pub_glfs_h_readlink(struct glfs *fs, struct glfs_object *object, char *buf,
+                    size_t bufsiz)
+{
+    loc_t loc = {
+        0,
+    };
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    char *linkval = NULL;
+
+    DECLARE_OLD_THIS;
+
+    /* validate in args */
+    if ((object == NULL) || (buf == NULL)) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, object);
+    if (!inode) {
+        errno = ESTALE;
+        goto out;
+    }
+
+    GLFS_LOC_FILL_INODE(inode, loc, out);
+
+    /* fop/op */
+    ret = syncop_readlink(subvol, &loc, &linkval, bufsiz, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    /* populate out args */
+    if (ret > 0)
+        memcpy(buf, linkval, ret);
+
+out:
+    loc_wipe(&loc);
+
+    if (inode)
+        inode_unref(inode);
+
+    if (linkval)
+        GF_FREE(linkval);
+
+    glfs_subvol_done(fs, subvol);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_link, 3.4.2)
+int
+pub_glfs_h_link(struct glfs *fs, struct glfs_object *linksrc,
+                struct glfs_object *parent, const char *name)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    inode_t *pinode = NULL;
+    loc_t oldloc = {
+        0,
+    };
+    loc_t newloc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+
+    DECLARE_OLD_THIS;
+
+    /* validate in args */
+    if ((linksrc == NULL) || (parent == NULL) || (name == NULL)) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, linksrc);
+    if (!inode) {
+        errno = ESTALE;
+        goto out;
+    }
+
+    if (inode->ia_type == IA_IFDIR) {
+        ret = -1;
+        errno = EISDIR;
+        goto out;
+    }
+
+    GLFS_LOC_FILL_INODE(inode, oldloc, out);
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    pinode = glfs_resolve_inode(fs, subvol, parent);
+    if (!pinode) {
+        errno = ESTALE;
+        goto out;
+    }
+
+    /* setup newloc based on parent */
+    newloc.parent = inode_ref(pinode);
+    newloc.name = name;
+    ret = glfs_loc_touchup(&newloc);
+    if (ret != 0) {
+        errno = EINVAL;
+        goto out;
+    }
+
+    /* Filling the inode of the hard link to be same as that of the
+     * original file
+     */
+    newloc.inode = inode_ref(inode);
+
+    /* fop/op */
+    ret = syncop_link(subvol, &oldloc, &newloc, &iatt, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    if (ret == 0)
+        ret = glfs_loc_link(&newloc, &iatt);
+out:
+    loc_wipe(&oldloc);
+    loc_wipe(&newloc);
+
+    if (inode)
+        inode_unref(inode);
+
+    if (pinode)
+        inode_unref(pinode);
+
+    glfs_subvol_done(fs, subvol);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_rename, 3.4.2)
+int
+pub_glfs_h_rename(struct glfs *fs, struct glfs_object *olddir,
+                  const char *oldname, struct glfs_object *newdir,
+                  const char *newname)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    inode_t *oldpinode = NULL;
+    inode_t *newpinode = NULL;
+    loc_t oldloc = {
+        0,
+    };
+    loc_t newloc = {
+        0,
+    };
+    struct iatt oldiatt = {
+        0,
+    };
+    struct iatt newiatt = {
+        0,
+    };
+
+    DECLARE_OLD_THIS;
+
+    /* validate in args */
+    if ((olddir == NULL) || (oldname == NULL) || (newdir == NULL) ||
+        (newname == NULL)) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    oldpinode = glfs_resolve_inode(fs, subvol, olddir);
+    if (!oldpinode) {
+        errno = ESTALE;
+        goto out;
+    }
+
+    ret = glfs_resolve_at(fs, subvol, oldpinode, oldname, &oldloc, &oldiatt, 0,
+                          0);
+    if (ret != 0) {
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    newpinode = glfs_resolve_inode(fs, subvol, newdir);
+    if (!newpinode) {
+        errno = ESTALE;
+        goto out;
+    }
+
+    ret = glfs_resolve_at(fs, subvol, newpinode, newname, &newloc, &newiatt, 0,
+                          0);
+
+    if (ret && errno != ENOENT && newloc.parent)
+        goto out;
+
+    if (newiatt.ia_type != IA_INVAL) {
+        if ((oldiatt.ia_type == IA_IFDIR) != (newiatt.ia_type == IA_IFDIR)) {
+            /* Either both old and new must be dirs,
+             * or both must be non-dirs. Else, fail.
+             */
+            ret = -1;
+            errno = EEXIST;
+            goto out;
+        }
+    }
+
+    /* TODO: check if new or old is a prefix of the other, and fail EINVAL */
+
+    ret = syncop_rename(subvol, &oldloc, &newloc, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    if (ret == 0) {
+        inode_rename(oldloc.parent->table, oldloc.parent, oldloc.name,
+                     newloc.parent, newloc.name, oldloc.inode, &oldiatt);
+
+        if (newloc.inode && !inode_has_dentry(newloc.inode))
+            inode_forget(newloc.inode, 0);
+    }
+
+out:
+    loc_wipe(&oldloc);
+    loc_wipe(&newloc);
+
+    if (oldpinode)
+        inode_unref(oldpinode);
+
+    if (newpinode)
+        inode_unref(newpinode);
+
+    glfs_subvol_done(fs, subvol);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
+
+/*
+ * Given a handle/gfid, find if the corresponding inode is present in
+ * the inode table. If yes create and return the corresponding glfs_object.
+ */
+struct glfs_object *
+glfs_h_find_handle(struct glfs *fs, unsigned char *handle, int len)
+{
+    inode_t *newinode = NULL;
+    xlator_t *subvol = NULL;
+    struct glfs_object *object = NULL;
+    uuid_t gfid;
+
+    /* validate in args */
+    if ((fs == NULL) || (handle == NULL) || (len != GFAPI_HANDLE_LENGTH)) {
+        errno = EINVAL;
+        return NULL;
+    }
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        errno = EIO;
+        goto out;
+    }
+
+    memcpy(gfid, handle, GFAPI_HANDLE_LENGTH);
+
+    /* make sure the gfid received is valid */
+    GF_VALIDATE_OR_GOTO("glfs_h_find_handle", !(gf_uuid_is_null(gfid)), out);
+
+    newinode = inode_find(subvol->itable, gfid);
+    if (!newinode) {
+        goto out;
+    }
+
+    object = GF_CALLOC(1, sizeof(struct glfs_object), glfs_mt_glfs_object_t);
+    if (object == NULL) {
+        errno = ENOMEM;
+        goto out;
+    }
+
+    /* populate the return object. The ref taken here
+     * is un'refed when the application does glfs_h_close() */
+    object->inode = inode_ref(newinode);
+    gf_uuid_copy(object->gfid, object->inode->gfid);
+
+out:
+    /* inode_find takes a reference. Unref it. */
+    if (newinode)
+        inode_unref(newinode);
+
+    glfs_subvol_done(fs, subvol);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return object;
+}
+
+static void
+glfs_free_upcall_inode(void *to_free)
+{
+    struct glfs_upcall_inode *arg = to_free;
+
+    if (!arg)
+        return;
+
+    if (arg->object)
+        glfs_h_close(arg->object);
+    if (arg->p_object)
+        glfs_h_close(arg->p_object);
+    if (arg->oldp_object)
+        glfs_h_close(arg->oldp_object);
+
+    GF_FREE(arg);
+}
+
+int
+glfs_h_poll_cache_invalidation(struct glfs *fs, struct glfs_upcall *up_arg,
+                               struct gf_upcall *upcall_data)
+{
+    int ret = -1;
+    struct glfs_object *p_object = NULL;
+    struct glfs_object *oldp_object = NULL;
+    struct glfs_object *object = NULL;
+    struct gf_upcall_cache_invalidation *ca_data = NULL;
+    struct glfs_upcall_inode *up_inode_arg = NULL;
+
+    ca_data = upcall_data->data;
+    GF_VALIDATE_OR_GOTO("glfs_h_poll_cache_invalidation", ca_data, out);
+
+    object = glfs_h_find_handle(fs, upcall_data->gfid, GFAPI_HANDLE_LENGTH);
+    if (!object) {
+        /* The reason handle creation will fail is because we
+         * couldn't find the inode in the gfapi inode table.
+         *
+         * But since application would have taken inode_ref, the
+         * only case when this can happen is when it has closed
+         * the handle and hence will no more be interested in
+         * the upcall for this particular gfid.
+         */
+        gf_smsg(THIS->name, GF_LOG_DEBUG, errno, API_MSG_CREATE_HANDLE_FAILED,
+                "gfid=%s", uuid_utoa(upcall_data->gfid), NULL);
+        errno = ESTALE;
+        goto out;
+    }
+
+    up_inode_arg = GF_CALLOC(1, sizeof(struct glfs_upcall_inode),
+                             glfs_mt_upcall_inode_t);
+    GF_VALIDATE_OR_GOTO("glfs_h_poll_cache_invalidation", up_inode_arg, out);
+
+    up_inode_arg->object = object;
+    up_inode_arg->flags = ca_data->flags;
+    up_inode_arg->expire_time_attr = ca_data->expire_time_attr;
+
+    /* XXX: Update stat as well in case of UP_*_TIMES.
+     * This will be addressed as part of INODE_UPDATE */
+    if (ca_data->flags & GFAPI_INODE_UPDATE_FLAGS) {
+        glfs_iatt_to_stat(fs, &ca_data->stat, &up_inode_arg->buf);
+    }
+
+    if (ca_data->flags & GFAPI_UP_PARENT_TIMES) {
+        p_object = glfs_h_find_handle(fs, ca_data->p_stat.ia_gfid,
+                                      GFAPI_HANDLE_LENGTH);
+        if (!p_object) {
+            gf_smsg(THIS->name, GF_LOG_DEBUG, errno,
+                    API_MSG_CREATE_HANDLE_FAILED, "gfid=%s",
+                    uuid_utoa(ca_data->p_stat.ia_gfid), NULL);
+            errno = ESTALE;
+            goto out;
+        }
+
+        glfs_iatt_to_stat(fs, &ca_data->p_stat, &up_inode_arg->p_buf);
+    }
+    up_inode_arg->p_object = p_object;
+
+    /* In case of RENAME, update old parent as well */
+    if (ca_data->flags & GFAPI_UP_RENAME) {
+        oldp_object = glfs_h_find_handle(fs, ca_data->oldp_stat.ia_gfid,
+                                         GFAPI_HANDLE_LENGTH);
+        if (!oldp_object) {
+            gf_smsg(THIS->name, GF_LOG_DEBUG, errno,
+                    API_MSG_CREATE_HANDLE_FAILED, "gfid=%s",
+                    uuid_utoa(ca_data->oldp_stat.ia_gfid), NULL);
+            errno = ESTALE;
+            /* By the time we receive upcall old parent_dir may
+             * have got removed. We still need to send upcall
+             * for the file/dir and current parent handles. */
+            up_inode_arg->oldp_object = NULL;
+            ret = 0;
+        }
+
+        glfs_iatt_to_stat(fs, &ca_data->oldp_stat, &up_inode_arg->oldp_buf);
+    }
+    up_inode_arg->oldp_object = oldp_object;
+
+    up_arg->reason = GLFS_UPCALL_INODE_INVALIDATE;
+    up_arg->event = up_inode_arg;
+    up_arg->free_event = glfs_free_upcall_inode;
+
+    ret = 0;
+
+out:
+    if (ret) {
+        /* Close p_object and oldp_object as well if being referenced.*/
+        if (object)
+            glfs_h_close(object);
+
+        /* Set reason to prevent applications from using ->event */
+        up_arg->reason = GLFS_UPCALL_EVENT_NULL;
+        GF_FREE(up_inode_arg);
+    }
+    return ret;
+}
+
+void
+glfs_release_upcall(void *ptr)
+{
+    struct glfs_upcall *to_free = ptr;
+
+    if (to_free->event)
+        to_free->free_event(to_free->event);
+}
+
+/*
+ * This API is used to poll for upcall events stored in the upcall list.
+ * Current users of this API is NFS-Ganesha. In case of any event received, it
+ * will be mapped appropriately into 'glfs_upcall' along with the handle object
+ * to be passed to NFS-Ganesha.
+ *
+ * On success, applications need to check if up_arg is not-NULL or errno is not
+ * ENOENT. glfs_upcall_get_reason() can be used to decide what kind of event
+ * has been received.
+ *
+ * Current supported upcall_events:
+ *      GLFS_UPCALL_INODE_INVALIDATE
+ *
+ * After processing the event, applications need to free 'up_arg' by calling
+ * glfs_free().
+ *
+ * Also similar to I/Os, the application should ideally stop polling before
+ * calling glfs_fini(..). Hence making an assumption that 'fs' & ctx structures
+ * cannot be freed while in this routine.
+ */
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_poll_upcall, 3.7.16)
+int
+pub_glfs_h_poll_upcall(struct glfs *fs, struct glfs_upcall **up_arg)
+{
+    upcall_entry *u_list = NULL;
+    upcall_entry *tmp = NULL;
+    xlator_t *subvol = NULL;
+    glusterfs_ctx_t *ctx = NULL;
+    int ret = -1;
+    struct gf_upcall *upcall_data = NULL;
+
+    DECLARE_OLD_THIS;
+
+    if (!up_arg) {
+        errno = EINVAL;
+        goto err;
+    }
+
+    __GLFS_ENTRY_VALIDATE_FS(fs, err);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        errno = EIO;
+        goto restore;
+    }
+
+    /* Ideally applications should stop polling before calling
+     * 'glfs_fini'. Yet cross check if cleanup has started. */
+    pthread_mutex_lock(&fs->mutex);
+    {
+        ctx = fs->ctx;
+
+        if (ctx->cleanup_started) {
+            pthread_mutex_unlock(&fs->mutex);
+            goto out;
+        }
+
+        fs->pin_refcnt++;
+
+        /* once we call this function, the applications seems to be
+         * interested in events, enable caching them */
+        fs->cache_upcalls = _gf_true;
+    }
+    pthread_mutex_unlock(&fs->mutex);
+
+    pthread_mutex_lock(&fs->upcall_list_mutex);
+    {
+        list_for_each_entry_safe(u_list, tmp, &fs->upcall_list, upcall_list)
+        {
+            list_del_init(&u_list->upcall_list);
+            upcall_data = &u_list->upcall_data;
+            break;
+        }
+    }
+    /* No other thread can delete this entry. So unlock it */
+    pthread_mutex_unlock(&fs->upcall_list_mutex);
+
+    if (upcall_data) {
+        switch (upcall_data->event_type) {
+            case GF_UPCALL_CACHE_INVALIDATION:
+                *up_arg = GLFS_CALLOC(1, sizeof(struct gf_upcall),
+                                      glfs_release_upcall,
+                                      glfs_mt_upcall_entry_t);
+                if (!*up_arg) {
+                    errno = ENOMEM;
+                    break; /* goto free u_list */
+                }
+
+                /* XXX: Need to revisit this to support
+                 * GLFS_UPCALL_INODE_UPDATE if required. */
+                ret = glfs_h_poll_cache_invalidation(fs, *up_arg, upcall_data);
+                if (ret || (*up_arg)->reason == GLFS_UPCALL_EVENT_NULL) {
+                    /* It could so happen that the file which got
+                     * upcall notification may have got deleted by
+                     * the same client. Irrespective of the error,
+                     * return with an error or success+ENOENT. */
+                    if ((*up_arg)->reason == GLFS_UPCALL_EVENT_NULL)
+                        errno = ENOENT;
+
+                    GLFS_FREE(*up_arg);
+                    *up_arg = NULL;
+                }
+                break;
+            case GF_UPCALL_RECALL_LEASE:
+                gf_log("glfs_h_poll_upcall", GF_LOG_DEBUG,
+                       "UPCALL_RECALL_LEASE is not implemented yet");
+            /* fallthrough till we support leases */
+            case GF_UPCALL_EVENT_NULL:
+                /* no 'default:' label, to force handling all upcall events */
+                errno = ENOENT;
+                break;
+        }
+
+        GF_FREE(u_list->upcall_data.data);
+        GF_FREE(u_list);
+    } else {
+        /* fs->upcall_list was empty, no upcall events cached */
+        errno = ENOENT;
+    }
+
+    ret = 0;
+
+out:
+    pthread_mutex_lock(&fs->mutex);
+    {
+        fs->pin_refcnt--;
+    }
+    pthread_mutex_unlock(&fs->mutex);
+
+    glfs_subvol_done(fs, subvol);
+
+restore:
+    __GLFS_EXIT_FS;
+err:
+    return ret;
+}
+
+static gf_boolean_t log_upcall370 = _gf_true; /* log once */
+
+/* The old glfs_h_poll_upcall interface requires intimate knowledge of the
+ * structures that are returned to the calling application. This is not
+ * recommended, as the returned structures need to returned correctly (handles
+ * closed, memory free'd with the unavailable GF_FREE(), and possibly more.)
+ *
+ * To the best of our knowledge, only NFS-Ganesha uses the upcall events
+ * through gfapi. We keep this backwards compatibility function around so that
+ * applications using the existing implementation do not break.
+ *
+ * WARNING: this function will be removed in the future.
+ */
+GFAPI_SYMVER_PUBLIC(glfs_h_poll_upcall370, glfs_h_poll_upcall, 3.7.0)
+int
+pub_glfs_h_poll_upcall370(struct glfs *fs, struct glfs_callback_arg *up_arg)
+{
+    struct glfs_upcall *upcall = NULL;
+    int ret = -1;
+
+    if (log_upcall370) {
+        log_upcall370 = _gf_false;
+        gf_log(THIS->name, GF_LOG_WARNING,
+               "this application is "
+               "compiled against an old version of libgfapi, it "
+               "should use glfs_free() to release the structure "
+               "returned by glfs_h_poll_upcall() - for more details, "
+               "see http://review.gluster.org/14701");
+    }
+
+    ret = pub_glfs_h_poll_upcall(fs, &upcall);
+    if (ret == 0) {
+        up_arg->fs = fs;
+        if ((errno == ENOENT) || !upcall || !upcall->event) {
+            up_arg->reason = GLFS_UPCALL_EVENT_NULL;
+            goto out;
+        }
+
+        up_arg->reason = upcall->reason;
+
+        if (upcall->reason == GLFS_UPCALL_INODE_INVALIDATE) {
+            struct glfs_callback_inode_arg *cb_inode = NULL;
+            struct glfs_upcall_inode *up_inode = NULL;
+
+            cb_inode = GF_CALLOC(1, sizeof(struct glfs_callback_inode_arg),
+                                 glfs_mt_upcall_inode_t);
+            if (!cb_inode) {
+                errno = ENOMEM;
+                ret = -1;
+                goto out;
+            }
+
+            up_inode = upcall->event;
+
+            /* copy attributes one by one, the memory layout might
+             * be different between the old glfs_callback_inode_arg
+             * and new glfs_upcall_inode */
+            cb_inode->object = up_inode->object;
+            cb_inode->flags = up_inode->flags;
+            memcpy(&cb_inode->buf, &up_inode->buf, sizeof(struct stat));
+            cb_inode->expire_time_attr = up_inode->expire_time_attr;
+            cb_inode->p_object = up_inode->p_object;
+            memcpy(&cb_inode->p_buf, &up_inode->p_buf, sizeof(struct stat));
+            cb_inode->oldp_object = up_inode->oldp_object;
+            memcpy(&cb_inode->oldp_buf, &up_inode->oldp_buf,
+                   sizeof(struct stat));
+
+            up_arg->event_arg = cb_inode;
+        }
+    }
+
+out:
+    if (upcall) {
+        /* we can not use glfs_free() here, objects need to stay */
+        GF_FREE(upcall->event);
+        GF_FREE(upcall);
+    }
+
+    return ret;
+}
+
+#ifdef HAVE_ACL_LIBACL_H
+#include <glusterfs/glusterfs-acl.h>
+#include <acl/libacl.h>
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_acl_set, 3.7.0)
+int
+pub_glfs_h_acl_set(struct glfs *fs, struct glfs_object *object,
+                   const acl_type_t type, const acl_t acl)
+{
+    int ret = -1;
+    char *acl_s = NULL;
+    const char *acl_key = NULL;
+    struct glfs_object *new_object = NULL;
+
+    DECLARE_OLD_THIS;
+
+    if (!object || !acl) {
+        errno = EINVAL;
+        return ret;
+    }
+
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    acl_key = gf_posix_acl_get_key(type);
+    if (!acl_key)
+        goto out;
+
+    acl_s = acl_to_any_text(acl, NULL, ',', TEXT_ABBREVIATE | TEXT_NUMERIC_IDS);
+    if (!acl_s)
+        goto out;
+
+    if (IA_ISLNK(object->inode->ia_type)) {
+        new_object = glfs_h_resolve_symlink(fs, object);
+        if (new_object == NULL)
+            goto out;
+    } else
+        new_object = object;
+
+    ret = pub_glfs_h_setxattrs(fs, new_object, acl_key, acl_s,
+                               strlen(acl_s) + 1, 0);
+
+    acl_free(acl_s);
+
+out:
+    if (IA_ISLNK(object->inode->ia_type) && new_object)
+        glfs_h_close(new_object);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_acl_get, 3.7.0)
+acl_t
+pub_glfs_h_acl_get(struct glfs *fs, struct glfs_object *object,
+                   const acl_type_t type)
+{
+    int ret = 0;
+    acl_t acl = NULL;
+    char *acl_s = NULL;
+    dict_t *xattr = NULL;
+    const char *acl_key = NULL;
+    struct glfs_object *new_object = NULL;
+
+    DECLARE_OLD_THIS;
+
+    if (!object) {
+        errno = EINVAL;
+        return NULL;
+    }
+
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    acl_key = gf_posix_acl_get_key(type);
+    if (!acl_key)
+        goto out;
+
+    if (IA_ISLNK(object->inode->ia_type)) {
+        new_object = glfs_h_resolve_symlink(fs, object);
+        if (new_object == NULL)
+            goto out;
+    } else
+        new_object = object;
+
+    ret = glfs_h_getxattrs_common(fs, new_object, &xattr, acl_key, _gf_false);
+    if (ret)
+        goto out;
+
+    ret = dict_get_str(xattr, (char *)acl_key, &acl_s);
+    if (ret)
+        goto out;
+
+    acl = acl_from_text(acl_s);
+
+out:
+    if (xattr)
+        dict_unref(xattr);
+
+    if (IA_ISLNK(object->inode->ia_type) && new_object)
+        glfs_h_close(new_object);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return acl;
+}
+#else /* !HAVE_ACL_LIBACL_H */
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_acl_get, 3.7.0)
+acl_t
+pub_glfs_h_acl_get(struct glfs *fs, struct glfs_object *object,
+                   const acl_type_t type)
+{
+    errno = ENOTSUP;
+    return NULL;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_acl_set, 3.7.0)
+int
+pub_glfs_h_acl_set(struct glfs *fs, struct glfs_object *object,
+                   const acl_type_t type, const acl_t acl)
+{
+    errno = ENOTSUP;
+    return -1;
+}
+#endif
+
+/* The API to perform read using anonymous fd */
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_anonymous_read, 3.7.0)
+ssize_t
+pub_glfs_h_anonymous_read(struct glfs *fs, struct glfs_object *object,
+                          const void *buf, size_t count, off_t offset)
+{
+    struct iovec iov = {
+        0,
+    };
+    ssize_t ret = 0;
+
+    /* validate in args */
+    if ((fs == NULL) || (object == NULL)) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    iov.iov_base = (void *)buf;
+    iov.iov_len = count;
+
+    ret = glfs_anonymous_preadv(fs, object, &iov, 1, offset, 0);
+
+    return ret;
+}
+
+/* The API to perform write using anonymous fd */
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_anonymous_write, 3.7.0)
+ssize_t
+pub_glfs_h_anonymous_write(struct glfs *fs, struct glfs_object *object,
+                           const void *buf, size_t count, off_t offset)
+{
+    struct iovec iov = {
+        0,
+    };
+    ssize_t ret = 0;
+
+    /* validate in args */
+    if ((fs == NULL) || (object == NULL)) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    iov.iov_base = (void *)buf;
+    iov.iov_len = count;
+
+    ret = glfs_anonymous_pwritev(fs, object, &iov, 1, offset, 0);
+
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_object_copy, 3.11.0)
+struct glfs_object *
+pub_glfs_object_copy(struct glfs_object *src)
+{
+    struct glfs_object *object = NULL;
+
+    GF_VALIDATE_OR_GOTO("glfs_dup_object", src, out);
+
+    object = GF_CALLOC(1, sizeof(struct glfs_object), glfs_mt_glfs_object_t);
+    if (object == NULL) {
+        errno = ENOMEM;
+        gf_smsg(THIS->name, GF_LOG_WARNING, errno, API_MSG_CREATE_HANDLE_FAILED,
+                "glfs_dup_object gfid=%s", uuid_utoa(src->inode->gfid), NULL);
+        return NULL;
+    }
+
+    object->inode = inode_ref(src->inode);
+    gf_uuid_copy(object->gfid, src->inode->gfid);
+
+out:
+    return object;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_xreaddirplus_get_object, 3.11.0)
+struct glfs_object *
+pub_glfs_xreaddirplus_get_object(struct glfs_xreaddirp_stat *xstat)
+{
+    GF_VALIDATE_OR_GOTO("glfs_xreaddirplus_get_object", xstat, out);
+
+    if (!(xstat->flags_handled & GFAPI_XREADDIRP_HANDLE))
+        gf_smsg(THIS->name, GF_LOG_ERROR, errno, API_MSG_HANDLE_NOT_SET,
+                "GFAPI_XREADDIRP_HANDLE xstat=%p", xstat, "handle=%x",
+                xstat->flags_handled, NULL);
+
+    return xstat->object;
+
+out:
+    return NULL;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_lease, 4.0.0)
+int
+pub_glfs_h_lease(struct glfs *fs, struct glfs_object *object,
+                 struct glfs_lease *lease)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct gf_lease gf_lease = {
+        0,
+    };
+
+    /* validate in args */
+    if ((fs == NULL) || (object == NULL)) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, object);
+    if (!inode) {
+        errno = ESTALE;
+        goto out;
+    }
+
+    /* populate loc */
+    GLFS_LOC_FILL_INODE(inode, loc, out);
+
+    glfs_lease_to_gf_lease(lease, &gf_lease);
+
+    ret = syncop_lease(subvol, &loc, &gf_lease, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    gf_lease_to_glfs_lease(&gf_lease, lease);
+
+out:
+    loc_wipe(&loc);
+
+    if (inode)
+        inode_unref(inode);
+
+    glfs_subvol_done(fs, subvol);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
diff --git a/api/src/glfs-handles.h b/api/src/glfs-handles.h
new file mode 100644
index 00000000000..4d039b9c76b
--- /dev/null
+++ b/api/src/glfs-handles.h
@@ -0,0 +1,355 @@
+/*
+  Copyright (c) 2013-2018 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _GLFS_HANDLES_H
+#define _GLFS_HANDLES_H
+
+#include "glfs.h"
+
+/* GLFS OBJECT BASED OPERATIONS
+ *
+ * The following APIs are introduced to provide an API framework that can work
+ * with gluster objects (files and directories), instead of absolute paths.
+ *
+ * The following API set can be related to the POSIX *at interfaces (like
+ * openat (2)). The intention of these APIs is to be able to operate based
+ * on parent object and looking up or creating child objects within, OR to be
+ * used on the actual object thus looked up or created, and retrieve information
+ * regarding the same.
+ *
+ * The APIs also provide for generating an opaque invariant handle to the
+ * object, that can later be used to lookup the object, instead of the regular
+ * glfs_h_* variants. The APIs that provide this behaviour are,
+ * glfs_h_extract_handle and glfs_h_create_from_handle.
+ *
+ * The object handles can be transitioned to fd based operations as supported
+ * by glfs.h calls, using the glfs_h_open call. This provides a way to move
+ * from objects to fd's akin to moving from path to fd for required operations.
+ *
+ * NOTE: The opaque invariant handle is the GFID of the object in reality, but
+ * maintained as an opaque data value, for potential internal changes to the
+ * same without impacting the caller.
+ *
+ * NOTE: Currently looking up an object can create multiple object handles to
+ * the same, i.e distinct glfs_object *. Hence each such looked up or received
+ * handle from other calls, would need to be closed. In the future, for a given
+ * object these pointers would be the same, and an ease of use API to forget all
+ * instances of this bject would be provided (instead of a per lookup close).
+ * This should not change the APIs in their current form.
+ *
+ */
+
+/* Handle length for object handles returned from glfs_h_extract_handle or
+ * glfs_h_create_from_handle */
+#define GFAPI_HANDLE_LENGTH 16
+
+/* These flags should be in sync to the ones defined in upcall.h */
+#define GFAPI_UP_NLINK 0x00000001 /* update nlink */
+#define GFAPI_UP_MODE 0x00000002  /* update mode and ctime */
+#define GFAPI_UP_OWN 0x00000004   /* update mode,uid,gid and ctime */
+#define GFAPI_UP_SIZE 0x00000008  /* update fsize */
+#define GFAPI_UP_TIMES 0x00000010 /* update all times */
+#define GFAPI_UP_ATIME 0x00000020 /* update atime only */
+#define GFAPI_UP_PERM                                                          \
+    0x00000040 /* update fields needed for                                     \
+                  permission checking */
+#define GFAPI_UP_RENAME                                                        \
+    0x00000080 /* this is a rename op -                                        \
+                  delete the cache entry */
+#define GFAPI_UP_FORGET                                                        \
+    0x00000100                           /* inode_forget on server side -      \
+                                            invalidate the cache entry */
+#define GFAPI_UP_PARENT_TIMES 0x00000200 /* update parent dir times */
+
+#define GFAPI_INODE_UPDATE_FLAGS                                               \
+    (GFAPI_UP_NLINK | GFAPI_UP_MODE | GFAPI_UP_OWN | GFAPI_UP_SIZE |           \
+     GFAPI_UP_TIMES | GFAPI_UP_ATIME)
+
+/* Portability non glibc c++ build systems */
+#ifndef __THROW
+#if defined __cplusplus
+#define __THROW throw()
+#else
+#define __THROW
+#endif
+#endif
+
+__BEGIN_DECLS
+
+/*
+ * Notes:
+ *
+ * The file object handle. One per looked up, created file/directory
+ *
+ * This had been introduced to facilitate gfid/inode based gfapi
+ * - a requirement introduced by nfs-ganesha
+ */
+struct glfs_object;
+typedef struct glfs_object glfs_object_t;
+
+/* Functions for getting details about the glfs_upcall_inode
+ *
+ * None of the pointers returned by the below functions should be free()'d,
+ * glfs_free()'d or glfs_h_close()'d by the application.
+ *
+ * Releasing of the structures is done by passing the glfs_upcall pointer
+ * to glfs_free().
+ */
+struct glfs_upcall_inode;
+typedef struct glfs_upcall_inode glfs_upcall_inode_t;
+
+glfs_object_t *
+glfs_upcall_inode_get_object(glfs_upcall_inode_t *arg) __THROW
+    GFAPI_PUBLIC(glfs_upcall_inode_get_object, 3.7.16);
+
+uint64_t
+glfs_upcall_inode_get_flags(glfs_upcall_inode_t *arg) __THROW
+    GFAPI_PUBLIC(glfs_upcall_inode_get_flags, 3.7.16);
+
+struct stat *
+glfs_upcall_inode_get_stat(glfs_upcall_inode_t *arg) __THROW
+    GFAPI_PUBLIC(glfs_upcall_inode_get_stat, 3.7.16);
+
+uint64_t
+glfs_upcall_inode_get_expire(glfs_upcall_inode_t *arg) __THROW
+    GFAPI_PUBLIC(glfs_upcall_inode_get_expire, 3.7.16);
+
+glfs_object_t *
+glfs_upcall_inode_get_pobject(glfs_upcall_inode_t *arg) __THROW
+    GFAPI_PUBLIC(glfs_upcall_inode_get_pobject, 3.7.16);
+
+struct stat *
+glfs_upcall_inode_get_pstat(glfs_upcall_inode_t *arg) __THROW
+    GFAPI_PUBLIC(glfs_upcall_inode_get_pstat, 3.7.16);
+
+glfs_object_t *
+glfs_upcall_inode_get_oldpobject(glfs_upcall_inode_t *arg) __THROW
+    GFAPI_PUBLIC(glfs_upcall_inode_get_oldpobject, 3.7.16);
+
+struct stat *
+glfs_upcall_inode_get_oldpstat(glfs_upcall_inode_t *arg) __THROW
+    GFAPI_PUBLIC(glfs_upcall_inode_get_oldpstat, 3.7.16);
+
+/* Handle based operations */
+/* Operations that generate handles */
+glfs_object_t *
+glfs_h_lookupat(glfs_t *fs, glfs_object_t *parent, const char *path,
+                struct stat *stat, int follow) __THROW
+    GFAPI_PUBLIC(glfs_h_lookupat, 3.7.4);
+
+glfs_object_t *
+glfs_h_creat(glfs_t *fs, glfs_object_t *parent, const char *path, int flags,
+             mode_t mode, struct stat *sb) __THROW
+    GFAPI_PUBLIC(glfs_h_creat, 3.4.2);
+
+glfs_object_t *
+glfs_h_mkdir(glfs_t *fs, glfs_object_t *parent, const char *path, mode_t flags,
+             struct stat *sb) __THROW GFAPI_PUBLIC(glfs_h_mkdir, 3.4.2);
+
+glfs_object_t *
+glfs_h_mknod(glfs_t *fs, glfs_object_t *parent, const char *path, mode_t mode,
+             dev_t dev, struct stat *sb) __THROW
+    GFAPI_PUBLIC(glfs_h_mknod, 3.4.2);
+
+glfs_object_t *
+glfs_h_symlink(glfs_t *fs, glfs_object_t *parent, const char *name,
+               const char *data, struct stat *stat) __THROW
+    GFAPI_PUBLIC(glfs_h_symlink, 3.4.2);
+
+/* Operations on the actual objects */
+int
+glfs_h_unlink(glfs_t *fs, glfs_object_t *parent, const char *path) __THROW
+    GFAPI_PUBLIC(glfs_h_unlink, 3.4.2);
+
+int
+glfs_h_close(glfs_object_t *object) __THROW GFAPI_PUBLIC(glfs_h_close, 3.4.2);
+
+int
+glfs_caller_specific_init(void *uid_caller_key, void *gid_caller_key,
+                          void *future) __THROW
+    GFAPI_PUBLIC(glfs_caller_specific_init, 3.5.0);
+
+int
+glfs_h_truncate(glfs_t *fs, glfs_object_t *object, off_t offset) __THROW
+    GFAPI_PUBLIC(glfs_h_truncate, 3.4.2);
+
+int
+glfs_h_stat(glfs_t *fs, glfs_object_t *object, struct stat *stat) __THROW
+    GFAPI_PUBLIC(glfs_h_stat, 3.4.2);
+
+int
+glfs_h_statfs(glfs_t *fs, glfs_object_t *object, struct statvfs *stat) __THROW
+    GFAPI_PUBLIC(glfs_h_statfs, 3.7.0);
+
+int
+glfs_h_getattrs(glfs_t *fs, glfs_object_t *object, struct stat *stat) __THROW
+    GFAPI_PUBLIC(glfs_h_getattrs, 3.4.2);
+
+int
+glfs_h_getxattrs(glfs_t *fs, glfs_object_t *object, const char *name,
+                 void *value, size_t size) __THROW
+    GFAPI_PUBLIC(glfs_h_getxattrs, 3.5.1);
+
+int
+glfs_h_setattrs(glfs_t *fs, glfs_object_t *object, struct stat *sb,
+                int valid) __THROW GFAPI_PUBLIC(glfs_h_setattrs, 3.4.2);
+
+int
+glfs_h_setxattrs(glfs_t *fs, glfs_object_t *object, const char *name,
+                 const void *value, size_t size, int flags) __THROW
+    GFAPI_PUBLIC(glfs_h_setxattrs, 3.5.0);
+
+int
+glfs_h_readlink(glfs_t *fs, glfs_object_t *object, char *buf,
+                size_t bufsiz) __THROW GFAPI_PUBLIC(glfs_h_readlink, 3.4.2);
+
+int
+glfs_h_link(glfs_t *fs, glfs_object_t *linktgt, glfs_object_t *parent,
+            const char *name) __THROW GFAPI_PUBLIC(glfs_h_link, 3.4.2);
+
+int
+glfs_h_rename(glfs_t *fs, glfs_object_t *olddir, const char *oldname,
+              glfs_object_t *newdir, const char *newname) __THROW
+    GFAPI_PUBLIC(glfs_h_rename, 3.4.2);
+
+int
+glfs_h_removexattrs(glfs_t *fs, glfs_object_t *object, const char *name) __THROW
+    GFAPI_PUBLIC(glfs_h_removexattrs, 3.5.1);
+
+/* Operations enabling opaque invariant handle to object transitions */
+ssize_t
+glfs_h_extract_handle(glfs_object_t *object, unsigned char *handle,
+                      int len) __THROW
+    GFAPI_PUBLIC(glfs_h_extract_handle, 3.4.2);
+
+/* Given a handle, looks up the inode and creates glfs_object.
+ * In addition, if provided 'stat', copies the inode attributes
+ */
+glfs_object_t *
+glfs_h_create_from_handle(glfs_t *fs, unsigned char *handle, int len,
+                          struct stat *stat) __THROW
+    GFAPI_PUBLIC(glfs_h_create_from_handle, 3.4.2);
+
+/* Operations enabling object handles to fd transitions */
+glfs_fd_t *
+glfs_h_opendir(glfs_t *fs, glfs_object_t *object) __THROW
+    GFAPI_PUBLIC(glfs_h_opendir, 3.4.2);
+
+glfs_fd_t *
+glfs_h_open(glfs_t *fs, glfs_object_t *object, int flags) __THROW
+    GFAPI_PUBLIC(glfs_h_open, 3.4.2);
+
+int
+glfs_h_access(glfs_t *fs, glfs_object_t *object, int mask) __THROW
+    GFAPI_PUBLIC(glfs_h_access, 3.6.0);
+
+struct glfs_object *
+glfs_h_creat_open(struct glfs *fs, struct glfs_object *parent, const char *path,
+                  int flags, mode_t mode, struct stat *stat,
+                  struct glfs_fd **out_fd) __THROW
+    GFAPI_PUBLIC(glfs_h_creat_open, 6.6);
+/*
+  SYNOPSIS
+
+  glfs_h_poll_upcall: Poll for upcall events given a 'glfs' object.
+
+  DESCRIPTION
+
+  This API is used to poll for upcall events stored in the
+  upcall list. Current users of this API is NFS-Ganesha.
+  In case of any event received, it will be mapped appropriately
+  into 'glfs_upcall' along with the handle('glfs_object') to be
+  passed to NFS-Ganesha.
+
+  In case of success, applications need to check the value of
+  cbk->handle to be NON NULL before processing the upcall
+  events.
+
+  PARAMETERS
+
+  @fs: glfs object to poll the upcall events for
+  @cbk: Pointer that will contain an upcall event for use by the application.
+        Application is responsible for free'ing the structure with glfs_free().
+
+  RETURN VALUES
+
+  0   : Success.
+  -1  : Error condition, mostly due to out of memory.
+
+*/
+
+int
+glfs_h_poll_upcall(glfs_t *fs, glfs_upcall_t **cbk) __THROW
+    GFAPI_PUBLIC(glfs_h_poll_upcall, 3.7.16);
+
+int
+glfs_h_acl_set(glfs_t *fs, glfs_object_t *object, const acl_type_t type,
+               const acl_t acl) __THROW GFAPI_PUBLIC(glfs_h_acl_set, 3.7.0);
+
+acl_t
+glfs_h_acl_get(glfs_t *fs, glfs_object_t *object, const acl_type_t type) __THROW
+    GFAPI_PUBLIC(glfs_h_acl_get, 3.7.0);
+
+size_t
+glfs_h_anonymous_write(glfs_t *fs, glfs_object_t *object, const void *buf,
+                       size_t count, off_t offset) __THROW
+    GFAPI_PUBLIC(glfs_h_anonymous_write, 3.7.0);
+
+ssize_t
+glfs_h_anonymous_read(glfs_t *fs, glfs_object_t *object, const void *buf,
+                      size_t count, off_t offset) __THROW
+    GFAPI_PUBLIC(glfs_h_anonymous_read, 3.7.0);
+
+/*
+ * Caution: The object returned by this object gets freed as part
+ * of 'glfs_free(xstat)'. Make sure to have a copy using 'glfs_object_copy()'
+ * to use post that.
+ */
+glfs_object_t *
+glfs_xreaddirplus_get_object(struct glfs_xreaddirp_stat *xstat) __THROW
+    GFAPI_PUBLIC(glfs_xreaddirplus_get_object, 3.11.0);
+
+/* Applications should close the object returned by this routine
+ * explicitly using 'glfs_h_close()'
+ */
+glfs_object_t *
+glfs_object_copy(glfs_object_t *src) __THROW
+    GFAPI_PUBLIC(glfs_object_copy, 3.11.0);
+
+int
+glfs_h_lease(glfs_t *fs, glfs_object_t *object, glfs_lease_t *lease) __THROW
+    GFAPI_PUBLIC(glfs_h_lease, 4.0.0);
+
+glfs_object_t *
+glfs_h_find_handle(glfs_t *fs, unsigned char *handle, int len) __THROW
+    GFAPI_PUBLIC(glfs_h_lease, 4.0.0);
+
+/* Functions for getting details about the glfs_upcall_lease
+ *
+ * None of the pointers returned by the below functions should be free()'d,
+ * glfs_free()'d or glfs_h_close()'d by the application.
+ *
+ * Releasing of the structures is done by passing the glfs_upcall pointer
+ * to glfs_free().
+ */
+struct glfs_upcall_lease;
+typedef struct glfs_upcall_lease glfs_upcall_lease_t;
+
+glfs_object_t *
+glfs_upcall_lease_get_object(glfs_upcall_lease_t *arg) __THROW
+    GFAPI_PUBLIC(glfs_upcall_lease_get_object, 4.1.6);
+
+uint32_t
+glfs_upcall_lease_get_lease_type(glfs_upcall_lease_t *arg) __THROW
+    GFAPI_PUBLIC(glfs_upcall_lease_get_lease_type, 4.1.6);
+
+__END_DECLS
+
+#endif /* !_GLFS_HANDLES_H */
diff --git a/api/src/glfs-internal.h b/api/src/glfs-internal.h
index c7fdf75f541..7cc3b18a104 100644
--- a/api/src/glfs-internal.h
+++ b/api/src/glfs-internal.h
@@ -1,5 +1,5 @@
 /*
-  Copyright (c) 2012 Red Hat, Inc. <http://www.redhat.com>
+  Copyright (c) 2012-2018 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
   This file is licensed to you under your choice of the GNU Lesser
@@ -8,88 +8,484 @@
   cases as published by the Free Software Foundation.
 */
 
-
 #ifndef _GLFS_INTERNAL_H
 #define _GLFS_INTERNAL_H
 
-#include "xlator.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/upcall-utils.h>
+#include "glfs-handles.h"
+#include <glusterfs/refcount.h>
+#include <glusterfs/syncop.h>
 
 #define GLFS_SYMLINK_MAX_FOLLOW 2048
 
+#define DEFAULT_REVAL_COUNT 1
+
+/*
+ * According to  pthread mutex and conditional variable ( cond,
+ * child_down_count, upcall mutex and mutex) initialization of struct glfs
+ * members, below GLFS_INIT_* flags are set in 'pthread_flags' member of struct
+ * glfs. The flags are set from glfs_init() and  glfs_new_from_ctx() functions
+ * as part of fs inititialization.
+ *
+ * These flag bits are validated in glfs_fini() to destroy all or partially
+ * initialized mutex and conditional variables of glfs object.
+ * If you introduce new pthread mutex or conditional variable in glfs object,
+ * please make sure you have a flag bit intorduced here for proper cleanup
+ * in glfs_fini().
+ *
+ */
+
+#define PTHREAD_MUTEX_INIT(mutex, attr, flags, mask, label)                    \
+    do {                                                                       \
+        int __ret = -1;                                                        \
+        __ret = pthread_mutex_init(mutex, attr);                               \
+        if (__ret == 0)                                                        \
+            flags |= mask;                                                     \
+        else                                                                   \
+            goto label;                                                        \
+    } while (0)
+
+#define PTHREAD_MUTEX_DESTROY(mutex, flags, mask)                              \
+    do {                                                                       \
+        if (flags & mask)                                                      \
+            (void)pthread_mutex_destroy(mutex);                                \
+    } while (0)
+
+#define PTHREAD_COND_INIT(cond, attr, flags, mask, label)                      \
+    do {                                                                       \
+        int __ret = -1;                                                        \
+        __ret = pthread_cond_init(cond, attr);                                 \
+        if (__ret == 0)                                                        \
+            flags |= mask;                                                     \
+        else                                                                   \
+            goto label;                                                        \
+    } while (0)
+
+#define PTHREAD_COND_DESTROY(cond, flags, mask)                                \
+    do {                                                                       \
+        if (flags & mask)                                                      \
+            (void)pthread_cond_destroy(cond);                                  \
+    } while (0)
+
+#define GLFS_INIT_MUTEX 0x00000001        /* pthread_mutex_flag */
+#define GLFS_INIT_COND 0x00000002         /* pthread_cond_flag */
+#define GLFS_INIT_COND_CHILD 0x00000004   /* pthread_cond_child_down_flag */
+#define GLFS_INIT_MUTEX_UPCALL 0x00000008 /* pthread_mutex_upcall_flag */
+
+#ifndef GF_DARWIN_HOST_OS
+#ifndef GFAPI_PUBLIC
+#define GFAPI_PUBLIC(sym, ver) /**/
+#endif
+#ifndef GFAPI_PRIVATE
+#define GFAPI_PRIVATE(sym, ver) /**/
+#endif
+#if __GNUC__ >= 10
+#define GFAPI_SYMVER_PUBLIC_DEFAULT(fn, ver)                                   \
+    __attribute__((__symver__(STR(fn) "@@GFAPI_" STR(ver))))
+
+#define GFAPI_SYMVER_PRIVATE_DEFAULT(fn, ver)                                  \
+    __attribute__((__symver__(STR(fn) "@@GFAPI_PRIVATE_" STR(ver))))
+
+#define GFAPI_SYMVER_PUBLIC(fn1, fn2, ver)                                     \
+    __attribute__((__symver__(STR(fn2) "@GFAPI_" STR(ver))))
+
+#define GFAPI_SYMVER_PRIVATE(fn1, fn2, ver)                                    \
+    __attribute__((__symver__(STR(fn2) "@GFAPI_PRIVATE_" STR(ver))))
+
+#else
+#define GFAPI_SYMVER_PUBLIC_DEFAULT(fn, ver)                                   \
+    asm(".symver pub_" STR(fn) ", " STR(fn) "@@GFAPI_" STR(ver));
+
+#define GFAPI_SYMVER_PRIVATE_DEFAULT(fn, ver)                                  \
+    asm(".symver priv_" STR(fn) ", " STR(fn) "@@GFAPI_PRIVATE_" STR(ver));
+
+#define GFAPI_SYMVER_PUBLIC(fn1, fn2, ver)                                     \
+    asm(".symver pub_" STR(fn1) ", " STR(fn2) "@GFAPI_" STR(ver));
+
+#define GFAPI_SYMVER_PRIVATE(fn1, fn2, ver)                                    \
+    asm(".symver priv_" STR(fn1) ", " STR(fn2) "@GFAPI_PRIVATE_" STR(ver));
+#endif
+#define STR(str) #str
+#else
+#ifndef GFAPI_PUBLIC
+#define GFAPI_PUBLIC(sym, ver) __asm("_" __STRING(sym) "$GFAPI_" __STRING(ver));
+#endif
+#ifndef GFAPI_PRIVATE
+#define GFAPI_PRIVATE(sym, ver)                                                \
+    __asm("_" __STRING(sym) "$GFAPI_PRIVATE_" __STRING(ver));
+#endif
+#define GFAPI_SYMVER_PUBLIC_DEFAULT(fn, dotver)  /**/
+#define GFAPI_SYMVER_PRIVATE_DEFAULT(fn, dotver) /**/
+#define GFAPI_SYMVER_PUBLIC(fn1, fn2, dotver)    /**/
+#define GFAPI_SYMVER_PRIVATE(fn1, fn2, dotver)   /**/
+#endif
+
+#define ESTALE_RETRY(ret, errno, reval, loc, label)                            \
+    do {                                                                       \
+        if (ret == -1 && errno == ESTALE) {                                    \
+            if (reval < DEFAULT_REVAL_COUNT) {                                 \
+                reval++;                                                       \
+                loc_wipe(loc);                                                 \
+                goto label;                                                    \
+            }                                                                  \
+        }                                                                      \
+    } while (0)
+
+#define GLFS_LOC_FILL_INODE(oinode, loc, label)                                \
+    do {                                                                       \
+        loc.inode = inode_ref(oinode);                                         \
+        gf_uuid_copy(loc.gfid, oinode->gfid);                                  \
+        ret = glfs_loc_touchup(&loc);                                          \
+        if (ret != 0) {                                                        \
+            errno = EINVAL;                                                    \
+            goto label;                                                        \
+        }                                                                      \
+    } while (0)
+
+#define GLFS_LOC_FILL_PINODE(pinode, loc, ret, errno, label, path)             \
+    do {                                                                       \
+        loc.inode = inode_new(pinode->table);                                  \
+        if (!loc.inode) {                                                      \
+            ret = -1;                                                          \
+            errno = ENOMEM;                                                    \
+            goto label;                                                        \
+        }                                                                      \
+        loc.parent = inode_ref(pinode);                                        \
+        loc.name = path;                                                       \
+        ret = glfs_loc_touchup(&loc);                                          \
+        if (ret != 0) {                                                        \
+            errno = EINVAL;                                                    \
+            goto label;                                                        \
+        }                                                                      \
+    } while (0)
+
 struct glfs;
 
-typedef int (*glfs_init_cbk) (struct glfs *fs, int ret);
+struct _upcall_entry {
+    struct list_head upcall_list;
+    struct gf_upcall upcall_data;
+};
+typedef struct _upcall_entry upcall_entry;
+
+typedef int (*glfs_init_cbk)(struct glfs *fs, int ret);
 
 struct glfs {
-	char               *volname;
-
-	glusterfs_ctx_t    *ctx;
+    char *volname;
+    uuid_t vol_uuid;
+
+    glusterfs_ctx_t *ctx;
+
+    pthread_t poller;
+
+    glfs_init_cbk init_cbk;
+    pthread_mutex_t mutex;
+    pthread_cond_t cond;
+    pthread_cond_t child_down_cond; /* for broadcasting CHILD_DOWN */
+    int init;
+    int ret;
+    int err;
+
+    xlator_t *active_subvol; /* active graph */
+    xlator_t *mip_subvol;    /* graph for which migration is in
+                              * progress */
+    xlator_t *next_subvol;   /* Any new graph is put to
+                              * next_subvol, the graph in
+                              * next_subvol can either be moved
+                              * to mip_subvol (if any IO picks it
+                              * up for migration), or be
+                              * destroyed (if there is a new
+                              * graph, and this was never picked
+                              * for migration) */
+    xlator_t *old_subvol;
+
+    char *oldvolfile;
+    ssize_t oldvollen;
+
+    inode_t *cwd;
+
+    uint32_t dev_id; /* Used to fill st_dev in struct stat */
+
+    struct list_head openfds;
+
+    gf_boolean_t migration_in_progress;
+
+    gf_boolean_t cache_upcalls; /* add upcalls to the upcall_list? */
+    struct list_head upcall_list;
+    pthread_mutex_t upcall_list_mutex; /* mutex for upcall entry list */
+
+    uint32_t pin_refcnt;
+    uint32_t pthread_flags; /* GLFS_INIT_* # defines set this flag */
+
+    uint32_t upcall_events; /* Mask of upcall events application
+                             * is interested in */
+    glfs_upcall_cbk up_cbk; /* upcall cbk function to be registered */
+    void *up_data;          /* Opaque data provided by application
+                             * during upcall registration */
+    struct list_head waitq; /* waiting synctasks */
+};
 
-	pthread_t           poller;
+/* This enum is used to maintain the state of glfd. In case of async fops
+ * fd might be closed before the actual fop is complete. Therefore we need
+ * to track whether the fd is closed or not, instead actually closing it.*/
+enum glfs_fd_state { GLFD_INIT, GLFD_OPEN, GLFD_CLOSE };
 
-	glfs_init_cbk       init_cbk;
-	pthread_mutex_t     mutex;
-	pthread_cond_t      cond;
-	int                 init;
-	int                 ret;
-	int                 err;
+struct glfs_fd {
+    struct list_head openfds;
+    struct list_head list;
+    GF_REF_DECL;
+    struct glfs *fs;
+    enum glfs_fd_state state;
+    off_t offset;
+    fd_t *fd; /* Currently guared by @fs->mutex. TODO: per-glfd lock */
+    struct list_head entries;
+    gf_dirent_t *next;
+    struct dirent *readdirbuf;
+    gf_lkowner_t lk_owner;
+    glfs_leaseid_t lease_id; /* Stores lease_id of client in glfd */
+    gf_lock_t lock;          /* lock taken before updating fd state */
+    glfs_recall_cbk cbk;
+    void *cookie;
+};
 
-	xlator_t           *active_subvol;
-	xlator_t           *next_subvol;
-	xlator_t           *old_subvol;
+/* glfs object handle introduced for the alternate gfapi implementation based
+   on glfs handles/gfid/inode
+*/
+struct glfs_object {
+    inode_t *inode;
+    uuid_t gfid;
+};
 
-	char               *oldvolfile;
-	ssize_t             oldvollen;
+struct glfs_upcall {
+    struct glfs *fs;                /* glfs object */
+    enum glfs_upcall_reason reason; /* Upcall event type */
+    void *event;                    /* changes based in the event type */
+    void (*free_event)(void *);     /* free event after the usage */
+};
 
-	inode_t            *cwd;
+struct glfs_upcall_inode {
+    struct glfs_object *object;      /* Object which need to be acted upon */
+    int flags;                       /* Cache UPDATE/INVALIDATE flags */
+    struct stat buf;                 /* Latest stat of this entry */
+    unsigned int expire_time_attr;   /* the amount of time for which
+                                      * the application need to cache
+                                      * this entry */
+    struct glfs_object *p_object;    /* parent Object to be updated */
+    struct stat p_buf;               /* Latest stat of parent dir handle */
+    struct glfs_object *oldp_object; /* Old parent Object to be updated */
+    struct stat oldp_buf;            /* Latest stat of old parent dir handle */
+};
 
-	uint32_t            dev_id; /* Used to fill st_dev in struct stat */
+struct glfs_upcall_lease {
+    struct glfs_object *object; /* Object which need to be acted upon */
+    uint32_t lease_type;        /* Lease type to which client can downgrade to*/
+};
 
-	struct list_head    openfds;
+struct glfs_upcall_lease_fd {
+    uint32_t lease_type; /* Lease type to which client can downgrade to*/
+    void *fd_cookie;     /* Object which need to be acted upon */
+};
 
-	gf_boolean_t        migration_in_progress;
+struct glfs_xreaddirp_stat {
+    struct stat
+        st; /* Stat for that dirent - corresponds to GFAPI_XREADDIRP_STAT */
+    struct glfs_object *object; /* handled for GFAPI_XREADDIRP_HANDLE */
+    uint32_t flags_handled;     /* final set of flags successfulyy handled */
 };
 
-struct glfs_fd {
-	struct list_head   openfds;
-	struct glfs       *fs;
-	off_t              offset;
-	fd_t              *fd; /* Currently guared by @fs->mutex. TODO: per-glfd lock */
-	struct list_head   entries;
-	gf_dirent_t       *next;
+#define DEFAULT_EVENT_POOL_SIZE 16384
+#define GF_MEMPOOL_COUNT_OF_DICT_T 4096
+#define GF_MEMPOOL_COUNT_OF_DATA_T (GF_MEMPOOL_COUNT_OF_DICT_T * 4)
+#define GF_MEMPOOL_COUNT_OF_DATA_PAIR_T (GF_MEMPOOL_COUNT_OF_DICT_T * 4)
+
+#define GF_MEMPOOL_COUNT_OF_LRU_BUF_T 256
+
+typedef void(glfs_mem_release_t)(void *ptr);
+
+struct glfs_mem_header {
+    uint32_t magic;
+    size_t nmemb;
+    size_t size;
+    glfs_mem_release_t *release;
 };
 
-#define DEFAULT_EVENT_POOL_SIZE           16384
-#define GF_MEMPOOL_COUNT_OF_DICT_T        4096
-#define GF_MEMPOOL_COUNT_OF_DATA_T        (GF_MEMPOOL_COUNT_OF_DICT_T * 4)
-#define GF_MEMPOOL_COUNT_OF_DATA_PAIR_T   (GF_MEMPOOL_COUNT_OF_DICT_T * 4)
+#define GLFS_MEM_HEADER_SIZE (sizeof(struct glfs_mem_header))
+#define GLFS_MEM_HEADER_MAGIC 0x20170830
 
-int glfs_mgmt_init (struct glfs *fs);
-void glfs_init_done (struct glfs *fs, int ret);
-int glfs_process_volfp (struct glfs *fs, FILE *fp);
-int glfs_resolve (struct glfs *fs, xlator_t *subvol, const char *path, loc_t *loc,
-		  struct iatt *iatt, int reval);
-int glfs_lresolve (struct glfs *fs, xlator_t *subvol, const char *path, loc_t *loc,
-		   struct iatt *iatt, int reval);
-fd_t *glfs_resolve_fd (struct glfs *fs, xlator_t *subvol, struct glfs_fd *glfd);
+static inline void *
+__glfs_calloc(size_t nmemb, size_t size, glfs_mem_release_t release,
+              uint32_t type, const char *typestr)
+{
+    struct glfs_mem_header *header = NULL;
 
-fd_t *__glfs_migrate_fd (struct glfs *fs, xlator_t *subvol, struct glfs_fd *glfd);
+    header = __gf_calloc(nmemb, (size + GLFS_MEM_HEADER_SIZE), type, typestr);
+    if (!header)
+        return NULL;
 
-int glfs_first_lookup (xlator_t *subvol);
+    header->magic = GLFS_MEM_HEADER_MAGIC;
+    header->nmemb = nmemb;
+    header->size = size;
+    header->release = release;
 
-static inline void
-__glfs_entry_fs (struct glfs *fs)
+    return header + 1;
+}
+
+static inline void *
+__glfs_malloc(size_t size, glfs_mem_release_t release, uint32_t type,
+              const char *typestr)
 {
-	THIS = fs->ctx->master;
+    struct glfs_mem_header *header = NULL;
+
+    header = __gf_malloc((size + GLFS_MEM_HEADER_SIZE), type, typestr);
+    if (!header)
+        return NULL;
+
+    header->magic = GLFS_MEM_HEADER_MAGIC;
+    header->nmemb = 1;
+    header->size = size;
+    header->release = release;
+
+    return header + 1;
 }
 
+static inline void *
+__glfs_realloc(void *ptr, size_t size)
+{
+    struct glfs_mem_header *old_header = NULL;
+    struct glfs_mem_header *new_header = NULL;
+    struct glfs_mem_header tmp_header;
+    void *new_ptr = NULL;
+
+    GF_ASSERT(NULL != ptr);
+
+    old_header = (struct glfs_mem_header *)(ptr - GLFS_MEM_HEADER_SIZE);
+    GF_ASSERT(old_header->magic == GLFS_MEM_HEADER_MAGIC);
+    tmp_header = *old_header;
+
+    new_ptr = __gf_realloc(old_header, (size + GLFS_MEM_HEADER_SIZE));
+    if (!new_ptr)
+        return NULL;
+
+    new_header = (struct glfs_mem_header *)new_ptr;
+    *new_header = tmp_header;
+    new_header->size = size;
+
+    return new_header + 1;
+}
 
 static inline void
-__glfs_entry_fd (struct glfs_fd *fd)
+__glfs_free(void *free_ptr)
 {
-	THIS = fd->fd->inode->table->xl->ctx->master;
+    struct glfs_mem_header *header = NULL;
+    void *release_ptr = NULL;
+    int i = 0;
+
+    if (!free_ptr)
+        return;
+
+    header = (struct glfs_mem_header *)(free_ptr - GLFS_MEM_HEADER_SIZE);
+    GF_ASSERT(header->magic == GLFS_MEM_HEADER_MAGIC);
+
+    if (header->release) {
+        release_ptr = free_ptr;
+        for (i = 0; i < header->nmemb; i++) {
+            header->release(release_ptr);
+            release_ptr += header->size;
+        }
+    }
+
+    __gf_free(header);
 }
 
+#define GLFS_CALLOC(nmemb, size, release, type)                                \
+    __glfs_calloc(nmemb, size, release, type, #type)
+
+#define GLFS_MALLOC(size, release, type)                                       \
+    __glfs_malloc(size, release, type, #type)
+
+#define GLFS_REALLOC(ptr, size) __glfs_realloc(ptr, size)
+
+#define GLFS_FREE(free_ptr) __glfs_free(free_ptr)
+
+int
+glfs_mgmt_init(struct glfs *fs);
+void
+glfs_init_done(struct glfs *fs, int ret) GFAPI_PRIVATE(glfs_init_done, 3.4.0);
+int
+glfs_process_volfp(struct glfs *fs, FILE *fp);
+int
+glfs_resolve(struct glfs *fs, xlator_t *subvol, const char *path, loc_t *loc,
+             struct iatt *iatt, int reval) GFAPI_PRIVATE(glfs_resolve, 3.7.0);
+int
+glfs_lresolve(struct glfs *fs, xlator_t *subvol, const char *path, loc_t *loc,
+              struct iatt *iatt, int reval);
+fd_t *
+glfs_resolve_fd(struct glfs *fs, xlator_t *subvol, struct glfs_fd *glfd);
+
+fd_t *
+__glfs_migrate_fd(struct glfs *fs, xlator_t *subvol, struct glfs_fd *glfd);
+
+int
+glfs_first_lookup(xlator_t *subvol);
+
+void
+glfs_process_upcall_event(struct glfs *fs, void *data)
+    GFAPI_PRIVATE(glfs_process_upcall_event, 3.7.0);
+
+#define __GLFS_ENTRY_VALIDATE_FS(fs, label)                                    \
+    do {                                                                       \
+        if (!fs) {                                                             \
+            errno = EINVAL;                                                    \
+            goto label;                                                        \
+        }                                                                      \
+        old_THIS = THIS;                                                       \
+        THIS = fs->ctx->master;                                                \
+    } while (0)
+
+#define __GLFS_EXIT_FS                                                         \
+    do {                                                                       \
+        THIS = old_THIS;                                                       \
+    } while (0)
+
+#define __GLFS_ENTRY_VALIDATE_FD(glfd, label)                                  \
+    do {                                                                       \
+        if (!glfd || !glfd->fd || !glfd->fd->inode ||                          \
+            glfd->state != GLFD_OPEN) {                                        \
+            errno = EBADF;                                                     \
+            goto label;                                                        \
+        }                                                                      \
+        old_THIS = THIS;                                                       \
+        THIS = glfd->fd->inode->table->xl->ctx->master;                        \
+    } while (0)
+
+#define __GLFS_LOCK_WAIT(fs)                                                   \
+    do {                                                                       \
+        struct synctask *task = NULL;                                          \
+                                                                               \
+        task = synctask_get();                                                 \
+                                                                               \
+        if (task) {                                                            \
+            list_add_tail(&task->waitq, &fs->waitq);                           \
+            pthread_mutex_unlock(&fs->mutex);                                  \
+            synctask_yield(task, NULL);                                        \
+            pthread_mutex_lock(&fs->mutex);                                    \
+        } else {                                                               \
+            /* non-synctask */                                                 \
+            pthread_cond_wait(&fs->cond, &fs->mutex);                          \
+        }                                                                      \
+    } while (0)
+
+#define __GLFS_SYNCTASK_WAKE(fs)                                               \
+    do {                                                                       \
+        struct synctask *waittask = NULL;                                      \
+                                                                               \
+        while (!list_empty(&fs->waitq)) {                                      \
+            waittask = list_entry(fs->waitq.next, struct synctask, waitq);     \
+            list_del_init(&waittask->waitq);                                   \
+            synctask_wake(waittask);                                           \
+        }                                                                      \
+    } while (0)
 
 /*
   By default all lock attempts from user context must
@@ -98,43 +494,263 @@ __glfs_entry_fd (struct glfs_fd *fd)
   we can give up the mutex during syncop calls so
   that bottom up calls (particularly CHILD_UP notify)
   can do a mutex_lock() on @glfs without deadlocking
-  the filesystem
+  the filesystem.
+
+  All the fops should wait for graph migration to finish
+  before starting the fops. Therefore these functions should
+  call glfs_lock with wait_for_migration as true. But waiting
+  for migration to finish in call-back path can result thread
+  dead-locks. The reason for this is we only have finite
+  number of epoll threads. so if we wait on epoll threads
+  there will not be any thread left to handle outstanding
+  rpc replies.
 */
 static inline int
-glfs_lock (struct glfs *fs)
+glfs_lock(struct glfs *fs, gf_boolean_t wait_for_migration)
 {
-	pthread_mutex_lock (&fs->mutex);
+    pthread_mutex_lock(&fs->mutex);
 
-	while (!fs->init)
-		pthread_cond_wait (&fs->cond, &fs->mutex);
+    while (!fs->init)
+        __GLFS_LOCK_WAIT(fs);
 
-	while (fs->migration_in_progress)
-		pthread_cond_wait (&fs->cond, &fs->mutex);
+    while (wait_for_migration && fs->migration_in_progress)
+        __GLFS_LOCK_WAIT(fs);
 
-	return 0;
+    return 0;
 }
 
-
 static inline void
-glfs_unlock (struct glfs *fs)
+glfs_unlock(struct glfs *fs)
 {
-	pthread_mutex_unlock (&fs->mutex);
+    pthread_mutex_unlock(&fs->mutex);
 }
 
+struct glfs_fd *
+glfs_fd_new(struct glfs *fs);
+void
+glfs_fd_bind(struct glfs_fd *glfd);
+void
+glfd_set_state_bind(struct glfs_fd *glfd);
+
+xlator_t *
+glfs_active_subvol(struct glfs *fs) GFAPI_PRIVATE(glfs_active_subvol, 3.4.0);
+xlator_t *
+__glfs_active_subvol(struct glfs *fs);
+void
+glfs_subvol_done(struct glfs *fs, xlator_t *subvol)
+    GFAPI_PRIVATE(glfs_subvol_done, 3.4.0);
+
+inode_t *
+glfs_refresh_inode(xlator_t *subvol, inode_t *inode);
+
+inode_t *
+glfs_cwd_get(struct glfs *fs);
+int
+glfs_cwd_set(struct glfs *fs, inode_t *inode);
+inode_t *
+glfs_resolve_inode(struct glfs *fs, xlator_t *subvol,
+                   struct glfs_object *object);
+int
+glfs_create_object(loc_t *loc, struct glfs_object **retobject);
+int
+__glfs_cwd_set(struct glfs *fs, inode_t *inode);
+
+int
+glfs_resolve_base(struct glfs *fs, xlator_t *subvol, inode_t *inode,
+                  struct iatt *iatt);
+
+int
+glfs_resolve_at(struct glfs *fs, xlator_t *subvol, inode_t *at,
+                const char *origpath, loc_t *loc, struct iatt *iatt, int follow,
+                int reval) GFAPI_PRIVATE(glfs_resolve_at, 3.4.0);
+int
+glfs_loc_touchup(loc_t *loc) GFAPI_PRIVATE(glfs_loc_touchup, 3.4.0);
+void
+glfs_iatt_to_stat(struct glfs *fs, struct iatt *iatt, struct stat *stat);
+void
+glfs_iatt_from_stat(struct stat *stat, int valid, struct iatt *iatt,
+                    int *gvalid);
+int
+glfs_loc_link(loc_t *loc, struct iatt *iatt);
+int
+glfs_loc_unlink(loc_t *loc);
+int
+glfs_getxattr_process(void *value, size_t size, dict_t *xattr,
+                      const char *name);
+
+/* Sends RPC call to glusterd to fetch required volume info */
+int
+glfs_get_volume_info(struct glfs *fs);
 
-void glfs_fd_destroy (struct glfs_fd *glfd);
+/*
+  SYNOPSIS
+
+       glfs_new_from_ctx: Creates a virtual mount object by taking a
+       glusterfs_ctx_t object.
+
+  DESCRIPTION
+
+       glfs_new_from_ctx() is not same as glfs_new(). It takes the
+       glusterfs_ctx_t object instead of creating one by glusterfs_ctx_new().
+       Again the usage is restricted to NFS MOUNT over UDP i.e. in
+       glfs_resolve_at() which would take fs object as input but never use
+       (purpose is not to change the ABI of glfs_resolve_at()).
+
+  PARAMETERS
+
+       @ctx: glusterfs_ctx_t object
+
+  RETURN VALUES
+
+       fs     : Pointer to the newly created glfs_t object.
+       NULL   : Otherwise.
+*/
+
+struct glfs *
+glfs_new_from_ctx(glusterfs_ctx_t *ctx) GFAPI_PRIVATE(glfs_new_from_ctx, 3.7.0);
+
+/*
+  SYNOPSIS
+
+       glfs_free_from_ctx: Free up the memory occupied by glfs_t object
+       created by glfs_new_from_ctx().
+
+  DESCRIPTION
+
+       The glfs_t object allocated by glfs_new_from_ctx() must be released
+       by the caller using this routine. The usage can be found
+       at glfs_fini() or NFS, MOUNT over UDP i.e.
+                        __mnt3udp_get_export_subdir_inode ()
+                                => glfs_resolve_at().
+
+  PARAMETERS
+
+       @fs: The glfs_t object to be deallocated.
+
+  RETURN VALUES
+
+       void
+*/
+
+void
+glfs_free_from_ctx(struct glfs *fs) GFAPI_PRIVATE(glfs_free_from_ctx, 3.7.0);
+
+int
+glfs_recall_lease_fd(struct glfs *fs, struct gf_upcall *up_data);
+
+int
+glfs_get_upcall_cache_invalidation(struct gf_upcall *to_up_data,
+                                   struct gf_upcall *from_up_data);
+int
+glfs_h_poll_cache_invalidation(struct glfs *fs, struct glfs_upcall *up_arg,
+                               struct gf_upcall *upcall_data);
+
+ssize_t
+glfs_anonymous_preadv(struct glfs *fs, struct glfs_object *object,
+                      const struct iovec *iovec, int iovcnt, off_t offset,
+                      int flags);
+ssize_t
+glfs_anonymous_pwritev(struct glfs *fs, struct glfs_object *object,
+                       const struct iovec *iovec, int iovcnt, off_t offset,
+                       int flags);
+
+struct glfs_object *
+glfs_h_resolve_symlink(struct glfs *fs, struct glfs_object *object);
+
+/* Deprecated structures that were passed to client applications, replaced by
+ * accessor functions. Do not use these in new applications, and update older
+ * usage.
+ *
+ * See http://review.gluster.org/14701 for more details.
+ *
+ * WARNING: These structures will be removed in the future.
+ */
+struct glfs_callback_arg {
+    struct glfs *fs;
+    enum glfs_upcall_reason reason;
+    void *event_arg;
+};
+
+struct glfs_callback_inode_arg {
+    struct glfs_object *object;      /* Object which need to be acted upon */
+    int flags;                       /* Cache UPDATE/INVALIDATE flags */
+    struct stat buf;                 /* Latest stat of this entry */
+    unsigned int expire_time_attr;   /* the amount of time for which
+                                      * the application need to cache
+                                      * this entry
+                                      */
+    struct glfs_object *p_object;    /* parent Object to be updated */
+    struct stat p_buf;               /* Latest stat of parent dir handle */
+    struct glfs_object *oldp_object; /* Old parent Object
+                                      * to be updated */
+    struct stat oldp_buf;            /* Latest stat of old parent
+                                      * dir handle */
+};
+struct dirent *
+glfs_readdirbuf_get(struct glfs_fd *glfd);
+
+gf_dirent_t *
+glfd_entry_next(struct glfs_fd *glfd, int plus);
+
+void
+gf_dirent_to_dirent(gf_dirent_t *gf_dirent, struct dirent *dirent);
+
+void
+gf_lease_to_glfs_lease(struct gf_lease *gf_lease, struct glfs_lease *lease);
+
+void
+glfs_lease_to_gf_lease(struct glfs_lease *lease, struct gf_lease *gf_lease);
+
+void
+glfs_release_upcall(void *ptr);
 
-struct glfs_fd *glfs_fd_new (struct glfs *fs);
-void glfs_fd_bind (struct glfs_fd *glfd);
+int
+get_fop_attr_glfd(dict_t **fop_attr, struct glfs_fd *glfd);
 
-xlator_t * glfs_active_subvol (struct glfs *fs);
-xlator_t * __glfs_active_subvol (struct glfs *fs);
-void glfs_subvol_done (struct glfs *fs, xlator_t *subvol);
+int
+set_fop_attr_glfd(struct glfs_fd *glfd);
 
-inode_t * glfs_refresh_inode (xlator_t *subvol, inode_t *inode);
+int
+get_fop_attr_thrd_key(dict_t **fop_attr);
 
-inode_t *glfs_cwd_get (struct glfs *fs);
-int glfs_cwd_set (struct glfs *fs, inode_t *inode);
-int __glfs_cwd_set (struct glfs *fs, inode_t *inode);
+void
+unset_fop_attr(dict_t **fop_attr);
 
+/*
+  SYNOPSIS
+  glfs_statx: Fetch extended file attributes for the given path.
+
+  DESCRIPTION
+  This function fetches extended file attributes for the given path.
+
+  PARAMETERS
+  @fs: The 'virtual mount' object referencing a volume, under which file exists.
+  @path: Path of the file within the virtual mount.
+  @mask: Requested extended file attributes mask, (See mask defines above)
+
+  RETURN VALUES
+  -1 : Failure. @errno will be set with the type of failure.
+  0  : Filled in statxbuf with appropriate masks for valid items in the
+       structure.
+
+  ERRNO VALUES
+    EINVAL: fs is invalid
+    EINVAL: mask has unsupported bits set
+    Other errors as returned by stat(2)
+ */
+
+int
+glfs_statx(struct glfs *fs, const char *path, unsigned int mask,
+           struct glfs_stat *statxbuf) GFAPI_PRIVATE(glfs_statx, 6.0);
+
+void
+glfs_iatt_from_statx(struct iatt *, const struct glfs_stat *)
+    GFAPI_PRIVATE(glfs_iatt_from_statx, 6.0);
+
+/*
+ * This API is a per thread setting, similar to glfs_setfs{u/g}id, because of
+ * the call to syncopctx_setfspid.
+ */
+int
+glfs_setfspid(struct glfs *, pid_t) GFAPI_PRIVATE(glfs_setfspid, 6.1);
 #endif /* !_GLFS_INTERNAL_H */
diff --git a/api/src/glfs-master.c b/api/src/glfs-master.c
index 09a92b7f060..100dcc16cc0 100644
--- a/api/src/glfs-master.c
+++ b/api/src/glfs-master.c
@@ -1,5 +1,5 @@
 /*
-  Copyright (c) 2012 Red Hat, Inc. <http://www.redhat.com>
+  Copyright (c) 2012-2016 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
   This file is licensed to you under your choice of the GNU Lesser
@@ -8,134 +8,176 @@
   cases as published by the Free Software Foundation.
 */
 
-#include <unistd.h>
-#include <string.h>
-#include <stdlib.h>
 #include <stdio.h>
-#include <inttypes.h>
-#include <limits.h>
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "xlator.h"
-#include "glusterfs.h"
+#include <glusterfs/glusterfs.h>
 
 #include "glfs-internal.h"
-
+#include "glfs-mem-types.h"
+#include "gfapi-messages.h"
 
 int
-glfs_graph_setup (struct glfs *fs, glusterfs_graph_t *graph)
+graph_setup(struct glfs *fs, glusterfs_graph_t *graph)
 {
-	xlator_t      *new_subvol = NULL;
-	xlator_t      *old_subvol = NULL;
-	inode_table_t *itable = NULL;
-	int            ret = -1;
-
-	new_subvol = graph->top;
-
-	/* This is called in a bottom-up context, it should specifically
-	   NOT be glfs_lock()
-	*/
-	pthread_mutex_lock (&fs->mutex);
-	{
-		if (new_subvol->switched ||
-		    new_subvol == fs->active_subvol ||
-		    new_subvol == fs->next_subvol) {
-			/* Spurious CHILD_UP event on old graph */
-			ret = 0;
-			goto unlock;
-		}
-
-		if (!new_subvol->itable) {
-			itable = inode_table_new (131072, new_subvol);
-			if (!itable) {
-				errno = ENOMEM;
-				ret = -1;
-				goto unlock;
-			}
-
-			new_subvol->itable = itable;
-		}
-
-		old_subvol = fs->next_subvol;
-		fs->next_subvol = new_subvol;
-		fs->next_subvol->winds++; /* first ref */
-		ret = 0;
-	}
+    xlator_t *new_subvol = NULL;
+    xlator_t *old_subvol = NULL;
+    inode_table_t *itable = NULL;
+    int ret = -1;
+
+    new_subvol = graph->top;
+
+    /* This is called in a bottom-up context, it should specifically
+       NOT be glfs_lock()
+    */
+    pthread_mutex_lock(&fs->mutex);
+    {
+        if (new_subvol->switched || new_subvol == fs->active_subvol ||
+            new_subvol == fs->next_subvol || new_subvol == fs->mip_subvol) {
+            /* Spurious CHILD_UP event on old graph */
+            ret = 0;
+            goto unlock;
+        }
+
+        if (!new_subvol->itable) {
+            itable = inode_table_new(131072, new_subvol);
+            if (!itable) {
+                errno = ENOMEM;
+                ret = -1;
+                goto unlock;
+            }
+
+            new_subvol->itable = itable;
+        }
+
+        old_subvol = fs->next_subvol;
+        fs->next_subvol = new_subvol;
+        fs->next_subvol->winds++; /* first ref */
+        ret = 0;
+    }
 unlock:
-	pthread_mutex_unlock (&fs->mutex);
+    pthread_mutex_unlock(&fs->mutex);
 
-	if (old_subvol)
-		/* wasn't picked up so far, skip */
-		glfs_subvol_done (fs, old_subvol);
+    if (old_subvol)
+        /* wasn't picked up so far, skip */
+        glfs_subvol_done(fs, old_subvol);
 
-	return ret;
+    return ret;
 }
 
-
 int
-notify (xlator_t *this, int event, void *data, ...)
+notify(xlator_t *this, int event, void *data, ...)
 {
-	glusterfs_graph_t   *graph = NULL;
-	struct glfs	    *fs = NULL;
-
-	graph = data;
-	fs = this->private;
-
-	switch (event) {
-	case GF_EVENT_GRAPH_NEW:
-		gf_log (this->name, GF_LOG_INFO, "New graph %s (%d) coming up",
-			uuid_utoa ((unsigned char *)graph->graph_uuid),
-			graph->id);
-		break;
-	case GF_EVENT_CHILD_UP:
-		glfs_graph_setup (fs, graph);
-		glfs_init_done (fs, 0);
-		break;
-	case GF_EVENT_CHILD_DOWN:
-		glfs_graph_setup (fs, graph);
-		glfs_init_done (fs, 1);
-		break;
-	case GF_EVENT_CHILD_CONNECTING:
-		break;
-	default:
-		gf_log (this->name, GF_LOG_DEBUG,
-			"got notify event %d", event);
-		break;
-	}
-
-	return 0;
+    glusterfs_graph_t *graph = NULL;
+    struct glfs *fs = NULL;
+
+    graph = data;
+    fs = this->private;
+
+    switch (event) {
+        case GF_EVENT_GRAPH_NEW:
+            gf_smsg(this->name, GF_LOG_INFO, 0, API_MSG_NEW_GRAPH,
+                    "graph-uuid=%s",
+                    uuid_utoa((unsigned char *)graph->graph_uuid), "id=%d",
+                    graph->id, NULL);
+            break;
+        case GF_EVENT_CHILD_UP:
+            pthread_mutex_lock(&fs->mutex);
+            {
+                graph->used = 1;
+            }
+            pthread_mutex_unlock(&fs->mutex);
+            graph_setup(fs, graph);
+            glfs_init_done(fs, 0);
+            break;
+        case GF_EVENT_CHILD_DOWN:
+            pthread_mutex_lock(&fs->mutex);
+            {
+                graph->used = 0;
+                pthread_cond_broadcast(&fs->child_down_cond);
+            }
+            pthread_mutex_unlock(&fs->mutex);
+            glfs_init_done(fs, 1);
+            break;
+        case GF_EVENT_CHILD_CONNECTING:
+            break;
+        case GF_EVENT_UPCALL:
+            glfs_process_upcall_event(fs, data);
+            break;
+        default:
+            gf_msg_debug(this->name, 0, "got notify event %d", event);
+            break;
+    }
+
+    return 0;
 }
 
-
 int
-mem_acct_init (xlator_t *this)
+mem_acct_init(xlator_t *this)
 {
-	return 0;
-}
+    int ret = -1;
+
+    if (!this)
+        return ret;
 
+    ret = xlator_mem_acct_init(this, glfs_mt_end + 1);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, ENOMEM, API_MSG_MEM_ACCT_INIT_FAILED,
+                NULL);
+        return ret;
+    }
+
+    return 0;
+}
 
 int
-init (xlator_t *this)
+init(xlator_t *this)
 {
-	return 0;
+    return 0;
 }
 
-
 void
-fini (xlator_t *this)
+fini(xlator_t *this)
 {
+}
 
+/* place-holder fops */
+int
+glfs_forget(xlator_t *this, inode_t *inode)
+{
+    return 0;
 }
 
+int
+glfs_release(xlator_t *this, fd_t *fd)
+{
+    return 0;
+}
 
-struct xlator_dumpops dumpops;
+int
+glfs_releasedir(xlator_t *this, fd_t *fd)
+{
+    return 0;
+}
 
+struct xlator_dumpops dumpops;
 
 struct xlator_fops fops;
 
-
-struct xlator_cbks cbks;
+struct xlator_cbks cbks = {
+    .forget = glfs_forget,
+    .release = glfs_release,
+    .releasedir = glfs_releasedir,
+};
+
+xlator_api_t xlator_api = {
+    .init = init,
+    .fini = fini,
+    .notify = notify,
+    .mem_acct_init = mem_acct_init,
+    .op_version = {1},
+    .dumpops = &dumpops,
+    .fops = &fops,
+    .cbks = &cbks,
+    .identifier = "glfs-api",
+    .category = GF_MAINTAINED,
+};
diff --git a/api/src/glfs-mem-types.h b/api/src/glfs-mem-types.h
index 590acd03f11..bfa325a3ad9 100644
--- a/api/src/glfs-mem-types.h
+++ b/api/src/glfs-mem-types.h
@@ -1,5 +1,5 @@
 /*
-  Copyright (c) 2012 Red Hat, Inc. <http://www.redhat.com>
+  Copyright (c) 2012-2017 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
   This file is licensed to you under your choice of the GNU Lesser
@@ -11,20 +11,25 @@
 #ifndef _GLFS_MEM_TYPES_H
 #define _GLFS_MEM_TYPES_H
 
-#include "mem-types.h"
+#include <glusterfs/mem-types.h>
 
 #define GF_MEM_TYPE_START (gf_common_mt_end + 1)
 
 enum glfs_mem_types_ {
-        glfs_mt_glfs_t,
-        glfs_mt_call_pool_t,
-        glfs_mt_xlator_t,
-	glfs_mt_glfs_fd_t,
-	glfs_mt_glfs_io_t,
-	glfs_mt_volfile_t,
-	glfs_mt_xlator_cmdline_option_t,
-        glfs_mt_end
-
+    glfs_mt_call_pool_t = GF_MEM_TYPE_START,
+    glfs_mt_xlator_t,
+    glfs_mt_glfs_fd_t,
+    glfs_mt_glfs_io_t,
+    glfs_mt_volfile_t,
+    glfs_mt_xlator_cmdline_option_t,
+    glfs_mt_server_cmdline_t,
+    glfs_mt_glfs_object_t,
+    glfs_mt_readdirbuf_t,
+    glfs_mt_upcall_entry_t,
+    glfs_mt_acl_t,
+    glfs_mt_upcall_inode_t,
+    glfs_mt_realpath_t,
+    glfs_mt_xreaddirp_stat_t,
+    glfs_mt_end
 };
 #endif
-
diff --git a/api/src/glfs-mgmt.c b/api/src/glfs-mgmt.c
index 20362543e62..7c82b8cd162 100644
--- a/api/src/glfs-mgmt.c
+++ b/api/src/glfs-mgmt.c
@@ -1,5 +1,5 @@
 /*
-  Copyright (c) 2012 Red Hat, Inc. <http://www.redhat.com>
+  Copyright (c) 2012-2018 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
   This file is licensed to you under your choice of the GNU Lesser
@@ -8,7 +8,6 @@
   cases as published by the Free Software Foundation.
 */
 
-
 #include <stdio.h>
 #include <sys/types.h>
 #include <sys/wait.h>
@@ -16,683 +15,1035 @@
 #include <signal.h>
 #include <pthread.h>
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif /* _CONFIG_H */
-
-#include "glusterfs.h"
-#include "stack.h"
-#include "dict.h"
-#include "event.h"
-#include "defaults.h"
+#include <glusterfs/glusterfs.h>
+#include "glfs.h"
+#include <glusterfs/dict.h>
 
 #include "rpc-clnt.h"
 #include "protocol-common.h"
-#include "glusterfs3.h"
-#include "portmap-xdr.h"
 #include "xdr-generic.h"
+#include "rpc-common-xdr.h"
 
-#include "syncop.h"
-#include "xlator.h"
+#include <glusterfs/syncop.h>
 
 #include "glfs-internal.h"
-#include "glfs-mem-types.h"
-
+#include "gfapi-messages.h"
+#include <glusterfs/syscall.h>
 
-int glfs_volfile_fetch (struct glfs *fs);
+int
+glfs_volfile_fetch(struct glfs *fs);
+int32_t
+glfs_get_volume_info_rpc(call_frame_t *frame, xlator_t *this, struct glfs *fs);
 
 int
-glfs_process_volfp (struct glfs *fs, FILE *fp)
+glfs_process_volfp(struct glfs *fs, FILE *fp)
 {
-	glusterfs_graph_t  *graph = NULL;
-	int		    ret = -1;
-	xlator_t	   *trav = NULL;
-	glusterfs_ctx_t	   *ctx = NULL;
-
-	ctx = fs->ctx;
-	graph = glusterfs_graph_construct (fp);
-	if (!graph) {
-		gf_log ("glfs", GF_LOG_ERROR, "failed to construct the graph");
-		goto out;
-	}
-
-	for (trav = graph->first; trav; trav = trav->next) {
-		if (strcmp (trav->type, "mount/fuse") == 0) {
-			gf_log ("glfs", GF_LOG_ERROR,
-				"fuse xlator cannot be specified "
-				"in volume file");
-			goto out;
-		}
-	}
-
-	ret = glusterfs_graph_prepare (graph, ctx);
-	if (ret) {
-		glusterfs_graph_destroy (graph);
-		goto out;
-	}
-
-	ret = glusterfs_graph_activate (graph, ctx);
-
-	if (ret) {
-		glusterfs_graph_destroy (graph);
-		goto out;
-	}
-
-	ret = 0;
+    glusterfs_graph_t *graph = NULL;
+    int ret = -1;
+    xlator_t *trav = NULL;
+    glusterfs_ctx_t *ctx = NULL;
+
+    ctx = fs->ctx;
+    graph = glusterfs_graph_construct(fp);
+    if (!graph) {
+        gf_smsg("glfs", GF_LOG_ERROR, errno, API_MSG_GRAPH_CONSTRUCT_FAILED,
+                NULL);
+        goto out;
+    }
+
+    for (trav = graph->first; trav; trav = trav->next) {
+        if (strcmp(trav->type, "mount/api") == 0) {
+            gf_smsg("glfs", GF_LOG_ERROR, EINVAL, API_MSG_API_XLATOR_ERROR,
+                    NULL);
+            goto out;
+        }
+    }
+
+    ret = glusterfs_graph_prepare(graph, ctx, fs->volname);
+    if (ret) {
+        glusterfs_graph_destroy(graph);
+        goto out;
+    }
+
+    ret = glusterfs_graph_activate(graph, ctx);
+
+    if (ret) {
+        glusterfs_graph_destroy(graph);
+        goto out;
+    }
+
+    gf_log_dump_graph(fp, graph);
+
+    ret = 0;
 out:
-	if (fp)
-		fclose (fp);
+    if (fp)
+        fclose(fp);
 
-	if (!ctx->active) {
-		ret = -1;
-	}
+    if (!ctx->active) {
+        ret = -1;
+    }
 
-	return ret;
+    return ret;
 }
 
-
 int
-mgmt_cbk_spec (struct rpc_clnt *rpc, void *mydata, void *data)
+mgmt_cbk_spec(struct rpc_clnt *rpc, void *mydata, void *data)
 {
-	struct glfs *fs = NULL;
-	xlator_t    *this = NULL;
+    struct glfs *fs = NULL;
+    xlator_t *this = NULL;
 
-	this = mydata;
-	fs = this->private;
+    this = mydata;
+    fs = this->private;
 
-	glfs_volfile_fetch (fs);
+    glfs_volfile_fetch(fs);
 
-	return 0;
+    return 0;
 }
 
-
 int
-mgmt_cbk_event (struct rpc_clnt *rpc, void *mydata, void *data)
+mgmt_cbk_event(struct rpc_clnt *rpc, void *mydata, void *data)
 {
-	return 0;
+    return 0;
 }
 
+static int
+mgmt_cbk_statedump(struct rpc_clnt *rpc, void *mydata, void *data)
+{
+    struct glfs *fs = NULL;
+    xlator_t *this = NULL;
+    gf_statedump target_pid = {
+        0,
+    };
+    struct iovec *iov = NULL;
+    int ret = -1;
+
+    this = mydata;
+    if (!this) {
+        gf_smsg("glfs", GF_LOG_ERROR, EINVAL, API_MSG_NULL, "mydata", NULL);
+        errno = EINVAL;
+        goto out;
+    }
+
+    fs = this->private;
+    if (!fs) {
+        gf_smsg("glfs", GF_LOG_ERROR, EINVAL, API_MSG_NULL, "glfs", NULL);
+        errno = EINVAL;
+        goto out;
+    }
+
+    iov = (struct iovec *)data;
+    if (!iov) {
+        gf_smsg("glfs", GF_LOG_ERROR, EINVAL, API_MSG_NULL, "iovec data", NULL);
+        errno = EINVAL;
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &target_pid, (xdrproc_t)xdr_gf_statedump);
+    if (ret < 0) {
+        gf_smsg("glfs", GF_LOG_ERROR, EINVAL, API_MSG_DECODE_XDR_FAILED, NULL);
+        goto out;
+    }
+
+    gf_msg_trace("glfs", 0, "statedump requested for pid: %d", target_pid.pid);
+
+    if ((uint64_t)getpid() == target_pid.pid) {
+        gf_msg_debug("glfs", 0, "Taking statedump for pid: %d", target_pid.pid);
+
+        ret = glfs_sysrq(fs, GLFS_SYSRQ_STATEDUMP);
+        if (ret < 0) {
+            gf_smsg("glfs", GF_LOG_INFO, 0, API_MSG_STATEDUMP_FAILED, NULL);
+        }
+    }
+out:
+    return ret;
+}
 
-rpcclnt_cb_actor_t mgmt_cbk_actors[] = {
-	[GF_CBK_FETCHSPEC] = {"FETCHSPEC", GF_CBK_FETCHSPEC, mgmt_cbk_spec },
-	[GF_CBK_EVENT_NOTIFY] = {"EVENTNOTIFY", GF_CBK_EVENT_NOTIFY,
-				 mgmt_cbk_event},
+static rpcclnt_cb_actor_t mgmt_cbk_actors[GF_CBK_MAXVALUE] = {
+    [GF_CBK_FETCHSPEC] = {"FETCHSPEC", mgmt_cbk_spec, GF_CBK_FETCHSPEC},
+    [GF_CBK_EVENT_NOTIFY] = {"EVENTNOTIFY", mgmt_cbk_event,
+                             GF_CBK_EVENT_NOTIFY},
+    [GF_CBK_STATEDUMP] = {"STATEDUMP", mgmt_cbk_statedump, GF_CBK_STATEDUMP},
 };
 
-
-struct rpcclnt_cb_program mgmt_cbk_prog = {
-	.progname  = "GlusterFS Callback",
-	.prognum   = GLUSTER_CBK_PROGRAM,
-	.progver   = GLUSTER_CBK_VERSION,
-	.actors	   = mgmt_cbk_actors,
-	.numactors = GF_CBK_MAXVALUE,
+static struct rpcclnt_cb_program mgmt_cbk_prog = {
+    .progname = "GlusterFS Callback",
+    .prognum = GLUSTER_CBK_PROGRAM,
+    .progver = GLUSTER_CBK_VERSION,
+    .actors = mgmt_cbk_actors,
+    .numactors = GF_CBK_MAXVALUE,
 };
 
-char *clnt_handshake_procs[GF_HNDSK_MAXVALUE] = {
-	[GF_HNDSK_NULL]		= "NULL",
-	[GF_HNDSK_SETVOLUME]	= "SETVOLUME",
-	[GF_HNDSK_GETSPEC]	= "GETSPEC",
-	[GF_HNDSK_PING]		= "PING",
-	[GF_HNDSK_EVENT_NOTIFY] = "EVENTNOTIFY",
+static char *clnt_handshake_procs[GF_HNDSK_MAXVALUE] = {
+    [GF_HNDSK_NULL] = "NULL",
+    [GF_HNDSK_SETVOLUME] = "SETVOLUME",
+    [GF_HNDSK_GETSPEC] = "GETSPEC",
+    [GF_HNDSK_PING] = "PING",
+    [GF_HNDSK_EVENT_NOTIFY] = "EVENTNOTIFY",
+    [GF_HNDSK_GET_VOLUME_INFO] = "GETVOLUMEINFO",
 };
 
-rpc_clnt_prog_t clnt_handshake_prog = {
-	.progname  = "GlusterFS Handshake",
-	.prognum   = GLUSTER_HNDSK_PROGRAM,
-	.progver   = GLUSTER_HNDSK_VERSION,
-	.procnames = clnt_handshake_procs,
+static rpc_clnt_prog_t clnt_handshake_prog = {
+    .progname = "GlusterFS Handshake",
+    .prognum = GLUSTER_HNDSK_PROGRAM,
+    .progver = GLUSTER_HNDSK_VERSION,
+    .procnames = clnt_handshake_procs,
 };
 
-
 int
-mgmt_submit_request (void *req, call_frame_t *frame,
-		     glusterfs_ctx_t *ctx,
-		     rpc_clnt_prog_t *prog, int procnum,
-		     fop_cbk_fn_t cbkfn, xdrproc_t xdrproc)
+mgmt_submit_request(void *req, call_frame_t *frame, glusterfs_ctx_t *ctx,
+                    rpc_clnt_prog_t *prog, int procnum, fop_cbk_fn_t cbkfn,
+                    xdrproc_t xdrproc)
 {
-	int			ret	    = -1;
-	int			count	   = 0;
-	struct iovec		iov	    = {0, };
-	struct iobuf		*iobuf = NULL;
-	struct iobref		*iobref = NULL;
-	ssize_t			xdr_size = 0;
-
-	iobref = iobref_new ();
-	if (!iobref) {
-		goto out;
-	}
-
-	if (req) {
-		xdr_size = xdr_sizeof (xdrproc, req);
-
-		iobuf = iobuf_get2 (ctx->iobuf_pool, xdr_size);
-		if (!iobuf) {
-			goto out;
-		};
-
-		iobref_add (iobref, iobuf);
-
-		iov.iov_base = iobuf->ptr;
-		iov.iov_len  = iobuf_pagesize (iobuf);
-
-		/* Create the xdr payload */
-		ret = xdr_serialize_generic (iov, req, xdrproc);
-		if (ret == -1) {
-			gf_log (THIS->name, GF_LOG_WARNING,
-				"failed to create XDR payload");
-			goto out;
-		}
-		iov.iov_len = ret;
-		count = 1;
-	}
-
-	/* Send the msg */
-	ret = rpc_clnt_submit (ctx->mgmt, prog, procnum, cbkfn,
-			       &iov, count,
-			       NULL, 0, iobref, frame, NULL, 0, NULL, 0, NULL);
+    int ret = -1;
+    int count = 0;
+    struct iovec iov = {
+        0,
+    };
+    struct iobuf *iobuf = NULL;
+    struct iobref *iobref = NULL;
+    ssize_t xdr_size = 0;
+
+    iobref = iobref_new();
+    if (!iobref) {
+        goto out;
+    }
+
+    if (req) {
+        xdr_size = xdr_sizeof(xdrproc, req);
+
+        iobuf = iobuf_get2(ctx->iobuf_pool, xdr_size);
+        if (!iobuf) {
+            goto out;
+        };
+
+        iobref_add(iobref, iobuf);
+
+        iov.iov_base = iobuf->ptr;
+        iov.iov_len = iobuf_pagesize(iobuf);
+
+        /* Create the xdr payload */
+        ret = xdr_serialize_generic(iov, req, xdrproc);
+        if (ret == -1) {
+            gf_smsg(THIS->name, GF_LOG_WARNING, 0, API_MSG_XDR_PAYLOAD_FAILED,
+                    NULL);
+            goto out;
+        }
+        iov.iov_len = ret;
+        count = 1;
+    }
+
+    /* Send the msg */
+    ret = rpc_clnt_submit(ctx->mgmt, prog, procnum, cbkfn, &iov, count, NULL, 0,
+                          iobref, frame, NULL, 0, NULL, 0, NULL);
 
 out:
-	if (iobref)
-		iobref_unref (iobref);
+    if (iobref)
+        iobref_unref(iobref);
 
-	if (iobuf)
-		iobuf_unref (iobuf);
-	return ret;
+    if (iobuf)
+        iobuf_unref(iobuf);
+    return ret;
 }
 
-
-static int
-xlator_equal_rec (xlator_t *xl1, xlator_t *xl2)
+/*
+ * Callback routine for 'GF_HNDSK_GET_VOLUME_INFO' rpc request
+ */
+int
+mgmt_get_volinfo_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                     void *myframe)
 {
-	xlator_list_t *trav1 = NULL;
-	xlator_list_t *trav2 = NULL;
-	int	       ret   = 0;
-
-	if (xl1 == NULL || xl2 == NULL) {
-		gf_log ("xlator", GF_LOG_DEBUG, "invalid argument");
-		return -1;
-	}
-
-	trav1 = xl1->children;
-	trav2 = xl2->children;
-
-	while (trav1 && trav2) {
-		ret = xlator_equal_rec (trav1->xlator, trav2->xlator);
-		if (ret) {
-			gf_log ("glfs-mgmt", GF_LOG_DEBUG,
-				"xlators children not equal");
-			goto out;
-		}
-
-		trav1 = trav1->next;
-		trav2 = trav2->next;
-	}
-
-	if (trav1 || trav2) {
-		ret = -1;
-		goto out;
-	}
-
-	if (strcmp (xl1->name, xl2->name)) {
-		ret = -1;
-		goto out;
-	}
-out :
-	return ret;
-}
+    int ret = 0;
+    char *volume_id_str = NULL;
+    dict_t *dict = NULL;
+    gf_get_volume_info_rsp rsp = {
+        0,
+    };
+    call_frame_t *frame = NULL;
+    glusterfs_ctx_t *ctx = NULL;
+    struct glfs *fs = NULL;
+    struct syncargs *args;
+
+    frame = myframe;
+    ctx = frame->this->ctx;
+    args = frame->local;
+
+    if (!ctx) {
+        gf_smsg(frame->this->name, GF_LOG_ERROR, EINVAL, API_MSG_NULL,
+                "context", NULL);
+        errno = EINVAL;
+        ret = -1;
+        goto out;
+    }
+
+    fs = ((xlator_t *)ctx->master)->private;
+
+    if (-1 == req->rpc_status) {
+        gf_smsg(frame->this->name, GF_LOG_ERROR, EINVAL,
+                API_MSG_CALL_NOT_SUCCESSFUL, NULL);
+        errno = EINVAL;
+        ret = -1;
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_get_volume_info_rsp);
+
+    if (ret < 0) {
+        gf_smsg(frame->this->name, GF_LOG_ERROR, 0,
+                API_MSG_XDR_RESPONSE_DECODE_FAILED, NULL);
+        goto out;
+    }
+
+    gf_msg_debug(frame->this->name, 0,
+                 "Received resp to GET_VOLUME_INFO "
+                 "RPC: %d",
+                 rsp.op_ret);
+
+    if (rsp.op_ret == -1) {
+        errno = rsp.op_errno;
+        ret = -1;
+        goto out;
+    }
+
+    if (!rsp.dict.dict_len) {
+        gf_smsg(frame->this->name, GF_LOG_ERROR, EINVAL, API_MSG_CALL_NOT_VALID,
+                NULL);
+        ret = -1;
+        errno = EINVAL;
+        goto out;
+    }
+
+    dict = dict_new();
+
+    if (!dict) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len, &dict);
+
+    if (ret) {
+        errno = ENOMEM;
+        goto out;
+    }
+
+    ret = dict_get_str_sizen(dict, "volume_id", &volume_id_str);
+    if (ret) {
+        errno = EINVAL;
+        goto out;
+    }
+
+    ret = 0;
+out:
+    if (volume_id_str) {
+        gf_msg_debug(frame->this->name, 0, "Volume Id: %s", volume_id_str);
+        pthread_mutex_lock(&fs->mutex);
+        gf_uuid_parse(volume_id_str, fs->vol_uuid);
+        pthread_mutex_unlock(&fs->mutex);
+    }
 
+    if (ret) {
+        gf_smsg(frame->this->name, GF_LOG_ERROR, errno,
+                API_MSG_GET_VOLINFO_CBK_FAILED, "error=%s", strerror(errno),
+                NULL);
+    }
 
-static gf_boolean_t
-is_graph_topology_equal (glusterfs_graph_t *graph1,
-			 glusterfs_graph_t *graph2)
-{
-	xlator_t    *trav1    = NULL;
-	xlator_t    *trav2    = NULL;
-	gf_boolean_t ret      = _gf_true;
+    if (dict)
+        dict_unref(dict);
 
-	trav1 = graph1->first;
-	trav2 = graph2->first;
+    if (rsp.dict.dict_val)
+        free(rsp.dict.dict_val);
 
-	ret = xlator_equal_rec (trav1, trav2);
+    if (rsp.op_errstr)
+        free(rsp.op_errstr);
 
-	if (ret) {
-		gf_log ("glfs-mgmt", GF_LOG_DEBUG,
-			"graphs are not equal");
-		ret = _gf_false;
-		goto out;
-	}
+    gf_msg_debug(frame->this->name, 0, "Returning: %d", ret);
 
-	ret = _gf_true;
-	gf_log ("glfs-mgmt", GF_LOG_DEBUG,
-		"graphs are equal");
+    __wake(args);
 
-out:
-	return ret;
+    return ret;
 }
 
-
-/* Function has 3types of return value 0, -ve , 1
- *   return 0	       =======> reconfiguration of options has succeeded
- *   return 1	       =======> the graph has to be reconstructed and all the xlators should be inited
- *   return -1(or -ve) =======> Some Internal Error occurred during the operation
- */
-static int
-glusterfs_volfile_reconfigure (struct glfs *fs, FILE *newvolfile_fp)
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_get_volumeid, 3.5.0)
+int
+pub_glfs_get_volumeid(struct glfs *fs, char *volid, size_t size)
 {
-	glusterfs_graph_t *oldvolfile_graph = NULL;
-	glusterfs_graph_t *newvolfile_graph = NULL;
-	FILE		  *oldvolfile_fp    = NULL;
-	glusterfs_ctx_t	  *ctx		    = NULL;
-
-	int ret = -1;
-
-	oldvolfile_fp = tmpfile ();
-	if (!oldvolfile_fp)
-		goto out;
-
-	if (!fs->oldvollen) {
-		ret = 1; // Has to call INIT for the whole graph
-		goto out;
-	}
-	fwrite (fs->oldvolfile, fs->oldvollen, 1, oldvolfile_fp);
-	fflush (oldvolfile_fp);
-	if (ferror (oldvolfile_fp)) {
-		goto out;
-	}
-
-	oldvolfile_graph = glusterfs_graph_construct (oldvolfile_fp);
-	if (!oldvolfile_graph) {
-		goto out;
-	}
-
-	newvolfile_graph = glusterfs_graph_construct (newvolfile_fp);
-	if (!newvolfile_graph) {
-		goto out;
-	}
-
-	if (!is_graph_topology_equal (oldvolfile_graph,
-				      newvolfile_graph)) {
-
-		ret = 1;
-		gf_log ("glfs-mgmt", GF_LOG_DEBUG,
-			"Graph topology not equal(should call INIT)");
-		goto out;
-	}
-
-	gf_log ("glfs-mgmt", GF_LOG_DEBUG,
-		"Only options have changed in the new "
-		"graph");
-
-	ctx = fs->ctx;
-
-	if (!ctx) {
-		gf_log ("glfs-mgmt", GF_LOG_ERROR,
-			"glusterfs_ctx_get() returned NULL");
-		goto out;
-	}
-
-	oldvolfile_graph = ctx->active;
-
-	if (!oldvolfile_graph) {
-		gf_log ("glfs-mgmt", GF_LOG_ERROR,
-			"glusterfs_ctx->active is NULL");
-		goto out;
-	}
-
-	/* */
-	ret = glusterfs_graph_reconfigure (oldvolfile_graph,
-					   newvolfile_graph);
-	if (ret) {
-		gf_log ("glfs-mgmt", GF_LOG_DEBUG,
-			"Could not reconfigure new options in old graph");
-		goto out;
-	}
-
-	ret = 0;
-out:
-	if (oldvolfile_fp)
-		fclose (oldvolfile_fp);
+    /* TODO: Define a global macro to store UUID size */
+    size_t uuid_size = 16;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    pthread_mutex_lock(&fs->mutex);
+    {
+        /* check if the volume uuid is initialized */
+        if (!gf_uuid_is_null(fs->vol_uuid)) {
+            pthread_mutex_unlock(&fs->mutex);
+            goto done;
+        }
+    }
+    pthread_mutex_unlock(&fs->mutex);
 
-	return ret;
-}
+    /* Need to fetch volume_uuid */
+    glfs_get_volume_info(fs);
 
+    if (gf_uuid_is_null(fs->vol_uuid)) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, EINVAL, API_MSG_FETCH_VOLUUID_FAILED,
+                NULL);
+        goto out;
+    }
 
-static int
-glusterfs_oldvolfile_update (struct glfs *fs, char *volfile, ssize_t size)
-{
-	int ret = -1;
-
-	fs->oldvollen = size;
-	if (!fs->oldvolfile) {
-		fs->oldvolfile = GF_CALLOC (1, size+1, glfs_mt_volfile_t);
-	} else {
-		fs->oldvolfile = GF_REALLOC (fs->oldvolfile, size+1);
-	}
-
-	if (!fs->oldvolfile) {
-		fs->oldvollen = 0;
-	} else {
-		memcpy (fs->oldvolfile, volfile, size);
-		fs->oldvollen = size;
-		ret = 0;
-	}
-
-	return ret;
-}
+done:
+    if (!volid || !size) {
+        gf_msg_debug(THIS->name, 0, "volumeid/size is null");
+        __GLFS_EXIT_FS;
+        return uuid_size;
+    }
+
+    if (size < uuid_size) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, ERANGE, API_MSG_INSUFF_SIZE, NULL);
+        errno = ERANGE;
+        goto out;
+    }
+
+    memcpy(volid, fs->vol_uuid, uuid_size);
 
+    __GLFS_EXIT_FS;
+
+    return uuid_size;
+
+out:
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return -1;
+}
 
 int
-mgmt_getspec_cbk (struct rpc_req *req, struct iovec *iov, int count,
-		  void *myframe)
+glfs_get_volume_info(struct glfs *fs)
 {
-	gf_getspec_rsp		 rsp   = {0,};
-	call_frame_t		*frame = NULL;
-	glusterfs_ctx_t		*ctx = NULL;
-	int			 ret   = 0;
-	ssize_t			 size = 0;
-	FILE			*tmpfp = NULL;
-	int                      need_retry = 0;
-	struct glfs		*fs = NULL;
-
-	frame = myframe;
-	ctx = frame->this->ctx;
-	fs = ((xlator_t *)ctx->master)->private;
-
-	if (-1 == req->rpc_status) {
-		ret = -1;
-		need_retry = 1;
-		goto out;
-	}
-
-	ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_getspec_rsp);
-	if (ret < 0) {
-		gf_log (frame->this->name, GF_LOG_ERROR, "XDR decoding error");
-		ret   = -1;
-		goto out;
-	}
-
-	if (-1 == rsp.op_ret) {
-		gf_log (frame->this->name, GF_LOG_ERROR,
-			"failed to get the 'volume file' from server");
-		ret = -1;
-                errno = rsp.op_errno;
-		goto out;
-	}
-
-	ret = 0;
-	size = rsp.op_ret;
-
-	if ((size == fs->oldvollen) &&
-	    (memcmp (fs->oldvolfile, rsp.spec, size) == 0)) {
-		gf_log (frame->this->name, GF_LOG_INFO,
-			"No change in volfile, continuing");
-		goto out;
-	}
-
-	tmpfp = tmpfile ();
-	if (!tmpfp) {
-		ret = -1;
-		goto out;
-	}
-
-	fwrite (rsp.spec, size, 1, tmpfp);
-	fflush (tmpfp);
-	if (ferror (tmpfp)) {
-		ret = -1;
-		goto out;
-	}
-
-	/*  Check if only options have changed. No need to reload the
-	*  volfile if topology hasn't changed.
-	*  glusterfs_volfile_reconfigure returns 3 possible return states
-	*  return 0	     =======> reconfiguration of options has succeeded
-	*  return 1	     =======> the graph has to be reconstructed and all the xlators should be inited
-	*  return -1(or -ve) =======> Some Internal Error occurred during the operation
-	*/
-
-	ret = glusterfs_volfile_reconfigure (fs, tmpfp);
-	if (ret == 0) {
-		gf_log ("glusterfsd-mgmt", GF_LOG_DEBUG,
-			"No need to re-load volfile, reconfigure done");
-		ret = glusterfs_oldvolfile_update (fs, rsp.spec, size);
-		goto out;
-	}
-
-	if (ret < 0) {
-		gf_log ("glusterfsd-mgmt", GF_LOG_DEBUG,
-			"Reconfigure failed !!");
-		goto out;
-	}
-
-	ret = glfs_process_volfp (fs, tmpfp);
-	/* tmpfp closed */
-	tmpfp = NULL;
-	if (ret)
-		goto out;
-
-	ret = glusterfs_oldvolfile_update (fs, rsp.spec, size);
-out:
-	STACK_DESTROY (frame->root);
+    call_frame_t *frame = NULL;
+    glusterfs_ctx_t *ctx = NULL;
+    struct syncargs args = {
+        0,
+    };
+    int ret = 0;
 
-	if (rsp.spec)
-		free (rsp.spec);
+    ctx = fs->ctx;
+    frame = create_frame(THIS, ctx->pool);
+    if (!frame) {
+        gf_smsg("glfs", GF_LOG_ERROR, ENOMEM, API_MSG_FRAME_CREAT_FAILED, NULL);
+        ret = -1;
+        goto out;
+    }
 
-        // Stop if server is running at an unsupported op-version
-        if (ENOTSUP == ret) {
-                gf_log ("mgmt", GF_LOG_ERROR, "Server is operating at an "
-                        "op-version which is not supported");
-                errno = ENOTSUP;
-                glfs_init_done (fs, -1);
-        }
+    frame->local = &args;
 
-	if (ret && ctx && !ctx->active) {
-		/* Do it only for the first time */
-		/* Failed to get the volume file, something wrong,
-		   restart the process */
-		gf_log ("glfs-mgmt", GF_LOG_ERROR,
-			"failed to fetch volume file (key:%s)",
-			ctx->cmd_args.volfile_id);
-		if (!need_retry) {
-                        if (!errno)
-                                errno = EINVAL;
-			glfs_init_done (fs, -1);
-                }
-	}
+    __yawn((&args));
+
+    ret = glfs_get_volume_info_rpc(frame, THIS, fs);
+    if (ret)
+        goto out;
 
-	if (tmpfp)
-		fclose (tmpfp);
+    __yield((&args));
 
-	return 0;
+    frame->local = NULL;
+    STACK_DESTROY(frame->root);
+
+out:
+    return ret;
 }
 
+int32_t
+glfs_get_volume_info_rpc(call_frame_t *frame, xlator_t *this, struct glfs *fs)
+{
+    gf_get_volume_info_req req = {{
+        0,
+    }};
+    int ret = 0;
+    glusterfs_ctx_t *ctx = NULL;
+    dict_t *dict = NULL;
+    int32_t flags = 0;
+
+    if (!frame || !this || !fs) {
+        ret = -1;
+        goto out;
+    }
+
+    ctx = fs->ctx;
+
+    dict = dict_new();
+    if (!dict) {
+        ret = -1;
+        goto out;
+    }
+
+    if (fs->volname) {
+        ret = dict_set_str(dict, "volname", fs->volname);
+        if (ret)
+            goto out;
+    }
+
+    // Set the flags for the fields which we are interested in
+    flags = (int32_t)GF_GET_VOLUME_UUID;  // ctx->flags;
+    ret = dict_set_int32(dict, "flags", flags);
+    if (ret) {
+        gf_smsg(frame->this->name, GF_LOG_ERROR, EINVAL,
+                API_MSG_DICT_SET_FAILED, "flags", NULL);
+        goto out;
+    }
+
+    ret = dict_allocate_and_serialize(dict, &req.dict.dict_val,
+                                      &req.dict.dict_len);
+
+    ret = mgmt_submit_request(&req, frame, ctx, &clnt_handshake_prog,
+                              GF_HNDSK_GET_VOLUME_INFO, mgmt_get_volinfo_cbk,
+                              (xdrproc_t)xdr_gf_get_volume_info_req);
+out:
+    if (dict) {
+        dict_unref(dict);
+    }
 
-int
-glfs_volfile_fetch (struct glfs *fs)
+    GF_FREE(req.dict.dict_val);
+
+    return ret;
+}
+
+static int
+glusterfs_oldvolfile_update(struct glfs *fs, char *volfile, ssize_t size)
 {
-	cmd_args_t	 *cmd_args = NULL;
-	gf_getspec_req	  req = {0, };
-	int		  ret = 0;
-	call_frame_t	 *frame = NULL;
-	glusterfs_ctx_t	 *ctx = NULL;
-        dict_t           *dict = NULL;
+    int ret = -1;
 
-	ctx = fs->ctx;
-	cmd_args = &ctx->cmd_args;
+    pthread_mutex_lock(&fs->mutex);
 
-	frame = create_frame (THIS, ctx->pool);
+    fs->oldvollen = size;
+    if (!fs->oldvolfile) {
+        fs->oldvolfile = CALLOC(1, size + 1);
+    } else {
+        fs->oldvolfile = REALLOC(fs->oldvolfile, size + 1);
+    }
 
-	req.key = cmd_args->volfile_id;
-	req.flags = 0;
+    if (!fs->oldvolfile) {
+        fs->oldvollen = 0;
+    } else {
+        memcpy(fs->oldvolfile, volfile, size);
+        fs->oldvollen = size;
+        ret = 0;
+    }
 
-        dict = dict_new ();
-        if (!dict) {
-                ret = -1;
-                goto out;
-        }
+    pthread_mutex_unlock(&fs->mutex);
 
-        // Set the supported min and max op-versions, so glusterd can make a
-        // decision
-        ret = dict_set_int32 (dict, "min-op-version", GD_OP_VERSION_MIN);
-        if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "Failed to set min-op-version"
-                        " in request dict");
-                goto out;
-        }
+    return ret;
+}
 
-        ret = dict_set_int32 (dict, "max-op-version", GD_OP_VERSION_MAX);
-        if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "Failed to set max-op-version"
-                        " in request dict");
-                goto out;
+int
+glfs_mgmt_getspec_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                      void *myframe)
+{
+    gf_getspec_rsp rsp = {
+        0,
+    };
+    call_frame_t *frame = NULL;
+    glusterfs_ctx_t *ctx = NULL;
+    int ret = 0;
+    ssize_t size = 0;
+    FILE *tmpfp = NULL;
+    int need_retry = 0;
+    struct glfs *fs = NULL;
+    dict_t *dict = NULL;
+    char *servers_list = NULL;
+    int tmp_fd = -1;
+    char template[] = "/tmp/gfapi.volfile.XXXXXX";
+
+    frame = myframe;
+    ctx = frame->this->ctx;
+
+    if (!ctx) {
+        gf_smsg(frame->this->name, GF_LOG_ERROR, EINVAL, API_MSG_NULL,
+                "context", NULL);
+        errno = EINVAL;
+        ret = -1;
+        goto out;
+    }
+
+    fs = ((xlator_t *)ctx->master)->private;
+
+    if (-1 == req->rpc_status) {
+        ret = -1;
+        need_retry = 1;
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_getspec_rsp);
+    if (ret < 0) {
+        gf_smsg(frame->this->name, GF_LOG_ERROR, 0, API_MSG_XDR_DECODE_FAILED,
+                NULL);
+        ret = -1;
+        goto out;
+    }
+
+    if (-1 == rsp.op_ret) {
+        gf_smsg(frame->this->name, GF_LOG_ERROR, rsp.op_errno,
+                API_MSG_GET_VOLFILE_FAILED, "from server", NULL);
+        ret = -1;
+        errno = rsp.op_errno;
+        goto out;
+    }
+
+    if (!rsp.xdata.xdata_len) {
+        goto volfile;
+    }
+
+    dict = dict_new();
+    if (!dict) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    ret = dict_unserialize(rsp.xdata.xdata_val, rsp.xdata.xdata_len, &dict);
+    if (ret) {
+        gf_log(frame->this->name, GF_LOG_ERROR,
+               "failed to unserialize xdata to dictionary");
+        goto out;
+    }
+    dict->extra_stdfree = rsp.xdata.xdata_val;
+
+    /* glusterd2 only */
+    ret = dict_get_str(dict, "servers-list", &servers_list);
+    if (ret) {
+        goto volfile;
+    }
+
+    gf_log(frame->this->name, GF_LOG_INFO,
+           "Received list of available volfile servers: %s", servers_list);
+
+    ret = gf_process_getspec_servers_list(&ctx->cmd_args, servers_list);
+    if (ret) {
+        gf_log(frame->this->name, GF_LOG_ERROR,
+               "Failed (%s) to process servers list: %s", strerror(errno),
+               servers_list);
+    }
+
+volfile:
+    ret = 0;
+    size = rsp.op_ret;
+
+    pthread_mutex_lock(&fs->mutex);
+    if ((size == fs->oldvollen) &&
+        (memcmp(fs->oldvolfile, rsp.spec, size) == 0)) {
+        pthread_mutex_unlock(&fs->mutex);
+        gf_smsg(frame->this->name, GF_LOG_INFO, 0, API_MSG_VOLFILE_INFO, NULL);
+        goto out;
+    }
+    pthread_mutex_unlock(&fs->mutex);
+
+    /* coverity[secure_temp] mkstemp uses 0600 as the mode and is safe */
+    tmp_fd = mkstemp(template);
+    if (-1 == tmp_fd) {
+        ret = -1;
+        goto out;
+    }
+
+    /* Calling unlink so that when the file is closed or program
+     * terminates the temporary file is deleted.
+     */
+    ret = sys_unlink(template);
+    if (ret < 0) {
+        gf_smsg(frame->this->name, GF_LOG_INFO, 0, API_MSG_UNABLE_TO_DEL,
+                "template=%s", template, NULL);
+        ret = 0;
+    }
+
+    tmpfp = fdopen(tmp_fd, "w+b");
+    if (!tmpfp) {
+        ret = -1;
+        goto out;
+    }
+
+    fwrite(rsp.spec, size, 1, tmpfp);
+    fflush(tmpfp);
+    if (ferror(tmpfp)) {
+        ret = -1;
+        goto out;
+    }
+
+    /*  Check if only options have changed. No need to reload the
+     *  volfile if topology hasn't changed.
+     *  glusterfs_volfile_reconfigure returns 3 possible return states
+     *  return 0	     =======> reconfiguration of options has succeeded
+     *  return 1	     =======> the graph has to be reconstructed and all
+     * the xlators should be inited return -1(or -ve) =======> Some Internal
+     * Error occurred during the operation
+     */
+
+    pthread_mutex_lock(&fs->mutex);
+    ret = gf_volfile_reconfigure(fs->oldvollen, tmpfp, fs->ctx, fs->oldvolfile);
+    pthread_mutex_unlock(&fs->mutex);
+
+    if (ret == 0) {
+        gf_msg_debug("glusterfsd-mgmt", 0,
+                     "No need to re-load "
+                     "volfile, reconfigure done");
+        ret = glusterfs_oldvolfile_update(fs, rsp.spec, size);
+        goto out;
+    }
+
+    if (ret < 0) {
+        gf_msg_debug("glusterfsd-mgmt", 0, "Reconfigure failed !!");
+        goto out;
+    }
+
+    ret = glfs_process_volfp(fs, tmpfp);
+    /* tmpfp closed */
+    tmpfp = NULL;
+    tmp_fd = -1;
+    if (ret)
+        goto out;
+
+    ret = glusterfs_oldvolfile_update(fs, rsp.spec, size);
+out:
+    STACK_DESTROY(frame->root);
+
+    if (rsp.spec)
+        free(rsp.spec);
+
+    if (dict)
+        dict_unref(dict);
+
+    // Stop if server is running at an unsupported op-version
+    if (ENOTSUP == ret) {
+        gf_smsg("mgmt", GF_LOG_ERROR, ENOTSUP, API_MSG_WRONG_OPVERSION, NULL);
+        errno = ENOTSUP;
+        glfs_init_done(fs, -1);
+    }
+
+    if (ret && ctx && !ctx->active) {
+        /* Do it only for the first time */
+        /* Failed to get the volume file, something wrong,
+           restart the process */
+        gf_smsg("glfs-mgmt", GF_LOG_ERROR, EINVAL, API_MSG_GET_VOLFILE_FAILED,
+                "key=%s", ctx->cmd_args.volfile_id, NULL);
+        if (!need_retry) {
+            if (!errno)
+                errno = EINVAL;
+            glfs_init_done(fs, -1);
         }
+    }
 
-        ret = dict_allocate_and_serialize (dict, &req.xdata.xdata_val,
-                                           &req.xdata.xdata_len);
-        if (ret < 0) {
-                gf_log (THIS->name, GF_LOG_ERROR,
-                        "Failed to serialize dictionary");
-                goto out;
-        }
+    if (tmpfp)
+        fclose(tmpfp);
+    else if (tmp_fd != -1)
+        sys_close(tmp_fd);
 
-	ret = mgmt_submit_request (&req, frame, ctx, &clnt_handshake_prog,
-				   GF_HNDSK_GETSPEC, mgmt_getspec_cbk,
-				   (xdrproc_t)xdr_gf_getspec_req);
-out:
-        return ret;
+    return 0;
 }
 
+int
+glfs_volfile_fetch(struct glfs *fs)
+{
+    cmd_args_t *cmd_args = NULL;
+    gf_getspec_req req = {
+        0,
+    };
+    int ret = -1;
+    call_frame_t *frame = NULL;
+    glusterfs_ctx_t *ctx = NULL;
+    dict_t *dict = NULL;
+
+    ctx = fs->ctx;
+    cmd_args = &ctx->cmd_args;
+
+    req.key = cmd_args->volfile_id;
+    req.flags = 0;
+
+    dict = dict_new();
+    if (!dict) {
+        goto out;
+    }
+
+    // Set the supported min and max op-versions, so glusterd can make a
+    // decision
+    ret = dict_set_int32(dict, "min-op-version", GD_OP_VERSION_MIN);
+    if (ret) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, EINVAL, API_MSG_DICT_SET_FAILED,
+                "min-op-version", NULL);
+        goto out;
+    }
+
+    ret = dict_set_int32(dict, "max-op-version", GD_OP_VERSION_MAX);
+    if (ret) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, EINVAL, API_MSG_DICT_SET_FAILED,
+                "max-op-version", NULL);
+        goto out;
+    }
+
+    /* Ask for a list of volfile (glusterd2 only) servers */
+    if (GF_CLIENT_PROCESS == ctx->process_mode) {
+        req.flags = req.flags | GF_GETSPEC_FLAG_SERVERS_LIST;
+    }
+
+    ret = dict_allocate_and_serialize(dict, &req.xdata.xdata_val,
+                                      &req.xdata.xdata_len);
+    if (ret < 0) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, 0, API_MSG_DICT_SERIALIZE_FAILED,
+                NULL);
+        goto out;
+    }
+
+    frame = create_frame(THIS, ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = mgmt_submit_request(&req, frame, ctx, &clnt_handshake_prog,
+                              GF_HNDSK_GETSPEC, glfs_mgmt_getspec_cbk,
+                              (xdrproc_t)xdr_gf_getspec_req);
+out:
+    if (req.xdata.xdata_val)
+        GF_FREE(req.xdata.xdata_val);
+    if (dict)
+        dict_unref(dict);
+
+    return ret;
+}
 
 static int
-mgmt_rpc_notify (struct rpc_clnt *rpc, void *mydata, rpc_clnt_event_t event,
-		 void *data)
+mgmt_rpc_notify(struct rpc_clnt *rpc, void *mydata, rpc_clnt_event_t event,
+                void *data)
 {
-	xlator_t	*this = NULL;
-	cmd_args_t	*cmd_args = NULL;
-	glusterfs_ctx_t *ctx = NULL;
-	struct glfs	 *fs = NULL;
-	int		 ret = 0;
-
-	this = mydata;
-	ctx = this->ctx;
-	fs = ((xlator_t *)ctx->master)->private;
-	cmd_args = &ctx->cmd_args;
-
-	switch (event) {
-	case RPC_CLNT_DISCONNECT:
-		if (!ctx->active) {
-			cmd_args->max_connect_attempts--;
-			gf_log ("glfs-mgmt", GF_LOG_ERROR,
-				"failed to connect with remote-host: %s",
-				strerror (errno));
-			gf_log ("glfs-mgmt", GF_LOG_INFO,
-				"%d connect attempts left",
-				cmd_args->max_connect_attempts);
-			if (0 >= cmd_args->max_connect_attempts) {
-                                errno = ENOTCONN;
-				glfs_init_done (fs, -1);
-                        }
-		}
-		break;
-	case RPC_CLNT_CONNECT:
-		rpc_clnt_set_connected (&((struct rpc_clnt*)ctx->mgmt)->conn);
-
-		ret = glfs_volfile_fetch (fs);
-		if (ret && ctx && (ctx->active == NULL)) {
-			/* Do it only for the first time */
-			/* Exit the process.. there are some wrong options */
-			gf_log ("glfs-mgmt", GF_LOG_ERROR,
-				"failed to fetch volume file (key:%s)",
-				ctx->cmd_args.volfile_id);
-                        errno = EINVAL;
-			glfs_init_done (fs, -1);
-		}
-
-		break;
-	default:
-		break;
-	}
-
-	return 0;
-}
+    xlator_t *this = NULL;
+    glusterfs_ctx_t *ctx = NULL;
+    server_cmdline_t *server = NULL;
+    rpc_transport_t *rpc_trans = NULL;
+    struct glfs *fs = NULL;
+    int ret = 0;
+    struct dnscache6 *dnscache = NULL;
+
+    this = mydata;
+    rpc_trans = rpc->conn.trans;
+
+    ctx = this->ctx;
+    if (!ctx)
+        goto out;
+
+    fs = ((xlator_t *)ctx->master)->private;
+
+    switch (event) {
+        case RPC_CLNT_DISCONNECT:
+            if (!ctx->active) {
+                if (rpc_trans->connect_failed)
+                    gf_smsg("glfs-mgmt", GF_LOG_ERROR, 0,
+                            API_MSG_REMOTE_HOST_CONN_FAILED, "server=%s",
+                            ctx->cmd_args.volfile_server, NULL);
+                else
+                    gf_smsg("glfs-mgmt", GF_LOG_INFO, 0,
+                            API_MSG_REMOTE_HOST_CONN_FAILED, "server=%s",
+                            ctx->cmd_args.volfile_server, NULL);
+
+                if (!rpc->disabled) {
+                    /*
+                     * Check if dnscache is exhausted for current server
+                     * and continue until cache is exhausted
+                     */
+                    dnscache = rpc_trans->dnscache;
+                    if (dnscache && dnscache->next) {
+                        break;
+                    }
+                }
+                server = ctx->cmd_args.curr_server;
+                if (server->list.next == &ctx->cmd_args.volfile_servers) {
+                    errno = ENOTCONN;
+                    gf_smsg("glfs-mgmt", GF_LOG_INFO, ENOTCONN,
+                            API_MSG_VOLFILE_SERVER_EXHAUST, NULL);
+                    glfs_init_done(fs, -1);
+                    break;
+                }
+                server = list_entry(server->list.next, typeof(*server), list);
+                ctx->cmd_args.curr_server = server;
+                ctx->cmd_args.volfile_server_port = server->port;
+                ctx->cmd_args.volfile_server = server->volfile_server;
+                ctx->cmd_args.volfile_server_transport = server->transport;
+
+                ret = dict_set_str(rpc_trans->options, "transport-type",
+                                   server->transport);
+                if (ret != 0) {
+                    gf_smsg("glfs-mgmt", GF_LOG_ERROR, ENOTCONN,
+                            API_MSG_DICT_SET_FAILED, "transport-type=%s",
+                            server->transport, NULL);
+                    errno = ENOTCONN;
+                    glfs_init_done(fs, -1);
+                    break;
+                }
+
+                if (strcmp(server->transport, "unix") == 0) {
+                    ret = dict_set_str(rpc_trans->options,
+                                       "transport.socket.connect-path",
+                                       server->volfile_server);
+                    if (ret != 0) {
+                        gf_smsg("glfs-mgmt", GF_LOG_ERROR, ENOTCONN,
+                                API_MSG_DICT_SET_FAILED,
+                                "socket.connect-path=%s",
+                                server->volfile_server, NULL);
+                        errno = ENOTCONN;
+                        glfs_init_done(fs, -1);
+                        break;
+                    }
+                    /* delete the remote-host and remote-port keys
+                     * in case they were set while looping through
+                     * list of volfile servers previously
+                     */
+                    dict_del(rpc_trans->options, "remote-host");
+                    dict_del(rpc_trans->options, "remote-port");
+                } else {
+                    ret = dict_set_int32(rpc_trans->options, "remote-port",
+                                         server->port);
+                    if (ret != 0) {
+                        gf_smsg("glfs-mgmt", GF_LOG_ERROR, ENOTCONN,
+                                API_MSG_DICT_SET_FAILED, "remote-port=%d",
+                                server->port, NULL);
+                        errno = ENOTCONN;
+                        glfs_init_done(fs, -1);
+                        break;
+                    }
+
+                    ret = dict_set_str(rpc_trans->options, "remote-host",
+                                       server->volfile_server);
+                    if (ret != 0) {
+                        gf_smsg("glfs-mgmt", GF_LOG_ERROR, ENOTCONN,
+                                API_MSG_DICT_SET_FAILED, "remote-host=%s",
+                                server->volfile_server, NULL);
+                        errno = ENOTCONN;
+                        glfs_init_done(fs, -1);
+                        break;
+                    }
+                    /* delete the "transport.socket.connect-path"
+                     * key in case if it was set while looping
+                     * through list of volfile servers previously
+                     */
+                    dict_del(rpc_trans->options,
+                             "transport.socket.connect-path");
+                }
 
+                gf_smsg("glfs-mgmt", GF_LOG_INFO, 0, API_MSG_VOLFILE_CONNECTING,
+                        "server=%s", server->volfile_server, "port=%d",
+                        server->port, "transport=%s", server->transport, NULL);
+            }
+            break;
+        case RPC_CLNT_CONNECT:
+            ret = glfs_volfile_fetch(fs);
+            if (ret && (ctx->active == NULL)) {
+                /* Do it only for the first time */
+                /* Exit the process.. there are some wrong options */
+                gf_smsg("glfs-mgmt", GF_LOG_ERROR, EINVAL,
+                        API_MSG_GET_VOLFILE_FAILED, "key=%s",
+                        ctx->cmd_args.volfile_id, NULL);
+                errno = EINVAL;
+                glfs_init_done(fs, -1);
+            }
+
+            break;
+        default:
+            break;
+    }
+out:
+    return 0;
+}
 
 int
-glusterfs_mgmt_notify (int32_t op, void *data, ...)
+glusterfs_mgmt_notify(int32_t op, void *data, ...)
 {
-	int ret = 0;
+    int ret = 0;
 
-	switch (op)
-	{
-		case GF_EN_DEFRAG_STATUS:
-			break;
+    switch (op) {
+        case GF_EN_DEFRAG_STATUS:
+            break;
 
-		default:
-			break;
-	}
+        default:
+            break;
+    }
 
-	return ret;
+    return ret;
 }
 
-
 int
-glfs_mgmt_init (struct glfs *fs)
+glfs_mgmt_init(struct glfs *fs)
 {
-	cmd_args_t		*cmd_args = NULL;
-	struct rpc_clnt		*rpc = NULL;
-	dict_t			*options = NULL;
-	int			ret = -1;
-	int			port = GF_DEFAULT_BASE_PORT;
-	char			*host = NULL;
-	glusterfs_ctx_t		*ctx = NULL;
-
-	ctx = fs->ctx;
-	cmd_args = &ctx->cmd_args;
-
-	if (ctx->mgmt)
-		return 0;
-
-	if (cmd_args->volfile_server_port)
-		port = cmd_args->volfile_server_port;
-
-	host = "localhost";
-	if (cmd_args->volfile_server)
-		host = cmd_args->volfile_server;
-
-	ret = rpc_transport_inet_options_build (&options, host, port);
-	if (ret)
-		goto out;
-
-	rpc = rpc_clnt_new (options, THIS->ctx, THIS->name, 8);
-	if (!rpc) {
-		ret = -1;
-		gf_log (THIS->name, GF_LOG_WARNING,
-			"failed to create rpc clnt");
-		goto out;
-	}
-
-	ret = rpc_clnt_register_notify (rpc, mgmt_rpc_notify, THIS);
-	if (ret) {
-		gf_log (THIS->name, GF_LOG_WARNING,
-			"failed to register notify function");
-		goto out;
-	}
-
-	ret = rpcclnt_cbk_program_register (rpc, &mgmt_cbk_prog, THIS);
-	if (ret) {
-		gf_log (THIS->name, GF_LOG_WARNING,
-			"failed to register callback function");
-		goto out;
-	}
-
-	ctx->notify = glusterfs_mgmt_notify;
-
-	/* This value should be set before doing the 'rpc_clnt_start()' as
-	   the notify function uses this variable */
-	ctx->mgmt = rpc;
-
-	ret = rpc_clnt_start (rpc);
+    cmd_args_t *cmd_args = NULL;
+    struct rpc_clnt *rpc = NULL;
+    dict_t *options = NULL;
+    int ret = -1;
+    int port = GF_DEFAULT_BASE_PORT;
+    char *host = NULL;
+    glusterfs_ctx_t *ctx = NULL;
+
+    ctx = fs->ctx;
+    cmd_args = &ctx->cmd_args;
+
+    if (ctx->mgmt)
+        return 0;
+
+    options = dict_new();
+    if (!options)
+        goto out;
+
+    if (cmd_args->volfile_server_port)
+        port = cmd_args->volfile_server_port;
+
+    if (cmd_args->volfile_server) {
+        host = cmd_args->volfile_server;
+    } else if (cmd_args->volfile_server_transport &&
+               !strcmp(cmd_args->volfile_server_transport, "unix")) {
+        host = DEFAULT_GLUSTERD_SOCKFILE;
+    } else {
+        host = "localhost";
+    }
+
+    if (cmd_args->volfile_server_transport &&
+        !strcmp(cmd_args->volfile_server_transport, "unix")) {
+        ret = rpc_transport_unix_options_build(options, host, 0);
+    } else {
+        xlator_cmdline_option_t *opt = find_xlator_option_in_cmd_args_t(
+            "address-family", cmd_args);
+        ret = rpc_transport_inet_options_build(options, host, port,
+                                               (opt ? opt->value : NULL));
+    }
+
+    if (ret)
+        goto out;
+
+    rpc = rpc_clnt_new(options, THIS, THIS->name, 8);
+    if (!rpc) {
+        ret = -1;
+        gf_smsg(THIS->name, GF_LOG_WARNING, 0, API_MSG_CREATE_RPC_CLIENT_FAILED,
+                NULL);
+        goto out;
+    }
+
+    ret = rpc_clnt_register_notify(rpc, mgmt_rpc_notify, THIS);
+    if (ret) {
+        gf_smsg(THIS->name, GF_LOG_WARNING, 0, API_MSG_REG_NOTIFY_FUNC_FAILED,
+                NULL);
+        goto out;
+    }
+
+    ret = rpcclnt_cbk_program_register(rpc, &mgmt_cbk_prog, THIS);
+    if (ret) {
+        gf_smsg(THIS->name, GF_LOG_WARNING, 0, API_MSG_REG_CBK_FUNC_FAILED,
+                NULL);
+        goto out;
+    }
+
+    ctx->notify = glusterfs_mgmt_notify;
+
+    /* This value should be set before doing the 'rpc_clnt_start()' as
+       the notify function uses this variable */
+    ctx->mgmt = rpc;
+
+    ret = rpc_clnt_start(rpc);
 out:
-	return ret;
+    if (options)
+        dict_unref(options);
+    return ret;
 }
-
diff --git a/api/src/glfs-resolve.c b/api/src/glfs-resolve.c
index 3179af22c2d..8a393ecb464 100644
--- a/api/src/glfs-resolve.c
+++ b/api/src/glfs-resolve.c
@@ -1,5 +1,5 @@
 /*
-  Copyright (c) 2012 Red Hat, Inc. <http://www.redhat.com>
+  Copyright (c) 2012-2018 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
   This file is licensed to you under your choice of the GNU Lesser
@@ -8,7 +8,6 @@
   cases as published by the Free Software Foundation.
 */
 
-
 #include <unistd.h>
 #include <string.h>
 #include <stdlib.h>
@@ -16,887 +15,1185 @@
 #include <inttypes.h>
 #include <limits.h>
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "glusterfs.h"
-#include "logging.h"
-#include "stack.h"
-#include "event.h"
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/stack.h>
+#include <glusterfs/gf-event.h>
 #include "glfs-mem-types.h"
-#include "common-utils.h"
-#include "syncop.h"
-#include "call-stub.h"
-
+#include <glusterfs/common-utils.h>
+#include <glusterfs/syncop.h>
+#include <glusterfs/call-stub.h>
+#include "gfapi-messages.h"
+#include <glusterfs/inode.h>
 #include "glfs-internal.h"
 
-#define graphid_str(subvol) (uuid_utoa((unsigned char *)subvol->graph->graph_uuid))
-
-
+#define graphid_str(subvol)                                                    \
+    (uuid_utoa((unsigned char *)subvol->graph->graph_uuid))
 int
-glfs_first_lookup_safe (xlator_t *subvol)
+glfs_first_lookup_safe(xlator_t *subvol)
 {
-	loc_t  loc = {0, };
-	int    ret = -1;
+    loc_t loc = {
+        0,
+    };
+    int ret = -1;
 
-	loc.inode = subvol->itable->root;
-	memset (loc.gfid, 0, 16);
-	loc.gfid[15] = 1;
-	loc.path = "/";
-	loc.name = "";
+    loc.inode = subvol->itable->root;
+    memset(loc.gfid, 0, 16);
+    loc.gfid[15] = 1;
+    loc.path = "/";
+    loc.name = "";
 
-	ret = syncop_lookup (subvol, &loc, 0, 0, 0, 0);
+    ret = syncop_lookup(subvol, &loc, 0, 0, 0, 0);
+    DECODE_SYNCOP_ERR(ret);
 
-	gf_log (subvol->name, GF_LOG_DEBUG, "first lookup complete %d", ret);
+    gf_msg_debug(subvol->name, 0, "first lookup complete %d", ret);
 
-	return ret;
+    return ret;
 }
 
-
 int
-__glfs_first_lookup (struct glfs *fs, xlator_t *subvol)
+__glfs_first_lookup(struct glfs *fs, xlator_t *subvol)
 {
-	int ret = -1;
+    int ret = -1;
 
-	fs->migration_in_progress = 1;
-	pthread_mutex_unlock (&fs->mutex);
-	{
-		ret = glfs_first_lookup_safe (subvol);
-	}
-	pthread_mutex_lock (&fs->mutex);
-	fs->migration_in_progress = 0;
+    fs->migration_in_progress = 1;
+    pthread_mutex_unlock(&fs->mutex);
+    {
+        ret = glfs_first_lookup_safe(subvol);
+    }
+    pthread_mutex_lock(&fs->mutex);
+    fs->migration_in_progress = 0;
+    pthread_cond_broadcast(&fs->cond);
 
-	return ret;
-}
+    /* wake up other waiting tasks */
+    __GLFS_SYNCTASK_WAKE(fs);
 
+    return ret;
+}
 
+/**
+ * We have to check if need_lookup flag is set in both old and the new inodes.
+ * If its set in oldinode, then directly go ahead and do an explicit lookup.
+ * But if its not set in the oldinode, then check if the newinode is linked
+ * via readdirp. If so an explicit lookup is needed on the new inode, so that
+ * below xlators can set their respective contexts.
+ */
 inode_t *
-glfs_refresh_inode_safe (xlator_t *subvol, inode_t *oldinode)
+glfs_refresh_inode_safe(xlator_t *subvol, inode_t *oldinode,
+                        gf_boolean_t need_lookup)
 {
-	loc_t        loc = {0, };
-	int          ret = -1;
-	struct iatt  iatt = {0, };
-	inode_t     *newinode = NULL;
-
-
-	if (!oldinode)
-		return NULL;
-
-	if (oldinode->table->xl == subvol)
-		return inode_ref (oldinode);
+    loc_t loc = {
+        0,
+    };
+    int ret = -1;
+    struct iatt iatt = {
+        0,
+    };
+    inode_t *newinode = NULL;
+    gf_boolean_t lookup_needed = _gf_false;
+    uint64_t ctx_value = LOOKUP_NOT_NEEDED;
+
+    if (!oldinode)
+        return NULL;
+
+    if (!need_lookup && oldinode->table->xl == subvol)
+        return inode_ref(oldinode);
+
+    newinode = inode_find(subvol->itable, oldinode->gfid);
+    if (!need_lookup && newinode) {
+        lookup_needed = inode_needs_lookup(newinode, THIS);
+        if (!lookup_needed)
+            return newinode;
+    }
+
+    gf_uuid_copy(loc.gfid, oldinode->gfid);
+    if (!newinode)
+        loc.inode = inode_new(subvol->itable);
+    else
+        loc.inode = newinode;
+
+    if (!loc.inode)
+        return NULL;
+
+    ret = syncop_lookup(subvol, &loc, &iatt, 0, 0, 0);
+    DECODE_SYNCOP_ERR(ret);
+
+    if (ret) {
+        gf_smsg(subvol->name, GF_LOG_WARNING, errno,
+                API_MSG_INODE_REFRESH_FAILED, "gfid=%s",
+                uuid_utoa(oldinode->gfid), "err=%s", strerror(errno), NULL);
+        loc_wipe(&loc);
+        return NULL;
+    }
+
+    newinode = inode_link(loc.inode, 0, 0, &iatt);
+    if (newinode) {
+        if (newinode == loc.inode)
+            inode_ctx_set(newinode, THIS, &ctx_value);
+        inode_lookup(newinode);
+    } else {
+        gf_smsg(subvol->name, GF_LOG_WARNING, errno, API_MSG_INODE_LINK_FAILED,
+                "gfid=%s", uuid_utoa((unsigned char *)&iatt.ia_gfid), NULL);
+    }
+
+    loc_wipe(&loc);
+
+    return newinode;
+}
 
-	newinode = inode_find (subvol->itable, oldinode->gfid);
-	if (newinode)
-		return newinode;
+inode_t *
+__glfs_refresh_inode(struct glfs *fs, xlator_t *subvol, inode_t *inode,
+                     gf_boolean_t need_lookup)
+{
+    inode_t *newinode = NULL;
 
-	uuid_copy (loc.gfid, oldinode->gfid);
-	loc.inode = inode_new (subvol->itable);
-	if (!loc.inode)
-		return NULL;
+    fs->migration_in_progress = 1;
+    pthread_mutex_unlock(&fs->mutex);
+    {
+        newinode = glfs_refresh_inode_safe(subvol, inode, need_lookup);
+    }
+    pthread_mutex_lock(&fs->mutex);
+    fs->migration_in_progress = 0;
+    pthread_cond_broadcast(&fs->cond);
 
-	ret = syncop_lookup (subvol, &loc, 0, &iatt, 0, 0);
+    /* wake up other waiting tasks */
+    __GLFS_SYNCTASK_WAKE(fs);
 
-	if (ret) {
-		gf_log (subvol->name, GF_LOG_WARNING,
-			"inode refresh of %s failed: %s",
-			uuid_utoa (oldinode->gfid), strerror (errno));
-		loc_wipe (&loc);
-		return NULL;
-	}
+    return newinode;
+}
 
-	newinode = inode_link (loc.inode, 0, 0, &iatt);
-	if (newinode)
-		inode_lookup (newinode);
+GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_loc_touchup, 3.4.0)
+int
+priv_glfs_loc_touchup(loc_t *loc)
+{
+    int ret = 0;
 
-	loc_wipe (&loc);
+    ret = loc_touchup(loc, loc->name);
+    if (ret < 0) {
+        errno = -ret;
+        ret = -1;
+    }
 
-	return newinode;
+    return ret;
 }
 
-
-inode_t *
-__glfs_refresh_inode (struct glfs *fs, xlator_t *subvol, inode_t *inode)
+int
+glfs_resolve_symlink(struct glfs *fs, xlator_t *subvol, inode_t *inode,
+                     char **lpath)
 {
-	inode_t *newinode = NULL;
-
-	fs->migration_in_progress = 1;
-	pthread_mutex_unlock (&fs->mutex);
-	{
-		newinode = glfs_refresh_inode_safe (subvol, inode);
-	}
-	pthread_mutex_lock (&fs->mutex);
-	fs->migration_in_progress = 0;
-
-	return newinode;
+    loc_t loc = {
+        0,
+    };
+    char *path = NULL;
+    char *rpath = NULL;
+    int ret = -1;
+
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
+    ret = inode_path(inode, NULL, &rpath);
+    if (ret < 0)
+        goto out;
+    loc.path = rpath;
+
+    ret = syncop_readlink(subvol, &loc, &path, 4096, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    if (ret < 0)
+        goto out;
+
+    if (lpath)
+        *lpath = path;
+out:
+    loc_wipe(&loc);
+    return ret;
 }
 
 int
-glfs_loc_touchup (loc_t *loc)
+glfs_resolve_base(struct glfs *fs, xlator_t *subvol, inode_t *inode,
+                  struct iatt *iatt)
 {
-	char *path = NULL;
-	int   ret = -1;
-	char *bn = NULL;
-
-	if (loc->parent)
-		ret = inode_path (loc->parent, loc->name, &path);
-	else
-		ret = inode_path (loc->inode, 0, &path);
-
-	loc->path = path;
-
-	if (ret < 0 || !path) {
-		ret = -1;
-		errno = ENOMEM;
-		goto out;
-	}
-
-	bn = strrchr (path, '/');
-	if (bn)
-		bn++;
-	loc->name = bn;
-	ret = 0;
+    loc_t loc = {
+        0,
+    };
+    int ret = -1;
+    char *path = NULL;
+
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
+
+    ret = inode_path(loc.inode, NULL, &path);
+    loc.path = path;
+    if (ret < 0)
+        goto out;
+
+    ret = syncop_lookup(subvol, &loc, iatt, NULL, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
 out:
-	return ret;
+    loc_wipe(&loc);
+
+    return ret;
 }
+/*
+ * This function can be used to call named lookup on root.
+ * If you use glfs_resolve_base, that will be a nameless lookup.
+ */
+static int
+glfs_resolve_root(struct glfs *fs, xlator_t *subvol, inode_t *inode,
+                  struct iatt *iatt)
+{
+    loc_t loc = {
+        0,
+    };
+    int ret = -1;
+    char *path = NULL;
+
+    loc.inode = inode_ref(inode);
+
+    ret = inode_path(loc.inode, ".", &path);
+    loc.path = path;
+    loc.name = ".";
+    /* Having a value in loc.name will help to bypass md-cache check for
+     * nameless lookup.
+     * TODO: Re-visit on nameless lookup and md-cache.
+     * Github issue : https://github.com/gluster/glusterfs/issues/232
+     */
+    loc.parent = inode_ref(inode);
+    if (ret < 0)
+        goto out;
+
+    ret = syncop_lookup(subvol, &loc, iatt, NULL, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+out:
+    loc_wipe(&loc);
 
+    return ret;
+}
 
-int
-glfs_resolve_symlink (struct glfs *fs, xlator_t *subvol, inode_t *inode,
-		      char **lpath)
+inode_t *
+glfs_resolve_component(struct glfs *fs, xlator_t *subvol, inode_t *parent,
+                       const char *component, struct iatt *iatt,
+                       int force_lookup)
 {
-	loc_t  loc = {0, };
-	char  *path = NULL;
-	char  *rpath = NULL;
-	int    ret = -1;
+    loc_t loc = {
+        0,
+    };
+    inode_t *inode = NULL;
+    inode_t *temp_parent = NULL;
+    int reval = 0;
+    int ret = -1;
+    int glret = -1;
+    struct iatt ciatt = {
+        0,
+    };
+    uuid_t gfid;
+    dict_t *xattr_req = NULL;
+    uint64_t ctx_value = LOOKUP_NOT_NEEDED;
+
+    loc.parent = inode_ref(parent);
+    gf_uuid_copy(loc.pargfid, parent->gfid);
+
+    if (__is_root_gfid(parent->gfid) &&
+        ((strcmp(component, ".") == 0) || (strcmp(component, "..") == 0) ||
+         (strcmp(component, "") == 0))) {
+        if (!force_lookup) {
+            inode = inode_ref(parent);
+        } else {
+            ret = glfs_resolve_root(fs, subvol, parent, &ciatt);
+            if (!ret)
+                inode = inode_ref(parent);
+        }
+        goto found;
+    }
+    /* *
+     * if the component name is either "." or "..", it will try to
+     * resolve that if inode has a proper parent (named lookup).
+     *
+     * Below condition works like this
+     *
+     * Example 1 :
+     * Path /out_dir/dir/in_dir/.
+     *     In put values :
+     *         parent = in_dir
+     *         component : "."
+     *
+     *      Out put values:
+     *         parent : dir
+     *         component : "in_dir"
+     *
+     * Example 2 :
+     * Path /out_dir/dir/in_dir/..
+     *     In put values :
+     *         parent = in_dir
+     *         component : ".."
+     *
+     *     Out put values:
+     *         parent : output_dir
+     *         component : "dir"
+     *
+     * In case of nameless lookup, both "." and ".." retained
+     */
+
+    if (strcmp(component, ".") == 0) {
+        loc.inode = inode_ref(parent);
+        temp_parent = inode_parent(loc.inode, 0, 0);
+        if (temp_parent) {
+            inode_unref(loc.parent);
+            loc.parent = temp_parent;
+            gf_uuid_copy(loc.pargfid, temp_parent->gfid);
+            inode_find_directory_name(loc.inode, &loc.name);
+        }
 
-	loc.inode = inode_ref (inode);
-	uuid_copy (loc.gfid, inode->gfid);
-	ret = inode_path (inode, NULL, &rpath);
-	if (ret < 0)
-		goto out;
-	loc.path = rpath;
+    } else if (strcmp(component, "..") == 0) {
+        loc.inode = inode_parent(parent, 0, 0);
+        if (loc.inode) {
+            temp_parent = inode_parent(loc.inode, 0, 0);
+            if (temp_parent) {
+                inode_unref(loc.parent);
+                loc.parent = temp_parent;
+                gf_uuid_copy(loc.pargfid, temp_parent->gfid);
+                inode_find_directory_name(loc.inode, &loc.name);
+            } else if (__is_root_gfid(loc.inode->gfid)) {
+                inode_unref(loc.parent);
+                loc.parent = inode_ref(loc.inode);
+                gf_uuid_copy(loc.pargfid, loc.inode->gfid);
+                loc.name = ".";
+            } else {
+                inode_unref(loc.inode);
+                loc.inode = NULL;
+            }
+        }
+    } else
+        loc.inode = inode_grep(parent->table, parent, component);
 
-	ret = syncop_readlink (subvol, &loc, &path, 4096);
+    if (!loc.name)
+        loc.name = component;
 
-	if (ret < 0)
-		goto out;
+    if (loc.inode) {
+        gf_uuid_copy(loc.gfid, loc.inode->gfid);
+        reval = 1;
 
-	if (lpath)
-		*lpath = path;
-out:
-	loc_wipe (&loc);
-	return ret;
-}
+        if (!(force_lookup || inode_needs_lookup(loc.inode, THIS))) {
+            inode = inode_ref(loc.inode);
+            goto found;
+        }
+    } else {
+        gf_uuid_generate(gfid);
+        loc.inode = inode_new(parent->table);
+        if (!loc.inode) {
+            errno = ENOMEM;
+            goto out;
+        }
 
+        xattr_req = dict_new();
+        if (!xattr_req) {
+            errno = ENOMEM;
+            goto out;
+        }
 
-void
-glfs_resolve_base (struct glfs *fs, xlator_t *subvol, inode_t *inode,
-		   struct iatt *iatt)
-{
-	loc_t       loc = {0, };
-	int         ret = -1;
-	char       *path = NULL;
+        ret = dict_set_gfuuid(xattr_req, "gfid-req", gfid, true);
+        if (ret) {
+            errno = ENOMEM;
+            goto out;
+        }
+    }
+
+    glret = priv_glfs_loc_touchup(&loc);
+    if (glret < 0) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = syncop_lookup(subvol, &loc, &ciatt, NULL, xattr_req, NULL);
+    if (ret && reval) {
+        /*
+         * A stale mapping might exist for a dentry/inode that has been
+         * removed from another client.
+         */
+        if (-ret == ENOENT) {
+            inode_unlink(loc.inode, loc.parent, loc.name);
+            if (!inode_has_dentry(loc.inode))
+                inode_forget(loc.inode, 0);
+        }
 
-	loc.inode = inode_ref (inode);
-	uuid_copy (loc.gfid, inode->gfid);
+        inode_unref(loc.inode);
+        gf_uuid_clear(loc.gfid);
+        loc.inode = inode_new(parent->table);
+        if (!loc.inode) {
+            errno = ENOMEM;
+            goto out;
+        }
 
-	ret = inode_path (loc.inode, NULL, &path);
-	loc.path = path;
-	if (ret < 0)
-		goto out;
+        xattr_req = dict_new();
+        if (!xattr_req) {
+            errno = ENOMEM;
+            goto out;
+        }
 
-	ret = syncop_lookup (subvol, &loc, NULL, iatt, NULL, NULL);
-out:
-	loc_wipe (&loc);
-}
+        gf_uuid_generate(gfid);
 
+        ret = dict_set_gfuuid(xattr_req, "gfid-req", gfid, true);
+        if (ret) {
+            errno = ENOMEM;
+            goto out;
+        }
 
-inode_t *
-glfs_resolve_component (struct glfs *fs, xlator_t *subvol, inode_t *parent,
-			const char *component, struct iatt *iatt,
-			int force_lookup)
-{
-	loc_t        loc = {0, };
-	inode_t     *inode = NULL;
-	int          reval = 0;
-	int          ret = -1;
-	int          glret = -1;
-	struct iatt  ciatt = {0, };
-	uuid_t       gfid;
-	dict_t      *xattr_req = NULL;
-
-	loc.name = component;
-
-	loc.parent = inode_ref (parent);
-	uuid_copy (loc.pargfid, parent->gfid);
-
-
-	if (strcmp (component, ".") == 0)
-		loc.inode = inode_ref (parent);
-	else if (strcmp (component, "..") == 0)
-		loc.inode = inode_parent (parent, 0, 0);
-	else
-		loc.inode = inode_grep (parent->table, parent, component);
-
-	if (loc.inode) {
-		uuid_copy (loc.gfid, loc.inode->gfid);
-		reval = 1;
-
-		if (!force_lookup) {
-			inode = inode_ref (loc.inode);
-			ciatt.ia_type = inode->ia_type;
-			goto found;
-		}
-	} else {
-		uuid_generate (gfid);
-		loc.inode = inode_new (parent->table);
-	}
-
-	if (!loc.inode)
-		goto out;
-
-	glret = glfs_loc_touchup (&loc);
-	if (glret < 0) {
-		ret = -1;
-		goto out;
-	}
-
-	ret = syncop_lookup (subvol, &loc, NULL, &ciatt, NULL, NULL);
-	if (ret && reval) {
-		inode_unref (loc.inode);
-		loc.inode = inode_new (parent->table);
-		if (!loc.inode) {
-			errno = ENOMEM;
-			goto out;
-		}
-
-		xattr_req = dict_new ();
-		if (!xattr_req) {
-			errno = ENOMEM;
-			goto out;
-		}
-
-		uuid_generate (gfid);
-
-		ret = dict_set_static_bin (xattr_req, "gfid-req", gfid, 16);
-		if (ret) {
-			errno = ENOMEM;
-			goto out;
-		}
-
-		ret = syncop_lookup (subvol, &loc, xattr_req, &ciatt,
-				     NULL, NULL);
-	}
-	if (ret)
-		goto out;
-
-	inode = inode_link (loc.inode, loc.parent, component, &ciatt);
+        ret = syncop_lookup(subvol, &loc, &ciatt, NULL, xattr_req, NULL);
+    }
+    DECODE_SYNCOP_ERR(ret);
+    if (ret)
+        goto out;
+
+    inode = inode_link(loc.inode, loc.parent, component, &ciatt);
+
+    if (!inode) {
+        gf_smsg(subvol->name, GF_LOG_WARNING, errno, API_MSG_INODE_LINK_FAILED,
+                "gfid=%s", uuid_utoa((unsigned char *)&ciatt.ia_gfid), NULL);
+        goto out;
+    } else if (inode == loc.inode)
+        inode_ctx_set(inode, THIS, &ctx_value);
 found:
-	if (inode)
-		inode_lookup (inode);
-	if (iatt)
-		*iatt = ciatt;
+    if (inode) {
+        ciatt.ia_type = inode->ia_type;
+        inode_lookup(inode);
+    }
+    if (iatt)
+        *iatt = ciatt;
 out:
-	if (xattr_req)
-		dict_unref (xattr_req);
-
-	loc_wipe (&loc);
+    if (xattr_req)
+        dict_unref(xattr_req);
+    loc_wipe(&loc);
 
-	return inode;
+    return inode;
 }
 
-
+GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_resolve_at, 3.4.0)
 int
-glfs_resolve_at (struct glfs *fs, xlator_t *subvol, inode_t *at,
-		 const char *origpath, loc_t *loc, struct iatt *iatt,
-		 int follow, int reval)
+priv_glfs_resolve_at(struct glfs *fs, xlator_t *subvol, inode_t *at,
+                     const char *origpath, loc_t *loc, struct iatt *iatt,
+                     int follow, int reval)
 {
-	inode_t    *inode = NULL;
-	inode_t    *parent = NULL;
-	char       *saveptr = NULL;
-	char       *path = NULL;
-	char       *component = NULL;
-	char       *next_component = NULL;
-	int         ret = -1;
-	struct iatt ciatt = {0, };
-
-	path = gf_strdup (origpath);
-	if (!path) {
-		errno = ENOMEM;
-		return -1;
-	}
-
-	parent = NULL;
-	if (at && path[0] != '/') {
-		/* A relative resolution of a path which starts with '/'
-		   is equal to an absolute path resolution.
-		*/
-		inode = inode_ref (at);
-	} else {
-		inode = inode_ref (subvol->itable->root);
-
-		glfs_resolve_base (fs, subvol, inode, &ciatt);
-	}
-
-	for (component = strtok_r (path, "/", &saveptr);
-	     component; component = next_component) {
-
-		next_component = strtok_r (NULL, "/", &saveptr);
-
-		if (parent)
-			inode_unref (parent);
-
-		parent = inode;
-
-		inode = glfs_resolve_component (fs, subvol, parent,
-						component, &ciatt,
-						/* force hard lookup on the last
-						   component, as the caller
-						   wants proper iatt filled
-						*/
-						(reval || !next_component));
-		if (!inode)
-			break;
-
-		if (IA_ISLNK (ciatt.ia_type) && (next_component || follow)) {
-			/* If the component is not the last piece,
-			   then following it is necessary even if
-			   not requested by the caller
-			*/
-			char *lpath = NULL;
-			loc_t sym_loc = {0,};
-
-			if (follow > GLFS_SYMLINK_MAX_FOLLOW) {
-				errno = ELOOP;
-				ret = -1;
-				if (inode) {
-					inode_unref (inode);
-					inode = NULL;
-				}
-				break;
-			}
-
-			ret = glfs_resolve_symlink (fs, subvol, inode, &lpath);
-			inode_unref (inode);
-			inode = NULL;
-			if (ret < 0)
-				break;
-
-			ret = glfs_resolve_at (fs, subvol, parent, lpath,
-					       &sym_loc,
-					       /* followed iatt becomes the
-						  component iatt
-					       */
-					       &ciatt,
-					       /* always recurisvely follow while
-						  following symlink
-					       */
-					       follow + 1, reval);
-			if (ret == 0)
-				inode = inode_ref (sym_loc.inode);
-			loc_wipe (&sym_loc);
-			GF_FREE (lpath);
-		}
-
-		if (!next_component)
-			break;
-
-		if (!IA_ISDIR (ciatt.ia_type)) {
-			/* next_component exists and this component is
-			   not a directory
-			*/
-			inode_unref (inode);
-			inode = NULL;
-			ret = -1;
-			errno = ENOTDIR;
-			break;
-		}
-	}
-
-	if (parent && next_component)
-		/* resolution failed mid-way */
-		goto out;
-
-	/* At this point, all components up to the last parent directory
-	   have been resolved successfully (@parent). Resolution of basename
-	   might have failed (@inode) if at all.
-	*/
-
-	loc->parent = parent;
-	if (parent) {
-		uuid_copy (loc->pargfid, parent->gfid);
-		loc->name = component;
-	}
-
-	loc->inode = inode;
-	if (inode) {
-		uuid_copy (loc->gfid, inode->gfid);
-		if (iatt)
-			*iatt = ciatt;
-		ret = 0;
-	}
-
-	glfs_loc_touchup (loc);
-out:
-	GF_FREE (path);
+    inode_t *inode = NULL;
+    inode_t *parent = NULL;
+    char *saveptr = NULL;
+    char *path = NULL;
+    char *component = NULL;
+    char *next_component = NULL;
+    int ret = -1;
+    struct iatt ciatt = {
+        0,
+    };
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    if (origpath[0] == '\0') {
+        errno = EINVAL;
+        goto invalid_fs;
+    }
+
+    parent = NULL;
+    if (at && origpath[0] != '/') {
+        /* A relative resolution of a path which starts with '/'
+           is equal to an absolute path resolution.
+        */
+        inode = inode_ref(at);
+    } else {
+        inode = inode_ref(subvol->itable->root);
+
+        if (strcmp(origpath, "/") == 0)
+            glfs_resolve_root(fs, subvol, inode, &ciatt);
+    }
+
+    path = gf_strdup(origpath);
+    if (!path)
+        goto invalid_fs;
+
+    for (component = strtok_r(path, "/", &saveptr); component;
+         component = next_component) {
+        next_component = strtok_r(NULL, "/", &saveptr);
+
+        if (parent)
+            inode_unref(parent);
+        parent = inode;
+        inode = glfs_resolve_component(fs, subvol, parent, component, &ciatt,
+                                       /* force hard lookup on the last
+                                          component, as the caller
+                                          wants proper iatt filled
+                                       */
+                                       (reval || (!next_component && iatt)));
+        if (!inode) {
+            ret = -1;
+            break;
+        }
 
-	/* do NOT loc_wipe here as only last component might be missing */
+        if (IA_ISLNK(ciatt.ia_type) && (next_component || follow)) {
+            /* If the component is not the last piece,
+               then following it is necessary even if
+               not requested by the caller
+            */
+            char *lpath = NULL;
+            loc_t sym_loc = {
+                0,
+            };
+
+            if (follow > GLFS_SYMLINK_MAX_FOLLOW) {
+                errno = ELOOP;
+                ret = -1;
+                if (inode) {
+                    inode_unref(inode);
+                    inode = NULL;
+                }
+                break;
+            }
+
+            ret = glfs_resolve_symlink(fs, subvol, inode, &lpath);
+            inode_unref(inode);
+            inode = NULL;
+            if (ret < 0)
+                break;
+
+            ret = priv_glfs_resolve_at(fs, subvol, parent, lpath, &sym_loc,
+                                       /* followed iatt becomes the
+                                          component iatt
+                                       */
+                                       &ciatt,
+                                       /* always recurisvely follow while
+                                          following symlink
+                                       */
+                                       follow + 1, reval);
+            if (ret == 0)
+                inode = inode_ref(sym_loc.inode);
+            loc_wipe(&sym_loc);
+            GF_FREE(lpath);
+        }
 
-	return ret;
-}
+        if (!next_component)
+            break;
+
+        if (!IA_ISDIR(ciatt.ia_type)) {
+            /* next_component exists and this component is
+               not a directory
+            */
+            inode_unref(inode);
+            inode = NULL;
+            ret = -1;
+            errno = ENOTDIR;
+            break;
+        }
+    }
+
+    if (parent && next_component)
+        /* resolution failed mid-way */
+        goto out;
+
+    /* At this point, all components up to the last parent directory
+       have been resolved successfully (@parent). Resolution of basename
+       might have failed (@inode) if at all.
+    */
+
+    loc->parent = parent;
+    if (parent) {
+        gf_uuid_copy(loc->pargfid, parent->gfid);
+        loc->name = component;
+    }
+
+    loc->inode = inode;
+    if (inode) {
+        gf_uuid_copy(loc->gfid, inode->gfid);
+        if (iatt)
+            *iatt = ciatt;
+        ret = 0;
+    }
+
+    if (priv_glfs_loc_touchup(loc) < 0) {
+        ret = -1;
+    }
+out:
+    GF_FREE(path);
+    __GLFS_EXIT_FS;
 
+    /* do NOT loc_wipe here as only last component might be missing */
+invalid_fs:
+    return ret;
+}
 
 int
-glfs_resolve_path (struct glfs *fs, xlator_t *subvol, const char *origpath,
-		   loc_t *loc, struct iatt *iatt, int follow, int reval)
+glfs_resolve_path(struct glfs *fs, xlator_t *subvol, const char *origpath,
+                  loc_t *loc, struct iatt *iatt, int follow, int reval)
 {
-	int ret = -1;
-	inode_t *cwd = NULL;
-
-	if (origpath[0] == '/')
-		return glfs_resolve_at (fs, subvol, NULL, origpath, loc, iatt,
-					follow, reval);
+    int ret = -1;
+    inode_t *cwd = NULL;
+
+    if (origpath[0] == '/')
+        return priv_glfs_resolve_at(fs, subvol, NULL, origpath, loc, iatt,
+                                    follow, reval);
+
+    cwd = glfs_cwd_get(fs);
+    if (NULL == cwd) {
+        gf_smsg(subvol->name, GF_LOG_WARNING, EIO, API_MSG_GET_CWD_FAILED,
+                NULL);
+        errno = EIO;
+        goto out;
+    }
+
+    ret = priv_glfs_resolve_at(fs, subvol, cwd, origpath, loc, iatt, follow,
+                               reval);
+    if (cwd)
+        inode_unref(cwd);
 
-	cwd = glfs_cwd_get (fs);
-
-	ret = glfs_resolve_at (fs, subvol, cwd, origpath, loc, iatt,
-			       follow, reval);
-	if (cwd)
-		inode_unref (cwd);
-
-	return ret;
+out:
+    return ret;
 }
 
-
+GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_resolve, 3.7.0)
 int
-glfs_resolve (struct glfs *fs, xlator_t *subvol, const char *origpath,
-	      loc_t *loc, struct iatt *iatt, int reval)
+priv_glfs_resolve(struct glfs *fs, xlator_t *subvol, const char *origpath,
+                  loc_t *loc, struct iatt *iatt, int reval)
 {
-	int ret = -1;
+    int ret = -1;
 
-	ret = glfs_resolve_path (fs, subvol, origpath, loc, iatt, 1, reval);
+    ret = glfs_resolve_path(fs, subvol, origpath, loc, iatt, 1, reval);
 
-	return ret;
+    return ret;
 }
 
-
 int
-glfs_lresolve (struct glfs *fs, xlator_t *subvol, const char *origpath,
-	       loc_t *loc, struct iatt *iatt, int reval)
+glfs_lresolve(struct glfs *fs, xlator_t *subvol, const char *origpath,
+              loc_t *loc, struct iatt *iatt, int reval)
 {
-	int ret = -1;
+    int ret = -1;
 
-	ret = glfs_resolve_path (fs, subvol, origpath, loc, iatt, 0, reval);
+    ret = glfs_resolve_path(fs, subvol, origpath, loc, iatt, 0, reval);
 
-	return ret;
+    return ret;
 }
 
-
 int
-glfs_migrate_fd_locks_safe (struct glfs *fs, xlator_t *oldsubvol, fd_t *oldfd,
-			    xlator_t *newsubvol, fd_t *newfd)
+glfs_migrate_fd_locks_safe(struct glfs *fs, xlator_t *oldsubvol, fd_t *oldfd,
+                           xlator_t *newsubvol, fd_t *newfd)
 {
-	dict_t *lockinfo = NULL;
-	int ret = 0;
-	char uuid1[64];
-
-	if (!oldfd->lk_ctx || fd_lk_ctx_empty (oldfd->lk_ctx))
-		return 0;
-
-	newfd->lk_ctx = fd_lk_ctx_ref (oldfd->lk_ctx);
-
-	ret = syncop_fgetxattr (oldsubvol, oldfd, &lockinfo,
-				GF_XATTR_LOCKINFO_KEY);
-	if (ret < 0) {
-		gf_log (fs->volname, GF_LOG_WARNING,
-			"fgetxattr (%s) failed (%s) on graph %s (%d)",
-			uuid_utoa_r (oldfd->inode->gfid, uuid1),
-			strerror (errno),
-			graphid_str (oldsubvol), oldsubvol->graph->id);
-		goto out;
-	}
-
-	if (!dict_get (lockinfo, GF_XATTR_LOCKINFO_KEY)) {
-		gf_log (fs->volname, GF_LOG_WARNING,
-			"missing lokinfo key (%s) on graph %s (%d)",
-			uuid_utoa_r (oldfd->inode->gfid, uuid1),
-			graphid_str (oldsubvol), oldsubvol->graph->id);
-		goto out;
-	}
-
-	ret = syncop_fsetxattr (newsubvol, newfd, lockinfo, 0);
-	if (ret < 0) {
-		gf_log (fs->volname, GF_LOG_WARNING,
-			"fsetxattr (%s) failed (%s) on graph %s (%d)",
-			uuid_utoa_r (newfd->inode->gfid, uuid1),
-			strerror (errno),
-			graphid_str (newsubvol), newsubvol->graph->id);
-		goto out;
-	}
+    dict_t *lockinfo = NULL;
+    int ret = 0;
+    char uuid1[64];
+
+    if (!oldfd->lk_ctx || fd_lk_ctx_empty(oldfd->lk_ctx))
+        return 0;
+
+    newfd->lk_ctx = fd_lk_ctx_ref(oldfd->lk_ctx);
+
+    ret = syncop_fgetxattr(oldsubvol, oldfd, &lockinfo, GF_XATTR_LOCKINFO_KEY,
+                           NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+    if (ret < 0) {
+        gf_smsg(fs->volname, GF_LOG_WARNING, errno, API_MSG_FGETXATTR_FAILED,
+                "gfid=%s", uuid_utoa_r(oldfd->inode->gfid, uuid1), "err=%s",
+                strerror(errno), "subvol=%s", graphid_str(oldsubvol), "id=%d",
+                oldsubvol->graph->id, NULL);
+        goto out;
+    }
+
+    if (!dict_get(lockinfo, GF_XATTR_LOCKINFO_KEY)) {
+        gf_smsg(fs->volname, GF_LOG_WARNING, 0, API_MSG_LOCKINFO_KEY_MISSING,
+                "gfid=%s", uuid_utoa_r(oldfd->inode->gfid, uuid1), "subvol=%s",
+                graphid_str(oldsubvol), "id=%d", oldsubvol->graph->id, NULL);
+        goto out;
+    }
+
+    ret = syncop_fsetxattr(newsubvol, newfd, lockinfo, 0, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+    if (ret < 0) {
+        gf_smsg(fs->volname, GF_LOG_WARNING, 0, API_MSG_FSETXATTR_FAILED,
+                "gfid=%s", uuid_utoa_r(newfd->inode->gfid, uuid1), "err=%s",
+                strerror(errno), "subvol=%s", graphid_str(newsubvol), "id=%d",
+                newsubvol->graph->id, NULL);
+        goto out;
+    }
 out:
-	if (lockinfo)
-		dict_unref (lockinfo);
-	return ret;
+    if (lockinfo)
+        dict_unref(lockinfo);
+    return ret;
 }
 
-
 fd_t *
-glfs_migrate_fd_safe (struct glfs *fs, xlator_t *newsubvol, fd_t *oldfd)
+glfs_migrate_fd_safe(struct glfs *fs, xlator_t *newsubvol, fd_t *oldfd)
 {
-	fd_t *newfd = NULL;
-	inode_t *oldinode = NULL;
-	inode_t *newinode = NULL;
-	xlator_t *oldsubvol = NULL;
-	int ret = -1;
-	loc_t loc = {0, };
-	char uuid1[64];
-
-
-	oldinode = oldfd->inode;
-	oldsubvol = oldinode->table->xl;
-
-	if (oldsubvol == newsubvol)
-		return fd_ref (oldfd);
-
-	if (!oldsubvol->switched) {
-		ret = syncop_fsync (oldsubvol, oldfd, 0);
-		if (ret) {
-			gf_log (fs->volname, GF_LOG_WARNING,
-				"fsync() failed (%s) on %s graph %s (%d)",
-				strerror (errno),
-				uuid_utoa_r (oldfd->inode->gfid, uuid1),
-				graphid_str (oldsubvol), oldsubvol->graph->id);
-		}
-	}
-
-	newinode = glfs_refresh_inode_safe (newsubvol, oldinode);
-	if (!newinode) {
-		gf_log (fs->volname, GF_LOG_WARNING,
-			"inode (%s) refresh failed (%s) on graph %s (%d)",
-			uuid_utoa_r (oldinode->gfid, uuid1),
-			strerror (errno),
-			graphid_str (newsubvol), newsubvol->graph->id);
-		goto out;
-	}
-
-	newfd = fd_create (newinode, getpid());
-	if (!newfd) {
-		gf_log (fs->volname, GF_LOG_WARNING,
-			"fd_create (%s) failed (%s) on graph %s (%d)",
-			uuid_utoa_r (newinode->gfid, uuid1),
-			strerror (errno),
-			graphid_str (newsubvol), newsubvol->graph->id);
-		goto out;
-	}
-
-	loc.inode = inode_ref (newinode);
-
-        ret = inode_path (oldfd->inode, NULL, (char **)&loc.path);
-        if (ret < 0) {
-                gf_log (fs->volname, GF_LOG_INFO, "inode_path failed");
-                goto out;
+    fd_t *newfd = NULL;
+    inode_t *oldinode = NULL;
+    inode_t *newinode = NULL;
+    xlator_t *oldsubvol = NULL;
+    int ret = -1;
+    loc_t loc = {
+        0,
+    };
+    char uuid1[64];
+    dict_t *xdata = NULL;
+
+    oldinode = oldfd->inode;
+    oldsubvol = oldinode->table->xl;
+
+    if (oldsubvol == newsubvol)
+        return fd_ref(oldfd);
+
+    if (!oldsubvol->switched) {
+        xdata = dict_new();
+        if (!xdata || dict_set_int8(xdata, "last-fsync", 1)) {
+            gf_smsg(fs->volname, GF_LOG_WARNING, ENOMEM, API_MSG_FSYNC_FAILED,
+                    "err=%s", "last-fsync set failed", "gfid=%s",
+                    uuid_utoa_r(oldfd->inode->gfid, uuid1), "subvol=%s",
+                    graphid_str(oldsubvol), "id=%d", oldsubvol->graph->id,
+                    NULL);
         }
 
-        uuid_copy (loc.gfid, oldinode->gfid);
-
-
-	if (IA_ISDIR (oldinode->ia_type))
-		ret = syncop_opendir (newsubvol, &loc, newfd);
-	else
-		ret = syncop_open (newsubvol, &loc,
-				   oldfd->flags & ~(O_TRUNC|O_EXCL|O_CREAT),
-				   newfd);
-	loc_wipe (&loc);
-
-	if (ret) {
-		gf_log (fs->volname, GF_LOG_WARNING,
-			"syncop_open%s (%s) failed (%s) on graph %s (%d)",
-			IA_ISDIR (oldinode->ia_type) ? "dir" : "",
-			uuid_utoa_r (newinode->gfid, uuid1),
-			strerror (errno),
-			graphid_str (newsubvol), newsubvol->graph->id);
-		goto out;
-	}
-
-	ret = glfs_migrate_fd_locks_safe (fs, oldsubvol, oldfd, newsubvol,
-					  newfd);
-
-	if (ret) {
-		gf_log (fs->volname, GF_LOG_WARNING,
-			"lock migration (%s) failed (%s) on graph %s (%d)",
-			uuid_utoa_r (newinode->gfid, uuid1),
-			strerror (errno),
-			graphid_str (newsubvol), newsubvol->graph->id);
-		goto out;
-	}
-
-        newfd->flags = oldfd->flags;
-	fd_bind (newfd);
+        ret = syncop_fsync(oldsubvol, oldfd, 0, NULL, NULL, xdata, NULL);
+        DECODE_SYNCOP_ERR(ret);
+        if (ret) {
+            gf_smsg(fs->volname, GF_LOG_WARNING, errno, API_MSG_FSYNC_FAILED,
+                    "err=%s", strerror(errno), "gfid=%s",
+                    uuid_utoa_r(oldfd->inode->gfid, uuid1), "subvol=%s",
+                    graphid_str(oldsubvol), "id=%d", oldsubvol->graph->id,
+                    NULL);
+        }
+    }
+
+    newinode = glfs_refresh_inode_safe(newsubvol, oldinode, _gf_false);
+    if (!newinode) {
+        gf_smsg(fs->volname, GF_LOG_WARNING, errno,
+                API_MSG_INODE_REFRESH_FAILED, "gfid=%s",
+                uuid_utoa_r(oldinode->gfid, uuid1), "err=%s", strerror(errno),
+                "subvol=%s", graphid_str(newsubvol), "id=%d",
+                newsubvol->graph->id, NULL);
+        goto out;
+    }
+
+    newfd = fd_create(newinode, getpid());
+    if (!newfd) {
+        gf_smsg(fs->volname, GF_LOG_WARNING, errno,
+                API_MSG_FDCREATE_FAILED_ON_GRAPH, "gfid=%s",
+                uuid_utoa_r(newinode->gfid, uuid1), "err=%s", strerror(errno),
+                "subvol=%s", graphid_str(newsubvol), "id=%d",
+                newsubvol->graph->id, NULL);
+        goto out;
+    }
+
+    loc.inode = inode_ref(newinode);
+
+    ret = inode_path(oldfd->inode, NULL, (char **)&loc.path);
+    if (ret < 0) {
+        gf_smsg(fs->volname, GF_LOG_INFO, 0, API_MSG_INODE_PATH_FAILED, NULL);
+        goto out;
+    }
+
+    gf_uuid_copy(loc.gfid, oldinode->gfid);
+
+    if (IA_ISDIR(oldinode->ia_type))
+        ret = syncop_opendir(newsubvol, &loc, newfd, NULL, NULL);
+    else
+        ret = syncop_open(newsubvol, &loc,
+                          oldfd->flags & ~(O_TRUNC | O_EXCL | O_CREAT), newfd,
+                          NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+    loc_wipe(&loc);
+
+    if (ret) {
+        gf_smsg(fs->volname, GF_LOG_WARNING, errno, API_MSG_SYNCOP_OPEN_FAILED,
+                "type=%s", IA_ISDIR(oldinode->ia_type) ? "dir" : "", "gfid=%s",
+                uuid_utoa_r(newinode->gfid, uuid1), "err=%s", strerror(errno),
+                "subvol=%s", graphid_str(newsubvol), "id=%d",
+                newsubvol->graph->id, NULL);
+        goto out;
+    }
+
+    ret = glfs_migrate_fd_locks_safe(fs, oldsubvol, oldfd, newsubvol, newfd);
+
+    if (ret) {
+        gf_smsg(fs->volname, GF_LOG_WARNING, errno, API_MSG_LOCK_MIGRATE_FAILED,
+                "gfid=%s", uuid_utoa_r(newinode->gfid, uuid1), "err=%s",
+                strerror(errno), "subvol=%s", graphid_str(newsubvol), "id=%d",
+                newsubvol->graph->id, NULL);
+        goto out;
+    }
+
+    newfd->flags = oldfd->flags;
+    fd_bind(newfd);
 out:
-	if (newinode)
-		inode_unref (newinode);
+    if (newinode)
+        inode_unref(newinode);
 
-	if (ret) {
-		fd_unref (newfd);
-		newfd = NULL;
-	}
+    if (ret) {
+        fd_unref(newfd);
+        newfd = NULL;
+    }
 
-	return newfd;
-}
+    if (xdata)
+        dict_unref(xdata);
 
+    return newfd;
+}
 
 fd_t *
-__glfs_migrate_fd (struct glfs *fs, xlator_t *newsubvol, struct glfs_fd *glfd)
+__glfs_migrate_fd(struct glfs *fs, xlator_t *newsubvol, struct glfs_fd *glfd)
 {
-	fd_t *oldfd = NULL;
-	fd_t *newfd = NULL;
+    fd_t *oldfd = NULL;
+    fd_t *newfd = NULL;
 
-	oldfd = glfd->fd;
+    oldfd = glfd->fd;
 
-	fs->migration_in_progress = 1;
-	pthread_mutex_unlock (&fs->mutex);
-	{
-		newfd = glfs_migrate_fd_safe (fs, newsubvol, oldfd);
-	}
-	pthread_mutex_lock (&fs->mutex);
-	fs->migration_in_progress = 0;
+    fs->migration_in_progress = 1;
+    pthread_mutex_unlock(&fs->mutex);
+    {
+        newfd = glfs_migrate_fd_safe(fs, newsubvol, oldfd);
+    }
+    pthread_mutex_lock(&fs->mutex);
+    fs->migration_in_progress = 0;
+    pthread_cond_broadcast(&fs->cond);
 
-	return newfd;
-}
+    /* wake up other waiting tasks */
+    __GLFS_SYNCTASK_WAKE(fs);
 
+    return newfd;
+}
 
 fd_t *
-__glfs_resolve_fd (struct glfs *fs, xlator_t *subvol, struct glfs_fd *glfd)
+__glfs_resolve_fd(struct glfs *fs, xlator_t *subvol, struct glfs_fd *glfd)
 {
-	fd_t *fd = NULL;
+    fd_t *fd = NULL;
 
-	if (glfd->fd->inode->table->xl == subvol)
-		return fd_ref (glfd->fd);
+    if (glfd->fd->inode->table->xl == subvol)
+        return fd_ref(glfd->fd);
 
-	fd = __glfs_migrate_fd (fs, subvol, glfd);
-	if (!fd)
-		return NULL;
+    fd = __glfs_migrate_fd(fs, subvol, glfd);
+    if (!fd)
+        return NULL;
 
-	if (subvol == fs->active_subvol) {
-		fd_unref (glfd->fd);
-		glfd->fd = fd_ref (fd);
-	}
+    if (subvol == fs->active_subvol) {
+        fd_unref(glfd->fd);
+        glfd->fd = fd_ref(fd);
+    }
 
-	return fd;
+    return fd;
 }
 
-
 fd_t *
-glfs_resolve_fd (struct glfs *fs, xlator_t *subvol, struct glfs_fd *glfd)
+glfs_resolve_fd(struct glfs *fs, xlator_t *subvol, struct glfs_fd *glfd)
 {
-	fd_t *fd = NULL;
+    fd_t *fd = NULL;
 
-	glfs_lock (fs);
-	{
-		fd = __glfs_resolve_fd (fs, subvol, glfd);
-	}
-	glfs_unlock (fs);
+    glfs_lock(fs, _gf_true);
+    {
+        fd = __glfs_resolve_fd(fs, subvol, glfd);
+    }
+    glfs_unlock(fs);
 
-	return fd;
+    return fd;
 }
 
-
 void
-__glfs_migrate_openfds (struct glfs *fs, xlator_t *subvol)
+__glfs_migrate_openfds(struct glfs *fs, xlator_t *subvol)
 {
-	struct glfs_fd *glfd = NULL;
-	fd_t *fd = NULL;
-
-	list_for_each_entry (glfd, &fs->openfds, openfds) {
-		if (uuid_is_null (glfd->fd->inode->gfid)) {
-			gf_log (fs->volname, GF_LOG_INFO,
-				"skipping openfd %p/%p in graph %s (%d)",
-				glfd, glfd->fd,	graphid_str(subvol),
-				subvol->graph->id);
-			/* create in progress, defer */
-			continue;
-		}
-
-		fd = __glfs_migrate_fd (fs, subvol, glfd);
-		if (fd) {
-			fd_unref (glfd->fd);
-			glfd->fd = fd;
-		}
-	}
-}
-
+    struct glfs_fd *glfd = NULL;
+    fd_t *fd = NULL;
+
+    list_for_each_entry(glfd, &fs->openfds, openfds)
+    {
+        if (gf_uuid_is_null(glfd->fd->inode->gfid)) {
+            gf_smsg(fs->volname, GF_LOG_INFO, 0, API_MSG_OPENFD_SKIPPED,
+                    "glfd=%p", glfd, "glfd->fd=%p", glfd->fd, "subvol=%s",
+                    graphid_str(subvol), "id=%d", subvol->graph->id, NULL);
+            /* create in progress, defer */
+            continue;
+        }
 
-xlator_t *
-__glfs_active_subvol (struct glfs *fs)
-{
-	xlator_t      *new_subvol = NULL;
-	int            ret = -1;
-	inode_t       *new_cwd = NULL;
-
-	if (!fs->next_subvol)
-		return fs->active_subvol;
-
-	new_subvol = fs->next_subvol;
-
-	ret = __glfs_first_lookup (fs, new_subvol);
-	if (ret) {
-		gf_log (fs->volname, GF_LOG_INFO,
-			"first lookup on graph %s (%d) failed (%s)",
-			graphid_str (new_subvol), new_subvol->graph->id,
-			strerror (errno));
-		return NULL;
-	}
-
-	if (fs->cwd) {
-		new_cwd = __glfs_refresh_inode (fs, new_subvol, fs->cwd);
-
-		if (!new_cwd) {
-			char buf1[64];
-			gf_log (fs->volname, GF_LOG_INFO,
-				"cwd refresh of %s graph %s (%d) failed (%s)",
-				uuid_utoa_r (fs->cwd->gfid, buf1),
-				graphid_str (new_subvol),
-				new_subvol->graph->id, strerror (errno));
-			return NULL;
-		}
-	}
-
-	__glfs_migrate_openfds (fs, new_subvol);
-
-	/* switching @active_subvol and @cwd
-	   should be atomic
-	*/
-	fs->old_subvol = fs->active_subvol;
-	fs->active_subvol = fs->next_subvol;
-	fs->next_subvol = NULL;
-
-	if (new_cwd) {
-		__glfs_cwd_set (fs, new_cwd);
-		inode_unref (new_cwd);
-	}
-
-	gf_log (fs->volname, GF_LOG_INFO, "switched to graph %s (%d)",
-		graphid_str (new_subvol), new_subvol->graph->id);
-
-	return new_subvol;
+        fd = __glfs_migrate_fd(fs, subvol, glfd);
+        if (fd) {
+            fd_unref(glfd->fd);
+            glfd->fd = fd;
+        }
+    }
 }
 
+/* Note that though it appears that this function executes under fs->mutex,
+ * it is not fully executed under fs->mutex. i.e. there are functions like
+ * __glfs_first_lookup, __glfs_refresh_inode, __glfs_migrate_openfds which
+ * unlocks fs->mutex before sending any network fop, and reacquire fs->mutex
+ * once the fop is complete. Hence the variable read from fs at the start of the
+ * function need not have the same value by the end of the function.
+ */
 xlator_t *
-glfs_active_subvol (struct glfs *fs)
+__glfs_active_subvol(struct glfs *fs)
 {
-	xlator_t      *subvol = NULL;
-	xlator_t      *old_subvol = NULL;
+    xlator_t *new_subvol = NULL;
+    int ret = -1;
+    inode_t *new_cwd = NULL;
+
+    if (!fs->next_subvol)
+        return fs->active_subvol;
+
+    new_subvol = fs->mip_subvol = fs->next_subvol;
+    fs->next_subvol = NULL;
+
+    ret = __glfs_first_lookup(fs, new_subvol);
+    if (ret) {
+        gf_smsg(fs->volname, GF_LOG_INFO, errno,
+                API_MSG_FIRST_LOOKUP_GRAPH_FAILED, "subvol=%s",
+                graphid_str(new_subvol), "id=%d", new_subvol->graph->id,
+                "err=%s", strerror(errno), NULL);
+        return NULL;
+    }
+
+    if (fs->cwd) {
+        new_cwd = __glfs_refresh_inode(fs, new_subvol, fs->cwd, _gf_false);
+
+        if (!new_cwd) {
+            char buf1[64];
+            gf_smsg(fs->volname, GF_LOG_INFO, errno,
+                    API_MSG_CWD_GRAPH_REF_FAILED, "buf=%s",
+                    uuid_utoa_r(fs->cwd->gfid, buf1), "subvol=%s",
+                    graphid_str(new_subvol), "id=%d", new_subvol->graph->id,
+                    "err=%s", strerror(errno), NULL);
+            return NULL;
+        }
+    }
 
-	glfs_lock (fs);
-	{
-		subvol = __glfs_active_subvol (fs);
+    __glfs_migrate_openfds(fs, new_subvol);
+    /* TODO: Migrate the fds and inodes which have leases to the new graph
+     * (issue #350)*/
 
-		if (subvol)
-			subvol->winds++;
+    /* switching @active_subvol and @cwd
+       should be atomic
+    */
+    fs->old_subvol = fs->active_subvol;
+    fs->active_subvol = fs->mip_subvol;
+    fs->mip_subvol = NULL;
 
-		if (fs->old_subvol) {
-			old_subvol = fs->old_subvol;
-			fs->old_subvol = NULL;
-			old_subvol->switched = 1;
-		}
-	}
-	glfs_unlock (fs);
+    if (new_cwd) {
+        __glfs_cwd_set(fs, new_cwd);
+        inode_unref(new_cwd);
+    }
 
-	if (old_subvol)
-		glfs_subvol_done (fs, old_subvol);
+    gf_smsg(fs->volname, GF_LOG_INFO, 0, API_MSG_SWITCHED_GRAPH, "subvol=%s",
+            graphid_str(new_subvol), "id=%d", new_subvol->graph->id, NULL);
 
-	return subvol;
+    return new_subvol;
 }
 
-
+GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_subvol_done, 3.4.0)
 void
-glfs_subvol_done (struct glfs *fs, xlator_t *subvol)
+priv_glfs_subvol_done(struct glfs *fs, xlator_t *subvol)
 {
-	int ref = 0;
-	xlator_t *active_subvol = NULL;
-
-	if (!subvol)
-		return;
-
-	glfs_lock (fs);
-	{
-		ref = (--subvol->winds);
-		active_subvol = fs->active_subvol;
-	}
-	glfs_unlock (fs);
-
-	if (ref == 0) {
-		assert (subvol != active_subvol);
-		xlator_notify (subvol, GF_EVENT_PARENT_DOWN, subvol, NULL);
-	}
+    int ref = 0;
+    xlator_t *active_subvol = NULL;
+
+    if (!subvol)
+        return;
+
+    /* For decrementing subvol->wind ref count we need not check/wait for
+     * migration-in-progress flag.
+     * Also glfs_subvol_done is called in call-back path therefore waiting
+     * for migration-in-progress flag can lead to dead-lock.
+     */
+    glfs_lock(fs, _gf_false);
+    {
+        ref = (--subvol->winds);
+        active_subvol = fs->active_subvol;
+    }
+    glfs_unlock(fs);
+
+    if (ref == 0) {
+        assert(subvol != active_subvol);
+        xlator_notify(subvol, GF_EVENT_PARENT_DOWN, subvol, NULL);
+    }
 }
 
+GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_active_subvol, 3.4.0)
+xlator_t *
+priv_glfs_active_subvol(struct glfs *fs)
+{
+    xlator_t *subvol = NULL;
+    xlator_t *old_subvol = NULL;
+
+    glfs_lock(fs, _gf_true);
+    {
+        subvol = __glfs_active_subvol(fs);
+
+        if (subvol)
+            subvol->winds++;
+
+        if (fs->old_subvol) {
+            old_subvol = fs->old_subvol;
+            fs->old_subvol = NULL;
+            old_subvol->switched = 1;
+        }
+    }
+    glfs_unlock(fs);
+
+    if (old_subvol)
+        priv_glfs_subvol_done(fs, old_subvol);
+
+    return subvol;
+}
 
 int
-__glfs_cwd_set (struct glfs *fs, inode_t *inode)
+__glfs_cwd_set(struct glfs *fs, inode_t *inode)
 {
-	if (inode->table->xl != fs->active_subvol) {
-		inode = __glfs_refresh_inode (fs, fs->active_subvol, inode);
-		if (!inode)
-			return -1;
-	} else {
-		inode_ref (inode);
-	}
+    if (inode->table->xl != fs->active_subvol) {
+        inode = __glfs_refresh_inode(fs, fs->active_subvol, inode, _gf_false);
+        if (!inode)
+            return -1;
+    } else {
+        inode_ref(inode);
+    }
 
-	if (fs->cwd)
-		inode_unref (fs->cwd);
+    if (fs->cwd)
+        inode_unref(fs->cwd);
 
-	fs->cwd = inode;
+    fs->cwd = inode;
 
-	return 0;
+    return 0;
 }
 
-
 int
-glfs_cwd_set (struct glfs *fs, inode_t *inode)
+glfs_cwd_set(struct glfs *fs, inode_t *inode)
 {
-	int ret = 0;
+    int ret = 0;
 
-	glfs_lock (fs);
-	{
-		ret = __glfs_cwd_set (fs, inode);
-	}
-	glfs_unlock (fs);
+    glfs_lock(fs, _gf_true);
+    {
+        ret = __glfs_cwd_set(fs, inode);
+    }
+    glfs_unlock(fs);
 
-	return ret;
+    return ret;
 }
 
+inode_t *
+__glfs_cwd_get(struct glfs *fs)
+{
+    inode_t *cwd = NULL;
+
+    if (!fs->cwd)
+        return NULL;
+
+    if (fs->cwd->table->xl == fs->active_subvol) {
+        cwd = inode_ref(fs->cwd);
+        return cwd;
+    }
+
+    cwd = __glfs_refresh_inode(fs, fs->active_subvol, fs->cwd, _gf_false);
+
+    return cwd;
+}
+
+inode_t *
+glfs_cwd_get(struct glfs *fs)
+{
+    inode_t *cwd = NULL;
+
+    glfs_lock(fs, _gf_true);
+    {
+        cwd = __glfs_cwd_get(fs);
+    }
+    glfs_unlock(fs);
+
+    return cwd;
+}
 
 inode_t *
-__glfs_cwd_get (struct glfs *fs)
+__glfs_resolve_inode(struct glfs *fs, xlator_t *subvol,
+                     struct glfs_object *object)
 {
-	inode_t *cwd = NULL;
+    inode_t *inode = NULL;
+    gf_boolean_t lookup_needed = _gf_false;
 
-	if (!fs->cwd)
-		return NULL;
+    lookup_needed = inode_needs_lookup(object->inode, THIS);
 
-	if (fs->cwd->table->xl == fs->active_subvol) {
-		cwd = inode_ref (fs->cwd);
-		return cwd;
-	}
+    if (!lookup_needed && object->inode->table->xl == subvol)
+        return inode_ref(object->inode);
 
-	cwd = __glfs_refresh_inode (fs, fs->active_subvol, fs->cwd);
+    inode = __glfs_refresh_inode(fs, fs->active_subvol, object->inode,
+                                 lookup_needed);
+    if (!inode)
+        return NULL;
 
-	return cwd;
+    if (subvol == fs->active_subvol) {
+        inode_unref(object->inode);
+        object->inode = inode_ref(inode);
+    }
+
+    return inode;
 }
 
 inode_t *
-glfs_cwd_get (struct glfs *fs)
+glfs_resolve_inode(struct glfs *fs, xlator_t *subvol,
+                   struct glfs_object *object)
+{
+    inode_t *inode = NULL;
+
+    glfs_lock(fs, _gf_true);
+    {
+        inode = __glfs_resolve_inode(fs, subvol, object);
+    }
+    glfs_unlock(fs);
+
+    return inode;
+}
+
+int
+glfs_create_object(loc_t *loc, struct glfs_object **retobject)
 {
-	inode_t *cwd = NULL;
+    struct glfs_object *object = NULL;
+
+    object = GF_CALLOC(1, sizeof(struct glfs_object), glfs_mt_glfs_object_t);
+    if (object == NULL) {
+        errno = ENOMEM;
+        return -1;
+    }
 
-	glfs_lock (fs);
-	{
-		cwd = __glfs_cwd_get (fs);
-	}
-	glfs_unlock (fs);
+    object->inode = loc->inode;
+    gf_uuid_copy(object->gfid, object->inode->gfid);
 
-	return cwd;
+    /* we hold the reference */
+    loc->inode = NULL;
+
+    *retobject = object;
+
+    return 0;
+}
+
+struct glfs_object *
+glfs_h_resolve_symlink(struct glfs *fs, struct glfs_object *object)
+{
+    xlator_t *subvol = NULL;
+    loc_t sym_loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    char *lpath = NULL;
+    int ret = 0;
+    struct glfs_object *target_object = NULL;
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    ret = glfs_resolve_symlink(fs, subvol, object->inode, &lpath);
+    if (ret < 0)
+        goto out;
+
+    ret = glfs_resolve_at(fs, subvol, NULL, lpath, &sym_loc, &iatt,
+                          /* always recurisvely follow while
+                            following symlink
+                          */
+                          1, 0);
+    if (ret == 0)
+        ret = glfs_create_object(&sym_loc, &target_object);
+
+out:
+    loc_wipe(&sym_loc);
+    GF_FREE(lpath);
+    return target_object;
 }
diff --git a/api/src/glfs.c b/api/src/glfs.c
index 1cb9088c0dc..b4bf1423f6d 100644
--- a/api/src/glfs.c
+++ b/api/src/glfs.c
@@ -1,5 +1,5 @@
 /*
-  Copyright (c) 2012 Red Hat, Inc. <http://www.redhat.com>
+  Copyright (c) 2012-2018 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
   This file is licensed to you under your choice of the GNU Lesser
@@ -8,10 +8,8 @@
   cases as published by the Free Software Foundation.
 */
 
-
 /*
   TODO:
-  - merge locks in glfs_posix_lock for lock self-healing
   - set proper pid/lk_owner to call frames (currently buried in syncop)
   - fix logging.c/h to store logfp and loglevel in glusterfs_ctx_t and
     reach it via THIS.
@@ -32,618 +30,1777 @@
 #include <sys/types.h>
 #include <unistd.h>
 #include <limits.h>
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
+#ifdef GF_LINUX_HOST_OS
+#include <sys/prctl.h>
 #endif
 
-#include "glusterfs.h"
-#include "logging.h"
-#include "stack.h"
-#include "event.h"
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/stack.h>
+#include <glusterfs/gf-event.h>
 #include "glfs-mem-types.h"
-#include "common-utils.h"
-#include "syncop.h"
-#include "call-stub.h"
+#include <glusterfs/common-utils.h>
+#include <glusterfs/syncop.h>
+#include <glusterfs/call-stub.h>
+#include <glusterfs/hashfn.h>
+#include "rpc-clnt.h"
+#include <glusterfs/statedump.h>
+#include <glusterfs/syscall.h>
 
+#include "gfapi-messages.h"
 #include "glfs.h"
 #include "glfs-internal.h"
-#include "hashfn.h"
-#include "rpc-clnt.h"
-
 
 static gf_boolean_t
-vol_assigned (cmd_args_t *args)
+vol_assigned(cmd_args_t *args)
 {
-	return args->volfile || args->volfile_server;
+    return args->volfile || args->volfile_server;
 }
 
-
 static int
-glusterfs_ctx_defaults_init (glusterfs_ctx_t *ctx)
-{
-	call_pool_t   *pool = NULL;
-	int	       ret = -1;
-
-	xlator_mem_acct_init (THIS, glfs_mt_end);
-
-	ctx->process_uuid = generate_glusterfs_ctx_id ();
-	if (!ctx->process_uuid) {
-		goto err;
-	}
-
-	ctx->page_size	= 128 * GF_UNIT_KB;
-
-	ctx->iobuf_pool = iobuf_pool_new ();
-	if (!ctx->iobuf_pool) {
-		goto err;
-	}
-
-	ctx->event_pool = event_pool_new (DEFAULT_EVENT_POOL_SIZE);
-	if (!ctx->event_pool) {
-		goto err;
-	}
-
-	ctx->env = syncenv_new (0, 0, 0);
-	if (!ctx->env) {
-		goto err;
-	}
-
-	pool = GF_CALLOC (1, sizeof (call_pool_t),
-			  glfs_mt_call_pool_t);
-	if (!pool) {
-		goto err;
-	}
-
-	/* frame_mem_pool size 112 * 4k */
-	pool->frame_mem_pool = mem_pool_new (call_frame_t, 4096);
-	if (!pool->frame_mem_pool) {
-		goto err;
-	}
-	/* stack_mem_pool size 256 * 1024 */
-	pool->stack_mem_pool = mem_pool_new (call_stack_t, 1024);
-	if (!pool->stack_mem_pool) {
-		goto err;
-	}
-
-	ctx->stub_mem_pool = mem_pool_new (call_stub_t, 1024);
-	if (!ctx->stub_mem_pool) {
-		goto err;
-	}
-
-	ctx->dict_pool = mem_pool_new (dict_t, GF_MEMPOOL_COUNT_OF_DICT_T);
-	if (!ctx->dict_pool)
-		goto err;
-
-	ctx->dict_pair_pool = mem_pool_new (data_pair_t,
-					    GF_MEMPOOL_COUNT_OF_DATA_PAIR_T);
-	if (!ctx->dict_pair_pool)
-		goto err;
-
-	ctx->dict_data_pool = mem_pool_new (data_t, GF_MEMPOOL_COUNT_OF_DATA_T);
-	if (!ctx->dict_data_pool)
-		goto err;
-
-	INIT_LIST_HEAD (&pool->all_frames);
-	INIT_LIST_HEAD (&ctx->cmd_args.xlator_options);
-	LOCK_INIT (&pool->lock);
-	ctx->pool = pool;
-
-	pthread_mutex_init (&(ctx->lock), NULL);
-
-	ret = 0;
-err:
-	if (ret && pool) {
-		if (pool->frame_mem_pool)
-			mem_pool_destroy (pool->frame_mem_pool);
-		if (pool->stack_mem_pool)
-			mem_pool_destroy (pool->stack_mem_pool);
-		GF_FREE (pool);
-	}
+glusterfs_ctx_defaults_init(glusterfs_ctx_t *ctx)
+{
+    call_pool_t *pool = NULL;
+    int ret = -1;
 
-	if (ret && ctx) {
-		if (ctx->stub_mem_pool)
-			mem_pool_destroy (ctx->stub_mem_pool);
-		if (ctx->dict_pool)
-			mem_pool_destroy (ctx->dict_pool);
-		if (ctx->dict_data_pool)
-			mem_pool_destroy (ctx->dict_data_pool);
-		if (ctx->dict_pair_pool)
-			mem_pool_destroy (ctx->dict_pair_pool);
-	}
+    if (!ctx) {
+        goto err;
+    }
 
-	return ret;
+    ret = xlator_mem_acct_init(THIS, glfs_mt_end + 1);
+    if (ret != 0) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, ENOMEM, API_MSG_MEM_ACCT_INIT_FAILED,
+                NULL);
+        return ret;
+    }
+
+    /* reset ret to -1 so that we don't need to explicitly
+     * set it in all error paths before "goto err"
+     */
+
+    ret = -1;
+
+    ctx->process_uuid = generate_glusterfs_ctx_id();
+    if (!ctx->process_uuid) {
+        goto err;
+    }
+
+    ctx->page_size = 128 * GF_UNIT_KB;
+
+    ctx->iobuf_pool = iobuf_pool_new();
+    if (!ctx->iobuf_pool) {
+        goto err;
+    }
+
+    ctx->event_pool = gf_event_pool_new(DEFAULT_EVENT_POOL_SIZE,
+                                        STARTING_EVENT_THREADS);
+    if (!ctx->event_pool) {
+        goto err;
+    }
+
+    ctx->env = syncenv_new(0, 0, 0);
+    if (!ctx->env) {
+        goto err;
+    }
+
+    pool = GF_CALLOC(1, sizeof(call_pool_t), glfs_mt_call_pool_t);
+    if (!pool) {
+        goto err;
+    }
+
+    /* frame_mem_pool size 112 * 4k */
+    pool->frame_mem_pool = mem_pool_new(call_frame_t, 4096);
+    if (!pool->frame_mem_pool) {
+        goto err;
+    }
+    /* stack_mem_pool size 256 * 1024 */
+    pool->stack_mem_pool = mem_pool_new(call_stack_t, 1024);
+    if (!pool->stack_mem_pool) {
+        goto err;
+    }
+
+    ctx->stub_mem_pool = mem_pool_new(call_stub_t, 1024);
+    if (!ctx->stub_mem_pool) {
+        goto err;
+    }
+
+    ctx->dict_pool = mem_pool_new(dict_t, GF_MEMPOOL_COUNT_OF_DICT_T);
+    if (!ctx->dict_pool)
+        goto err;
+
+    ctx->dict_pair_pool = mem_pool_new(data_pair_t,
+                                       GF_MEMPOOL_COUNT_OF_DATA_PAIR_T);
+    if (!ctx->dict_pair_pool)
+        goto err;
+
+    ctx->dict_data_pool = mem_pool_new(data_t, GF_MEMPOOL_COUNT_OF_DATA_T);
+    if (!ctx->dict_data_pool)
+        goto err;
+
+    ctx->logbuf_pool = mem_pool_new(log_buf_t, GF_MEMPOOL_COUNT_OF_LRU_BUF_T);
+    if (!ctx->logbuf_pool)
+        goto err;
+
+    INIT_LIST_HEAD(&pool->all_frames);
+    INIT_LIST_HEAD(&ctx->cmd_args.xlator_options);
+    INIT_LIST_HEAD(&ctx->cmd_args.volfile_servers);
+
+    LOCK_INIT(&pool->lock);
+    ctx->pool = pool;
+
+    ret = 0;
+err:
+    if (ret && pool) {
+        if (pool->frame_mem_pool)
+            mem_pool_destroy(pool->frame_mem_pool);
+        if (pool->stack_mem_pool)
+            mem_pool_destroy(pool->stack_mem_pool);
+        GF_FREE(pool);
+    }
+
+    if (ret && ctx) {
+        if (ctx->stub_mem_pool)
+            mem_pool_destroy(ctx->stub_mem_pool);
+        if (ctx->dict_pool)
+            mem_pool_destroy(ctx->dict_pool);
+        if (ctx->dict_data_pool)
+            mem_pool_destroy(ctx->dict_data_pool);
+        if (ctx->dict_pair_pool)
+            mem_pool_destroy(ctx->dict_pair_pool);
+        if (ctx->logbuf_pool)
+            mem_pool_destroy(ctx->logbuf_pool);
+    }
+
+    return ret;
 }
 
-
 static int
-create_master (struct glfs *fs)
+create_master(struct glfs *fs)
 {
-	int		 ret = 0;
-	xlator_t	*master = NULL;
+    int ret = 0;
+    xlator_t *master = NULL;
 
-	master = GF_CALLOC (1, sizeof (*master),
-			    glfs_mt_xlator_t);
-	if (!master)
-		goto err;
+    master = GF_CALLOC(1, sizeof(*master), glfs_mt_xlator_t);
+    if (!master)
+        goto err;
 
-	master->name = gf_strdup ("gfapi");
-	if (!master->name)
-		goto err;
+    master->name = gf_strdup("gfapi");
+    if (!master->name)
+        goto err;
 
-	if (xlator_set_type (master, "mount/api") == -1) {
-		gf_log ("glfs", GF_LOG_ERROR,
-			"master xlator for %s initialization failed",
-			fs->volname);
-		goto err;
-	}
+    if (xlator_set_type(master, "mount/api") == -1) {
+        gf_smsg("glfs", GF_LOG_ERROR, 0, API_MSG_MASTER_XLATOR_INIT_FAILED,
+                "name=%s", fs->volname, NULL);
+        goto err;
+    }
 
-	master->ctx	 = fs->ctx;
-	master->private	 = fs;
-	master->options	 = get_new_dict ();
-	if (!master->options)
-		goto err;
+    master->ctx = fs->ctx;
+    master->private = fs;
+    master->options = dict_new();
+    if (!master->options)
+        goto err;
 
+    ret = xlator_init(master);
+    if (ret) {
+        gf_smsg("glfs", GF_LOG_ERROR, 0, API_MSG_GFAPI_XLATOR_INIT_FAILED,
+                NULL);
+        goto err;
+    }
 
-	ret = xlator_init (master);
-	if (ret) {
-		gf_log ("glfs", GF_LOG_ERROR,
-			"failed to initialize gfapi translator");
-		goto err;
-	}
+    fs->ctx->master = master;
+    THIS = master;
 
-	fs->ctx->master = master;
-	THIS = master;
-
-	return 0;
+    return 0;
 
 err:
-	if (master) {
-		xlator_destroy (master);
-	}
+    if (master) {
+        xlator_destroy(master);
+    }
 
-	return -1;
+    return -1;
 }
 
-
 static FILE *
-get_volfp (struct glfs *fs)
+get_volfp(struct glfs *fs)
 {
-	int	     ret = 0;
-	cmd_args_t  *cmd_args = NULL;
-	FILE	    *specfp = NULL;
-	struct stat  statbuf;
+    cmd_args_t *cmd_args = NULL;
+    FILE *specfp = NULL;
 
-	cmd_args = &fs->ctx->cmd_args;
+    cmd_args = &fs->ctx->cmd_args;
 
-	ret = lstat (cmd_args->volfile, &statbuf);
-	if (ret == -1) {
-		gf_log ("glfs", GF_LOG_ERROR,
-			"%s: %s", cmd_args->volfile, strerror (errno));
-		return NULL;
-	}
+    if ((specfp = fopen(cmd_args->volfile, "r")) == NULL) {
+        gf_smsg("glfs", GF_LOG_ERROR, errno, API_MSG_VOLFILE_OPEN_FAILED,
+                "file=%s", cmd_args->volfile, "err=%s", strerror(errno), NULL);
+        return NULL;
+    }
 
-	if ((specfp = fopen (cmd_args->volfile, "r")) == NULL) {
-		gf_log ("glfs", GF_LOG_ERROR,
-			"volume file %s: %s",
-			cmd_args->volfile,
-			strerror (errno));
-		return NULL;
-	}
+    gf_msg_debug("glfs", 0, "loading volume file %s", cmd_args->volfile);
 
-	gf_log ("glfs", GF_LOG_DEBUG,
-		"loading volume file %s", cmd_args->volfile);
-
-	return specfp;
+    return specfp;
 }
 
-
 int
-glfs_volumes_init (struct glfs *fs)
+glfs_volumes_init(struct glfs *fs)
 {
-	FILE		   *fp = NULL;
-	cmd_args_t	   *cmd_args = NULL;
-	int		    ret = 0;
+    FILE *fp = NULL;
+    cmd_args_t *cmd_args = NULL;
+    int ret = 0;
 
-	cmd_args = &fs->ctx->cmd_args;
+    cmd_args = &fs->ctx->cmd_args;
 
-	if (!vol_assigned (cmd_args))
-		return -1;
+    if (!vol_assigned(cmd_args))
+        return -1;
 
-	if (cmd_args->volfile_server) {
-		ret = glfs_mgmt_init (fs);
-		goto out;
-	}
+    if (sys_access(SECURE_ACCESS_FILE, F_OK) == 0) {
+        fs->ctx->secure_mgmt = 1;
+        fs->ctx->ssl_cert_depth = glusterfs_read_secure_access_file();
+    }
 
-	fp = get_volfp (fs);
+    if (cmd_args->volfile_server) {
+        ret = glfs_mgmt_init(fs);
+        goto out;
+    }
 
-	if (!fp) {
-		gf_log ("glfs", GF_LOG_ERROR,
-			"Cannot reach volume specification file");
-		ret = -1;
-		goto out;
-	}
+    fp = get_volfp(fs);
 
-	ret = glfs_process_volfp (fs, fp);
-	if (ret)
-		goto out;
+    if (!fp) {
+        gf_smsg("glfs", GF_LOG_ERROR, ENOENT, API_MSG_VOL_SPEC_FILE_ERROR,
+                NULL);
+        ret = -1;
+        goto out;
+    }
+
+    ret = glfs_process_volfp(fs, fp);
+    if (ret)
+        goto out;
 
 out:
-	return ret;
+    return ret;
 }
 
-
 ///////////////////////////////////////////////////////////////////////////////
 
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_set_xlator_option, 3.4.0)
 int
-glfs_set_xlator_option (struct glfs *fs, const char *xlator, const char *key,
-			const char *value)
+pub_glfs_set_xlator_option(struct glfs *fs, const char *xlator, const char *key,
+                           const char *value)
 {
-	xlator_cmdline_option_t *option = NULL;
+    xlator_cmdline_option_t *option = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
 
-	option = GF_CALLOC (1, sizeof (*option),
-			    glfs_mt_xlator_cmdline_option_t);
-	if (!option)
-		goto enomem;
+    option = GF_CALLOC(1, sizeof(*option), glfs_mt_xlator_cmdline_option_t);
+    if (!option)
+        goto enomem;
 
-	INIT_LIST_HEAD (&option->cmd_args);
+    INIT_LIST_HEAD(&option->cmd_args);
 
-	option->volume = gf_strdup (xlator);
-	if (!option->volume)
-		goto enomem;
-	option->key = gf_strdup (key);
-	if (!option->key)
-		goto enomem;
-	option->value = gf_strdup (value);
-	if (!option->value)
-		goto enomem;
+    option->volume = gf_strdup(xlator);
+    if (!option->volume)
+        goto enomem;
+    option->key = gf_strdup(key);
+    if (!option->key)
+        goto enomem;
+    option->value = gf_strdup(value);
+    if (!option->value)
+        goto enomem;
 
-	list_add (&option->cmd_args, &fs->ctx->cmd_args.xlator_options);
+    list_add(&option->cmd_args, &fs->ctx->cmd_args.xlator_options);
 
-	return 0;
+    __GLFS_EXIT_FS;
+
+    return 0;
 enomem:
-	errno = ENOMEM;
+    errno = ENOMEM;
+
+    if (!option) {
+        __GLFS_EXIT_FS;
+        return -1;
+    }
 
-	if (!option)
-		return -1;
+    GF_FREE(option->volume);
+    GF_FREE(option->key);
+    GF_FREE(option->value);
+    GF_FREE(option);
 
-	GF_FREE (option->volume);
-	GF_FREE (option->key);
-	GF_FREE (option->value);
-	GF_FREE (option);
+    __GLFS_EXIT_FS;
 
-	return -1;
+invalid_fs:
+    return -1;
 }
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_unset_volfile_server, 3.5.1)
+int
+pub_glfs_unset_volfile_server(struct glfs *fs, const char *transport,
+                              const char *host, const int port)
+{
+    cmd_args_t *cmd_args = NULL;
+    server_cmdline_t *server = NULL;
+    server_cmdline_t *tmp = NULL;
+    char *transport_val = NULL;
+    int port_val = 0;
+    int ret = -1;
+
+    if (!fs || !host) {
+        errno = EINVAL;
+        return ret;
+    }
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    cmd_args = &fs->ctx->cmd_args;
+
+    if (transport) {
+        transport_val = gf_strdup(transport);
+    } else {
+        transport_val = gf_strdup(GF_DEFAULT_VOLFILE_TRANSPORT);
+    }
+
+    if (!transport_val) {
+        errno = ENOMEM;
+        goto out;
+    }
+
+    if (port) {
+        port_val = port;
+    } else {
+        port_val = GF_DEFAULT_BASE_PORT;
+    }
+
+    list_for_each_entry_safe(server, tmp, &cmd_args->curr_server->list, list)
+    {
+        if (!server->volfile_server || !server->transport)
+            continue;
+        if ((!strcmp(server->volfile_server, host) &&
+             !strcmp(server->transport, transport_val) &&
+             (server->port == port_val))) {
+            list_del(&server->list);
+            ret = 0;
+            goto out;
+        }
+    }
+
+out:
+    GF_FREE(transport_val);
+    __GLFS_EXIT_FS;
 
-struct glfs *
-glfs_from_glfd (struct glfs_fd *glfd)
+invalid_fs:
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_set_volfile_server, 3.4.0)
+int
+pub_glfs_set_volfile_server(struct glfs *fs, const char *transport,
+                            const char *host, int port)
 {
-	return glfd->fs;
+    cmd_args_t *cmd_args = NULL;
+    int ret = -1;
+    char *server_host = NULL;
+    char *server_transport = NULL;
+
+    if (!fs || !host) {
+        errno = EINVAL;
+        return ret;
+    }
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    cmd_args = &fs->ctx->cmd_args;
+    cmd_args->max_connect_attempts = 1;
+
+    server_host = gf_strdup(host);
+    if (!server_host) {
+        errno = ENOMEM;
+        goto out;
+    }
+
+    if (transport) {
+        /* volfile fetch support over tcp|unix only */
+        if (!strcmp(transport, "tcp") || !strcmp(transport, "unix")) {
+            server_transport = gf_strdup(transport);
+        } else if (!strcmp(transport, "rdma")) {
+            server_transport = gf_strdup(GF_DEFAULT_VOLFILE_TRANSPORT);
+            gf_smsg("glfs", GF_LOG_WARNING, EINVAL, API_MSG_TRANS_RDMA_DEP,
+                    NULL);
+        } else {
+            gf_smsg("glfs", GF_LOG_TRACE, EINVAL, API_MSG_TRANS_NOT_SUPPORTED,
+                    "transport=%s", transport, NULL);
+            goto out;
+        }
+    } else {
+        server_transport = gf_strdup(GF_DEFAULT_VOLFILE_TRANSPORT);
+    }
+
+    if (!server_transport) {
+        errno = ENOMEM;
+        goto out;
+    }
+
+    if (!port) {
+        port = GF_DEFAULT_BASE_PORT;
+    }
+
+    if (!strcmp(server_transport, "unix")) {
+        port = 0;
+    }
+
+    ret = gf_set_volfile_server_common(cmd_args, server_host, server_transport,
+                                       port);
+    if (ret) {
+        gf_log("glfs", GF_LOG_ERROR, "failed to set volfile server: %s",
+               strerror(errno));
+    }
+
+out:
+    if (server_host) {
+        GF_FREE(server_host);
+    }
+
+    if (server_transport) {
+        GF_FREE(server_transport);
+    }
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
 }
 
+/* *
+ * Used to free the arguments allocated by glfs_set_volfile_server()
+ */
+static void
+glfs_free_volfile_servers(cmd_args_t *cmd_args)
+{
+    server_cmdline_t *server = NULL;
+    server_cmdline_t *tmp = NULL;
+
+    GF_VALIDATE_OR_GOTO(THIS->name, cmd_args, out);
+
+    list_for_each_entry_safe(server, tmp, &cmd_args->volfile_servers, list)
+    {
+        list_del_init(&server->list);
+        GF_FREE(server->volfile_server);
+        GF_FREE(server->transport);
+        GF_FREE(server);
+    }
+    cmd_args->curr_server = NULL;
+out:
+    return;
+}
 
-struct glfs_fd *
-glfs_fd_new (struct glfs *fs)
+static void
+glfs_free_xlator_options(cmd_args_t *cmd_args)
 {
-	struct glfs_fd  *glfd = NULL;
+    xlator_cmdline_option_t *xo = NULL;
+    xlator_cmdline_option_t *tmp_xo = NULL;
+
+    if (!&(cmd_args->xlator_options))
+        return;
+
+    list_for_each_entry_safe(xo, tmp_xo, &cmd_args->xlator_options, cmd_args)
+    {
+        list_del_init(&xo->cmd_args);
+        GF_FREE(xo->volume);
+        GF_FREE(xo->key);
+        GF_FREE(xo->value);
+        GF_FREE(xo);
+    }
+}
 
-	glfd = GF_CALLOC (1, sizeof (*glfd), glfs_mt_glfs_fd_t);
-	if (!glfd)
-		return NULL;
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_setfsuid, 3.4.2)
+int
+pub_glfs_setfsuid(uid_t fsuid)
+{
+    /* TODO:
+     * - Set the THIS and restore it appropriately
+     */
+    return syncopctx_setfsuid(&fsuid);
+}
 
-	glfd->fs = fs;
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_setfsgid, 3.4.2)
+int
+pub_glfs_setfsgid(gid_t fsgid)
+{
+    /* TODO:
+     * - Set the THIS and restore it appropriately
+     */
+    return syncopctx_setfsgid(&fsgid);
+}
 
-	INIT_LIST_HEAD (&glfd->openfds);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_setfsgroups, 3.4.2)
+int
+pub_glfs_setfsgroups(size_t size, const gid_t *list)
+{
+    /* TODO:
+     * - Set the THIS and restore it appropriately
+     */
+    return syncopctx_setfsgroups(size, list);
+}
 
-	return glfd;
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_setfsleaseid, 4.0.0)
+int
+pub_glfs_setfsleaseid(glfs_leaseid_t leaseid)
+{
+    int ret = -1;
+    char *gleaseid = NULL;
+
+    gleaseid = gf_leaseid_get();
+    if (gleaseid) {
+        if (leaseid)
+            memcpy(gleaseid, leaseid, LEASE_ID_SIZE);
+        else /* reset leaseid */
+            memset(gleaseid, 0, LEASE_ID_SIZE);
+        ret = 0;
+    }
+
+    if (ret)
+        gf_log("glfs", GF_LOG_ERROR, "failed to set leaseid: %s",
+               strerror(errno));
+    return ret;
 }
 
+int
+get_fop_attr_glfd(dict_t **fop_attr, struct glfs_fd *glfd)
+{
+    char *leaseid = NULL;
+    int ret = 0;
+    gf_boolean_t dict_create = _gf_false;
+
+    leaseid = GF_MALLOC(LEASE_ID_SIZE, gf_common_mt_char);
+    GF_CHECK_ALLOC_AND_LOG("gfapi", leaseid, ret, "lease id alloc failed", out);
+    memcpy(leaseid, glfd->lease_id, LEASE_ID_SIZE);
+    if (*fop_attr == NULL) {
+        *fop_attr = dict_new();
+        dict_create = _gf_true;
+    }
+    GF_CHECK_ALLOC_AND_LOG("gfapi", *fop_attr, ret, "dict_new failed", out);
+    ret = dict_set_bin(*fop_attr, "lease-id", leaseid, LEASE_ID_SIZE);
+out:
+    if (ret) {
+        GF_FREE(leaseid);
+        if (dict_create) {
+            if (*fop_attr)
+                dict_unref(*fop_attr);
+            *fop_attr = NULL;
+        }
+    }
+    return ret;
+}
 
-void
-glfs_fd_bind (struct glfs_fd *glfd)
+int
+set_fop_attr_glfd(struct glfs_fd *glfd)
 {
-	struct glfs *fs = NULL;
+    char *lease_id = NULL;
+    int ret = -1;
+
+    lease_id = gf_existing_leaseid();
+    if (lease_id) {
+        memcpy(glfd->lease_id, lease_id, LEASE_ID_SIZE);
+        ret = 0;
+    }
+    return ret;
+}
 
-	fs = glfd->fs;
+int
+get_fop_attr_thrd_key(dict_t **fop_attr)
+{
+    char *existing_leaseid = NULL, *leaseid = NULL;
+    int ret = 0;
+    gf_boolean_t dict_create = _gf_false;
+
+    existing_leaseid = gf_existing_leaseid();
+    if (existing_leaseid) {
+        leaseid = GF_MALLOC(LEASE_ID_SIZE, gf_common_mt_char);
+        GF_CHECK_ALLOC_AND_LOG("gfapi", leaseid, ret, "lease id alloc failed",
+                               out);
+        memcpy(leaseid, existing_leaseid, LEASE_ID_SIZE);
+        if (*fop_attr == NULL) {
+            *fop_attr = dict_new();
+            dict_create = _gf_true;
+        }
+        GF_CHECK_ALLOC_AND_LOG("gfapi", *fop_attr, ret, "dict_new failed", out);
+        ret = dict_set_bin(*fop_attr, "lease-id", leaseid, LEASE_ID_SIZE);
+    }
 
-	glfs_lock (fs);
-	{
-		list_add_tail (&glfd->openfds, &fs->openfds);
-	}
-	glfs_unlock (fs);
+out:
+    if (ret) {
+        GF_FREE(leaseid);
+        if (dict_create) {
+            if (*fop_attr)
+                dict_unref(*fop_attr);
+            *fop_attr = NULL;
+        }
+    }
+    return ret;
 }
 
 void
-glfs_fd_destroy (struct glfs_fd *glfd)
+unset_fop_attr(dict_t **fop_attr)
 {
-	if (!glfd)
-		return;
+    char *lease_id = NULL;
+    lease_id = gf_existing_leaseid();
+    if (lease_id)
+        memset(lease_id, 0, LEASE_ID_SIZE);
+    if (*fop_attr) {
+        dict_unref(*fop_attr);
+        *fop_attr = NULL;
+    }
+}
 
-	glfs_lock (glfd->fs);
-	{
-		list_del_init (&glfd->openfds);
-	}
-	glfs_unlock (glfd->fs);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_from_glfd, 3.4.0)
+struct glfs *
+pub_glfs_from_glfd(struct glfs_fd *glfd)
+{
+    if (glfd == NULL) {
+        errno = EBADF;
+        return NULL;
+    }
 
-	if (glfd->fd)
-		fd_unref (glfd->fd);
-	GF_FREE (glfd);
+    return glfd->fs;
 }
 
+static void
+glfs_fd_destroy(struct glfs_fd *glfd)
+{
+    if (!glfd)
+        return;
+
+    glfs_lock(glfd->fs, _gf_true);
+    {
+        list_del_init(&glfd->openfds);
+    }
+    glfs_unlock(glfd->fs);
+
+    if (glfd->fd) {
+        fd_unref(glfd->fd);
+        glfd->fd = NULL;
+    }
+
+    GF_FREE(glfd->readdirbuf);
+
+    GF_FREE(glfd);
+}
+
+struct glfs_fd *
+glfs_fd_new(struct glfs *fs)
+{
+    struct glfs_fd *glfd = NULL;
+
+    glfd = GF_CALLOC(1, sizeof(*glfd), glfs_mt_glfs_fd_t);
+    if (!glfd)
+        return NULL;
+
+    glfd->fs = fs;
+
+    INIT_LIST_HEAD(&glfd->openfds);
+
+    GF_REF_INIT(glfd, glfs_fd_destroy);
+
+    return glfd;
+}
+
+void
+glfs_fd_bind(struct glfs_fd *glfd)
+{
+    struct glfs *fs = NULL;
+
+    fs = glfd->fs;
+
+    glfs_lock(fs, _gf_true);
+    {
+        list_add_tail(&glfd->openfds, &fs->openfds);
+    }
+    glfs_unlock(fs);
+}
 
 static void *
-glfs_poller (void *data)
+glfs_poller(void *data)
 {
-	struct glfs  *fs = NULL;
+    struct glfs *fs = NULL;
 
-	fs = data;
+    fs = data;
 
-	event_dispatch (fs->ctx->event_pool);
+    gf_event_dispatch(fs->ctx->event_pool);
 
-	return NULL;
+    return NULL;
 }
 
+static struct glfs *
+glfs_new_fs(const char *volname)
+{
+    struct glfs *fs = NULL;
+
+    fs = CALLOC(1, sizeof(*fs));
+    if (!fs)
+        return NULL;
+
+    INIT_LIST_HEAD(&fs->openfds);
+    INIT_LIST_HEAD(&fs->upcall_list);
+    INIT_LIST_HEAD(&fs->waitq);
+
+    PTHREAD_MUTEX_INIT(&fs->mutex, NULL, fs->pthread_flags, GLFS_INIT_MUTEX,
+                       err);
+
+    PTHREAD_COND_INIT(&fs->cond, NULL, fs->pthread_flags, GLFS_INIT_COND, err);
 
+    PTHREAD_COND_INIT(&fs->child_down_cond, NULL, fs->pthread_flags,
+                      GLFS_INIT_COND_CHILD, err);
+
+    PTHREAD_MUTEX_INIT(&fs->upcall_list_mutex, NULL, fs->pthread_flags,
+                       GLFS_INIT_MUTEX_UPCALL, err);
+
+    fs->volname = strdup(volname);
+    if (!fs->volname)
+        goto err;
+
+    fs->pin_refcnt = 0;
+    fs->upcall_events = 0;
+    fs->up_cbk = NULL;
+    fs->up_data = NULL;
+
+    return fs;
+
+err:
+    glfs_free_from_ctx(fs);
+    return NULL;
+}
+
+extern xlator_t global_xlator;
+extern glusterfs_ctx_t *global_ctx;
+extern pthread_mutex_t global_ctx_mutex;
+
+static int
+glfs_init_global_ctx()
+{
+    int ret = 0;
+    glusterfs_ctx_t *ctx = NULL;
+
+    pthread_mutex_lock(&global_ctx_mutex);
+    {
+        if (global_xlator.ctx)
+            goto unlock;
+
+        ctx = glusterfs_ctx_new();
+        if (!ctx) {
+            ret = -1;
+            goto unlock;
+        }
+
+        gf_log_globals_init(ctx, GF_LOG_NONE);
+
+        global_ctx = ctx;
+        global_xlator.ctx = global_ctx;
+
+        ret = glusterfs_ctx_defaults_init(ctx);
+        if (ret) {
+            global_ctx = NULL;
+            global_xlator.ctx = NULL;
+            goto unlock;
+        }
+    }
+unlock:
+    pthread_mutex_unlock(&global_ctx_mutex);
+
+    if (ret)
+        FREE(ctx);
+
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_new, 3.4.0)
 struct glfs *
-glfs_new (const char *volname)
+pub_glfs_new(const char *volname)
 {
-	struct glfs     *fs = NULL;
-	int              ret = -1;
-	glusterfs_ctx_t *ctx = NULL;
+    if (!volname) {
+        errno = EINVAL;
+        return NULL;
+    }
+
+    struct glfs *fs = NULL;
+    int i = 0;
+    int ret = -1;
+    glusterfs_ctx_t *ctx = NULL;
+    xlator_t *old_THIS = NULL;
+    char pname[16] = "";
+    char msg[32] = "";
+
+    if (volname[0] == '/' || volname[0] == '-') {
+        if (strncmp(volname, "/snaps/", 7) == 0) {
+            goto label;
+        }
+        errno = EINVAL;
+        return NULL;
+    }
+
+    for (i = 0; i < strlen(volname); i++) {
+        if (!isalnum(volname[i]) && (volname[i] != '_') &&
+            (volname[i] != '-')) {
+            errno = EINVAL;
+            return NULL;
+        }
+    }
+
+label:
+    /*
+     * Do this as soon as possible in case something else depends on
+     * pool allocations.
+     */
+    mem_pools_init();
+
+    fs = glfs_new_fs(volname);
+    if (!fs)
+        goto out;
+
+    ctx = glusterfs_ctx_new();
+    if (!ctx)
+        goto out;
+
+    /* first globals init, for gf_mem_acct_enable_set () */
+
+    ret = glusterfs_globals_init(ctx);
+    if (ret)
+        goto out;
+
+    old_THIS = THIS;
+    ret = glfs_init_global_ctx();
+    if (ret)
+        goto out;
+
+    /* then ctx_defaults_init, for xlator_mem_acct_init(THIS) */
+
+    ret = glusterfs_ctx_defaults_init(ctx);
+    if (ret)
+        goto out;
+
+    fs->ctx = ctx;
+    fs->ctx->process_mode = GF_CLIENT_PROCESS;
+
+    ret = glfs_set_logging(fs, "/dev/null", 0);
+    if (ret)
+        goto out;
 
-	ctx = glusterfs_ctx_new ();
-	if (!ctx) {
-		return NULL;
-	}
+    fs->ctx->cmd_args.volfile_id = gf_strdup(volname);
+    if (!(fs->ctx->cmd_args.volfile_id)) {
+        ret = -1;
+        goto out;
+    }
 
-#ifdef DEBUG
-        gf_mem_acct_enable_set (ctx);
+    ret = -1;
+#ifdef GF_LINUX_HOST_OS
+    ret = prctl(PR_GET_NAME, (unsigned long)pname, 0, 0, 0);
 #endif
+    if (ret)
+        fs->ctx->cmd_args.process_name = gf_strdup("gfapi");
+    else {
+        snprintf(msg, sizeof(msg), "gfapi.%s", pname);
+        fs->ctx->cmd_args.process_name = gf_strdup(msg);
+    }
+    ret = 0;
 
-	/* first globals init, for gf_mem_acct_enable_set () */
-	ret = glusterfs_globals_init (ctx);
-	if (ret)
-		return NULL;
+out:
+    if (ret) {
+        if (fs) {
+            glfs_fini(fs);
+            fs = NULL;
+        } else {
+            /* glfs_fini() calls mem_pools_fini() too */
+            mem_pools_fini();
+        }
+    }
+
+    if (old_THIS)
+        THIS = old_THIS;
+
+    return fs;
+}
+
+GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_new_from_ctx, 3.7.0)
+struct glfs *
+priv_glfs_new_from_ctx(glusterfs_ctx_t *ctx)
+{
+    struct glfs *fs = NULL;
 
-	THIS->ctx = ctx;
+    if (!ctx)
+        goto out;
+
+    fs = glfs_new_fs("");
+    if (!fs)
+        goto out;
+
+    fs->ctx = ctx;
+
+out:
+    return fs;
+}
+
+GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_free_from_ctx, 3.7.0)
+void
+priv_glfs_free_from_ctx(struct glfs *fs)
+{
+    upcall_entry *u_list = NULL;
+    upcall_entry *tmp = NULL;
 
-	/* then ctx_defaults_init, for xlator_mem_acct_init(THIS) */
-	ret = glusterfs_ctx_defaults_init (ctx);
-	if (ret)
-		return NULL;
+    if (!fs)
+        return;
 
-	fs = GF_CALLOC (1, sizeof (*fs), glfs_mt_glfs_t);
-	if (!fs)
-		return NULL;
-	fs->ctx = ctx;
+    /* cleanup upcall structures */
+    list_for_each_entry_safe(u_list, tmp, &fs->upcall_list, upcall_list)
+    {
+        list_del_init(&u_list->upcall_list);
+        GF_FREE(u_list->upcall_data.data);
+        GF_FREE(u_list);
+    }
 
-	glfs_set_logging (fs, "/dev/null", 0);
+    PTHREAD_MUTEX_DESTROY(&fs->mutex, fs->pthread_flags, GLFS_INIT_MUTEX);
 
-	fs->ctx->cmd_args.volfile_id = gf_strdup (volname);
+    PTHREAD_COND_DESTROY(&fs->cond, fs->pthread_flags, GLFS_INIT_COND);
 
-	fs->volname = gf_strdup (volname);
+    PTHREAD_COND_DESTROY(&fs->child_down_cond, fs->pthread_flags,
+                         GLFS_INIT_COND_CHILD);
 
-	pthread_mutex_init (&fs->mutex, NULL);
-	pthread_cond_init (&fs->cond, NULL);
+    PTHREAD_MUTEX_DESTROY(&fs->upcall_list_mutex, fs->pthread_flags,
+                          GLFS_INIT_MUTEX_UPCALL);
 
-	INIT_LIST_HEAD (&fs->openfds);
+    if (fs->oldvolfile)
+        FREE(fs->oldvolfile);
 
-	return fs;
+    FREE(fs->volname);
+
+    FREE(fs);
 }
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_set_volfile, 3.4.0)
+int
+pub_glfs_set_volfile(struct glfs *fs, const char *volfile)
+{
+    cmd_args_t *cmd_args = NULL;
+
+    cmd_args = &fs->ctx->cmd_args;
 
+    if (vol_assigned(cmd_args))
+        return -1;
+
+    cmd_args->volfile = gf_strdup(volfile);
+    if (!cmd_args->volfile)
+        return -1;
+    return 0;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_set_logging, 3.4.0)
 int
-glfs_set_volfile (struct glfs *fs, const char *volfile)
+pub_glfs_set_logging(struct glfs *fs, const char *logfile, int loglevel)
 {
-	cmd_args_t  *cmd_args = NULL;
+    int ret = -1;
+    char *tmplog = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
 
-	cmd_args = &fs->ctx->cmd_args;
+    if (!logfile) {
+        ret = gf_set_log_file_path(&fs->ctx->cmd_args, fs->ctx);
+        if (ret)
+            goto out;
+        tmplog = fs->ctx->cmd_args.log_file;
+    } else {
+        tmplog = (char *)logfile;
+    }
+
+    /* finish log set parameters before init */
+    if (loglevel >= 0)
+        gf_log_set_loglevel(fs->ctx, loglevel);
 
-	if (vol_assigned (cmd_args))
-		return -1;
+    ret = gf_log_init(fs->ctx, tmplog, NULL);
+    if (ret)
+        goto out;
 
-	cmd_args->volfile = gf_strdup (volfile);
+    ret = gf_log_inject_timer_event(fs->ctx);
+    if (ret)
+        goto out;
+
+out:
+    __GLFS_EXIT_FS;
 
-	return 0;
+invalid_fs:
+    return ret;
 }
 
+int
+glfs_init_wait(struct glfs *fs)
+{
+    int ret = -1;
+
+    /* Always a top-down call, use glfs_lock() */
+    glfs_lock(fs, _gf_true);
+    {
+        while (!fs->init)
+            pthread_cond_wait(&fs->cond, &fs->mutex);
+        ret = fs->ret;
+        errno = fs->err;
+    }
+    glfs_unlock(fs);
+
+    return ret;
+}
+
+GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_init_done, 3.4.0)
+void
+priv_glfs_init_done(struct glfs *fs, int ret)
+{
+    glfs_init_cbk init_cbk;
+
+    if (!fs) {
+        gf_smsg("glfs", GF_LOG_ERROR, EINVAL, API_MSG_GLFS_FSOBJ_NULL, NULL);
+        goto out;
+    }
+
+    init_cbk = fs->init_cbk;
+
+    /* Always a bottom-up call, use mutex_lock() */
+    pthread_mutex_lock(&fs->mutex);
+    {
+        fs->init = 1;
+        fs->ret = ret;
+        fs->err = errno;
+
+        if (!init_cbk)
+            pthread_cond_broadcast(&fs->cond);
+    }
+    pthread_mutex_unlock(&fs->mutex);
+
+    if (init_cbk)
+        init_cbk(fs, ret);
+out:
+    return;
+}
 
 int
-glfs_set_volfile_server (struct glfs *fs, const char *transport,
-			 const char *host, int port)
+glfs_init_common(struct glfs *fs)
 {
-	cmd_args_t  *cmd_args = NULL;
+    int ret = -1;
 
-	cmd_args = &fs->ctx->cmd_args;
+    ret = create_master(fs);
+    if (ret)
+        return ret;
 
-	if (vol_assigned (cmd_args))
-		return -1;
+    ret = gf_thread_create(&fs->poller, NULL, glfs_poller, fs, "glfspoll");
+    if (ret)
+        return ret;
 
-	cmd_args->volfile_server = gf_strdup (host);
-	cmd_args->volfile_server_transport = gf_strdup (transport);
-	cmd_args->volfile_server_port = port;
-	cmd_args->max_connect_attempts = 2;
+    ret = glfs_volumes_init(fs);
+    if (ret)
+        return ret;
 
-	return 0;
+    fs->dev_id = gf_dm_hashfn(fs->volname, strlen(fs->volname));
+    return ret;
 }
 
+int
+glfs_init_async(struct glfs *fs, glfs_init_cbk cbk)
+{
+    int ret = -1;
+
+    if (!fs || !fs->ctx) {
+        gf_smsg("glfs", GF_LOG_ERROR, EINVAL, API_MSG_FS_NOT_INIT, NULL);
+        errno = EINVAL;
+        return ret;
+    }
+
+    fs->init_cbk = cbk;
+
+    ret = glfs_init_common(fs);
+
+    return ret;
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_init, 3.4.0)
 int
-glfs_set_logging (struct glfs *fs, const char *logfile, int loglevel)
+pub_glfs_init(struct glfs *fs)
 {
-	int  ret = 0;
-        char *tmplog = NULL;
+    int ret = -1;
 
-        if (!logfile) {
-                ret = gf_set_log_file_path (&fs->ctx->cmd_args);
-                if (ret)
-                        goto out;
-                tmplog = fs->ctx->cmd_args.log_file;
-        } else {
-                tmplog = (char *)logfile;
-        }
+    DECLARE_OLD_THIS;
 
-        ret = gf_log_init (fs->ctx, tmplog, NULL);
-        if (ret)
-                goto out;
+    if (!fs || !fs->ctx) {
+        gf_smsg("glfs", GF_LOG_ERROR, EINVAL, API_MSG_FS_NOT_INIT, NULL);
+        errno = EINVAL;
+        return ret;
+    }
 
-	if (loglevel >= 0)
-		gf_log_set_loglevel (loglevel);
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
 
+    ret = glfs_init_common(fs);
+    if (ret)
+        goto out;
+
+    ret = glfs_init_wait(fs);
 out:
-	return ret;
+    __GLFS_EXIT_FS;
+
+    /* Set the initial current working directory to "/" */
+    if (ret >= 0) {
+        ret = glfs_chdir(fs, "/");
+    }
+
+invalid_fs:
+    return ret;
 }
 
+static int
+glusterfs_ctx_destroy(glusterfs_ctx_t *ctx)
+{
+    call_pool_t *pool = NULL;
+    int ret = 0;
+    glusterfs_graph_t *trav_graph = NULL;
+    glusterfs_graph_t *tmp = NULL;
+
+    if (ctx == NULL)
+        return 0;
+
+    if (ctx->cmd_args.curr_server)
+        glfs_free_volfile_servers(&ctx->cmd_args);
+
+    glfs_free_xlator_options(&ctx->cmd_args);
+
+    /* For all the graphs, crawl through the xlator_t structs and free
+     * all its members except for the mem_acct member,
+     * as GF_FREE will be referencing it.
+     */
+    list_for_each_entry_safe(trav_graph, tmp, &ctx->graphs, list)
+    {
+        xlator_tree_free_members(trav_graph->first);
+    }
+
+    /* Free the memory pool */
+    if (ctx->stub_mem_pool)
+        mem_pool_destroy(ctx->stub_mem_pool);
+    if (ctx->dict_pool)
+        mem_pool_destroy(ctx->dict_pool);
+    if (ctx->dict_data_pool)
+        mem_pool_destroy(ctx->dict_data_pool);
+    if (ctx->dict_pair_pool)
+        mem_pool_destroy(ctx->dict_pair_pool);
+    if (ctx->logbuf_pool)
+        mem_pool_destroy(ctx->logbuf_pool);
+
+    pool = ctx->pool;
+    if (pool) {
+        if (pool->frame_mem_pool)
+            mem_pool_destroy(pool->frame_mem_pool);
+        if (pool->stack_mem_pool)
+            mem_pool_destroy(pool->stack_mem_pool);
+        LOCK_DESTROY(&pool->lock);
+        GF_FREE(pool);
+    }
+
+    /* Free the event pool */
+    ret = gf_event_pool_destroy(ctx->event_pool);
+
+    /* Free the iobuf pool */
+    iobuf_pool_destroy(ctx->iobuf_pool);
+
+    GF_FREE(ctx->process_uuid);
+    GF_FREE(ctx->cmd_args.volfile_id);
+    GF_FREE(ctx->cmd_args.process_name);
+
+    LOCK_DESTROY(&ctx->lock);
+    pthread_mutex_destroy(&ctx->notify_lock);
+    pthread_cond_destroy(&ctx->notify_cond);
+
+    /* Free all the graph structs and its containing xlator_t structs
+     * from this point there should be no reference to GF_FREE/GF_CALLOC
+     * as it will try to access mem_acct and the below function would
+     * have freed the same.
+     */
+    list_for_each_entry_safe(trav_graph, tmp, &ctx->graphs, list)
+    {
+        glusterfs_graph_destroy_residual(trav_graph);
+    }
+
+    GF_FREE(ctx->statedump_path);
+    FREE(ctx);
+
+    return ret;
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fini, 3.4.0)
 int
-glfs_init_wait (struct glfs *fs)
+pub_glfs_fini(struct glfs *fs)
 {
-	int   ret = -1;
+    int ret = -1;
+    int countdown = 100;
+    xlator_t *subvol = NULL;
+    glusterfs_ctx_t *ctx = NULL;
+    glusterfs_graph_t *graph = NULL;
+    call_pool_t *call_pool = NULL;
+    int fs_init = 0;
+    int err = -1;
+    struct synctask *waittask = NULL;
+
+    DECLARE_OLD_THIS;
+
+    if (!fs) {
+        errno = EINVAL;
+        goto invalid_fs;
+    }
+
+    ctx = fs->ctx;
+    if (!ctx) {
+        goto free_fs;
+    }
+
+    THIS = fs->ctx->master;
+
+    if (ctx->mgmt) {
+        rpc_clnt_disable(ctx->mgmt);
+    }
+
+    call_pool = fs->ctx->pool;
+
+    /* Wake up any suspended synctasks */
+    while (!list_empty(&fs->waitq)) {
+        waittask = list_entry(fs->waitq.next, struct synctask, waitq);
+        list_del_init(&waittask->waitq);
+        synctask_wake(waittask);
+    }
+
+    while (countdown--) {
+        /* give some time for background frames to finish */
+        pthread_mutex_lock(&fs->mutex);
+        {
+            /* Do we need to increase countdown? */
+            if ((!call_pool->cnt) && (!fs->pin_refcnt)) {
+                gf_msg_trace("glfs", 0,
+                             "call_pool_cnt - %" PRId64
+                             ","
+                             "pin_refcnt - %d",
+                             call_pool->cnt, fs->pin_refcnt);
+
+                ctx->cleanup_started = 1;
+                pthread_mutex_unlock(&fs->mutex);
+                break;
+            }
+        }
+        pthread_mutex_unlock(&fs->mutex);
+        gf_nanosleep(100000 * GF_US_IN_NS);
+    }
 
-	/* Always a top-down call, use glfs_lock() */
-	glfs_lock (fs);
-	{
-		while (!fs->init)
-			pthread_cond_wait (&fs->cond,
-					   &fs->mutex);
-		ret = fs->ret;
-		errno = fs->err;
-	}
-	glfs_unlock (fs);
+    /* leaked frames may exist, we ignore */
 
-	return ret;
-}
+    /*We deem glfs_fini as successful if there are no pending frames in the call
+     *pool*/
+    ret = (call_pool->cnt == 0) ? 0 : -1;
 
+    pthread_mutex_lock(&fs->mutex);
+    {
+        fs_init = fs->init;
+    }
+    pthread_mutex_unlock(&fs->mutex);
 
-void
-glfs_init_done (struct glfs *fs, int ret)
+    if (fs_init != 0) {
+        subvol = glfs_active_subvol(fs);
+        if (subvol) {
+            /* PARENT_DOWN within glfs_subvol_done() is issued
+               only on graph switch (new graph should activiate
+               and decrement the extra @winds count taken in
+               glfs_graph_setup()
+
+               Since we are explicitly destroying,
+               PARENT_DOWN is necessary
+            */
+            xlator_notify(subvol, GF_EVENT_PARENT_DOWN, subvol, 0);
+            /* Here we wait for GF_EVENT_CHILD_DOWN before exiting,
+               in case of asynchrnous cleanup
+            */
+            graph = subvol->graph;
+            err = pthread_mutex_lock(&fs->mutex);
+            if (err != 0) {
+                gf_smsg("glfs", GF_LOG_ERROR, err, API_MSG_FSMUTEX_LOCK_FAILED,
+                        "error=%s", strerror(err), NULL);
+                goto fail;
+            }
+            /* check and wait for CHILD_DOWN for active subvol*/
+            {
+                while (graph->used) {
+                    err = pthread_cond_wait(&fs->child_down_cond, &fs->mutex);
+                    if (err != 0)
+                        gf_smsg("glfs", GF_LOG_INFO, err,
+                                API_MSG_COND_WAIT_FAILED, "name=%s",
+                                subvol->name, "err=%s", strerror(err), NULL);
+                }
+            }
+
+            err = pthread_mutex_unlock(&fs->mutex);
+            if (err != 0) {
+                gf_smsg("glfs", GF_LOG_ERROR, err,
+                        API_MSG_FSMUTEX_UNLOCK_FAILED, "error=%s",
+                        strerror(err), NULL);
+                goto fail;
+            }
+        }
+        glfs_subvol_done(fs, subvol);
+    }
+
+    ctx->cleanup_started = 1;
+
+    if (fs_init != 0) {
+        /* Destroy all the inode tables of all the graphs.
+         * NOTE:
+         * - inode objects should be destroyed before calling fini()
+         *   of each xlator, as fini() and forget() of the xlators
+         *   can share few common locks or data structures, calling
+         *   fini first might destroy those required by forget
+         *   ( eg: in quick-read)
+         * - The call to inode_table_destroy_all is not required when
+         *   the cleanup during graph switch is implemented to perform
+         *   inode table destroy.
+         */
+        inode_table_destroy_all(ctx);
+
+        /* Call fini() of all the xlators in the active graph
+         * NOTE:
+         * - xlator fini() should be called before destroying any of
+         *   the threads. (eg: fini() in protocol-client uses timer
+         *   thread) */
+        glusterfs_graph_deactivate(ctx->active);
+
+        /* Join the syncenv_processor threads and cleanup
+         * syncenv resources*/
+        syncenv_destroy(ctx->env);
+
+        /* Join the poller thread */
+        if (gf_event_dispatch_destroy(ctx->event_pool) < 0)
+            ret = -1;
+    }
+
+    /* Avoid dispatching events to mgmt after freed,
+     * unreference mgmt after the event_dispatch_destroy */
+    if (ctx->mgmt) {
+        rpc_clnt_unref(ctx->mgmt);
+        ctx->mgmt = NULL;
+    }
+
+    /* log infra has to be brought down before destroying
+     * timer registry, as logging uses timer infra
+     */
+    if (gf_log_fini(ctx) != 0)
+        ret = -1;
+
+    /* Join the timer thread */
+    if (fs_init != 0) {
+        gf_timer_registry_destroy(ctx);
+    }
+
+    /* Destroy the context and the global pools */
+    if (glusterfs_ctx_destroy(ctx) != 0)
+        ret = -1;
+
+free_fs:
+    glfs_free_from_ctx(fs);
+
+    /*
+     * Do this as late as possible in case anything else has (or
+     * grows) a dependency on mem-pool allocations.
+     */
+    mem_pools_fini();
+
+fail:
+    if (!ret)
+        ret = err;
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_get_volfile, 3.6.0)
+ssize_t
+pub_glfs_get_volfile(struct glfs *fs, void *buf, size_t len)
 {
-	glfs_init_cbk init_cbk;
+    ssize_t res = -1;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    glfs_lock(fs, _gf_true);
+    if (len >= fs->oldvollen) {
+        gf_msg_trace("glfs", 0, "copying %zu to %p", len, buf);
+        memcpy(buf, fs->oldvolfile, len);
+        res = len;
+    } else {
+        res = len - fs->oldvollen;
+        gf_msg_trace("glfs", 0, "buffer is %zd too short", -res);
+    }
+    glfs_unlock(fs);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return res;
+}
 
-	if (!fs) {
-		gf_log ("glfs", GF_LOG_ERROR,
-			"fs is NULL");
-		goto out;
-	}
+GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_ipc, 3.12.0)
+int
+priv_glfs_ipc(struct glfs *fs, int opcode, void *xd_in, void **xd_out)
+{
+    xlator_t *subvol = NULL;
+    int ret = -1;
 
-	init_cbk = fs->init_cbk;
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
 
-	/* Always a bottom-up call, use mutex_lock() */
-	pthread_mutex_lock (&fs->mutex);
-	{
-		fs->init = 1;
-		fs->ret = ret;
-		fs->err = errno;
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
 
-		if (!init_cbk)
-			pthread_cond_broadcast (&fs->cond);
-	}
-	pthread_mutex_unlock (&fs->mutex);
+    ret = syncop_ipc(subvol, opcode, (dict_t *)xd_in, (dict_t **)xd_out);
+    DECODE_SYNCOP_ERR(ret);
 
-	if (init_cbk)
-		init_cbk (fs, ret);
 out:
-	return;
-}
+    glfs_subvol_done(fs, subvol);
+    __GLFS_EXIT_FS;
 
+invalid_fs:
+    return ret;
+}
 
+GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_setfspid, 6.1)
 int
-glfs_init_common (struct glfs *fs)
+priv_glfs_setfspid(struct glfs *fs, pid_t pid)
 {
-	int  ret = -1;
+    cmd_args_t *cmd_args = NULL;
+    int ret = 0;
 
-	ret = create_master (fs);
-	if (ret)
-		return ret;
+    cmd_args = &fs->ctx->cmd_args;
+    cmd_args->client_pid = pid;
+    cmd_args->client_pid_set = 1;
+    ret = syncopctx_setfspid(&pid);
+
+    return ret;
+}
 
-	ret = pthread_create (&fs->poller, NULL, glfs_poller, fs);
-	if (ret)
-		return ret;
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_free, 3.7.16)
+void
+pub_glfs_free(void *ptr)
+{
+    GLFS_FREE(ptr);
+}
 
-	ret = glfs_volumes_init (fs);
-	if (ret)
-		return ret;
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_get_fs, 3.7.16)
+struct glfs *
+pub_glfs_upcall_get_fs(struct glfs_upcall *arg)
+{
+    return arg->fs;
+}
 
-	fs->dev_id = gf_dm_hashfn (fs->volname, strlen (fs->volname));
-	return ret;
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_get_reason, 3.7.16)
+enum glfs_upcall_reason
+pub_glfs_upcall_get_reason(struct glfs_upcall *arg)
+{
+    return arg->reason;
 }
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_get_event, 3.7.16)
+void *
+pub_glfs_upcall_get_event(struct glfs_upcall *arg)
+{
+    return arg->event;
+}
 
-int
-glfs_init_async (struct glfs *fs, glfs_init_cbk cbk)
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_inode_get_object, 3.7.16)
+struct glfs_object *
+pub_glfs_upcall_inode_get_object(struct glfs_upcall_inode *arg)
 {
-	int  ret = -1;
+    return arg->object;
+}
 
-	fs->init_cbk = cbk;
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_inode_get_flags, 3.7.16)
+uint64_t
+pub_glfs_upcall_inode_get_flags(struct glfs_upcall_inode *arg)
+{
+    return arg->flags;
+}
 
-	ret = glfs_init_common (fs);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_inode_get_stat, 3.7.16)
+struct stat *
+pub_glfs_upcall_inode_get_stat(struct glfs_upcall_inode *arg)
+{
+    return &arg->buf;
+}
 
-	return ret;
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_inode_get_expire, 3.7.16)
+uint64_t
+pub_glfs_upcall_inode_get_expire(struct glfs_upcall_inode *arg)
+{
+    return arg->expire_time_attr;
 }
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_inode_get_pobject, 3.7.16)
+struct glfs_object *
+pub_glfs_upcall_inode_get_pobject(struct glfs_upcall_inode *arg)
+{
+    return arg->p_object;
+}
 
-int
-glfs_init (struct glfs *fs)
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_inode_get_pstat, 3.7.16)
+struct stat *
+pub_glfs_upcall_inode_get_pstat(struct glfs_upcall_inode *arg)
+{
+    return &arg->p_buf;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_inode_get_oldpobject, 3.7.16)
+struct glfs_object *
+pub_glfs_upcall_inode_get_oldpobject(struct glfs_upcall_inode *arg)
 {
-	int  ret = -1;
+    return arg->oldp_object;
+}
 
-	ret = glfs_init_common (fs);
-	if (ret)
-		return ret;
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_inode_get_oldpstat, 3.7.16)
+struct stat *
+pub_glfs_upcall_inode_get_oldpstat(struct glfs_upcall_inode *arg)
+{
+    return &arg->oldp_buf;
+}
 
-	ret = glfs_init_wait (fs);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_lease_get_object, 4.1.6)
+struct glfs_object *
+pub_glfs_upcall_lease_get_object(struct glfs_upcall_lease *arg)
+{
+    return arg->object;
+}
 
-	return ret;
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_lease_get_lease_type, 4.1.6)
+uint32_t
+pub_glfs_upcall_lease_get_lease_type(struct glfs_upcall_lease *arg)
+{
+    return arg->lease_type;
 }
 
+/* definitions of the GLFS_SYSRQ_* chars are in glfs.h */
+static struct glfs_sysrq_help {
+    char sysrq;
+    char *msg;
+} glfs_sysrq_help[] = {{GLFS_SYSRQ_HELP, "(H)elp"},
+                       {GLFS_SYSRQ_STATEDUMP, "(S)tatedump"},
+                       {0, NULL}};
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_sysrq, 3.10.0)
 int
-glfs_fini (struct glfs *fs)
+pub_glfs_sysrq(struct glfs *fs, char sysrq)
 {
-        int  ret = -1;
-        xlator_t *subvol = NULL;
-        glusterfs_ctx_t *ctx = NULL;
-        call_pool_t *call_pool = NULL;
-        int countdown = 100;
-
-        ctx = fs->ctx;
+    glusterfs_ctx_t *ctx = NULL;
+    int ret = 0;
+    int msg_len;
+    char msg[1024] = {
+        0,
+    }; /* should not exceed 1024 chars */
+
+    if (!fs || !fs->ctx) {
+        ret = -1;
+        errno = EINVAL;
+        goto out;
+    }
+
+    ctx = fs->ctx;
+
+    switch (sysrq) {
+        case GLFS_SYSRQ_HELP: {
+            struct glfs_sysrq_help *usage = NULL;
+
+            for (usage = glfs_sysrq_help; usage->sysrq; usage++) {
+                msg_len = strlen(msg);
+                snprintf(msg + msg_len, /* append to msg */
+                         sizeof(msg) - msg_len - 2,
+                         /* - 2 for the " " + terminating \0 */
+                         " %s", usage->msg);
+            }
+
+            /* not really an 'error', but make sure it gets logged */
+            gf_log("glfs", GF_LOG_ERROR, "available events: %s", msg);
+
+            break;
+        }
+        case GLFS_SYSRQ_STATEDUMP:
+            gf_proc_dump_info(SIGUSR1, ctx);
+            break;
+        default:
+            gf_smsg("glfs", GF_LOG_ERROR, ENOTSUP, API_MSG_INVALID_SYSRQ,
+                    "sysrq=%c", sysrq, NULL);
+            errno = ENOTSUP;
+            ret = -1;
+    }
+out:
+    return ret;
+}
 
-        if (ctx->mgmt) {
-                rpc_clnt_disable (ctx->mgmt);
-                ctx->mgmt = NULL;
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_register, 3.13.0)
+int
+pub_glfs_upcall_register(struct glfs *fs, uint32_t event_list,
+                         glfs_upcall_cbk cbk, void *data)
+{
+    int ret = 0;
+
+    /* list of supported upcall events */
+    uint32_t up_events = (GLFS_EVENT_INODE_INVALIDATE |
+                          GLFS_EVENT_RECALL_LEASE);
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    GF_VALIDATE_OR_GOTO(THIS->name, cbk, out);
+
+    /* Event list should be either GLFS_EVENT_ANY
+     * or list of supported individual events (up_events)
+     */
+    if ((event_list != GLFS_EVENT_ANY) && (event_list & ~up_events)) {
+        errno = EINVAL;
+        ret = -1;
+        gf_smsg(THIS->name, GF_LOG_ERROR, errno, API_MSG_INVALID_ARG,
+                "event_list=(0x%08x)", event_list, NULL);
+        goto out;
+    }
+
+    /* in case other thread does unregister */
+    pthread_mutex_lock(&fs->mutex);
+    {
+        if (event_list & GLFS_EVENT_INODE_INVALIDATE) {
+            /* @todo: Check if features.cache-invalidation is
+             * enabled.
+             */
+            fs->upcall_events |= GF_UPCALL_CACHE_INVALIDATION;
+            ret |= GLFS_EVENT_INODE_INVALIDATE;
         }
+        if (event_list & GLFS_EVENT_RECALL_LEASE) {
+            /* @todo: Check if features.leases is enabled */
+            fs->upcall_events |= GF_UPCALL_RECALL_LEASE;
+            ret |= GLFS_EVENT_RECALL_LEASE;
+        }
+        /* Override cbk function if existing */
+        fs->up_cbk = cbk;
+        fs->up_data = data;
+        fs->cache_upcalls = _gf_true;
+    }
+    pthread_mutex_unlock(&fs->mutex);
 
-        __glfs_entry_fs (fs);
+out:
+    __GLFS_EXIT_FS;
 
-        call_pool = fs->ctx->pool;
+invalid_fs:
+    return ret;
+}
 
-        while (countdown--) {
-                /* give some time for background frames to finish */
-                if (!call_pool->cnt)
-                        break;
-                usleep (100000);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_unregister, 3.13.0)
+int
+pub_glfs_upcall_unregister(struct glfs *fs, uint32_t event_list)
+{
+    int ret = 0;
+    /* list of supported upcall events */
+    uint32_t up_events = (GLFS_EVENT_INODE_INVALIDATE |
+                          GLFS_EVENT_RECALL_LEASE);
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* Event list should be either GLFS_EVENT_ANY
+     * or list of supported individual events (up_events)
+     */
+    if ((event_list != GLFS_EVENT_ANY) && (event_list & ~up_events)) {
+        errno = EINVAL;
+        ret = -1;
+        gf_smsg(THIS->name, GF_LOG_ERROR, errno, API_MSG_INVALID_ARG,
+                "event_list=(0x%08x)", event_list, NULL);
+        goto out;
+    }
+
+    pthread_mutex_lock(&fs->mutex);
+    {
+        /* We already checked if event_list contains list of supported
+         * upcall events. No other specific checks needed as of now for
+         * unregister */
+        fs->upcall_events &= ~(event_list);
+        ret |= ((event_list == GLFS_EVENT_ANY) ? up_events : event_list);
+
+        /* If there are no upcall events registered, reset cbk */
+        if (fs->upcall_events == 0) {
+            fs->up_cbk = NULL;
+            fs->up_data = NULL;
+            fs->cache_upcalls = _gf_false;
         }
-        /* leaked frames may exist, we ignore */
+    }
+    pthread_mutex_unlock(&fs->mutex);
 
-        subvol = glfs_active_subvol (fs);
-        if (subvol) {
-                /* PARENT_DOWN within glfs_subvol_done() is issued only
-                   on graph switch (new graph should activiate and
-                   decrement the extra @winds count taken in glfs_graph_setup()
-
-                   Since we are explicitly destroying, PARENT_DOWN is necessary
-                */
-                xlator_notify (subvol, GF_EVENT_PARENT_DOWN, subvol, 0);
-                /* TBD: wait for CHILD_DOWN before exiting, in case of
-                   asynchronous cleanup like graceful socket disconnection
-                   in the future.
-                */
+out:
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_set_statedump_path, 7.0)
+int
+pub_glfs_set_statedump_path(struct glfs *fs, const char *path)
+{
+    struct stat st;
+    int ret;
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    if (!path) {
+        gf_log("glfs", GF_LOG_ERROR, "path is NULL");
+        errno = EINVAL;
+        goto err;
+    }
+
+    /* If path is not present OR, if it is directory AND has enough permission
+     * to create files, then proceed */
+    ret = sys_stat(path, &st);
+    if (ret && errno != ENOENT) {
+        gf_log("glfs", GF_LOG_ERROR, "%s: not a valid path (%s)", path,
+               strerror(errno));
+        errno = EINVAL;
+        goto err;
+    }
+
+    if (!ret) {
+        /* file is present, now check other things */
+        if (!S_ISDIR(st.st_mode)) {
+            gf_log("glfs", GF_LOG_ERROR, "%s: path is not directory", path);
+            errno = EINVAL;
+            goto err;
         }
+        if (sys_access(path, W_OK | X_OK) < 0) {
+            gf_log("glfs", GF_LOG_ERROR,
+                   "%s: path doesn't have write permission", path);
+            errno = EPERM;
+            goto err;
+        }
+    }
 
-        glfs_subvol_done (fs, subvol);
+    /* If set, it needs to be freed, so we don't have leak */
+    GF_FREE(fs->ctx->statedump_path);
 
-        return ret;
+    fs->ctx->statedump_path = gf_strdup(path);
+    if (!fs->ctx->statedump_path) {
+        gf_log("glfs", GF_LOG_ERROR,
+               "%s: failed to set statedump path, no memory", path);
+        errno = ENOMEM;
+        goto err;
+    }
+
+    __GLFS_EXIT_FS;
+
+    return 0;
+err:
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return -1;
 }
diff --git a/api/src/glfs.h b/api/src/glfs.h
index fd44c2fc885..279d11d58ee 100644
--- a/api/src/glfs.h
+++ b/api/src/glfs.h
@@ -1,5 +1,5 @@
 /*
-  Copyright (c) 2012 Red Hat, Inc. <http://www.redhat.com>
+  Copyright (c) 2012-2018 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
   This file is licensed to you under your choice of the GNU Lesser
@@ -8,7 +8,6 @@
   cases as published by the Free Software Foundation.
 */
 
-
 #ifndef _GLFS_H
 #define _GLFS_H
 
@@ -21,6 +20,17 @@
   both the library and the application.
 */
 
+/* Values for valid flags to be used when using XXXsetattr, to set multiple
+ attribute values passed via the related stat structure.
+ */
+
+#define GFAPI_SET_ATTR_MODE 0x1
+#define GFAPI_SET_ATTR_UID 0x2
+#define GFAPI_SET_ATTR_GID 0x4
+#define GFAPI_SET_ATTR_SIZE 0x8
+#define GFAPI_SET_ATTR_ATIME 0x10
+#define GFAPI_SET_ATTR_MTIME 0x20
+
 #ifndef _FILE_OFFSET_BITS
 #define _FILE_OFFSET_BITS 64
 #endif
@@ -41,6 +51,65 @@
 #include <sys/cdefs.h>
 #include <dirent.h>
 #include <sys/statvfs.h>
+#include <stdint.h>
+#include <sys/time.h>
+
+/*
+ * For off64_t to be defined, we need both
+ * __USE_LARGEFILE64 to be true and __off64_t_defnined to be
+ * false. But, making __USE_LARGEFILE64 true causes other issues
+ * such as redinition of stat and fstat to stat64 and fstat64
+ * respectively which again causes compilation issues.
+ * Without off64_t being defined, this will not compile as
+ * copy_file_range uses off64_t. Hence define it here. First
+ * check whether __off64_t_defined is true or not. <unistd.h>
+ * sets that flag when it defines off64_t. If __off64_t_defined
+ * is false and __USE_FILE_OFFSET64 is true, then go on to define
+ * off64_t using __off64_t.
+ */
+#ifndef GF_BSD_HOST_OS
+#if defined(__USE_FILE_OFFSET64) && !defined(__off64_t_defined)
+typedef __off64_t off64_t;
+#endif /* defined(__USE_FILE_OFFSET64) && !defined(__off64_t_defined) */
+#else
+#include <stdio.h>
+#ifndef _OFF64_T_DECLARED
+/*
+ * Including <stdio.h> (done above) should actually define
+ * _OFF64_T_DECLARED with off64_t data type being available
+ * for consumption. But, off64_t data type is not recognizable
+ * for FreeBSD versions less than 11. Hence, int64_t is typedefed
+ * to off64_t.
+ */
+#define _OFF64_T_DECLARED
+typedef int64_t off64_t;
+#endif /* _OFF64_T_DECLARED */
+#endif /* GF_BSD_HOST_OS */
+
+#if defined(HAVE_SYS_ACL_H) || (defined(USE_POSIX_ACLS) && USE_POSIX_ACLS)
+#include <sys/acl.h>
+#else
+typedef void *acl_t;
+typedef int acl_type_t;
+#endif
+
+/* Portability non glibc c++ build systems */
+#ifndef __THROW
+#if defined __cplusplus
+#define __THROW throw()
+#else
+#define __THROW
+#endif
+#endif
+
+#ifndef GF_DARWIN_HOST_OS
+#define GFAPI_PUBLIC(sym, ver)  /**/
+#define GFAPI_PRIVATE(sym, ver) /**/
+#else
+#define GFAPI_PUBLIC(sym, ver) __asm("_" __STRING(sym) "$GFAPI_" __STRING(ver))
+#define GFAPI_PRIVATE(sym, ver)                                                \
+    __asm("_" __STRING(sym) "$GFAPI_PRIVATE_" __STRING(ver))
+#endif
 
 __BEGIN_DECLS
 
@@ -48,7 +117,6 @@ __BEGIN_DECLS
 struct glfs;
 typedef struct glfs glfs_t;
 
-
 /*
   SYNOPSIS
 
@@ -69,8 +137,8 @@ typedef struct glfs glfs_t;
 
   @volname: Name of the volume. This identifies the server-side volume and
             the fetched volfile (equivalent of --volfile-id command line
-	    parameter to glusterfsd). When used with glfs_set_volfile() the
-	    @volname has no effect (except for appearing in log messages).
+            parameter to glusterfsd). When used with glfs_set_volfile() the
+            @volname has no effect (except for appearing in log messages).
 
   RETURN VALUES
 
@@ -79,8 +147,8 @@ typedef struct glfs glfs_t;
 
 */
 
-glfs_t *glfs_new (const char *volname);
-
+glfs_t *
+glfs_new(const char *volname) __THROW GFAPI_PUBLIC(glfs_new, 3.4.0);
 
 /*
   SYNOPSIS
@@ -109,18 +177,19 @@ glfs_t *glfs_new (const char *volname);
 
 */
 
-int glfs_set_volfile (glfs_t *fs, const char *volfile);
-
+int
+glfs_set_volfile(glfs_t *fs, const char *volfile) __THROW
+    GFAPI_PUBLIC(glfs_set_volfile, 3.4.0);
 
 /*
   SYNOPSIS
 
-  glfs_set_volfile_server: Specify the address of management server.
+  glfs_set_volfile_server: Specify the list of addresses for management server.
 
   DESCRIPTION
 
-  This function specifies the address of the management server (glusterd)
-  to connect, and establish the volume configuration. The @volname
+  This function specifies the list of addresses for the management server
+  (glusterd) to connect, and establish the volume configuration. The @volname
   parameter passed to glfs_new() is the volume which will be virtually
   mounted as the glfs_t object. All operations performed by the CLI at
   the management server will automatically be reflected in the 'virtual
@@ -135,20 +204,24 @@ int glfs_set_volfile (glfs_t *fs, const char *volfile);
        specification file.
 
   @transport: String specifying the transport used to connect to the
-              management daemon. Specifying NULL will result in the usage
-	      of the default (tcp) transport type. Permitted values
-	      are those what you specify as transport-type in a volume
-	      specification file (e.g "tcp", "rdma", "unix".)
+              management daemon. Specifying NULL will result in the
+              usage of the default (tcp) transport type. Permitted
+              values are "tcp" or "unix".
+
+  @host:      String specifying the address where to find the management daemon.
+              Socket path, while using Unix domain socket as transport type.
+              This would either be
+              - FQDN (e.g : "storage01.company.com") or
+              - ASCII (e.g : "192.168.22.1") or
+              - Socket path (e.g : "/var/run/glusterd.socket")
 
-  @host: String specifying the address of where to find the management
-         daemon. Depending on the transport type this would either be
-	 an FQDN (e.g: "storage01.company.com"), ASCII encoded IP
-	 address "192.168.22.1", or a UNIX domain socket path (e.g
-	 "/tmp/glusterd.socket".)
+  NOTE: This API is special, multiple calls to this function with different
+        volfile servers, port or transport-type would create a list of volfile
+        servers which would be polled during `volfile_fetch_attempts()`
 
   @port: The TCP port number where gluster management daemon is listening.
          Specifying 0 uses the default port number GF_DEFAULT_BASE_PORT.
-	 This parameter is unused if you are using a UNIX domain socket.
+         This parameter is unused if you are using a UNIX domain socket.
 
   RETURN VALUES
 
@@ -157,10 +230,15 @@ int glfs_set_volfile (glfs_t *fs, const char *volfile);
 
 */
 
-int glfs_set_volfile_server (glfs_t *fs, const char *transport,
-			     const char *host, int port);
-
+int
+glfs_set_volfile_server(glfs_t *fs, const char *transport, const char *host,
+                        int port) __THROW
+    GFAPI_PUBLIC(glfs_set_volfile_server, 3.4.0);
 
+int
+glfs_unset_volfile_server(glfs_t *fs, const char *transport, const char *host,
+                          int port) __THROW
+    GFAPI_PUBLIC(glfs_unset_volfile_server, 3.5.1);
 /*
   SYNOPSIS
 
@@ -190,8 +268,9 @@ int glfs_set_volfile_server (glfs_t *fs, const char *transport,
 
 */
 
-int glfs_set_logging (glfs_t *fs, const char *logfile, int loglevel);
-
+int
+glfs_set_logging(glfs_t *fs, const char *logfile, int loglevel) __THROW
+    GFAPI_PUBLIC(glfs_set_logging, 3.4.0);
 
 /*
   SYNOPSIS
@@ -217,8 +296,8 @@ int glfs_set_logging (glfs_t *fs, const char *logfile, int loglevel);
 
 */
 
-int glfs_init (glfs_t *fs);
-
+int
+glfs_init(glfs_t *fs) __THROW GFAPI_PUBLIC(glfs_init, 3.4.0);
 
 /*
   SYNOPSIS
@@ -250,7 +329,72 @@ int glfs_init (glfs_t *fs);
    0 : Success.
 */
 
-int glfs_fini (glfs_t *fs);
+int
+glfs_fini(glfs_t *fs) __THROW GFAPI_PUBLIC(glfs_fini, 3.4.0);
+
+/*
+  SYNOPSIS
+
+      glfs_getvol: Get the volfile associated with a 'virtual mount'
+
+  DESCRIPTION
+
+      Sometimes it's useful e.g. for scripts to see the volfile, so that they
+      can parse it and find subvolumes to do things like split-brain resolution
+      or custom layouts.  The API here was specifically intended to make access
+      e.g. from Python as simple as possible.
+
+      Note that the volume must be started (not necessarily mounted) for this
+      to work.
+
+  PARAMETERS
+
+      @fs:  The 'virtual mount' object for which a volfile is desired
+      @buf: Pointer to a place for the volfile length to be stored
+      @len: Length of @buf
+
+  RETURN VALUES
+
+      >0: filled N bytes of buffer
+       0: no volfile available
+      <0: volfile length exceeds @len by N bytes (@buf unchanged)
+*/
+
+ssize_t
+glfs_get_volfile(glfs_t *fs, void *buf, size_t len) __THROW
+    GFAPI_PUBLIC(glfs_get_volfile, 3.6.0);
+
+/*
+  SYNOPSIS
+
+       glfs_get_volumeid: Copy the Volume UUID stored in the glfs object fs.
+
+  DESCRIPTION
+
+       This function when invoked for the first time sends RPC call to the
+       the management server (glusterd) to fetch volume uuid and stores it
+       in the glusterfs_context linked to the glfs object fs which can be used
+       in the subsequent calls. Later it parses that UUID to convert it from
+       canonical string format into an opaque byte array and copy it into
+       the volid array. In case if either of the input parameters, volid or
+  size, is NULL, number of bytes required to copy the volume UUID is returned.
+
+  PARAMETERS
+
+       @fs: The 'virtual mount' object to be used to retrieve and store
+            volume's UUID.
+       @volid: Pointer to a place for the volume UUID to be stored
+       @size: Length of @volid
+
+  RETURN VALUES
+
+       -1 : Failure. @errno will be set with the type of failure.
+        Others : length of the volume UUID stored.
+*/
+
+int
+glfs_get_volumeid(glfs_t *fs, char *volid, size_t size) __THROW
+    GFAPI_PUBLIC(glfs_get_volumeid, 3.5.0);
 
 /*
  * FILE OPERATION
@@ -271,6 +415,157 @@ int glfs_fini (glfs_t *fs);
 struct glfs_fd;
 typedef struct glfs_fd glfs_fd_t;
 
+/*
+ * Mask for request/result items in the struct glfs_stat.
+ *
+ * Query request/result mask for glfs_stat() (family of functions) and
+ * struct glfs_stat::glfs_st_mask.
+ *
+ * These bits should be set in the mask argument of glfs_stat() (family of
+ * functions) to request particular items when calling glfs_stat().
+ *
+ * NOTE: Lower order 32 bits are used to reflect statx(2) bits. For Gluster
+ * specific attrs/extensions, use higher order 32 bits.
+ *
+ */
+#define GLFS_STAT_TYPE 0x0000000000000001U   /* Want/got stx_mode & S_IFMT */
+#define GLFS_STAT_MODE 0x0000000000000002U   /* Want/got stx_mode & ~S_IFMT */
+#define GLFS_STAT_NLINK 0x0000000000000004U  /* Want/got stx_nlink */
+#define GLFS_STAT_UID 0x0000000000000008U    /* Want/got stx_uid */
+#define GLFS_STAT_GID 0x0000000000000010U    /* Want/got stx_gid */
+#define GLFS_STAT_ATIME 0x0000000000000020U  /* Want/got stx_atime */
+#define GLFS_STAT_MTIME 0x0000000000000040U  /* Want/got stx_mtime */
+#define GLFS_STAT_CTIME 0x0000000000000080U  /* Want/got stx_ctime */
+#define GLFS_STAT_INO 0x0000000000000100U    /* Want/got stx_ino */
+#define GLFS_STAT_SIZE 0x0000000000000200U   /* Want/got stx_size */
+#define GLFS_STAT_BLOCKS 0x0000000000000400U /* Want/got stx_blocks */
+#define GLFS_STAT_BASIC_STATS                                                  \
+    0x00000000000007ffU /* Items in the normal stat struct */
+#define GLFS_STAT_BTIME 0x0000000000000800U /* Want/got stx_btime */
+#define GLFS_STAT_ALL 0x0000000000000fffU   /* All currently supported flags */
+#define GLFS_STAT_RESERVED                                                     \
+    0x8000000000000000U /* Reserved to denote future expansion */
+
+/* Macros for checking validity of struct glfs_stat members.*/
+#define GLFS_STAT_TYPE_VALID(stmask) (stmask & GLFS_STAT_TYPE)
+#define GLFS_STAT_MODE_VALID(stmask) (stmask & GLFS_STAT_MODE)
+#define GLFS_STAT_NLINK_VALID(stmask) (stmask & GLFS_STAT_NLINK)
+#define GLFS_STAT_UID_VALID(stmask) (stmask & GLFS_STAT_UID)
+#define GLFS_STAT_GID_VALID(stmask) (stmask & GLFS_STAT_GID)
+#define GLFS_STAT_ATIME_VALID(stmask) (stmask & GLFS_STAT_ATIME)
+#define GLFS_STAT_MTIME_VALID(stmask) (stmask & GLFS_STAT_MTIME)
+#define GLFS_STAT_CTIME_VALID(stmask) (stmask & GLFS_STAT_CTIME)
+#define GLFS_STAT_INO_VALID(stmask) (stmask & GLFS_STAT_INO)
+#define GLFS_STAT_SIZE_VALID(stmask) (stmask & GLFS_STAT_SIZE)
+#define GLFS_STAT_BLOCKS_VALID(stmask) (stmask & GLFS_STAT_BLOCKS)
+#define GLFS_STAT_BTIME_VALID(stmask) (stmask & GLFS_STAT_BTIME)
+#define GLFS_STAT_GFID_VALID(stmask) (stmask & GLFS_STAT_GFID)
+
+/*
+ * Attributes to be found in glfs_st_attributes and masked in
+ * glfs_st_attributes_mask.
+ *
+ * These give information about the features or the state of a file that might
+ * be of use to programs.
+ *
+ * NOTE: Lower order 32 bits are used to reflect statx(2) attribute bits. For
+ * Gluster specific attrs, use higher order 32 bits.
+ *
+ * NOTE: We do not support any file attributes or state as yet!
+ */
+#define GLFS_STAT_ATTR_RESERVED                                                \
+    0x8000000000000000U /* Reserved to denote future expansion */
+
+/* Extended file attribute structure.
+ *
+ * The caller passes a mask of what they're specifically interested in as a
+ * parameter to glfs_stat().  What glfs_stat() actually got will be indicated
+ * in glfs_st_mask upon return.
+ *
+ * For each bit in the mask argument:
+ *
+ * - if the datum is not supported:
+ *
+ *   - the bit will be cleared, and
+ *
+ *   - the datum value is undefined
+ *
+ * - otherwise, if explicitly requested:
+ *
+ *   - the field will be filled in and the bit will be set;
+ *
+ * - otherwise, if not requested, but available in, it will be filled in
+ * anyway, and the bit will be set upon return;
+ *
+ * - otherwise the field and the bit will be cleared before returning.
+ *
+ */
+
+struct glfs_stat {
+    uint64_t glfs_st_mask;       /* What results were written [uncond] */
+    uint64_t glfs_st_attributes; /* Flags conveying information about the file
+                                    [uncond] */
+    uint64_t glfs_st_attributes_mask; /* Mask to show what's supported in
+                                         st_attributes [ucond] */
+    struct timespec glfs_st_atime;    /* Last access time */
+    struct timespec glfs_st_btime;    /* File creation time */
+    struct timespec glfs_st_ctime;    /* Last attribute change time */
+    struct timespec glfs_st_mtime;    /* Last data modification time */
+    ino_t glfs_st_ino;                /* Inode number */
+    off_t glfs_st_size;               /* File size */
+    blkcnt_t glfs_st_blocks;          /* Number of 512-byte blocks allocated */
+    uint32_t glfs_st_rdev_major; /* Device ID of special file [if bdev/cdev] */
+    uint32_t glfs_st_rdev_minor;
+    uint32_t glfs_st_dev_major; /* ID of device containing file [uncond] */
+    uint32_t glfs_st_dev_minor;
+    blksize_t glfs_st_blksize; /* Preferred general I/O size [uncond] */
+    nlink_t glfs_st_nlink;     /* Number of hard links */
+    uid_t glfs_st_uid;         /* User ID of owner */
+    gid_t glfs_st_gid;         /* Group ID of owner */
+    mode_t glfs_st_mode;       /* File mode */
+};
+
+#define GLFS_LEASE_ID_SIZE 16 /* 128bits */
+typedef char glfs_leaseid_t[GLFS_LEASE_ID_SIZE];
+
+/*
+ * PER THREAD IDENTITY MODIFIERS
+ *
+ * The following operations enable to set a per thread identity context
+ * for the glfs APIs to perform operations as. The calls here are kept as close
+ * to POSIX equivalents as possible.
+ *
+ * NOTES:
+ *
+ *  - setgroups is a per thread setting, hence this is named as fsgroups to be
+ *    close in naming to the fs(u/g)id APIs
+ *  - Typical mode of operation is to set the IDs as required, with the
+ *    supplementary groups being optionally set, make the glfs call and post the
+ *    glfs operation set them back to eu/gid or uid/gid as appropriate to the
+ *    caller
+ *  - The groups once set, need to be unset by setting the size to 0 (in which
+ *    case the list argument is a do not care)
+ *  - In case of leases feature enables, setfsleaseid is used to set and reset
+ *    leaseid before and after every I/O operation.
+ *  - Once a process for a thread of operation choses to set the IDs, all glfs
+ *    calls made from that thread would default to the IDs set for the thread.
+ *    As a result use these APIs with care and ensure that the set IDs are
+ *    reverted to global process defaults as required.
+ *
+ */
+int
+glfs_setfsuid(uid_t fsuid) __THROW GFAPI_PUBLIC(glfs_setfsuid, 3.4.2);
+
+int
+glfs_setfsgid(gid_t fsgid) __THROW GFAPI_PUBLIC(glfs_setfsgid, 3.4.2);
+
+int
+glfs_setfsgroups(size_t size, const gid_t *list) __THROW
+    GFAPI_PUBLIC(glfs_setfsgroups, 3.4.2);
+
+int
+glfs_setfsleaseid(glfs_leaseid_t leaseid) __THROW
+    GFAPI_PUBLIC(glfs_setfsleaseid, 4.0.0);
 
 /*
   SYNOPSIS
@@ -297,8 +592,9 @@ typedef struct glfs_fd glfs_fd_t;
 
  */
 
-glfs_fd_t *glfs_open (glfs_t *fs, const char *path, int flags);
-
+glfs_fd_t *
+glfs_open(glfs_t *fs, const char *path, int flags) __THROW
+    GFAPI_PUBLIC(glfs_open, 3.4.0);
 
 /*
   SYNOPSIS
@@ -326,15 +622,20 @@ glfs_fd_t *glfs_open (glfs_t *fs, const char *path, int flags);
 
  */
 
-glfs_fd_t *glfs_creat (glfs_t *fs, const char *path, int flags,
-		       mode_t mode);
+glfs_fd_t *
+glfs_creat(glfs_t *fs, const char *path, int flags, mode_t mode) __THROW
+    GFAPI_PUBLIC(glfs_creat, 3.4.0);
 
-int glfs_close (glfs_fd_t *fd);
+int
+glfs_close(glfs_fd_t *fd) __THROW GFAPI_PUBLIC(glfs_close, 3.4.0);
 
-glfs_t *glfs_from_glfd (glfs_fd_t *fd);
+glfs_t *
+glfs_from_glfd(glfs_fd_t *fd) __THROW GFAPI_PUBLIC(glfs_from_glfd, 3.4.0);
 
-int glfs_set_xlator_option (glfs_t *fs, const char *xlator, const char *key,
-			    const char *value);
+int
+glfs_set_xlator_option(glfs_t *fs, const char *xlator, const char *key,
+                       const char *value) __THROW
+    GFAPI_PUBLIC(glfs_set_xlator_option, 3.4.0);
 
 /*
 
@@ -357,174 +658,828 @@ int glfs_set_xlator_option (glfs_t *fs, const char *xlator, const char *key,
   time of issuing the async IO call. This can be used by the
   caller to differentiate different instances of the async requests
   in a common callback function.
+
+  @prestat and @poststat are allocated on the stack, that are auto destroyed
+  post the callback function returns.
 */
 
-typedef void (*glfs_io_cbk) (glfs_fd_t *fd, ssize_t ret, void *data);
+typedef void (*glfs_io_cbk)(glfs_fd_t *fd, ssize_t ret,
+                            struct glfs_stat *prestat,
+                            struct glfs_stat *poststat, void *data);
 
 // glfs_{read,write}[_async]
 
-ssize_t glfs_read (glfs_fd_t *fd, void *buf, size_t count, int flags);
-ssize_t glfs_write (glfs_fd_t *fd, const void *buf, size_t count, int flags);
-int glfs_read_async (glfs_fd_t *fd, void *buf, size_t count, int flags,
-		     glfs_io_cbk fn, void *data);
-int glfs_write_async (glfs_fd_t *fd, const void *buf, size_t count, int flags,
-		      glfs_io_cbk fn, void *data);
+ssize_t
+glfs_read(glfs_fd_t *fd, void *buf, size_t count, int flags) __THROW
+    GFAPI_PUBLIC(glfs_read, 3.4.0);
+
+ssize_t
+glfs_write(glfs_fd_t *fd, const void *buf, size_t count, int flags) __THROW
+    GFAPI_PUBLIC(glfs_write, 3.4.0);
+
+int
+glfs_read_async(glfs_fd_t *fd, void *buf, size_t count, int flags,
+                glfs_io_cbk fn, void *data) __THROW
+    GFAPI_PUBLIC(glfs_read_async, 6.0);
+
+int
+glfs_write_async(glfs_fd_t *fd, const void *buf, size_t count, int flags,
+                 glfs_io_cbk fn, void *data) __THROW
+    GFAPI_PUBLIC(glfs_write_async, 6.0);
 
 // glfs_{read,write}v[_async]
 
-ssize_t glfs_readv (glfs_fd_t *fd, const struct iovec *iov, int iovcnt,
-		    int flags);
-ssize_t glfs_writev (glfs_fd_t *fd, const struct iovec *iov, int iovcnt,
-		     int flags);
-int glfs_readv_async (glfs_fd_t *fd, const struct iovec *iov, int count,
-		      int flags, glfs_io_cbk fn, void *data);
-int glfs_writev_async (glfs_fd_t *fd, const struct iovec *iov, int count,
-		       int flags, glfs_io_cbk fn, void *data);
+ssize_t
+glfs_readv(glfs_fd_t *fd, const struct iovec *iov, int iovcnt,
+           int flags) __THROW GFAPI_PUBLIC(glfs_readv, 3.4.0);
+
+ssize_t
+glfs_writev(glfs_fd_t *fd, const struct iovec *iov, int iovcnt,
+            int flags) __THROW GFAPI_PUBLIC(glfs_writev, 3.4.0);
+
+int
+glfs_readv_async(glfs_fd_t *fd, const struct iovec *iov, int count, int flags,
+                 glfs_io_cbk fn, void *data) __THROW
+    GFAPI_PUBLIC(glfs_readv_async, 6.0);
+
+int
+glfs_writev_async(glfs_fd_t *fd, const struct iovec *iov, int count, int flags,
+                  glfs_io_cbk fn, void *data) __THROW
+    GFAPI_PUBLIC(glfs_writev_async, 6.0);
 
 // glfs_p{read,write}[_async]
 
-ssize_t glfs_pread (glfs_fd_t *fd, void *buf, size_t count, off_t offset,
-		    int flags);
-ssize_t glfs_pwrite (glfs_fd_t *fd, const void *buf, size_t count,
-		     off_t offset, int flags);
-int glfs_pread_async (glfs_fd_t *fd, void *buf, size_t count, off_t offset,
-		      int flags, glfs_io_cbk fn, void *data);
-int glfs_pwrite_async (glfs_fd_t *fd, const void *buf, int count, off_t offset,
-		       int flags, glfs_io_cbk fn, void *data);
+ssize_t
+glfs_pread(glfs_fd_t *fd, void *buf, size_t count, off_t offset, int flags,
+           struct glfs_stat *poststat) __THROW GFAPI_PUBLIC(glfs_pread, 6.0);
+
+ssize_t
+glfs_pwrite(glfs_fd_t *fd, const void *buf, size_t count, off_t offset,
+            int flags, struct glfs_stat *prestat,
+            struct glfs_stat *poststat) __THROW GFAPI_PUBLIC(glfs_pwrite, 6.0);
+
+int
+glfs_pread_async(glfs_fd_t *fd, void *buf, size_t count, off_t offset,
+                 int flags, glfs_io_cbk fn, void *data) __THROW
+    GFAPI_PUBLIC(glfs_pread_async, 6.0);
+
+int
+glfs_pwrite_async(glfs_fd_t *fd, const void *buf, int count, off_t offset,
+                  int flags, glfs_io_cbk fn, void *data) __THROW
+    GFAPI_PUBLIC(glfs_pwrite_async, 6.0);
 
 // glfs_p{read,write}v[_async]
 
-ssize_t glfs_preadv (glfs_fd_t *fd, const struct iovec *iov, int iovcnt,
-		     off_t offset, int flags);
-ssize_t glfs_pwritev (glfs_fd_t *fd, const struct iovec *iov, int iovcnt,
-		      off_t offset, int flags);
-int glfs_preadv_async (glfs_fd_t *fd, const struct iovec *iov, int count,
-		       off_t offset, int flags, glfs_io_cbk fn, void *data);
-int glfs_pwritev_async (glfs_fd_t *fd, const struct iovec *iov, int count,
-			off_t offset, int flags, glfs_io_cbk fn, void *data);
+ssize_t
+glfs_preadv(glfs_fd_t *fd, const struct iovec *iov, int iovcnt, off_t offset,
+            int flags) __THROW GFAPI_PUBLIC(glfs_preadv, 3.4.0);
+
+ssize_t
+glfs_pwritev(glfs_fd_t *fd, const struct iovec *iov, int iovcnt, off_t offset,
+             int flags) __THROW GFAPI_PUBLIC(glfs_pwritev, 3.4.0);
+
+int
+glfs_preadv_async(glfs_fd_t *fd, const struct iovec *iov, int count,
+                  off_t offset, int flags, glfs_io_cbk fn, void *data) __THROW
+    GFAPI_PUBLIC(glfs_preadv_async, 6.0);
+
+int
+glfs_pwritev_async(glfs_fd_t *fd, const struct iovec *iov, int count,
+                   off_t offset, int flags, glfs_io_cbk fn, void *data) __THROW
+    GFAPI_PUBLIC(glfs_pwritev_async, 6.0);
+
+off_t
+glfs_lseek(glfs_fd_t *fd, off_t offset, int whence) __THROW
+    GFAPI_PUBLIC(glfs_lseek, 3.4.0);
+
+ssize_t
+glfs_copy_file_range(struct glfs_fd *glfd_in, off64_t *off_in,
+                     struct glfs_fd *glfd_out, off64_t *off_out, size_t len,
+                     unsigned int flags, struct glfs_stat *statbuf,
+                     struct glfs_stat *prestat,
+                     struct glfs_stat *poststat) __THROW
+    GFAPI_PUBLIC(glfs_copy_file_range, 6.0);
+
+int
+glfs_truncate(glfs_t *fs, const char *path, off_t length) __THROW
+    GFAPI_PUBLIC(glfs_truncate, 3.7.15);
+
+int
+glfs_ftruncate(glfs_fd_t *fd, off_t length, struct glfs_stat *prestat,
+               struct glfs_stat *poststat) __THROW
+    GFAPI_PUBLIC(glfs_ftruncate, 6.0);
+
+int
+glfs_ftruncate_async(glfs_fd_t *fd, off_t length, glfs_io_cbk fn,
+                     void *data) __THROW
+    GFAPI_PUBLIC(glfs_ftruncate_async, 6.0);
+
+int
+glfs_lstat(glfs_t *fs, const char *path, struct stat *buf) __THROW
+    GFAPI_PUBLIC(glfs_lstat, 3.4.0);
+
+int
+glfs_stat(glfs_t *fs, const char *path, struct stat *buf) __THROW
+    GFAPI_PUBLIC(glfs_stat, 3.4.0);
+
+int
+glfs_fstat(glfs_fd_t *fd, struct stat *buf) __THROW
+    GFAPI_PUBLIC(glfs_fstat, 3.4.0);
+
+int
+glfs_fsync(glfs_fd_t *fd, struct glfs_stat *prestat,
+           struct glfs_stat *poststat) __THROW GFAPI_PUBLIC(glfs_fsync, 6.0);
+
+int
+glfs_fsync_async(glfs_fd_t *fd, glfs_io_cbk fn, void *data) __THROW
+    GFAPI_PUBLIC(glfs_fsync_async, 6.0);
+
+int
+glfs_fdatasync(glfs_fd_t *fd, struct glfs_stat *prestat,
+               struct glfs_stat *poststat) __THROW
+    GFAPI_PUBLIC(glfs_fdatasync, 6.0);
+
+int
+glfs_fdatasync_async(glfs_fd_t *fd, glfs_io_cbk fn, void *data) __THROW
+    GFAPI_PUBLIC(glfs_fdatasync_async, 6.0);
+
+int
+glfs_access(glfs_t *fs, const char *path, int mode) __THROW
+    GFAPI_PUBLIC(glfs_access, 3.4.0);
+
+int
+glfs_symlink(glfs_t *fs, const char *oldpath, const char *newpath) __THROW
+    GFAPI_PUBLIC(glfs_symlink, 3.4.0);
+
+int
+glfs_readlink(glfs_t *fs, const char *path, char *buf, size_t bufsiz) __THROW
+    GFAPI_PUBLIC(glfs_readlink, 3.4.0);
+
+int
+glfs_mknod(glfs_t *fs, const char *path, mode_t mode, dev_t dev) __THROW
+    GFAPI_PUBLIC(glfs_mknod, 3.4.0);
+
+int
+glfs_mkdir(glfs_t *fs, const char *path, mode_t mode) __THROW
+    GFAPI_PUBLIC(glfs_mkdir, 3.4.0);
+
+int
+glfs_unlink(glfs_t *fs, const char *path) __THROW
+    GFAPI_PUBLIC(glfs_unlink, 3.4.0);
+
+int
+glfs_rmdir(glfs_t *fs, const char *path) __THROW
+    GFAPI_PUBLIC(glfs_rmdir, 3.4.0);
+
+int
+glfs_rename(glfs_t *fs, const char *oldpath, const char *newpath) __THROW
+    GFAPI_PUBLIC(glfs_rename, 3.4.0);
+
+int
+glfs_link(glfs_t *fs, const char *oldpath, const char *newpath) __THROW
+    GFAPI_PUBLIC(glfs_link, 3.4.0);
+
+glfs_fd_t *
+glfs_opendir(glfs_t *fs, const char *path) __THROW
+    GFAPI_PUBLIC(glfs_opendir, 3.4.0);
 
+/*
+ * @glfs_readdir_r and @glfs_readdirplus_r ARE thread safe AND re-entrant,
+ * but the interface has ambiguity about the size of @dirent to be allocated
+ * before calling the APIs. 512 byte buffer (for @dirent) is sufficient for
+ * all known systems which are tested againt glusterfs/gfapi, but may be
+ * insufficient in the future.
+ */
 
-off_t glfs_lseek (glfs_fd_t *fd, off_t offset, int whence);
+int
+glfs_readdir_r(glfs_fd_t *fd, struct dirent *dirent,
+               struct dirent **result) __THROW
+    GFAPI_PUBLIC(glfs_readdir_r, 3.4.0);
 
-int glfs_truncate (glfs_t *fs, const char *path, off_t length);
+int
+glfs_readdirplus_r(glfs_fd_t *fd, struct stat *stat, struct dirent *dirent,
+                   struct dirent **result) __THROW
+    GFAPI_PUBLIC(glfs_readdirplus_r, 3.4.0);
 
-int glfs_ftruncate (glfs_fd_t *fd, off_t length);
-int glfs_ftruncate_async (glfs_fd_t *fd, off_t length, glfs_io_cbk fn,
-			  void *data);
+/*
+ * @glfs_readdir and @glfs_readdirplus are NEITHER thread safe NOR re-entrant
+ * when called on the same directory handle. However they ARE thread safe
+ * AND re-entrant when called on different directory handles (which may be
+ * referring to the same directory too.)
+ */
 
-int glfs_lstat (glfs_t *fs, const char *path, struct stat *buf);
-int glfs_stat (glfs_t *fs, const char *path, struct stat *buf);
-int glfs_fstat (glfs_fd_t *fd, struct stat *buf);
+struct dirent *
+glfs_readdir(glfs_fd_t *fd) __THROW GFAPI_PUBLIC(glfs_readdir, 3.5.0);
 
-int glfs_fsync (glfs_fd_t *fd);
-int glfs_fsync_async (glfs_fd_t *fd, glfs_io_cbk fn, void *data);
+struct dirent *
+glfs_readdirplus(glfs_fd_t *fd, struct stat *stat) __THROW
+    GFAPI_PUBLIC(glfs_readdirplus, 3.5.0);
 
-int glfs_fdatasync (glfs_fd_t *fd);
-int glfs_fdatasync_async (glfs_fd_t *fd, glfs_io_cbk fn, void *data);
+long
+glfs_telldir(glfs_fd_t *fd) __THROW GFAPI_PUBLIC(glfs_telldir, 3.4.0);
 
-int glfs_access (glfs_t *fs, const char *path, int mode);
+void
+glfs_seekdir(glfs_fd_t *fd, long offset) __THROW
+    GFAPI_PUBLIC(glfs_seekdir, 3.4.0);
 
-int glfs_symlink (glfs_t *fs, const char *oldpath, const char *newpath);
+int
+glfs_closedir(glfs_fd_t *fd) __THROW GFAPI_PUBLIC(glfs_closedir, 3.4.0);
 
-int glfs_readlink (glfs_t *fs, const char *path, char *buf, size_t bufsiz);
+int
+glfs_statvfs(glfs_t *fs, const char *path, struct statvfs *buf) __THROW
+    GFAPI_PUBLIC(glfs_statvfs, 3.4.0);
 
-int glfs_mknod (glfs_t *fs, const char *path, mode_t mode, dev_t dev);
+int
+glfs_chmod(glfs_t *fs, const char *path, mode_t mode) __THROW
+    GFAPI_PUBLIC(glfs_chmod, 3.4.0);
 
-int glfs_mkdir (glfs_t *fs, const char *path, mode_t mode);
+int
+glfs_fchmod(glfs_fd_t *fd, mode_t mode) __THROW
+    GFAPI_PUBLIC(glfs_fchmod, 3.4.0);
 
-int glfs_unlink (glfs_t *fs, const char *path);
+int
+glfs_chown(glfs_t *fs, const char *path, uid_t uid, gid_t gid) __THROW
+    GFAPI_PUBLIC(glfs_chown, 3.4.0);
 
-int glfs_rmdir (glfs_t *fs, const char *path);
+int
+glfs_lchown(glfs_t *fs, const char *path, uid_t uid, gid_t gid) __THROW
+    GFAPI_PUBLIC(glfs_lchown, 3.4.0);
 
-int glfs_rename (glfs_t *fs, const char *oldpath, const char *newpath);
+int
+glfs_fchown(glfs_fd_t *fd, uid_t uid, gid_t gid) __THROW
+    GFAPI_PUBLIC(glfs_fchown, 3.4.0);
 
-int glfs_link (glfs_t *fs, const char *oldpath, const char *newpath);
+int
+glfs_utimens(glfs_t *fs, const char *path,
+             const struct timespec times[2]) __THROW
+    GFAPI_PUBLIC(glfs_utimens, 3.4.0);
 
-glfs_fd_t *glfs_opendir (glfs_t *fs, const char *path);
+int
+glfs_lutimens(glfs_t *fs, const char *path,
+              const struct timespec times[2]) __THROW
+    GFAPI_PUBLIC(glfs_lutimens, 3.4.0);
 
-int glfs_readdir_r (glfs_fd_t *fd, struct dirent *dirent,
-		    struct dirent **result);
+int
+glfs_futimens(glfs_fd_t *fd, const struct timespec times[2]) __THROW
+    GFAPI_PUBLIC(glfs_futimens, 3.4.0);
 
-int glfs_readdirplus_r (glfs_fd_t *fd, struct stat *stat, struct dirent *dirent,
-			struct dirent **result);
+ssize_t
+glfs_getxattr(glfs_t *fs, const char *path, const char *name, void *value,
+              size_t size) __THROW GFAPI_PUBLIC(glfs_getxattr, 3.4.0);
 
-long glfs_telldir (glfs_fd_t *fd);
+ssize_t
+glfs_lgetxattr(glfs_t *fs, const char *path, const char *name, void *value,
+               size_t size) __THROW GFAPI_PUBLIC(glfs_lgetxattr, 3.4.0);
 
-void glfs_seekdir (glfs_fd_t *fd, long offset);
+ssize_t
+glfs_fgetxattr(glfs_fd_t *fd, const char *name, void *value,
+               size_t size) __THROW GFAPI_PUBLIC(glfs_fgetxattr, 3.4.0);
 
-int glfs_closedir (glfs_fd_t *fd);
+ssize_t
+glfs_listxattr(glfs_t *fs, const char *path, void *value, size_t size) __THROW
+    GFAPI_PUBLIC(glfs_listxattr, 3.4.0);
 
-int glfs_statvfs (glfs_t *fs, const char *path, struct statvfs *buf);
+ssize_t
+glfs_llistxattr(glfs_t *fs, const char *path, void *value, size_t size) __THROW
+    GFAPI_PUBLIC(glfs_llistxattr, 3.4.0);
 
-int glfs_chmod (glfs_t *fs, const char *path, mode_t mode);
+ssize_t
+glfs_flistxattr(glfs_fd_t *fd, void *value, size_t size) __THROW
+    GFAPI_PUBLIC(glfs_flistxattr, 3.4.0);
 
-int glfs_fchmod (glfs_fd_t *fd, mode_t mode);
+int
+glfs_setxattr(glfs_t *fs, const char *path, const char *name, const void *value,
+              size_t size, int flags) __THROW
+    GFAPI_PUBLIC(glfs_setxattr, 3.4.0);
 
-int glfs_chown (glfs_t *fs, const char *path, uid_t uid, gid_t gid);
+int
+glfs_lsetxattr(glfs_t *fs, const char *path, const char *name,
+               const void *value, size_t size, int flags) __THROW
+    GFAPI_PUBLIC(glfs_lsetxattr, 3.4.0);
 
-int glfs_lchown (glfs_t *fs, const char *path, uid_t uid, gid_t gid);
+int
+glfs_fsetxattr(glfs_fd_t *fd, const char *name, const void *value, size_t size,
+               int flags) __THROW GFAPI_PUBLIC(glfs_fsetxattr, 3.4.0);
 
-int glfs_fchown (glfs_fd_t *fd, uid_t uid, gid_t gid);
+int
+glfs_removexattr(glfs_t *fs, const char *path, const char *name) __THROW
+    GFAPI_PUBLIC(glfs_removexattr, 3.4.0);
 
-int glfs_utimens (glfs_t *fs, const char *path, struct timespec times[2]);
+int
+glfs_lremovexattr(glfs_t *fs, const char *path, const char *name) __THROW
+    GFAPI_PUBLIC(glfs_lremovexattr, 3.4.0);
 
-int glfs_lutimens (glfs_t *fs, const char *path, struct timespec times[2]);
+int
+glfs_fremovexattr(glfs_fd_t *fd, const char *name) __THROW
+    GFAPI_PUBLIC(glfs_fremovexattr, 3.4.0);
 
-int glfs_futimens (glfs_fd_t *fd, struct timespec times[2]);
+int
+glfs_fallocate(glfs_fd_t *fd, int keep_size, off_t offset, size_t len) __THROW
+    GFAPI_PUBLIC(glfs_fallocate, 3.5.0);
 
-ssize_t glfs_getxattr (glfs_t *fs, const char *path, const char *name,
-		       void *value, size_t size);
+int
+glfs_discard(glfs_fd_t *fd, off_t offset, size_t len) __THROW
+    GFAPI_PUBLIC(glfs_discard, 3.5.0);
 
-ssize_t glfs_lgetxattr (glfs_t *fs, const char *path, const char *name,
-			void *value, size_t size);
+int
+glfs_discard_async(glfs_fd_t *fd, off_t length, size_t lent, glfs_io_cbk fn,
+                   void *data) __THROW GFAPI_PUBLIC(glfs_discard_async, 6.0);
 
-ssize_t glfs_fgetxattr (glfs_fd_t *fd, const char *name,
-			void *value, size_t size);
+int
+glfs_zerofill(glfs_fd_t *fd, off_t offset, off_t len) __THROW
+    GFAPI_PUBLIC(glfs_zerofill, 3.5.0);
 
-ssize_t glfs_listxattr (glfs_t *fs, const char *path, void *value, size_t size);
+int
+glfs_zerofill_async(glfs_fd_t *fd, off_t length, off_t len, glfs_io_cbk fn,
+                    void *data) __THROW GFAPI_PUBLIC(glfs_zerofill_async, 6.0);
 
-ssize_t glfs_llistxattr (glfs_t *fs, const char *path, void *value,
-			 size_t size);
+char *
+glfs_getcwd(glfs_t *fs, char *buf, size_t size) __THROW
+    GFAPI_PUBLIC(glfs_getcwd, 3.4.0);
 
-ssize_t glfs_flistxattr (glfs_fd_t *fd, void *value, size_t size);
+int
+glfs_chdir(glfs_t *fs, const char *path) __THROW
+    GFAPI_PUBLIC(glfs_chdir, 3.4.0);
 
-int glfs_setxattr (glfs_t *fs, const char *path, const char *name,
-		   const void *value, size_t size, int flags);
+int
+glfs_fchdir(glfs_fd_t *fd) __THROW GFAPI_PUBLIC(glfs_fchdir, 3.4.0);
 
-int glfs_lsetxattr (glfs_t *fs, const char *path, const char *name,
-		    const void *value, size_t size, int flags);
+char *
+glfs_realpath(glfs_t *fs, const char *path, char *resolved_path) __THROW
+    GFAPI_PUBLIC(glfs_realpath, 3.7.17);
 
-int glfs_fsetxattr (glfs_fd_t *fd, const char *name,
-		    const void *value, size_t size, int flags);
+/*
+ * @cmd and @flock are as specified in man fcntl(2).
+ */
+int
+glfs_posix_lock(glfs_fd_t *fd, int cmd, struct flock *flock) __THROW
+    GFAPI_PUBLIC(glfs_posix_lock, 3.4.0);
 
-int glfs_removexattr (glfs_t *fs, const char *path, const char *name);
+/*
+  SYNOPSIS
 
-int glfs_lremovexattr (glfs_t *fs, const char *path, const char *name);
+  glfs_file_lock: Request extended byte range lock on a file
 
-int glfs_fremovexattr (glfs_fd_t *fd, const char *name);
+  DESCRIPTION
+
+  This function is capable of requesting either advisory or mandatory type
+  byte range locks on a file.
+
+  Note: To set a unique owner key for locks based on a particular file
+  descriptor, make use of glfs_fd_set_lkowner() api to do so before
+  requesting lock via this api. This owner key will be further consumed
+  by other incoming data modifying file operations via the same file
+  descriptor.
+
+  PARAMETERS
+
+  @fd: File descriptor
 
-int glfs_fallocate(glfs_fd_t *fd, int keep_size, off_t offset, size_t len);
+  @cmd: As specified in man fcntl(2).
 
-int glfs_discard(glfs_fd_t *fd, off_t offset, size_t len);
+  @flock: As specified in man fcntl(2).
 
-int glfs_discard_async (glfs_fd_t *fd, off_t length, size_t lent,
-			glfs_io_cbk fn, void *data);
+  @lk_mode: Required lock type from options available with the
+            enum glfs_lock_mode_t defined below.
 
-char *glfs_getcwd (glfs_t *fs, char *buf, size_t size);
+  RETURN VALUES
 
-int glfs_chdir (glfs_t *fs, const char *path);
+  0   : Success. Lock has been granted.
+  -1  : Failure. @errno will be set indicating the type of failure.
 
-int glfs_fchdir (glfs_fd_t *fd);
+ */
 
-char *glfs_realpath (glfs_t *fs, const char *path, char *resolved_path);
+/* Lock modes used by glfs_file_lock() */
+enum glfs_lock_mode { GLFS_LK_ADVISORY = 0, GLFS_LK_MANDATORY };
+typedef enum glfs_lock_mode glfs_lock_mode_t;
+
+int
+glfs_file_lock(glfs_fd_t *fd, int cmd, struct flock *flock,
+               glfs_lock_mode_t lk_mode) __THROW
+    GFAPI_PUBLIC(glfs_file_lock, 3.13.0);
+
+glfs_fd_t *
+glfs_dup(glfs_fd_t *fd) __THROW GFAPI_PUBLIC(glfs_dup, 3.4.0);
+
+void
+glfs_free(void *ptr) __THROW GFAPI_PUBLIC(glfs_free, 3.7.16);
 
 /*
- * @cmd and @flock are as specified in man fcntl(2).
+ * glfs_sysrq: send a system request to the @fs instance
+ *
+ * Different commands for @sysrq are possible, the defines for these are listed
+ * below the function definition.
+ *
+ * This function always returns success if the @sysrq is recognized. The return
+ * value does not way anythin about the result of the @sysrq execution. Not all
+ * @sysrq command will be able to return a success/failure status.
  */
-int glfs_posix_lock (glfs_fd_t *fd, int cmd, struct flock *flock);
+int
+glfs_sysrq(glfs_t *fs, char sysrq) __THROW GFAPI_PUBLIC(glfs_sysrq, 3.10.0);
 
-glfs_fd_t *glfs_dup (glfs_fd_t *fd);
+#define GLFS_SYSRQ_HELP 'h' /* log a message with supported sysrq commands */
+#define GLFS_SYSRQ_STATEDUMP 's' /* create a statedump */
 
-__END_DECLS
+/*
+ * Structure returned as part of xreaddirplus
+ */
+struct glfs_xreaddirp_stat;
+typedef struct glfs_xreaddirp_stat glfs_xreaddirp_stat_t;
+
+/* Request flags to be used in XREADDIRP operation */
+#define GFAPI_XREADDIRP_NULL                                                   \
+    0x00000000 /* by default, no stat will be fetched */
+#define GFAPI_XREADDIRP_STAT 0x00000001   /* Get stat */
+#define GFAPI_XREADDIRP_HANDLE 0x00000002 /* Get object handle */
+
+/*
+ * This stat structure returned gets freed as part of glfs_free(xstat)
+ */
+struct stat *
+glfs_xreaddirplus_get_stat(glfs_xreaddirp_stat_t *xstat) __THROW
+    GFAPI_PUBLIC(glfs_xreaddirplus_get_stat, 3.11.0);
 
+/*
+ * SYNOPSIS
+ *
+ * glfs_xreaddirplus_r: Extended Readirplus operation
+ *
+ * DESCRIPTION
+ *
+ * This API does readdirplus operation, but along with stat it can fetch other
+ * extra information like object handles etc for each of the dirents returned
+ * based on requested flags. On success it returns the set of flags successfully
+ * processed.
+ *
+ * Note that there are chances that some of the requested information may not be
+ * available or returned (for example if reached EOD). Ensure to validate the
+ * returned value to determine what flags have been successfully processed
+ * & set.
+ *
+ * PARAMETERS
+ *
+ * INPUT:
+ * @glfd: GFAPI file descriptor of the directory
+ * @flags: Flags determining xreaddirp_stat requested
+ *         Current available values are:
+ *              GFAPI_XREADDIRP_NULL
+ *              GFAPI_XREADDIRP_STAT
+ *              GFAPI_XREADDIRP_HANDLE
+ * @ext: Dirent struture to copy the values to
+ *       (though optional recommended to be allocated by application
+ *        esp., in multi-threaded environment)
+ *
+ * OUTPUT:
+ * @res: to store the next dirent value. If NULL and return value is '0',
+ *       it means it reached end of the directory.
+ * @xstat_p: Pointer to contain all the requested data returned
+ *           for that dirent. Application should make use of glfs_free() API
+ *           to free this pointer and the variables returned by
+ *           glfs_xreaddirplus_get_*() APIs.
+ *
+ * RETURN VALUE:
+ * >=0: SUCCESS (value contains the flags successfully processed)
+ *  -1: FAILURE
+ */
+int
+glfs_xreaddirplus_r(glfs_fd_t *glfd, uint32_t flags,
+                    glfs_xreaddirp_stat_t **xstat_p, struct dirent *ext,
+                    struct dirent **res) __THROW
+    GFAPI_PUBLIC(glfs_xreaddirplus_r, 3.11.0);
+
+#define GFAPI_MAX_LOCK_OWNER_LEN 255
+
+/*
+ *
+ * DESCRIPTION
+ *
+ * This API allows application to set lk_owner on a fd.
+ * A glfd can be associated with only single lk_owner. In case if there
+ * is need to set another lk_owner, applications can make use of
+ * 'glfs_dup' to get duplicate glfd and set new lk_owner on that second
+ * glfd.
+ *
+ * Also its not recommended to override or clear lk_owner value as the
+ * same shall be used to flush any outstanding locks while closing the fd.
+ *
+ * PARAMETERS
+ *
+ * INPUT:
+ * @glfd: GFAPI file descriptor
+ * @len: Size of lk_owner buffer. Max value can be GFAPI_MAX_LOCK_OWNER_LEN
+ * @data: lk_owner data buffer.
+ *
+ * OUTPUT:
+ * 0: SUCCESS
+ * -1: FAILURE
+ */
+int
+glfs_fd_set_lkowner(glfs_fd_t *glfd, void *data, int len) __THROW
+    GFAPI_PUBLIC(glfs_fd_set_lkowner, 3.10.7);
+
+/*
+ * Applications (currently NFS-Ganesha) can make use of this
+ * structure to read upcall notifications sent by server either
+ * by polling or registering a callback function.
+ *
+ * On success, applications need to check for 'reason' to decide
+ * if any upcall event is received.
+ *
+ * Currently supported upcall_events -
+ *      GLFS_UPCALL_INODE_INVALIDATE -
+ *              'event_arg' - glfs_upcall_inode
+ *
+ * After processing the event, applications need to free 'event_arg' with
+ * glfs_free().
+ *
+ * Also similar to I/Os, the application should ideally stop polling
+ * or unregister upcall_cbk function before calling glfs_fini(..).
+ * Hence making an assumption that 'fs' & ctx structures cannot be
+ * freed while in this routine.
+ */
+struct glfs_upcall;
+typedef struct glfs_upcall glfs_upcall_t;
+
+glfs_t *
+glfs_upcall_get_fs(glfs_upcall_t *arg) __THROW
+    GFAPI_PUBLIC(glfs_upcall_get_fs, 3.7.16);
+
+enum glfs_upcall_reason {
+    GLFS_UPCALL_EVENT_NULL = 0,
+    GLFS_UPCALL_INODE_INVALIDATE, /* invalidate cache entry */
+    GLFS_UPCALL_RECALL_LEASE,     /* recall lease */
+};
+typedef enum glfs_upcall_reason glfs_upcall_reason_t;
+
+glfs_upcall_reason_t
+glfs_upcall_get_reason(glfs_upcall_t *arg) __THROW
+    GFAPI_PUBLIC(glfs_upcall_get_reason, 3.7.16);
+
+/*
+ * Applications first need to make use of above API i.e,
+ * "glfs_upcall_get_reason" to determine which upcall event it has
+ * received. Post that below API - "glfs_upcall_get_event" should
+ * be used to get corresponding upcall event object.
+ *
+ * Below are the upcall_reason and corresponding upcall_event objects:
+ *      ==========================================================
+ *      glfs_upcall_reason           -    event_object
+ *      ==========================================================
+ *      GLFS_UPCALL_EVENT_NULL       -    NULL
+ *      GLFS_UPCALL_INODE_INVALIDATE -    struct glfs_upcall_inode
+ *      GLFS_UPCALL_RECALL_LEASE     -    struct glfs_upcall_lease
+ *
+ * After processing upcall event, glfs_free() should be called on the
+ * glfs_upcall.
+ */
+void *
+glfs_upcall_get_event(glfs_upcall_t *arg) __THROW
+    GFAPI_PUBLIC(glfs_upcall_get_event, 3.7.16);
+
+/*
+ * SYNOPSIS
+ *
+ * glfs_upcall_cbk: Upcall callback definition
+ *
+ * This is function type definition of the callback function pointer
+ * which has to be provided by the caller while registering for any
+ * upcall events.
+ *
+ * This function is called whenever any upcall which the application
+ * has registered for is received from the server.
+ *
+ * @up_arg: Upcall structure whose contents need to be interpreted by
+ * making use of glfs_upcall_* helper routines.
+ *
+ * @data: The same context pointer provided by the caller at the time of
+ * registering of upcall events. This may be used by the caller for any
+ * of its internal use while processing upcalls.
+ */
+typedef void (*glfs_upcall_cbk)(glfs_upcall_t *up_arg, void *data);
+
+/*
+ * List of upcall events supported by gluster/gfapi
+ */
+#define GLFS_EVENT_INODE_INVALIDATE 0x00000001 /* invalidate cache entry */
+#define GLFS_EVENT_RECALL_LEASE 0x00000002     /* Recall lease */
+#define GLFS_EVENT_ANY 0xffffffff              /* for all the above events */
+
+/*
+ * SYNOPSIS
+ *
+ * glfs_upcall_register: Register for upcall events
+ *
+ * DESCRIPTION
+ *
+ * This function is used to register for various upcall events application
+ * is interested in and the callback function to be invoked when such
+ * events are triggered.
+ *
+ * Multiple calls of this routine shall override cbk function. That means
+ * only one cbk function can be used for all the upcall events registered
+ * and that shall be the one last updated.
+ *
+ * PARAMETERS:
+ *
+ * INPUT:
+ * @fs: The 'virtual mount' object
+ *
+ * @event_list: List of upcall events to be registered.
+ *              Current available values are:
+ *               - GLFS_EVENT_INODE_INVALIDATE
+ *               - GLFS_EVENT_RECALL_LEASE
+ *
+ * @cbk: The cbk routine to be invoked in case of any upcall received
+ * @data: Any opaque pointer provided by caller which shall be using while
+ * making cbk calls. This pointer may be used by caller for any of its
+ * internal use while processing upcalls. Can be NULL.
+ *
+ * RETURN VALUE:
+ * >0: SUCCESS (value contains the events successfully registered)
+ * -1: FAILURE
+ */
+int
+glfs_upcall_register(glfs_t *fs, uint32_t event_list, glfs_upcall_cbk cbk,
+                     void *data) __THROW
+    GFAPI_PUBLIC(glfs_upcall_register, 3.13.0);
+
+/*
+ * SYNOPSIS
+ *
+ * glfs_upcall_unregister: Unregister for upcall events
+ *
+ * DESCRIPTION
+ *
+ * This function is used to unregister the upcall events application
+ * is not interested in. In case if the caller unregisters all the events
+ * it has registered for, it shall no more receive any upcall event.
+ *
+ * PARAMETERS:
+ *
+ * INPUT:
+ * @fs: The 'virtual mount' object
+ *
+ * @event_list: List of upcall events to be unregistered.
+ *              Current available values are:
+ *               - GLFS_EVENT_INODE_INVALIDATE
+ *               - GLFS_EVENT_RECALL_LEASE
+ * RETURN VALUE:
+ * >0: SUCCESS (value contains the events successfully unregistered)
+ * -1: FAILURE
+ */
+int
+glfs_upcall_unregister(glfs_t *fs, uint32_t event_list) __THROW
+    GFAPI_PUBLIC(glfs_upcall_unregister, 3.13.0);
+
+/* Lease Types */
+enum glfs_lease_types {
+    GLFS_LEASE_NONE = 0,
+    GLFS_RD_LEASE = 1,
+    GLFS_RW_LEASE = 2,
+};
+typedef enum glfs_lease_types glfs_lease_types_t;
+
+/* Lease cmds */
+enum glfs_lease_cmds {
+    GLFS_GET_LEASE = 1,
+    GLFS_SET_LEASE = 2,
+    GLFS_UNLK_LEASE = 3,
+};
+typedef enum glfs_lease_cmds glfs_lease_cmds_t;
+
+struct glfs_lease {
+    glfs_lease_cmds_t cmd;
+    glfs_lease_types_t lease_type;
+    glfs_leaseid_t lease_id;
+    unsigned int lease_flags;
+};
+typedef struct glfs_lease glfs_lease_t;
+
+typedef void (*glfs_recall_cbk)(glfs_lease_t lease, void *data);
+
+/*
+  SYNOPSIS
+
+  glfs_lease: Takes a lease on a file.
+
+  DESCRIPTION
+
+  This function takes lease on an open file.
+
+  PARAMETERS
+
+  @glfd: The fd of the file on which lease should be taken,
+   this fd is returned by glfs_open/glfs_create.
+
+  @lease: Struct that defines the lease operation to be performed
+   on the file.
+      @lease.cmd - Can be one of the following values
+         GF_GET_LEASE:  Get the lease type currently present on the file,
+                        lease.lease_type will contain GF_RD_LEASE
+                        or GF_RW_LEASE or 0 if no leases.
+         GF_SET_LEASE:  Set the lease of given lease.lease_type on the file.
+         GF_UNLK_LEASE: Unlock the lease present on the given fd.
+                        Note that the every lease request should have
+                        a corresponding unlk_lease.
+
+      @lease.lease_type - Can be one of the following values
+         GF_RD_LEASE:   Read lease on a file, shared lease.
+         GF_RW_LEASE:   Read-Write lease on a file, exclusive lease.
+
+      @lease.lease_id - A unique identification of lease, 128bits.
+
+  @fn: This is the function that is invoked when the lease has to be recalled
+  @data: It is a cookie, this pointer is returned as a part of recall
+
+  fn and data field are stored as a part of glfs_fd, hence if there are multiple
+  glfs_lease calls, each of them updates the fn and data fields. glfs_recall_cbk
+  will be invoked with the last updated fn and data
+
+  RETURN VALUES
+  0:  Successful completion
+  <0: Failure. @errno will be set with the type of failure
+*/
+
+int
+glfs_lease(glfs_fd_t *glfd, glfs_lease_t *lease, glfs_recall_cbk fn,
+           void *data) __THROW GFAPI_PUBLIC(glfs_lease, 4.0.0);
+
+/*
+  SYNOPSIS
+
+  glfs_fsetattr: Function to set attributes.
+  glfs_setattr: Function to set attributes
+
+  DESCRIPTION
+
+  The functions are used to set attributes on the file.
+
+  PARAMETERS
+
+  @glfs_fsetattr
+
+     @glfd: The fd of the file for which the attributes are to be set,
+            this fd is returned by glfs_open/glfs_create.
+
+  @glfs_setattr
+
+         @fs: File object.
+
+         @path: The path of the file that is being operated on.
+
+         @follow: Flag used to resolve symlink.
+
+
+  @stat: Struct that has information about the file.
+
+  @valid: This is the mask bit, that accepts GFAPI_SET_ATTR* masks.
+          Refer glfs.h to see the mask definitions.
+
+  Both functions are similar in functionality, just that the
+  func setattr() uses file path whereas the func fsetattr()
+  uses the fd.
+
+  RETURN VALUES
+  0:  Successful completion
+  <0: Failure. @errno will be set with the type of failure
+
+ */
+
+int
+glfs_fsetattr(struct glfs_fd *glfd, struct glfs_stat *stat) __THROW
+    GFAPI_PUBLIC(glfs_fsetattr, 6.0);
+
+int
+glfs_setattr(struct glfs *fs, const char *path, struct glfs_stat *stat,
+             int follow) __THROW GFAPI_PUBLIC(glfs_setattr, 6.0);
+
+/*
+  SYNOPSIS
+
+  glfs_set_statedump_path: Function to set statedump path.
+
+  DESCRIPTION
+
+  This function is used to set statedump directory
+
+  PARAMETERS
+
+  @fs: The 'virtual mount' object to be configured with the volume
+       specification file.
+
+  @path: statedump path. Should be a directory. But the API won't fail if the
+  directory doesn't exist yet, as one may create it later.
+
+  RETURN VALUES
+
+   0 : Success.
+  -1 : Failure. @errno will be set with the type of failure.
+
+ */
+
+int
+glfs_set_statedump_path(struct glfs *fs, const char *path) __THROW
+    GFAPI_PUBLIC(glfs_set_statedump_path, 7.0);
+
+__END_DECLS
 #endif /* !_GLFS_H */
diff --git a/argp-standalone/Makefile.am b/argp-standalone/Makefile.am
deleted file mode 100644
index 4775d4876aa..00000000000
--- a/argp-standalone/Makefile.am
+++ /dev/null
@@ -1,38 +0,0 @@
-# From glibc
-
-# Copyright (C) 1997, 2003, 2004 Free Software Foundation, Inc.
-# This file is part of the GNU C Library.
-
-# The GNU C Library is free software; you can redistribute it and/or
-# modify it under the terms of the GNU Library General Public License as
-# published by the Free Software Foundation; either version 2 of the
-# License, or (at your option) any later version.
-
-# The GNU C Library is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-# Library General Public License for more details.
-
-# You should have received a copy of the GNU Library General Public
-# License along with the GNU C Library; see the file COPYING.LIB.  If
-# not, write to the Free Software Foundation, Inc.,
-# 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
-AUTOMAKE_OPTIONS = foreign
-SUBDIRS = .
-
-LIBOBJS = @LIBOBJS@
-
-noinst_LIBRARIES = libargp.a
-
-noinst_HEADERS = argp.h argp-fmtstream.h argp-namefrob.h
-
-EXTRA_DIST = mempcpy.c strchrnul.c strndup.c strcasecmp.c vsnprintf.c autogen.sh
-
-# Leaves out argp-fs-xinl.c and argp-xinl.c 
-libargp_a_SOURCES = argp-ba.c argp-eexst.c argp-fmtstream.c \
-		    argp-help.c argp-parse.c argp-pv.c \
-		    argp-pvh.c
-
-libargp_a_LIBADD = $(LIBOBJS)
-
-
diff --git a/argp-standalone/acinclude.m4 b/argp-standalone/acinclude.m4
deleted file mode 100644
index fb61e957dfa..00000000000
--- a/argp-standalone/acinclude.m4
+++ /dev/null
@@ -1,1084 +0,0 @@
-dnl Try to detect the type of the third arg to getsockname() et al
-AC_DEFUN([LSH_TYPE_SOCKLEN_T],
-[AH_TEMPLATE([socklen_t], [Length type used by getsockopt])
-AC_CACHE_CHECK([for socklen_t in sys/socket.h], ac_cv_type_socklen_t,
-[AC_EGREP_HEADER(socklen_t, sys/socket.h,
-  [ac_cv_type_socklen_t=yes], [ac_cv_type_socklen_t=no])])
-if test $ac_cv_type_socklen_t = no; then
-        AC_MSG_CHECKING(for AIX)
-        AC_EGREP_CPP(yes, [
-#ifdef _AIX
- yes
-#endif
-],[
-AC_MSG_RESULT(yes)
-AC_DEFINE(socklen_t, size_t)
-],[
-AC_MSG_RESULT(no)
-AC_DEFINE(socklen_t, int)
-])
-fi
-])
-
-dnl Choose cc flags for compiling position independent code
-AC_DEFUN([LSH_CCPIC],
-[AC_MSG_CHECKING(CCPIC)
-AC_CACHE_VAL(lsh_cv_sys_ccpic,[
-  if test -z "$CCPIC" ; then
-    if test "$GCC" = yes ; then
-      case `uname -sr` in
-	BSD/OS*)
-         case `uname -r` in
-           4.*) CCPIC="-fPIC";;
-           *) CCPIC="";;
-         esac
-	;;
-	Darwin*)
-	  CCPIC="-fPIC"
-	;;
-	SunOS\ 5.*)
-	  # Could also use -fPIC, if there are a large number of symbol reference
-	  CCPIC="-fPIC"
-	;;
-	CYGWIN*)
-	  CCPIC=""
-	;;
-	*)
-	  CCPIC="-fpic"
-	;;
-      esac
-    else
-      case `uname -sr` in
-	Darwin*)
-	  CCPIC="-fPIC"
-	;;
-        IRIX*)
-          CCPIC="-share"
-        ;;
-	hp*|HP*) CCPIC="+z"; ;;
-	FreeBSD*) CCPIC="-fpic";;
-	SCO_SV*) CCPIC="-KPIC -dy -Bdynamic";;
-        UnixWare*|OpenUNIX*) CCPIC="-KPIC -dy -Bdynamic";;
-	Solaris*) CCPIC="-KPIC -Bdynamic";;
-	Windows_NT*) CCPIC="-shared" ;;
-      esac
-    fi
-  fi
-  OLD_CFLAGS="$CFLAGS"
-  CFLAGS="$CFLAGS $CCPIC"
-  AC_TRY_COMPILE([], [exit(0);],
-    lsh_cv_sys_ccpic="$CCPIC", lsh_cv_sys_ccpic='')
-  CFLAGS="$OLD_CFLAGS"
-])
-CCPIC="$lsh_cv_sys_ccpic"
-AC_MSG_RESULT($CCPIC)
-AC_SUBST([CCPIC])])
-
-dnl LSH_PATH_ADD(path-id, directory)
-AC_DEFUN([LSH_PATH_ADD],
-[AC_MSG_CHECKING($2)
-ac_exists=no
-if test -d "$2/." ; then
-  ac_real_dir=`cd $2 && pwd`
-  if test -n "$ac_real_dir" ; then
-    ac_exists=yes
-    for old in $1_REAL_DIRS ; do
-      ac_found=no
-      if test x$ac_real_dir = x$old ; then
-        ac_found=yes;
-	break;
-      fi
-    done
-    if test $ac_found = yes ; then
-      AC_MSG_RESULT(already added)
-    else
-      AC_MSG_RESULT(added)
-      # LDFLAGS="$LDFLAGS -L $2"
-      $1_REAL_DIRS="$ac_real_dir [$]$1_REAL_DIRS"
-      $1_DIRS="$2 [$]$1_DIRS"
-    fi
-  fi
-fi
-if test $ac_exists = no ; then
-  AC_MSG_RESULT(not found)
-fi
-])
-
-dnl LSH_RPATH_ADD(dir)
-AC_DEFUN([LSH_RPATH_ADD], [LSH_PATH_ADD(RPATH_CANDIDATE, $1)])
-
-dnl LSH_RPATH_INIT(candidates)
-AC_DEFUN([LSH_RPATH_INIT],
-[AC_MSG_CHECKING([for -R flag])
-RPATHFLAG=''
-case `uname -sr` in
-  OSF1\ V4.*)
-    RPATHFLAG="-rpath "
-    ;;
-  IRIX\ 6.*)
-    RPATHFLAG="-rpath "
-    ;;
-  IRIX\ 5.*)
-    RPATHFLAG="-rpath "
-    ;;
-  SunOS\ 5.*)
-    if test "$TCC" = "yes"; then
-      # tcc doesn't know about -R
-      RPATHFLAG="-Wl,-R,"
-    else
-      RPATHFLAG=-R
-    fi
-    ;;
-  Linux\ 2.*)
-    RPATHFLAG="-Wl,-rpath,"
-    ;;
-  *)
-    :
-    ;;
-esac
-
-if test x$RPATHFLAG = x ; then
-  AC_MSG_RESULT(none)
-else
-  AC_MSG_RESULT([using $RPATHFLAG])
-fi
-
-RPATH_CANDIDATE_REAL_DIRS=''
-RPATH_CANDIDATE_DIRS=''
-
-AC_MSG_RESULT([Searching for libraries])
-
-for d in $1 ; do
-  LSH_RPATH_ADD($d)
-done
-])    
-
-dnl Try to execute a main program, and if it fails, try adding some
-dnl -R flag.
-dnl LSH_RPATH_FIX
-AC_DEFUN([LSH_RPATH_FIX],
-[if test $cross_compiling = no -a "x$RPATHFLAG" != x ; then
-  ac_success=no
-  AC_TRY_RUN([int main(int argc, char **argv) { return 0; }],
-    ac_success=yes, ac_success=no, :)
-  
-  if test $ac_success = no ; then
-    AC_MSG_CHECKING([Running simple test program failed. Trying -R flags])
-dnl echo RPATH_CANDIDATE_DIRS = $RPATH_CANDIDATE_DIRS
-    ac_remaining_dirs=''
-    ac_rpath_save_LDFLAGS="$LDFLAGS"
-    for d in $RPATH_CANDIDATE_DIRS ; do
-      if test $ac_success = yes ; then
-  	ac_remaining_dirs="$ac_remaining_dirs $d"
-      else
-  	LDFLAGS="$RPATHFLAG$d $LDFLAGS"
-dnl echo LDFLAGS = $LDFLAGS
-  	AC_TRY_RUN([int main(int argc, char **argv) { return 0; }],
-  	  [ac_success=yes
-  	  ac_rpath_save_LDFLAGS="$LDFLAGS"
-  	  AC_MSG_RESULT([adding $RPATHFLAG$d])
-  	  ],
-  	  [ac_remaining_dirs="$ac_remaining_dirs $d"], :)
-  	LDFLAGS="$ac_rpath_save_LDFLAGS"
-      fi
-    done
-    RPATH_CANDIDATE_DIRS=$ac_remaining_dirs
-  fi
-  if test $ac_success = no ; then
-    AC_MSG_RESULT(failed)
-  fi
-fi
-])
-
-dnl Like AC_CHECK_LIB, but uses $KRB_LIBS rather than $LIBS.
-dnl LSH_CHECK_KRB_LIB(LIBRARY, FUNCTION, [, ACTION-IF-FOUND [,
-dnl                  ACTION-IF-NOT-FOUND [, OTHER-LIBRARIES]]])
-
-AC_DEFUN([LSH_CHECK_KRB_LIB],
-[AC_CHECK_LIB([$1], [$2],
-  ifelse([$3], ,
-      [[ac_tr_lib=HAVE_LIB`echo $1 | sed -e 's/[^a-zA-Z0-9_]/_/g' \
-     	    -e 'y/abcdefghijklmnopqrstuvwxyz/ABCDEFGHIJKLMNOPQRSTUVWXYZ/'`
-        AC_DEFINE_UNQUOTED($ac_tr_lib)
-        KRB_LIBS="-l$1 $KRB_LIBS"
-      ]], [$3]),
-  ifelse([$4], , , [$4
-])dnl
-, [$5 $KRB_LIBS])
-])
-
-dnl LSH_LIB_ARGP(ACTION-IF-OK, ACTION-IF-BAD)
-AC_DEFUN([LSH_LIB_ARGP],
-[ ac_argp_save_LIBS="$LIBS"
-  ac_argp_save_LDFLAGS="$LDFLAGS"
-  ac_argp_ok=no
-  # First check if we can link with argp.
-  AC_SEARCH_LIBS(argp_parse, argp,
-  [ LSH_RPATH_FIX
-    AC_CACHE_CHECK([for working argp],
-      lsh_cv_lib_argp_works,
-      [ AC_TRY_RUN(
-[#include <argp.h>
-#include <stdlib.h>
-
-static const struct argp_option
-options[] =
-{
-  { NULL, 0, NULL, 0, NULL, 0 }
-};
-
-struct child_state
-{
-  int n;
-};
-
-static error_t
-child_parser(int key, char *arg, struct argp_state *state)
-{
-  struct child_state *input = (struct child_state *) state->input;
-  
-  switch(key)
-    {
-    default:
-      return ARGP_ERR_UNKNOWN;
-    case ARGP_KEY_END:
-      if (!input->n)
-	input->n = 1;
-      break;
-    }
-  return 0;
-}
-
-const struct argp child_argp =
-{
-  options,
-  child_parser,
-  NULL, NULL, NULL, NULL, NULL
-};
-
-struct main_state
-{
-  struct child_state child;
-  int m;
-};
-
-static error_t
-main_parser(int key, char *arg, struct argp_state *state)
-{
-  struct main_state *input = (struct main_state *) state->input;
-
-  switch(key)
-    {
-    default:
-      return ARGP_ERR_UNKNOWN;
-    case ARGP_KEY_INIT:
-      state->child_inputs[0] = &input->child;
-      break;
-    case ARGP_KEY_END:
-      if (!input->m)
-	input->m = input->child.n;
-      
-      break;
-    }
-  return 0;
-}
-
-static const struct argp_child
-main_children[] =
-{
-  { &child_argp, 0, "", 0 },
-  { NULL, 0, NULL, 0}
-};
-
-static const struct argp
-main_argp =
-{ options, main_parser, 
-  NULL,
-  NULL,
-  main_children,
-  NULL, NULL
-};
-
-int main(int argc, char **argv)
-{
-  struct main_state input = { { 0 }, 0 };
-  char *v[2] = { "foo", NULL };
-
-  argp_parse(&main_argp, 1, v, 0, NULL, &input);
-
-  if ( (input.m == 1) && (input.child.n == 1) )
-    return 0;
-  else
-    return 1;
-}
-], lsh_cv_lib_argp_works=yes,
-   lsh_cv_lib_argp_works=no,
-   lsh_cv_lib_argp_works=no)])
-
-  if test x$lsh_cv_lib_argp_works = xyes ; then
-    ac_argp_ok=yes
-  else
-    # Reset link flags
-    LIBS="$ac_argp_save_LIBS"
-    LDFLAGS="$ac_argp_save_LDFLAGS"
-  fi])
-
-  if test x$ac_argp_ok = xyes ; then
-    ifelse([$1],, true, [$1])
-  else
-    ifelse([$2],, true, [$2])
-  fi   
-])
-
-dnl LSH_GCC_ATTRIBUTES
-dnl Check for gcc's __attribute__ construction
-
-AC_DEFUN([LSH_GCC_ATTRIBUTES],
-[AC_CACHE_CHECK(for __attribute__,
-	       lsh_cv_c_attribute,
-[ AC_TRY_COMPILE([
-#include <stdlib.h>
-],
-[
-static void foo(void) __attribute__ ((noreturn));
-
-static void __attribute__ ((noreturn))
-foo(void)
-{
-  exit(1);
-}
-],
-lsh_cv_c_attribute=yes,
-lsh_cv_c_attribute=no)])
-
-AH_TEMPLATE([HAVE_GCC_ATTRIBUTE], [Define if the compiler understands __attribute__])
-if test "x$lsh_cv_c_attribute" = "xyes"; then
-  AC_DEFINE(HAVE_GCC_ATTRIBUTE)
-fi
-
-AH_BOTTOM(
-[#if __GNUC__ || HAVE_GCC_ATTRIBUTE
-# define NORETURN __attribute__ ((__noreturn__))
-# define PRINTF_STYLE(f, a) __attribute__ ((__format__ (__printf__, f, a)))
-# define UNUSED __attribute__ ((__unused__))
-#else
-# define NORETURN
-# define PRINTF_STYLE(f, a)
-# define UNUSED
-#endif
-])])
-
-AC_DEFUN([LSH_GCC_FUNCTION_NAME],
-[# Check for gcc's __FUNCTION__ variable
-AH_TEMPLATE([HAVE_GCC_FUNCTION],
-	    [Define if the compiler understands __FUNCTION__])
-AH_BOTTOM(
-[#if HAVE_GCC_FUNCTION
-# define FUNCTION_NAME __FUNCTION__
-#else
-# define FUNCTION_NAME "Unknown"
-#endif
-])
-
-AC_CACHE_CHECK(for __FUNCTION__,
-	       lsh_cv_c_FUNCTION,
-  [ AC_TRY_COMPILE(,
-      [ #if __GNUC__ == 3
-	#  error __FUNCTION__ is broken in gcc-3
-	#endif
-        void foo(void) { char c = __FUNCTION__[0]; } ],
-      lsh_cv_c_FUNCTION=yes,
-      lsh_cv_c_FUNCTION=no)])
-
-if test "x$lsh_cv_c_FUNCTION" = "xyes"; then
-  AC_DEFINE(HAVE_GCC_FUNCTION)
-fi
-])
-
-# Check for alloca, and include the standard blurb in config.h
-AC_DEFUN([LSH_FUNC_ALLOCA],
-[AC_FUNC_ALLOCA
-AC_CHECK_HEADERS([malloc.h])
-AH_BOTTOM(
-[/* AIX requires this to be the first thing in the file.  */
-#ifndef __GNUC__
-# if HAVE_ALLOCA_H
-#  include <alloca.h>
-# else
-#  ifdef _AIX
- #pragma alloca
-#  else
-#   ifndef alloca /* predefined by HP cc +Olibcalls */
-char *alloca ();
-#   endif
-#  endif
-# endif
-#else /* defined __GNUC__ */
-# if HAVE_ALLOCA_H
-#  include <alloca.h>
-# endif
-#endif
-/* Needed for alloca on windows */
-#if HAVE_MALLOC_H
-# include <malloc.h>
-#endif
-])])
-
-AC_DEFUN([LSH_FUNC_STRERROR],
-[AC_CHECK_FUNCS(strerror)
-AH_BOTTOM(
-[#if HAVE_STRERROR
-#define STRERROR strerror
-#else
-#define STRERROR(x) (sys_errlist[x])
-#endif
-])])
-
-AC_DEFUN([LSH_FUNC_STRSIGNAL],
-[AC_CHECK_FUNCS(strsignal)
-AC_CHECK_DECLS([sys_siglist, _sys_siglist])
-AH_BOTTOM(
-[#if HAVE_STRSIGNAL
-# define STRSIGNAL strsignal
-#else /* !HAVE_STRSIGNAL */
-# if HAVE_DECL_SYS_SIGLIST
-#  define STRSIGNAL(x) (sys_siglist[x])
-# else
-#  if HAVE_DECL__SYS_SIGLIST
-#   define STRSIGNAL(x) (_sys_siglist[x])
-#  else
-#   define STRSIGNAL(x) "Unknown signal"
-#   if __GNUC__
-#    warning Using dummy STRSIGNAL
-#   endif
-#  endif
-# endif
-#endif /* !HAVE_STRSIGNAL */
-])])
-
-dnl LSH_MAKE_CONDITIONAL(symbol, test)
-AC_DEFUN([LSH_MAKE_CONDITIONAL],
-[if $2 ; then
-  IF_$1=''
-  UNLESS_$1='# '
-else
-  IF_$1='# '
-  UNLESS_$1=''
-fi 
-AC_SUBST(IF_$1)
-AC_SUBST(UNLESS_$1)])
-
-dnl LSH_DEPENDENCY_TRACKING
-
-dnl Defines compiler flags DEP_FLAGS to generate dependency
-dnl information, and DEP_PROCESS that is any shell commands needed for
-dnl massaging the dependency information further. Dependencies are
-dnl generated as a side effect of compilation. Dependency files
-dnl themselves are not treated as targets.
-
-AC_DEFUN([LSH_DEPENDENCY_TRACKING],
-[AC_ARG_ENABLE(dependency_tracking,
-  AC_HELP_STRING([--disable-dependency-tracking],
-    [Disable dependency tracking. Dependency tracking doesn't work with BSD make]),,
-  [enable_dependency_tracking=yes])
-
-DEP_FLAGS=''
-DEP_PROCESS='true'
-if test x$enable_dependency_tracking = xyes ; then
-  if test x$GCC = xyes ; then
-    gcc_version=`gcc --version | head -1`
-    case "$gcc_version" in
-      2.*|*[[!0-9.]]2.*)
-        enable_dependency_tracking=no
-        AC_MSG_WARN([Dependency tracking disabled, gcc-3.x is needed])
-      ;;
-      *)
-        DEP_FLAGS='-MT $[]@ -MD -MP -MF $[]@.d'
-        DEP_PROCESS='true'
-      ;;
-    esac
-  else
-    enable_dependency_tracking=no
-    AC_MSG_WARN([Dependency tracking disabled])
-  fi
-fi
-
-if test x$enable_dependency_tracking = xyes ; then
-  DEP_INCLUDE='include '
-else
-  DEP_INCLUDE='# '
-fi
-
-AC_SUBST([DEP_INCLUDE])
-AC_SUBST([DEP_FLAGS])
-AC_SUBST([DEP_PROCESS])])
-
-dnl @synopsis AX_CREATE_STDINT_H [( HEADER-TO-GENERATE [, HEADERS-TO-CHECK])]
-dnl
-dnl the "ISO C9X: 7.18 Integer types <stdint.h>" section requires the
-dnl existence of an include file <stdint.h> that defines a set of 
-dnl typedefs, especially uint8_t,int32_t,uintptr_t.
-dnl Many older installations will not provide this file, but some will
-dnl have the very same definitions in <inttypes.h>. In other enviroments
-dnl we can use the inet-types in <sys/types.h> which would define the
-dnl typedefs int8_t and u_int8_t respectivly.
-dnl
-dnl This macros will create a local "_stdint.h" or the headerfile given as 
-dnl an argument. In many cases that file will just "#include <stdint.h>" 
-dnl or "#include <inttypes.h>", while in other environments it will provide 
-dnl the set of basic 'stdint's definitions/typedefs: 
-dnl   int8_t,uint8_t,int16_t,uint16_t,int32_t,uint32_t,intptr_t,uintptr_t
-dnl   int_least32_t.. int_fast32_t.. intmax_t
-dnl which may or may not rely on the definitions of other files,
-dnl or using the AC_CHECK_SIZEOF macro to determine the actual
-dnl sizeof each type.
-dnl
-dnl if your header files require the stdint-types you will want to create an
-dnl installable file mylib-int.h that all your other installable header
-dnl may include. So if you have a library package named "mylib", just use
-dnl      AX_CREATE_STDINT_H(mylib-int.h) 
-dnl in configure.ac and go to install that very header file in Makefile.am
-dnl along with the other headers (mylib.h) - and the mylib-specific headers
-dnl can simply use "#include <mylib-int.h>" to obtain the stdint-types.
-dnl
-dnl Remember, if the system already had a valid <stdint.h>, the generated
-dnl file will include it directly. No need for fuzzy HAVE_STDINT_H things...
-dnl
-dnl @, (status: used on new platforms) (see http://ac-archive.sf.net/gstdint/)
-dnl @version $Id: acinclude.m4,v 1.27 2004/11/23 21:27:35 nisse Exp $
-dnl @author  Guido Draheim <guidod@gmx.de> 
-
-AC_DEFUN([AX_CREATE_STDINT_H],
-[# ------ AX CREATE STDINT H -------------------------------------
-AC_MSG_CHECKING([for stdint types])
-ac_stdint_h=`echo ifelse($1, , _stdint.h, $1)`
-# try to shortcircuit - if the default include path of the compiler
-# can find a "stdint.h" header then we assume that all compilers can.
-AC_CACHE_VAL([ac_cv_header_stdint_t],[
-old_CXXFLAGS="$CXXFLAGS" ; CXXFLAGS=""
-old_CPPFLAGS="$CPPFLAGS" ; CPPFLAGS=""
-old_CFLAGS="$CFLAGS"     ; CFLAGS=""
-AC_TRY_COMPILE([#include <stdint.h>],[int_least32_t v = 0;],
-[ac_cv_stdint_result="(assuming C99 compatible system)"
- ac_cv_header_stdint_t="stdint.h"; ],
-[ac_cv_header_stdint_t=""])
-CXXFLAGS="$old_CXXFLAGS"
-CPPFLAGS="$old_CPPFLAGS"
-CFLAGS="$old_CFLAGS" ])
-
-v="... $ac_cv_header_stdint_h"
-if test "$ac_stdint_h" = "stdint.h" ; then
- AC_MSG_RESULT([(are you sure you want them in ./stdint.h?)])
-elif test "$ac_stdint_h" = "inttypes.h" ; then
- AC_MSG_RESULT([(are you sure you want them in ./inttypes.h?)])
-elif test "_$ac_cv_header_stdint_t" = "_" ; then
- AC_MSG_RESULT([(putting them into $ac_stdint_h)$v])
-else
- ac_cv_header_stdint="$ac_cv_header_stdint_t"
- AC_MSG_RESULT([$ac_cv_header_stdint (shortcircuit)])
-fi
-
-if test "_$ac_cv_header_stdint_t" = "_" ; then # can not shortcircuit..
-
-dnl .....intro message done, now do a few system checks.....
-dnl btw, all CHECK_TYPE macros do automatically "DEFINE" a type, therefore
-dnl we use the autoconf implementation detail _AC CHECK_TYPE_NEW instead
-
-inttype_headers=`echo $2 | sed -e 's/,/ /g'`
-
-ac_cv_stdint_result="(no helpful system typedefs seen)"
-AC_CACHE_CHECK([for stdint uintptr_t], [ac_cv_header_stdint_x],[
- ac_cv_header_stdint_x="" # the 1997 typedefs (inttypes.h)
-  AC_MSG_RESULT([(..)])
-  for i in stdint.h inttypes.h sys/inttypes.h $inttype_headers ; do
-   unset ac_cv_type_uintptr_t 
-   unset ac_cv_type_uint64_t
-   _AC_CHECK_TYPE_NEW(uintptr_t,[ac_cv_header_stdint_x=$i],dnl
-     continue,[#include <$i>])
-   AC_CHECK_TYPE(uint64_t,[and64="/uint64_t"],[and64=""],[#include<$i>])
-   ac_cv_stdint_result="(seen uintptr_t$and64 in $i)"
-   break;
-  done
-  AC_MSG_CHECKING([for stdint uintptr_t])
- ])
-
-if test "_$ac_cv_header_stdint_x" = "_" ; then
-AC_CACHE_CHECK([for stdint uint32_t], [ac_cv_header_stdint_o],[
- ac_cv_header_stdint_o="" # the 1995 typedefs (sys/inttypes.h)
-  AC_MSG_RESULT([(..)])
-  for i in inttypes.h sys/inttypes.h stdint.h $inttype_headers ; do
-   unset ac_cv_type_uint32_t
-   unset ac_cv_type_uint64_t
-   AC_CHECK_TYPE(uint32_t,[ac_cv_header_stdint_o=$i],dnl
-     continue,[#include <$i>])
-   AC_CHECK_TYPE(uint64_t,[and64="/uint64_t"],[and64=""],[#include<$i>])
-   ac_cv_stdint_result="(seen uint32_t$and64 in $i)"
-   break;
-  done
-  AC_MSG_CHECKING([for stdint uint32_t])
- ])
-fi
-
-if test "_$ac_cv_header_stdint_x" = "_" ; then
-if test "_$ac_cv_header_stdint_o" = "_" ; then
-AC_CACHE_CHECK([for stdint u_int32_t], [ac_cv_header_stdint_u],[
- ac_cv_header_stdint_u="" # the BSD typedefs (sys/types.h)
-  AC_MSG_RESULT([(..)])
-  for i in sys/types.h inttypes.h sys/inttypes.h $inttype_headers ; do
-   unset ac_cv_type_u_int32_t
-   unset ac_cv_type_u_int64_t
-   AC_CHECK_TYPE(u_int32_t,[ac_cv_header_stdint_u=$i],dnl
-     continue,[#include <$i>])
-   AC_CHECK_TYPE(u_int64_t,[and64="/u_int64_t"],[and64=""],[#include<$i>])
-   ac_cv_stdint_result="(seen u_int32_t$and64 in $i)"
-   break;
-  done
-  AC_MSG_CHECKING([for stdint u_int32_t])
- ])
-fi fi
-
-dnl if there was no good C99 header file, do some typedef checks...
-if test "_$ac_cv_header_stdint_x" = "_" ; then
-   AC_MSG_CHECKING([for stdint datatype model])
-   AC_MSG_RESULT([(..)])
-   AC_CHECK_SIZEOF(char)
-   AC_CHECK_SIZEOF(short)
-   AC_CHECK_SIZEOF(int)
-   AC_CHECK_SIZEOF(long)
-   AC_CHECK_SIZEOF(void*)
-   ac_cv_stdint_char_model=""
-   ac_cv_stdint_char_model="$ac_cv_stdint_char_model$ac_cv_sizeof_char"
-   ac_cv_stdint_char_model="$ac_cv_stdint_char_model$ac_cv_sizeof_short"
-   ac_cv_stdint_char_model="$ac_cv_stdint_char_model$ac_cv_sizeof_int"
-   ac_cv_stdint_long_model=""
-   ac_cv_stdint_long_model="$ac_cv_stdint_long_model$ac_cv_sizeof_int"
-   ac_cv_stdint_long_model="$ac_cv_stdint_long_model$ac_cv_sizeof_long"
-   ac_cv_stdint_long_model="$ac_cv_stdint_long_model$ac_cv_sizeof_voidp"
-   name="$ac_cv_stdint_long_model"
-   case "$ac_cv_stdint_char_model/$ac_cv_stdint_long_model" in
-    122/242)     name="$name,  IP16 (standard 16bit machine)" ;;
-    122/244)     name="$name,  LP32 (standard 32bit mac/win)" ;;
-    122/*)       name="$name        (unusual int16 model)" ;; 
-    124/444)     name="$name, ILP32 (standard 32bit unixish)" ;;
-    124/488)     name="$name,  LP64 (standard 64bit unixish)" ;;
-    124/448)     name="$name, LLP64 (unusual  64bit unixish)" ;;
-    124/*)       name="$name        (unusual int32 model)" ;; 
-    128/888)     name="$name, ILP64 (unusual  64bit numeric)" ;;
-    128/*)       name="$name        (unusual int64 model)" ;; 
-    222/*|444/*) name="$name        (unusual dsptype)" ;;
-     *)          name="$name        (very unusal model)" ;;
-   esac
-   AC_MSG_RESULT([combined for stdint datatype model...  $name])
-fi
-
-if test "_$ac_cv_header_stdint_x" != "_" ; then
-   ac_cv_header_stdint="$ac_cv_header_stdint_x"
-elif  test "_$ac_cv_header_stdint_o" != "_" ; then
-   ac_cv_header_stdint="$ac_cv_header_stdint_o"
-elif  test "_$ac_cv_header_stdint_u" != "_" ; then
-   ac_cv_header_stdint="$ac_cv_header_stdint_u"
-else
-   ac_cv_header_stdint="stddef.h"
-fi
-
-AC_MSG_CHECKING([for extra inttypes in chosen header])
-AC_MSG_RESULT([($ac_cv_header_stdint)])
-dnl see if int_least and int_fast types are present in _this_ header.
-unset ac_cv_type_int_least32_t
-unset ac_cv_type_int_fast32_t
-AC_CHECK_TYPE(int_least32_t,,,[#include <$ac_cv_header_stdint>])
-AC_CHECK_TYPE(int_fast32_t,,,[#include<$ac_cv_header_stdint>])
-AC_CHECK_TYPE(intmax_t,,,[#include <$ac_cv_header_stdint>])
-
-fi # shortcircut to system "stdint.h"
-# ------------------ PREPARE VARIABLES ------------------------------
-if test "$GCC" = "yes" ; then
-ac_cv_stdint_message="using gnu compiler "`$CC --version | head -1` 
-else
-ac_cv_stdint_message="using $CC"
-fi
-
-AC_MSG_RESULT([make use of $ac_cv_header_stdint in $ac_stdint_h dnl
-$ac_cv_stdint_result])
-
-# ----------------- DONE inttypes.h checks START header -------------
-AC_CONFIG_COMMANDS([$ac_stdint_h],[
-AC_MSG_NOTICE(creating $ac_stdint_h : $_ac_stdint_h)
-ac_stdint=$tmp/_stdint.h
-
-echo "#ifndef" $_ac_stdint_h >$ac_stdint
-echo "#define" $_ac_stdint_h "1" >>$ac_stdint
-echo "#ifndef" _GENERATED_STDINT_H >>$ac_stdint
-echo "#define" _GENERATED_STDINT_H '"'$PACKAGE $VERSION'"' >>$ac_stdint
-echo "/* generated $ac_cv_stdint_message */" >>$ac_stdint
-if test "_$ac_cv_header_stdint_t" != "_" ; then 
-echo "#define _STDINT_HAVE_STDINT_H" "1" >>$ac_stdint
-fi
-
-cat >>$ac_stdint <<STDINT_EOF
-
-/* ................... shortcircuit part ........................... */
-
-#if defined HAVE_STDINT_H || defined _STDINT_HAVE_STDINT_H
-#include <stdint.h>
-#else
-#include <stddef.h>
-
-/* .................... configured part ............................ */
-
-STDINT_EOF
-
-echo "/* whether we have a C99 compatible stdint header file */" >>$ac_stdint
-if test "_$ac_cv_header_stdint_x" != "_" ; then
-  ac_header="$ac_cv_header_stdint_x"
-  echo "#define _STDINT_HEADER_INTPTR" '"'"$ac_header"'"' >>$ac_stdint
-else
-  echo "/* #undef _STDINT_HEADER_INTPTR */" >>$ac_stdint
-fi
-
-echo "/* whether we have a C96 compatible inttypes header file */" >>$ac_stdint
-if  test "_$ac_cv_header_stdint_o" != "_" ; then
-  ac_header="$ac_cv_header_stdint_o"
-  echo "#define _STDINT_HEADER_UINT32" '"'"$ac_header"'"' >>$ac_stdint
-else
-  echo "/* #undef _STDINT_HEADER_UINT32 */" >>$ac_stdint
-fi
-
-echo "/* whether we have a BSD compatible inet types header */" >>$ac_stdint
-if  test "_$ac_cv_header_stdint_u" != "_" ; then
-  ac_header="$ac_cv_header_stdint_u"
-  echo "#define _STDINT_HEADER_U_INT32" '"'"$ac_header"'"' >>$ac_stdint
-else
-  echo "/* #undef _STDINT_HEADER_U_INT32 */" >>$ac_stdint
-fi
-
-echo "" >>$ac_stdint
-
-if test "_$ac_header" != "_" ; then if test "$ac_header" != "stddef.h" ; then
-  echo "#include <$ac_header>" >>$ac_stdint
-  echo "" >>$ac_stdint
-fi fi
-
-echo "/* which 64bit typedef has been found */" >>$ac_stdint
-if test "$ac_cv_type_uint64_t" = "yes" ; then
-echo "#define   _STDINT_HAVE_UINT64_T" "1"  >>$ac_stdint
-else
-echo "/* #undef _STDINT_HAVE_UINT64_T */" >>$ac_stdint
-fi
-if test "$ac_cv_type_u_int64_t" = "yes" ; then
-echo "#define   _STDINT_HAVE_U_INT64_T" "1"  >>$ac_stdint
-else
-echo "/* #undef _STDINT_HAVE_U_INT64_T */" >>$ac_stdint
-fi
-echo "" >>$ac_stdint
-
-echo "/* which type model has been detected */" >>$ac_stdint
-if test "_$ac_cv_stdint_char_model" != "_" ; then
-echo "#define   _STDINT_CHAR_MODEL" "$ac_cv_stdint_char_model" >>$ac_stdint
-echo "#define   _STDINT_LONG_MODEL" "$ac_cv_stdint_long_model" >>$ac_stdint
-else
-echo "/* #undef _STDINT_CHAR_MODEL // skipped */" >>$ac_stdint
-echo "/* #undef _STDINT_LONG_MODEL // skipped */" >>$ac_stdint
-fi
-echo "" >>$ac_stdint
-
-echo "/* whether int_least types were detected */" >>$ac_stdint
-if test "$ac_cv_type_int_least32_t" = "yes"; then
-echo "#define   _STDINT_HAVE_INT_LEAST32_T" "1"  >>$ac_stdint
-else
-echo "/* #undef _STDINT_HAVE_INT_LEAST32_T */" >>$ac_stdint
-fi
-echo "/* whether int_fast types were detected */" >>$ac_stdint
-if test "$ac_cv_type_int_fast32_t" = "yes"; then
-echo "#define   _STDINT_HAVE_INT_FAST32_T" "1" >>$ac_stdint
-else
-echo "/* #undef _STDINT_HAVE_INT_FAST32_T */" >>$ac_stdint
-fi
-echo "/* whether intmax_t type was detected */" >>$ac_stdint
-if test "$ac_cv_type_intmax_t" = "yes"; then
-echo "#define   _STDINT_HAVE_INTMAX_T" "1" >>$ac_stdint
-else
-echo "/* #undef _STDINT_HAVE_INTMAX_T */" >>$ac_stdint
-fi
-echo "" >>$ac_stdint
-
-  cat >>$ac_stdint <<STDINT_EOF
-/* .................... detections part ............................ */
-
-/* whether we need to define bitspecific types from compiler base types */
-#ifndef _STDINT_HEADER_INTPTR
-#ifndef _STDINT_HEADER_UINT32
-#ifndef _STDINT_HEADER_U_INT32
-#define _STDINT_NEED_INT_MODEL_T
-#else
-#define _STDINT_HAVE_U_INT_TYPES
-#endif
-#endif
-#endif
-
-#ifdef _STDINT_HAVE_U_INT_TYPES
-#undef _STDINT_NEED_INT_MODEL_T
-#endif
-
-#ifdef  _STDINT_CHAR_MODEL
-#if     _STDINT_CHAR_MODEL+0 == 122 || _STDINT_CHAR_MODEL+0 == 124
-#ifndef _STDINT_BYTE_MODEL
-#define _STDINT_BYTE_MODEL 12
-#endif
-#endif
-#endif
-
-#ifndef _STDINT_HAVE_INT_LEAST32_T
-#define _STDINT_NEED_INT_LEAST_T
-#endif
-
-#ifndef _STDINT_HAVE_INT_FAST32_T
-#define _STDINT_NEED_INT_FAST_T
-#endif
-
-#ifndef _STDINT_HEADER_INTPTR
-#define _STDINT_NEED_INTPTR_T
-#ifndef _STDINT_HAVE_INTMAX_T
-#define _STDINT_NEED_INTMAX_T
-#endif
-#endif
-
-
-/* .................... definition part ............................ */
-
-/* some system headers have good uint64_t */
-#ifndef _HAVE_UINT64_T
-#if     defined _STDINT_HAVE_UINT64_T  || defined HAVE_UINT64_T
-#define _HAVE_UINT64_T
-#elif   defined _STDINT_HAVE_U_INT64_T || defined HAVE_U_INT64_T
-#define _HAVE_UINT64_T
-typedef u_int64_t uint64_t;
-#endif
-#endif
-
-#ifndef _HAVE_UINT64_T
-/* .. here are some common heuristics using compiler runtime specifics */
-#if defined __STDC_VERSION__ && defined __STDC_VERSION__ >= 199901L
-#define _HAVE_UINT64_T
-typedef long long int64_t;
-typedef unsigned long long uint64_t;
-
-#elif !defined __STRICT_ANSI__
-#if defined _MSC_VER || defined __WATCOMC__ || defined __BORLANDC__
-#define _HAVE_UINT64_T
-typedef __int64 int64_t;
-typedef unsigned __int64 uint64_t;
-
-#elif defined __GNUC__ || defined __MWERKS__ || defined __ELF__
-/* note: all ELF-systems seem to have loff-support which needs 64-bit */
-#if !defined _NO_LONGLONG
-#define _HAVE_UINT64_T
-typedef long long int64_t;
-typedef unsigned long long uint64_t;
-#endif
-
-#elif defined __alpha || (defined __mips && defined _ABIN32)
-#if !defined _NO_LONGLONG
-typedef long int64_t;
-typedef unsigned long uint64_t;
-#endif
-  /* compiler/cpu type to define int64_t */
-#endif
-#endif
-#endif
-
-#if defined _STDINT_HAVE_U_INT_TYPES
-/* int8_t int16_t int32_t defined by inet code, redeclare the u_intXX types */
-typedef u_int8_t uint8_t;
-typedef u_int16_t uint16_t;
-typedef u_int32_t uint32_t;
-
-/* glibc compatibility */
-#ifndef __int8_t_defined
-#define __int8_t_defined
-#endif
-#endif
-
-#ifdef _STDINT_NEED_INT_MODEL_T
-/* we must guess all the basic types. Apart from byte-adressable system, */
-/* there a few 32-bit-only dsp-systems that we guard with BYTE_MODEL 8-} */
-/* (btw, those nibble-addressable systems are way off, or so we assume) */
-
-dnl   /* have a look at "64bit and data size neutrality" at */
-dnl   /* http://unix.org/version2/whatsnew/login_64bit.html */
-dnl   /* (the shorthand "ILP" types always have a "P" part) */
-
-#if defined _STDINT_BYTE_MODEL
-#if _STDINT_LONG_MODEL+0 == 242
-/* 2:4:2 =  IP16 = a normal 16-bit system                */
-typedef unsigned char   uint8_t;
-typedef unsigned short  uint16_t;
-typedef unsigned long   uint32_t;
-#ifndef __int8_t_defined
-#define __int8_t_defined
-typedef          char    int8_t;
-typedef          short   int16_t;
-typedef          long    int32_t;
-#endif
-#elif _STDINT_LONG_MODEL+0 == 244 || _STDINT_LONG_MODEL == 444
-/* 2:4:4 =  LP32 = a 32-bit system derived from a 16-bit */
-/* 4:4:4 = ILP32 = a normal 32-bit system                */
-typedef unsigned char   uint8_t;
-typedef unsigned short  uint16_t;
-typedef unsigned int    uint32_t;
-#ifndef __int8_t_defined
-#define __int8_t_defined
-typedef          char    int8_t;
-typedef          short   int16_t;
-typedef          int     int32_t;
-#endif
-#elif _STDINT_LONG_MODEL+0 == 484 || _STDINT_LONG_MODEL+0 == 488
-/* 4:8:4 =  IP32 = a 32-bit system prepared for 64-bit    */
-/* 4:8:8 =  LP64 = a normal 64-bit system                 */
-typedef unsigned char   uint8_t;
-typedef unsigned short  uint16_t;
-typedef unsigned int    uint32_t;
-#ifndef __int8_t_defined
-#define __int8_t_defined
-typedef          char    int8_t;
-typedef          short   int16_t;
-typedef          int     int32_t;
-#endif
-/* this system has a "long" of 64bit */
-#ifndef _HAVE_UINT64_T
-#define _HAVE_UINT64_T
-typedef unsigned long   uint64_t;
-typedef          long    int64_t;
-#endif
-#elif _STDINT_LONG_MODEL+0 == 448
-/*      LLP64   a 64-bit system derived from a 32-bit system */
-typedef unsigned char   uint8_t;
-typedef unsigned short  uint16_t;
-typedef unsigned int    uint32_t;
-#ifndef __int8_t_defined
-#define __int8_t_defined
-typedef          char    int8_t;
-typedef          short   int16_t;
-typedef          int     int32_t;
-#endif
-/* assuming the system has a "long long" */
-#ifndef _HAVE_UINT64_T
-#define _HAVE_UINT64_T
-typedef unsigned long long uint64_t;
-typedef          long long  int64_t;
-#endif
-#else
-#define _STDINT_NO_INT32_T
-#endif
-#else
-#define _STDINT_NO_INT8_T
-#define _STDINT_NO_INT32_T
-#endif
-#endif
-
-/*
- * quote from SunOS-5.8 sys/inttypes.h:
- * Use at your own risk.  As of February 1996, the committee is squarely
- * behind the fixed sized types; the "least" and "fast" types are still being
- * discussed.  The probability that the "fast" types may be removed before
- * the standard is finalized is high enough that they are not currently
- * implemented.
- */
-
-#if defined _STDINT_NEED_INT_LEAST_T
-typedef  int8_t    int_least8_t;
-typedef  int16_t   int_least16_t;
-typedef  int32_t   int_least32_t;
-#ifdef _HAVE_UINT64_T
-typedef  int64_t   int_least64_t;
-#endif
-
-typedef uint8_t   uint_least8_t;
-typedef uint16_t  uint_least16_t;
-typedef uint32_t  uint_least32_t;
-#ifdef _HAVE_UINT64_T
-typedef uint64_t  uint_least64_t;
-#endif
-  /* least types */
-#endif
-
-#if defined _STDINT_NEED_INT_FAST_T
-typedef  int8_t    int_fast8_t; 
-typedef  int       int_fast16_t;
-typedef  int32_t   int_fast32_t;
-#ifdef _HAVE_UINT64_T
-typedef  int64_t   int_fast64_t;
-#endif
-
-typedef uint8_t   uint_fast8_t; 
-typedef unsigned  uint_fast16_t;
-typedef uint32_t  uint_fast32_t;
-#ifdef _HAVE_UINT64_T
-typedef uint64_t  uint_fast64_t;
-#endif
-  /* fast types */
-#endif
-
-#ifdef _STDINT_NEED_INTMAX_T
-#ifdef _HAVE_UINT64_T
-typedef  int64_t       intmax_t;
-typedef uint64_t      uintmax_t;
-#else
-typedef          long  intmax_t;
-typedef unsigned long uintmax_t;
-#endif
-#endif
-
-#ifdef _STDINT_NEED_INTPTR_T
-#ifndef __intptr_t_defined
-#define __intptr_t_defined
-/* we encourage using "long" to store pointer values, never use "int" ! */
-#if   _STDINT_LONG_MODEL+0 == 242 || _STDINT_LONG_MODEL+0 == 484
-typedef  unsinged int   uintptr_t;
-typedef           int    intptr_t;
-#elif _STDINT_LONG_MODEL+0 == 244 || _STDINT_LONG_MODEL+0 == 444
-typedef  unsigned long  uintptr_t;
-typedef           long   intptr_t;
-#elif _STDINT_LONG_MODEL+0 == 448 && defined _HAVE_UINT64_T
-typedef        uint64_t uintptr_t;
-typedef         int64_t  intptr_t;
-#else /* matches typical system types ILP32 and LP64 - but not IP16 or LLP64 */
-typedef  unsigned long  uintptr_t;
-typedef           long   intptr_t;
-#endif
-#endif
-#endif
-
-  /* shortcircuit*/
-#endif
-  /* once */
-#endif
-#endif
-STDINT_EOF
-    if cmp -s $ac_stdint_h $ac_stdint 2>/dev/null; then
-      AC_MSG_NOTICE([$ac_stdint_h is unchanged])
-    else
-      ac_dir=`AS_DIRNAME(["$ac_stdint_h"])`
-      AS_MKDIR_P(["$ac_dir"])
-      rm -f $ac_stdint_h
-      mv $ac_stdint $ac_stdint_h
-    fi
-],[# variables for create stdint.h replacement
-PACKAGE="$PACKAGE"
-VERSION="$VERSION"
-ac_stdint_h="$ac_stdint_h"
-_ac_stdint_h=AS_TR_CPP(_$PACKAGE-$ac_stdint_h)
-ac_cv_stdint_message="$ac_cv_stdint_message"
-ac_cv_header_stdint_t="$ac_cv_header_stdint_t"
-ac_cv_header_stdint_x="$ac_cv_header_stdint_x"
-ac_cv_header_stdint_o="$ac_cv_header_stdint_o"
-ac_cv_header_stdint_u="$ac_cv_header_stdint_u"
-ac_cv_type_uint64_t="$ac_cv_type_uint64_t"
-ac_cv_type_u_int64_t="$ac_cv_type_u_int64_t"
-ac_cv_stdint_char_model="$ac_cv_stdint_char_model"
-ac_cv_stdint_long_model="$ac_cv_stdint_long_model"
-ac_cv_type_int_least32_t="$ac_cv_type_int_least32_t"
-ac_cv_type_int_fast32_t="$ac_cv_type_int_fast32_t"
-ac_cv_type_intmax_t="$ac_cv_type_intmax_t"
-])
-])
diff --git a/argp-standalone/argp-ba.c b/argp-standalone/argp-ba.c
deleted file mode 100644
index 0d3958c1151..00000000000
--- a/argp-standalone/argp-ba.c
+++ /dev/null
@@ -1,26 +0,0 @@
-/* Default definition for ARGP_PROGRAM_BUG_ADDRESS.
-   Copyright (C) 1996, 1997, 1999, 2004 Free Software Foundation, Inc.
-   This file is part of the GNU C Library.
-   Written by Miles Bader <miles@gnu.ai.mit.edu>.
-
-   The GNU C Library is free software; you can redistribute it and/or
-   modify it under the terms of the GNU Library General Public License as
-   published by the Free Software Foundation; either version 2 of the
-   License, or (at your option) any later version.
-
-   The GNU C Library is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Library General Public License for more details.
-
-   You should have received a copy of the GNU Library General Public
-   License along with the GNU C Library; see the file COPYING.LIB.  If not,
-   write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330,
-   Boston, MA 02111-1307, USA.  */
-
-/* If set by the user program, it should point to string that is the
-   bug-reporting address for the program.  It will be printed by argp_help if
-   the ARGP_HELP_BUG_ADDR flag is set (as it is by various standard help
-   messages), embedded in a sentence that says something like `Report bugs to
-   ADDR.'.  */
-const char *argp_program_bug_address = 0;
diff --git a/argp-standalone/argp-eexst.c b/argp-standalone/argp-eexst.c
deleted file mode 100644
index 46b27847ad4..00000000000
--- a/argp-standalone/argp-eexst.c
+++ /dev/null
@@ -1,36 +0,0 @@
-/* Default definition for ARGP_ERR_EXIT_STATUS
-   Copyright (C) 1997 Free Software Foundation, Inc.
-   This file is part of the GNU C Library.
-   Written by Miles Bader <miles@gnu.ai.mit.edu>.
-
-   The GNU C Library is free software; you can redistribute it and/or
-   modify it under the terms of the GNU Library General Public License as
-   published by the Free Software Foundation; either version 2 of the
-   License, or (at your option) any later version.
-
-   The GNU C Library is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Library General Public License for more details.
-
-   You should have received a copy of the GNU Library General Public
-   License along with the GNU C Library; see the file COPYING.LIB.  If not,
-   write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330,
-   Boston, MA 02111-1307, USA.  */
-
-#ifdef HAVE_CONFIG_H
-#include <config.h>
-#endif
-
-#if HAVE_SYSEXITS_H
-# include <sysexits.h>
-#else
-# define EX_USAGE 64
-#endif
-
-#include "argp.h"
-
-/* The exit status that argp will use when exiting due to a parsing error.
-   If not defined or set by the user program, this defaults to EX_USAGE from
-   <sysexits.h>.  */
-error_t argp_err_exit_status = EX_USAGE;
diff --git a/argp-standalone/argp-fmtstream.c b/argp-standalone/argp-fmtstream.c
deleted file mode 100644
index 7f792854fc3..00000000000
--- a/argp-standalone/argp-fmtstream.c
+++ /dev/null
@@ -1,475 +0,0 @@
-/* Word-wrapping and line-truncating streams
-   Copyright (C) 1997, 1998, 1999, 2001 Free Software Foundation, Inc.
-   This file is part of the GNU C Library.
-   Written by Miles Bader <miles@gnu.ai.mit.edu>.
-
-   The GNU C Library is free software; you can redistribute it and/or
-   modify it under the terms of the GNU Library General Public License as
-   published by the Free Software Foundation; either version 2 of the
-   License, or (at your option) any later version.
-
-   The GNU C Library is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Library General Public License for more details.
-
-   You should have received a copy of the GNU Library General Public
-   License along with the GNU C Library; see the file COPYING.LIB.  If not,
-   write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330,
-   Boston, MA 02111-1307, USA.  */
-
-/* This package emulates glibc `line_wrap_stream' semantics for systems that
-   don't have that.  */
-
-#ifdef HAVE_CONFIG_H
-#include <config.h>
-#endif
-
-#include <stdlib.h>
-#include <string.h>
-#include <errno.h>
-#include <stdarg.h>
-#include <ctype.h>
-
-#include "argp-fmtstream.h"
-#include "argp-namefrob.h"
-
-#ifndef ARGP_FMTSTREAM_USE_LINEWRAP
-
-#ifndef isblank
-#define isblank(ch) ((ch)==' ' || (ch)=='\t')
-#endif
-
-#if defined _LIBC && defined USE_IN_LIBIO
-# include <libio/libioP.h>
-# define __vsnprintf(s, l, f, a) _IO_vsnprintf (s, l, f, a)
-#endif
-
-#define INIT_BUF_SIZE 200
-#define PRINTF_SIZE_GUESS 150
-
-/* Return an argp_fmtstream that outputs to STREAM, and which prefixes lines
-   written on it with LMARGIN spaces and limits them to RMARGIN columns
-   total.  If WMARGIN >= 0, words that extend past RMARGIN are wrapped by
-   replacing the whitespace before them with a newline and WMARGIN spaces.
-   Otherwise, chars beyond RMARGIN are simply dropped until a newline.
-   Returns NULL if there was an error.  */
-argp_fmtstream_t
-__argp_make_fmtstream (FILE *stream,
-		       size_t lmargin, size_t rmargin, ssize_t wmargin)
-{
-  argp_fmtstream_t fs = malloc (sizeof (struct argp_fmtstream));
-  if (fs)
-    {
-      fs->stream = stream;
-
-      fs->lmargin = lmargin;
-      fs->rmargin = rmargin;
-      fs->wmargin = wmargin;
-      fs->point_col = 0;
-      fs->point_offs = 0;
-
-      fs->buf = malloc (INIT_BUF_SIZE);
-      if (! fs->buf)
-	{
-	  free (fs);
-	  fs = 0;
-	}
-      else
-	{
-	  fs->p = fs->buf;
-	  fs->end = fs->buf + INIT_BUF_SIZE;
-	}
-    }
-
-  return fs;
-}
-#ifdef weak_alias
-weak_alias (__argp_make_fmtstream, argp_make_fmtstream)
-#endif
-
-/* Flush FS to its stream, and free it (but don't close the stream).  */
-void
-__argp_fmtstream_free (argp_fmtstream_t fs)
-{
-  __argp_fmtstream_update (fs);
-  if (fs->p > fs->buf)
-    FWRITE_UNLOCKED (fs->buf, 1, fs->p - fs->buf, fs->stream);
-  free (fs->buf);
-  free (fs);
-}
-#ifdef weak_alias
-weak_alias (__argp_fmtstream_free, argp_fmtstream_free)
-#endif
-
-/* Process FS's buffer so that line wrapping is done from POINT_OFFS to the
-   end of its buffer.  This code is mostly from glibc stdio/linewrap.c.  */
-void
-__argp_fmtstream_update (argp_fmtstream_t fs)
-{
-  char *buf, *nl;
-  size_t len;
-
-  /* Scan the buffer for newlines.  */
-  buf = fs->buf + fs->point_offs;
-  while (buf < fs->p)
-    {
-      size_t r;
-
-      if (fs->point_col == 0 && fs->lmargin != 0)
-	{
-	  /* We are starting a new line.  Print spaces to the left margin.  */
-	  const size_t pad = fs->lmargin;
-	  if (fs->p + pad < fs->end)
-	    {
-	      /* We can fit in them in the buffer by moving the
-		 buffer text up and filling in the beginning.  */
-	      memmove (buf + pad, buf, fs->p - buf);
-	      fs->p += pad; /* Compensate for bigger buffer. */
-	      memset (buf, ' ', pad); /* Fill in the spaces.  */
-	      buf += pad; /* Don't bother searching them.  */
-	    }
-	  else
-	    {
-	      /* No buffer space for spaces.  Must flush.  */
-	      size_t i;
-	      for (i = 0; i < pad; i++)
-		PUTC_UNLOCKED (' ', fs->stream);
-	    }
-	  fs->point_col = pad;
-	}
-
-      len = fs->p - buf;
-      nl = memchr (buf, '\n', len);
-
-      if (fs->point_col < 0)
-	fs->point_col = 0;
-
-      if (!nl)
-	{
-	  /* The buffer ends in a partial line.  */
-
-	  if (fs->point_col + len < fs->rmargin)
-	    {
-	      /* The remaining buffer text is a partial line and fits
-		 within the maximum line width.  Advance point for the
-		 characters to be written and stop scanning.  */
-	      fs->point_col += len;
-	      break;
-	    }
-	  else
-	    /* Set the end-of-line pointer for the code below to
-	       the end of the buffer.  */
-	    nl = fs->p;
-	}
-      else if (fs->point_col + (nl - buf) < (ssize_t) fs->rmargin)
-	{
-	  /* The buffer contains a full line that fits within the maximum
-	     line width.  Reset point and scan the next line.  */
-	  fs->point_col = 0;
-	  buf = nl + 1;
-	  continue;
-	}
-
-      /* This line is too long.  */
-      r = fs->rmargin - 1;
-
-      if (fs->wmargin < 0)
-	{
-	  /* Truncate the line by overwriting the excess with the
-	     newline and anything after it in the buffer.  */
-	  if (nl < fs->p)
-	    {
-	      memmove (buf + (r - fs->point_col), nl, fs->p - nl);
-	      fs->p -= buf + (r - fs->point_col) - nl;
-	      /* Reset point for the next line and start scanning it.  */
-	      fs->point_col = 0;
-	      buf += r + 1; /* Skip full line plus \n. */
-	    }
-	  else
-	    {
-	      /* The buffer ends with a partial line that is beyond the
-		 maximum line width.  Advance point for the characters
-		 written, and discard those past the max from the buffer.  */
-	      fs->point_col += len;
-	      fs->p -= fs->point_col - r;
-	      break;
-	    }
-	}
-      else
-	{
-	  /* Do word wrap.  Go to the column just past the maximum line
-	     width and scan back for the beginning of the word there.
-	     Then insert a line break.  */
-
-	  char *p, *nextline;
-	  int i;
-
-	  p = buf + (r + 1 - fs->point_col);
-	  while (p >= buf && !isblank (*p))
-	    --p;
-	  nextline = p + 1;	/* This will begin the next line.  */
-
-	  if (nextline > buf)
-	    {
-	      /* Swallow separating blanks.  */
-	      if (p >= buf)
-		do
-		  --p;
-		while (p >= buf && isblank (*p));
-	      nl = p + 1;	/* The newline will replace the first blank. */
-	    }
-	  else
-	    {
-	      /* A single word that is greater than the maximum line width.
-		 Oh well.  Put it on an overlong line by itself.  */
-	      p = buf + (r + 1 - fs->point_col);
-	      /* Find the end of the long word.  */
-	      do
-		++p;
-	      while (p < nl && !isblank (*p));
-	      if (p == nl)
-		{
-		  /* It already ends a line.  No fussing required.  */
-		  fs->point_col = 0;
-		  buf = nl + 1;
-		  continue;
-		}
-	      /* We will move the newline to replace the first blank.  */
-	      nl = p;
-	      /* Swallow separating blanks.  */
-	      do
-		++p;
-	      while (isblank (*p));
-	      /* The next line will start here.  */
-	      nextline = p;
-	    }
-
-	  /* Note: There are a bunch of tests below for
-	     NEXTLINE == BUF + LEN + 1; this case is where NL happens to fall
-	     at the end of the buffer, and NEXTLINE is in fact empty (and so
-	     we need not be careful to maintain its contents).  */
-
-	  if (nextline == buf + len + 1
-	      ? fs->end - nl < fs->wmargin + 1
-	      : nextline - (nl + 1) < fs->wmargin)
-	    {
-	      /* The margin needs more blanks than we removed.  */
-	      if (fs->end - fs->p > fs->wmargin + 1)
-		/* Make some space for them.  */
-		{
-		  size_t mv = fs->p - nextline;
-		  memmove (nl + 1 + fs->wmargin, nextline, mv);
-		  nextline = nl + 1 + fs->wmargin;
-		  len = nextline + mv - buf;
-		  *nl++ = '\n';
-		}
-	      else
-		/* Output the first line so we can use the space.  */
-		{
-		  if (nl > fs->buf)
-		    FWRITE_UNLOCKED (fs->buf, 1, nl - fs->buf, fs->stream);
-		  PUTC_UNLOCKED ('\n', fs->stream);
-		  len += buf - fs->buf;
-		  nl = buf = fs->buf;
-		}
-	    }
-	  else
-	    /* We can fit the newline and blanks in before
-	       the next word.  */
-	    *nl++ = '\n';
-
-	  if (nextline - nl >= fs->wmargin
-	      || (nextline == buf + len + 1 && fs->end - nextline >= fs->wmargin))
-	    /* Add blanks up to the wrap margin column.  */
-	    for (i = 0; i < fs->wmargin; ++i)
-	      *nl++ = ' ';
-	  else
-	    for (i = 0; i < fs->wmargin; ++i)
-	      PUTC_UNLOCKED (' ', fs->stream);
-
-	  /* Copy the tail of the original buffer into the current buffer
-	     position.  */
-	  if (nl < nextline)
-	    memmove (nl, nextline, buf + len - nextline);
-	  len -= nextline - buf;
-
-	  /* Continue the scan on the remaining lines in the buffer.  */
-	  buf = nl;
-
-	  /* Restore bufp to include all the remaining text.  */
-	  fs->p = nl + len;
-
-	  /* Reset the counter of what has been output this line.  If wmargin
-	     is 0, we want to avoid the lmargin getting added, so we set
-	     point_col to a magic value of -1 in that case.  */
-	  fs->point_col = fs->wmargin ? fs->wmargin : -1;
-	}
-    }
-
-  /* Remember that we've scanned as far as the end of the buffer.  */
-  fs->point_offs = fs->p - fs->buf;
-}
-
-/* Ensure that FS has space for AMOUNT more bytes in its buffer, either by
-   growing the buffer, or by flushing it.  True is returned iff we succeed. */
-int
-__argp_fmtstream_ensure (struct argp_fmtstream *fs, size_t amount)
-{
-  if ((size_t) (fs->end - fs->p) < amount)
-    {
-      ssize_t wrote;
-
-      /* Flush FS's buffer.  */
-      __argp_fmtstream_update (fs);
-
-      wrote = FWRITE_UNLOCKED (fs->buf, 1, fs->p - fs->buf, fs->stream);
-      if (wrote == fs->p - fs->buf)
-	{
-	  fs->p = fs->buf;
-	  fs->point_offs = 0;
-	}
-      else
-	{
-	  fs->p -= wrote;
-	  fs->point_offs -= wrote;
-	  memmove (fs->buf, fs->buf + wrote, fs->p - fs->buf);
-	  return 0;
-	}
-
-      if ((size_t) (fs->end - fs->buf) < amount)
-	/* Gotta grow the buffer.  */
-	{
-	  size_t new_size = fs->end - fs->buf + amount;
-	  char *new_buf = realloc (fs->buf, new_size);
-
-	  if (! new_buf)
-	    {
-	      __set_errno (ENOMEM);
-	      return 0;
-	    }
-
-	  fs->buf = new_buf;
-	  fs->end = new_buf + new_size;
-	  fs->p = fs->buf;
-	}
-    }
-
-  return 1;
-}
-
-ssize_t
-__argp_fmtstream_printf (struct argp_fmtstream *fs, const char *fmt, ...)
-{
-  size_t out;
-  size_t avail;
-  size_t size_guess = PRINTF_SIZE_GUESS; /* How much space to reserve. */
-
-  do
-    {
-      va_list args;
-
-      if (! __argp_fmtstream_ensure (fs, size_guess))
-	return -1;
-
-      va_start (args, fmt);
-      avail = fs->end - fs->p;
-      out = __vsnprintf (fs->p, avail, fmt, args);
-      va_end (args);
-      if (out >= avail)
-	size_guess = out + 1;
-    }
-  while (out >= avail);
-
-  fs->p += out;
-
-  return out;
-}
-#ifdef weak_alias
-weak_alias (__argp_fmtstream_printf, argp_fmtstream_printf)
-#endif
-
-/* Duplicate the inline definitions in argp-fmtstream.h, for compilers
- * that don't do inlining. */
-size_t
-__argp_fmtstream_write (argp_fmtstream_t __fs,
-			__const char *__str, size_t __len)
-{
-  if (__fs->p + __len <= __fs->end || __argp_fmtstream_ensure (__fs, __len))
-    {
-      memcpy (__fs->p, __str, __len);
-      __fs->p += __len;
-      return __len;
-    }
-  else
-    return 0;
-}
-
-int
-__argp_fmtstream_puts (argp_fmtstream_t __fs, __const char *__str)
-{
-  size_t __len = strlen (__str);
-  if (__len)
-    {
-      size_t __wrote = __argp_fmtstream_write (__fs, __str, __len);
-      return __wrote == __len ? 0 : -1;
-    }
-  else
-    return 0;
-}
-
-int
-__argp_fmtstream_putc (argp_fmtstream_t __fs, int __ch)
-{
-  if (__fs->p < __fs->end || __argp_fmtstream_ensure (__fs, 1))
-    return *__fs->p++ = __ch;
-  else
-    return EOF;
-}
-
-/* Set __FS's left margin to __LMARGIN and return the old value.  */
-size_t
-__argp_fmtstream_set_lmargin (argp_fmtstream_t __fs, size_t __lmargin)
-{
-  size_t __old;
-  if ((size_t) (__fs->p - __fs->buf) > __fs->point_offs)
-    __argp_fmtstream_update (__fs);
-  __old = __fs->lmargin;
-  __fs->lmargin = __lmargin;
-  return __old;
-}
-
-/* Set __FS's right margin to __RMARGIN and return the old value.  */
-size_t
-__argp_fmtstream_set_rmargin (argp_fmtstream_t __fs, size_t __rmargin)
-{
-  size_t __old;
-  if ((size_t) (__fs->p - __fs->buf) > __fs->point_offs)
-    __argp_fmtstream_update (__fs);
-  __old = __fs->rmargin;
-  __fs->rmargin = __rmargin;
-  return __old;
-}
-
-/* Set FS's wrap margin to __WMARGIN and return the old value.  */
-size_t
-__argp_fmtstream_set_wmargin (argp_fmtstream_t __fs, size_t __wmargin)
-{
-  size_t __old;
-  if ((size_t) (__fs->p - __fs->buf) > __fs->point_offs)
-    __argp_fmtstream_update (__fs);
-  __old = __fs->wmargin;
-  __fs->wmargin = __wmargin;
-  return __old;
-}
-
-/* Return the column number of the current output point in __FS.  */
-size_t
-__argp_fmtstream_point (argp_fmtstream_t __fs)
-{
-  if ((size_t) (__fs->p - __fs->buf) > __fs->point_offs)
-    __argp_fmtstream_update (__fs);
-  return __fs->point_col >= 0 ? __fs->point_col : 0;
-}
-
-#endif /* !ARGP_FMTSTREAM_USE_LINEWRAP */
diff --git a/argp-standalone/argp-fmtstream.h b/argp-standalone/argp-fmtstream.h
deleted file mode 100644
index e797b119ebd..00000000000
--- a/argp-standalone/argp-fmtstream.h
+++ /dev/null
@@ -1,319 +0,0 @@
-/* Word-wrapping and line-truncating streams.
-   Copyright (C) 1997, 2003 Free Software Foundation, Inc.
-   This file is part of the GNU C Library.
-   Written by Miles Bader <miles@gnu.ai.mit.edu>.
-
-   The GNU C Library is free software; you can redistribute it and/or
-   modify it under the terms of the GNU Library General Public License as
-   published by the Free Software Foundation; either version 2 of the
-   License, or (at your option) any later version.
-
-   The GNU C Library is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Library General Public License for more details.
-
-   You should have received a copy of the GNU Library General Public
-   License along with the GNU C Library; see the file COPYING.LIB.  If not,
-   write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330,
-   Boston, MA 02111-1307, USA.  */
-
-/* This package emulates glibc `line_wrap_stream' semantics for systems that
-   don't have that.  If the system does have it, it is just a wrapper for
-   that.  This header file is only used internally while compiling argp, and
-   shouldn't be installed.  */
-
-#ifndef _ARGP_FMTSTREAM_H
-#define _ARGP_FMTSTREAM_H
-
-#include <stdio.h>
-#include <string.h>
-
-#if HAVE_UNISTD_H
-# include <unistd.h>
-#else
-/* This is a kludge to make the code compile on windows. Perhaps it
-   would be better to just replace ssize_t with int through out the
-   code. */
-# define ssize_t int
-#endif
-
-#if _LIBC || (defined (HAVE_FLOCKFILE) && defined(HAVE_PUTC_UNLOCKED) \
-     && defined (HAVE_FPUTS_UNLOCKED) && defined (HAVE_FWRITE_UNLOCKED) )
-/* Use locking funxtions */
-# define FLOCKFILE(f) flockfile(f)
-# define FUNLOCKFILE(f) funlockfile(f)
-# define PUTC_UNLOCKED(c, f) putc_unlocked((c), (f))
-# define FPUTS_UNLOCKED(s, f) fputs_unlocked((s), (f))
-# define FWRITE_UNLOCKED(b, s, n, f) fwrite_unlocked((b), (s), (n), (f))
-#else
-/* Disable stdio locking */
-# define FLOCKFILE(f)
-# define FUNLOCKFILE(f)
-# define PUTC_UNLOCKED(c, f) putc((c), (f))
-# define FPUTS_UNLOCKED(s, f) fputs((s), (f))
-# define FWRITE_UNLOCKED(b, s, n, f) fwrite((b), (s), (n), (f))
-#endif /* No thread safe i/o */
-
-#if    (_LIBC - 0 && !defined (USE_IN_LIBIO)) \
-    || (defined (__GNU_LIBRARY__) && defined (HAVE_LINEWRAP_H))
-/* line_wrap_stream is available, so use that.  */
-#define ARGP_FMTSTREAM_USE_LINEWRAP
-#endif
-
-#ifdef ARGP_FMTSTREAM_USE_LINEWRAP
-/* Just be a simple wrapper for line_wrap_stream; the semantics are
-   *slightly* different, as line_wrap_stream doesn't actually make a new
-   object, it just modifies the given stream (reversibly) to do
-   line-wrapping.  Since we control who uses this code, it doesn't matter.  */
-
-#include <linewrap.h>
-
-typedef FILE *argp_fmtstream_t;
-
-#define argp_make_fmtstream line_wrap_stream
-#define __argp_make_fmtstream line_wrap_stream
-#define argp_fmtstream_free line_unwrap_stream
-#define __argp_fmtstream_free line_unwrap_stream
-
-#define __argp_fmtstream_putc(fs,ch) putc(ch,fs)
-#define argp_fmtstream_putc(fs,ch) putc(ch,fs)
-#define __argp_fmtstream_puts(fs,str) fputs(str,fs)
-#define argp_fmtstream_puts(fs,str) fputs(str,fs)
-#define __argp_fmtstream_write(fs,str,len) fwrite(str,1,len,fs)
-#define argp_fmtstream_write(fs,str,len) fwrite(str,1,len,fs)
-#define __argp_fmtstream_printf fprintf
-#define argp_fmtstream_printf fprintf
-
-#define __argp_fmtstream_lmargin line_wrap_lmargin
-#define argp_fmtstream_lmargin line_wrap_lmargin
-#define __argp_fmtstream_set_lmargin line_wrap_set_lmargin
-#define argp_fmtstream_set_lmargin line_wrap_set_lmargin
-#define __argp_fmtstream_rmargin line_wrap_rmargin
-#define argp_fmtstream_rmargin line_wrap_rmargin
-#define __argp_fmtstream_set_rmargin line_wrap_set_rmargin
-#define argp_fmtstream_set_rmargin line_wrap_set_rmargin
-#define __argp_fmtstream_wmargin line_wrap_wmargin
-#define argp_fmtstream_wmargin line_wrap_wmargin
-#define __argp_fmtstream_set_wmargin line_wrap_set_wmargin
-#define argp_fmtstream_set_wmargin line_wrap_set_wmargin
-#define __argp_fmtstream_point line_wrap_point
-#define argp_fmtstream_point line_wrap_point
-
-#else /* !ARGP_FMTSTREAM_USE_LINEWRAP */
-/* Guess we have to define our own version.  */
-
-#ifndef __const
-#define __const const
-#endif
-
-
-struct argp_fmtstream
-{
-  FILE *stream;			/* The stream we're outputting to.  */
-
-  size_t lmargin, rmargin;	/* Left and right margins.  */
-  ssize_t wmargin;		/* Margin to wrap to, or -1 to truncate.  */
-
-  /* Point in buffer to which we've processed for wrapping, but not output.  */
-  size_t point_offs;
-  /* Output column at POINT_OFFS, or -1 meaning 0 but don't add lmargin.  */
-  ssize_t point_col;
-
-  char *buf;			/* Output buffer.  */
-  char *p;			/* Current end of text in BUF. */
-  char *end;			/* Absolute end of BUF.  */
-};
-
-typedef struct argp_fmtstream *argp_fmtstream_t;
-
-/* Return an argp_fmtstream that outputs to STREAM, and which prefixes lines
-   written on it with LMARGIN spaces and limits them to RMARGIN columns
-   total.  If WMARGIN >= 0, words that extend past RMARGIN are wrapped by
-   replacing the whitespace before them with a newline and WMARGIN spaces.
-   Otherwise, chars beyond RMARGIN are simply dropped until a newline.
-   Returns NULL if there was an error.  */
-extern argp_fmtstream_t __argp_make_fmtstream (FILE *__stream,
-					       size_t __lmargin,
-					       size_t __rmargin,
-					       ssize_t __wmargin);
-extern argp_fmtstream_t argp_make_fmtstream (FILE *__stream,
-					     size_t __lmargin,
-					     size_t __rmargin,
-					     ssize_t __wmargin);
-
-/* Flush __FS to its stream, and free it (but don't close the stream).  */
-extern void __argp_fmtstream_free (argp_fmtstream_t __fs);
-extern void argp_fmtstream_free (argp_fmtstream_t __fs);
-
-extern ssize_t __argp_fmtstream_printf (argp_fmtstream_t __fs,
-				       __const char *__fmt, ...)
-     PRINTF_STYLE(2,3);
-extern ssize_t argp_fmtstream_printf (argp_fmtstream_t __fs,
-				      __const char *__fmt, ...)
-     PRINTF_STYLE(2,3);
-
-extern int __argp_fmtstream_putc (argp_fmtstream_t __fs, int __ch);
-extern int argp_fmtstream_putc (argp_fmtstream_t __fs, int __ch);
-
-extern int __argp_fmtstream_puts (argp_fmtstream_t __fs, __const char *__str);
-extern int argp_fmtstream_puts (argp_fmtstream_t __fs, __const char *__str);
-
-extern size_t __argp_fmtstream_write (argp_fmtstream_t __fs,
-				      __const char *__str, size_t __len);
-extern size_t argp_fmtstream_write (argp_fmtstream_t __fs,
-				    __const char *__str, size_t __len);
-
-/* Access macros for various bits of state.  */
-#define argp_fmtstream_lmargin(__fs) ((__fs)->lmargin)
-#define argp_fmtstream_rmargin(__fs) ((__fs)->rmargin)
-#define argp_fmtstream_wmargin(__fs) ((__fs)->wmargin)
-#define __argp_fmtstream_lmargin argp_fmtstream_lmargin
-#define __argp_fmtstream_rmargin argp_fmtstream_rmargin
-#define __argp_fmtstream_wmargin argp_fmtstream_wmargin
-
-/* Set __FS's left margin to LMARGIN and return the old value.  */
-extern size_t argp_fmtstream_set_lmargin (argp_fmtstream_t __fs,
-					  size_t __lmargin);
-extern size_t __argp_fmtstream_set_lmargin (argp_fmtstream_t __fs,
-					    size_t __lmargin);
-
-/* Set __FS's right margin to __RMARGIN and return the old value.  */
-extern size_t argp_fmtstream_set_rmargin (argp_fmtstream_t __fs,
-					  size_t __rmargin);
-extern size_t __argp_fmtstream_set_rmargin (argp_fmtstream_t __fs,
-					    size_t __rmargin);
-
-/* Set __FS's wrap margin to __WMARGIN and return the old value.  */
-extern size_t argp_fmtstream_set_wmargin (argp_fmtstream_t __fs,
-					  size_t __wmargin);
-extern size_t __argp_fmtstream_set_wmargin (argp_fmtstream_t __fs,
-					    size_t __wmargin);
-
-/* Return the column number of the current output point in __FS.  */
-extern size_t argp_fmtstream_point (argp_fmtstream_t __fs);
-extern size_t __argp_fmtstream_point (argp_fmtstream_t __fs);
-
-/* Internal routines.  */
-extern void _argp_fmtstream_update (argp_fmtstream_t __fs);
-extern void __argp_fmtstream_update (argp_fmtstream_t __fs);
-extern int _argp_fmtstream_ensure (argp_fmtstream_t __fs, size_t __amount);
-extern int __argp_fmtstream_ensure (argp_fmtstream_t __fs, size_t __amount);
-
-#ifdef __OPTIMIZE__
-/* Inline versions of above routines.  */
-
-#if !_LIBC
-#define __argp_fmtstream_putc argp_fmtstream_putc
-#define __argp_fmtstream_puts argp_fmtstream_puts
-#define __argp_fmtstream_write argp_fmtstream_write
-#define __argp_fmtstream_set_lmargin argp_fmtstream_set_lmargin
-#define __argp_fmtstream_set_rmargin argp_fmtstream_set_rmargin
-#define __argp_fmtstream_set_wmargin argp_fmtstream_set_wmargin
-#define __argp_fmtstream_point argp_fmtstream_point
-#define __argp_fmtstream_update _argp_fmtstream_update
-#define __argp_fmtstream_ensure _argp_fmtstream_ensure
-#endif
-
-#ifndef ARGP_FS_EI
-#define ARGP_FS_EI extern inline
-#endif
-
-ARGP_FS_EI size_t
-__argp_fmtstream_write (argp_fmtstream_t __fs,
-			__const char *__str, size_t __len)
-{
-  if (__fs->p + __len <= __fs->end || __argp_fmtstream_ensure (__fs, __len))
-    {
-      memcpy (__fs->p, __str, __len);
-      __fs->p += __len;
-      return __len;
-    }
-  else
-    return 0;
-}
-
-ARGP_FS_EI int
-__argp_fmtstream_puts (argp_fmtstream_t __fs, __const char *__str)
-{
-  size_t __len = strlen (__str);
-  if (__len)
-    {
-      size_t __wrote = __argp_fmtstream_write (__fs, __str, __len);
-      return __wrote == __len ? 0 : -1;
-    }
-  else
-    return 0;
-}
-
-ARGP_FS_EI int
-__argp_fmtstream_putc (argp_fmtstream_t __fs, int __ch)
-{
-  if (__fs->p < __fs->end || __argp_fmtstream_ensure (__fs, 1))
-    return *__fs->p++ = __ch;
-  else
-    return EOF;
-}
-
-/* Set __FS's left margin to __LMARGIN and return the old value.  */
-ARGP_FS_EI size_t
-__argp_fmtstream_set_lmargin (argp_fmtstream_t __fs, size_t __lmargin)
-{
-  size_t __old;
-  if ((size_t) (__fs->p - __fs->buf) > __fs->point_offs)
-    __argp_fmtstream_update (__fs);
-  __old = __fs->lmargin;
-  __fs->lmargin = __lmargin;
-  return __old;
-}
-
-/* Set __FS's right margin to __RMARGIN and return the old value.  */
-ARGP_FS_EI size_t
-__argp_fmtstream_set_rmargin (argp_fmtstream_t __fs, size_t __rmargin)
-{
-  size_t __old;
-  if ((size_t) (__fs->p - __fs->buf) > __fs->point_offs)
-    __argp_fmtstream_update (__fs);
-  __old = __fs->rmargin;
-  __fs->rmargin = __rmargin;
-  return __old;
-}
-
-/* Set FS's wrap margin to __WMARGIN and return the old value.  */
-ARGP_FS_EI size_t
-__argp_fmtstream_set_wmargin (argp_fmtstream_t __fs, size_t __wmargin)
-{
-  size_t __old;
-  if ((size_t) (__fs->p - __fs->buf) > __fs->point_offs)
-    __argp_fmtstream_update (__fs);
-  __old = __fs->wmargin;
-  __fs->wmargin = __wmargin;
-  return __old;
-}
-
-/* Return the column number of the current output point in __FS.  */
-ARGP_FS_EI size_t
-__argp_fmtstream_point (argp_fmtstream_t __fs)
-{
-  if ((size_t) (__fs->p - __fs->buf) > __fs->point_offs)
-    __argp_fmtstream_update (__fs);
-  return __fs->point_col >= 0 ? __fs->point_col : 0;
-}
-
-#if !_LIBC
-#undef __argp_fmtstream_putc
-#undef __argp_fmtstream_puts
-#undef __argp_fmtstream_write
-#undef __argp_fmtstream_set_lmargin
-#undef __argp_fmtstream_set_rmargin
-#undef __argp_fmtstream_set_wmargin
-#undef __argp_fmtstream_point
-#undef __argp_fmtstream_update
-#undef __argp_fmtstream_ensure
-#endif
-
-#endif /* __OPTIMIZE__ */
-
-#endif /* ARGP_FMTSTREAM_USE_LINEWRAP */
-
-#endif /* argp-fmtstream.h */
diff --git a/argp-standalone/argp-help.c b/argp-standalone/argp-help.c
deleted file mode 100644
index ced78c4cb26..00000000000
--- a/argp-standalone/argp-help.c
+++ /dev/null
@@ -1,1849 +0,0 @@
-/* Hierarchial argument parsing help output
-   Copyright (C) 1995,96,97,98,99,2000, 2003 Free Software Foundation, Inc.
-   This file is part of the GNU C Library.
-   Written by Miles Bader <miles@gnu.ai.mit.edu>.
-
-   The GNU C Library is free software; you can redistribute it and/or
-   modify it under the terms of the GNU Library General Public License as
-   published by the Free Software Foundation; either version 2 of the
-   License, or (at your option) any later version.
-
-   The GNU C Library is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Library General Public License for more details.
-
-   You should have received a copy of the GNU Library General Public
-   License along with the GNU C Library; see the file COPYING.LIB.  If not,
-   write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330,
-   Boston, MA 02111-1307, USA.  */
-
-#ifndef _GNU_SOURCE
-# define _GNU_SOURCE	1
-#endif
-
-#ifdef HAVE_CONFIG_H
-#include <config.h>
-#endif
-
-#if HAVE_ALLOCA_H
-#include <alloca.h>
-#endif
-
-#include <stddef.h>
-#include <stdlib.h>
-#include <string.h>
-#include <assert.h>
-#include <stdarg.h>
-#include <ctype.h>
-#if HAVE_MALLOC_H
-/* Needed, for alloca on windows */
-# include <malloc.h>
-#endif
-
-#ifndef _
-/* This is for other GNU distributions with internationalized messages.  */
-# if defined HAVE_LIBINTL_H || defined _LIBC
-#  include <libintl.h>
-#  ifdef _LIBC
-#   undef dgettext
-#   define dgettext(domain, msgid) __dcgettext (domain, msgid, LC_MESSAGES)
-#  endif
-# else
-#  define dgettext(domain, msgid) (msgid)
-# endif
-#endif
-
-#include "argp.h"
-#include "argp-fmtstream.h"
-#include "argp-namefrob.h"
-
-
-#ifndef _LIBC
-# ifndef __strchrnul
-#  define __strchrnul strchrnul
-# endif
-# ifndef __mempcpy
-#  define __mempcpy mempcpy
-# endif
-/* We need to use a different name, as __strndup is likely a macro. */
-# define STRNDUP strndup
-# if HAVE_STRERROR
-#  define STRERROR strerror
-# else
-#  define STRERROR(x) (sys_errlist[x])
-# endif
-#else /* _LIBC */
-# define FLOCKFILE __flockfile
-# define FUNLOCKFILE __funlockfile
-# define STRNDUP __strndup
-# define STRERROR strerror
-#endif
-
-#if !_LIBC
-# if !HAVE_STRNDUP
-char *strndup (const char *s, size_t size);
-# endif /* !HAVE_STRNDUP */
-
-# if !HAVE_MEMPCPY
-void *mempcpy (void *to, const void *from, size_t size);
-# endif /* !HAVE_MEMPCPY */
-
-# if !HAVE_STRCHRNUL
-char *strchrnul(const char *s, int c);
-# endif /* !HAVE_STRCHRNUL */
-
-# if !HAVE_STRCASECMP
-int strcasecmp(const char *s1, const char *s2);
-#endif
-
-#endif /* !_LIBC */
-
-
-/* User-selectable (using an environment variable) formatting parameters.
-
-   These may be specified in an environment variable called `ARGP_HELP_FMT',
-   with a contents like:  VAR1=VAL1,VAR2=VAL2,BOOLVAR2,no-BOOLVAR2
-   Where VALn must be a positive integer.  The list of variables is in the
-   UPARAM_NAMES vector, below.  */
-
-/* Default parameters.  */
-#define DUP_ARGS      0		/* True if option argument can be duplicated. */
-#define DUP_ARGS_NOTE 1		/* True to print a note about duplicate args. */
-#define SHORT_OPT_COL 2		/* column in which short options start */
-#define LONG_OPT_COL  6		/* column in which long options start */
-#define DOC_OPT_COL   2		/* column in which doc options start */
-#define OPT_DOC_COL  29		/* column in which option text starts */
-#define HEADER_COL    1		/* column in which group headers are printed */
-#define USAGE_INDENT 12		/* indentation of wrapped usage lines */
-#define RMARGIN      79		/* right margin used for wrapping */
-
-/* User-selectable (using an environment variable) formatting parameters.
-   They must all be of type `int' for the parsing code to work.  */
-struct uparams
-{
-  /* If true, arguments for an option are shown with both short and long
-     options, even when a given option has both, e.g. `-x ARG, --longx=ARG'.
-     If false, then if an option has both, the argument is only shown with
-     the long one, e.g., `-x, --longx=ARG', and a message indicating that
-     this really means both is printed below the options.  */
-  int dup_args;
-
-  /* This is true if when DUP_ARGS is false, and some duplicate arguments have
-     been suppressed, an explanatory message should be printed.  */
-  int dup_args_note;
-
-  /* Various output columns.  */
-  int short_opt_col;
-  int long_opt_col;
-  int doc_opt_col;
-  int opt_doc_col;
-  int header_col;
-  int usage_indent;
-  int rmargin;
-
-  int valid;			/* True when the values in here are valid.  */
-};
-
-/* This is a global variable, as user options are only ever read once.  */
-static struct uparams uparams = {
-  DUP_ARGS, DUP_ARGS_NOTE,
-  SHORT_OPT_COL, LONG_OPT_COL, DOC_OPT_COL, OPT_DOC_COL, HEADER_COL,
-  USAGE_INDENT, RMARGIN,
-  0
-};
-
-/* A particular uparam, and what the user name is.  */
-struct uparam_name
-{
-  const char *name;		/* User name.  */
-  int is_bool;			/* Whether it's `boolean'.  */
-  size_t uparams_offs;		/* Location of the (int) field in UPARAMS.  */
-};
-
-/* The name-field mappings we know about.  */
-static const struct uparam_name uparam_names[] =
-{
-  { "dup-args",       1, offsetof (struct uparams, dup_args) },
-  { "dup-args-note",  1, offsetof (struct uparams, dup_args_note) },
-  { "short-opt-col",  0, offsetof (struct uparams, short_opt_col) },
-  { "long-opt-col",   0, offsetof (struct uparams, long_opt_col) },
-  { "doc-opt-col",    0, offsetof (struct uparams, doc_opt_col) },
-  { "opt-doc-col",    0, offsetof (struct uparams, opt_doc_col) },
-  { "header-col",     0, offsetof (struct uparams, header_col) },
-  { "usage-indent",   0, offsetof (struct uparams, usage_indent) },
-  { "rmargin",        0, offsetof (struct uparams, rmargin) },
-  { 0, 0, 0 }
-};
-
-/* Read user options from the environment, and fill in UPARAMS appropiately.  */
-static void
-fill_in_uparams (const struct argp_state *state)
-{
-
-  const char *var = getenv ("ARGP_HELP_FMT");
-
-#define SKIPWS(p) do { while (isspace (*p)) p++; } while (0);
-
-  if (var)
-    /* Parse var. */
-    while (*var)
-      {
-	SKIPWS (var);
-
-	if (isalpha (*var))
-	  {
-	    size_t var_len;
-	    const struct uparam_name *un;
-	    int unspec = 0, val = 0;
-	    const char *arg = var;
-
-	    while (isalnum (*arg) || *arg == '-' || *arg == '_')
-	      arg++;
-	    var_len = arg - var;
-
-	    SKIPWS (arg);
-
-	    if (*arg == '\0' || *arg == ',')
-	      unspec = 1;
-	    else if (*arg == '=')
-	      {
-		arg++;
-		SKIPWS (arg);
-	      }
-
-	    if (unspec)
-	      {
-		if (var[0] == 'n' && var[1] == 'o' && var[2] == '-')
-		  {
-		    val = 0;
-		    var += 3;
-		    var_len -= 3;
-		  }
-		else
-		  val = 1;
-	      }
-	    else if (isdigit (*arg))
-	      {
-		val = atoi (arg);
-		while (isdigit (*arg))
-		  arg++;
-		SKIPWS (arg);
-	      }
-
-	    for (un = uparam_names; un->name; un++)
-	      if (strlen (un->name) == var_len
-		  && strncmp (var, un->name, var_len) == 0)
-		{
-		  if (unspec && !un->is_bool)
-		    __argp_failure (state, 0, 0,
-				    dgettext (state->root_argp->argp_domain, "\
-%.*s: ARGP_HELP_FMT parameter requires a value"),
-				    (int) var_len, var);
-		  else
-		    *(int *)((char *)&uparams + un->uparams_offs) = val;
-		  break;
-		}
-	    if (! un->name)
-	      __argp_failure (state, 0, 0,
-			      dgettext (state->root_argp->argp_domain, "\
-%.*s: Unknown ARGP_HELP_FMT parameter"),
-			      (int) var_len, var);
-
-	    var = arg;
-	    if (*var == ',')
-	      var++;
-	  }
-	else if (*var)
-	  {
-	    __argp_failure (state, 0, 0,
-			    dgettext (state->root_argp->argp_domain,
-				      "Garbage in ARGP_HELP_FMT: %s"), var);
-	    break;
-	  }
-      }
-}
-
-/* Returns true if OPT hasn't been marked invisible.  Visibility only affects
-   whether OPT is displayed or used in sorting, not option shadowing.  */
-#define ovisible(opt) (! ((opt)->flags & OPTION_HIDDEN))
-
-/* Returns true if OPT is an alias for an earlier option.  */
-#define oalias(opt) ((opt)->flags & OPTION_ALIAS)
-
-/* Returns true if OPT is an documentation-only entry.  */
-#define odoc(opt) ((opt)->flags & OPTION_DOC)
-
-/* Returns true if OPT is the end-of-list marker for a list of options.  */
-#define oend(opt) __option_is_end (opt)
-
-/* Returns true if OPT has a short option.  */
-#define oshort(opt) __option_is_short (opt)
-
-/*
-   The help format for a particular option is like:
-
-     -xARG, -yARG, --long1=ARG, --long2=ARG        Documentation...
-
-   Where ARG will be omitted if there's no argument, for this option, or
-   will be surrounded by "[" and "]" appropiately if the argument is
-   optional.  The documentation string is word-wrapped appropiately, and if
-   the list of options is long enough, it will be started on a separate line.
-   If there are no short options for a given option, the first long option is
-   indented slighly in a way that's supposed to make most long options appear
-   to be in a separate column.
-
-   For example, the following output (from ps):
-
-     -p PID, --pid=PID          List the process PID
-	 --pgrp=PGRP            List processes in the process group PGRP
-     -P, -x, --no-parent        Include processes without parents
-     -Q, --all-fields           Don't elide unusable fields (normally if there's
-				some reason ps can't print a field for any
-				process, it's removed from the output entirely)
-     -r, --reverse, --gratuitously-long-reverse-option
-				Reverse the order of any sort
-	 --session[=SID]        Add the processes from the session SID (which
-				defaults to the sid of the current process)
-
-    Here are some more options:
-     -f ZOT, --foonly=ZOT       Glork a foonly
-     -z, --zaza                 Snit a zar
-
-     -?, --help                 Give this help list
-	 --usage                Give a short usage message
-     -V, --version              Print program version
-
-   The struct argp_option array for the above could look like:
-
-   {
-     {"pid",       'p',      "PID",  0, "List the process PID"},
-     {"pgrp",      OPT_PGRP, "PGRP", 0, "List processes in the process group PGRP"},
-     {"no-parent", 'P',	      0,     0, "Include processes without parents"},
-     {0,           'x',       0,     OPTION_ALIAS},
-     {"all-fields",'Q',       0,     0, "Don't elide unusable fields (normally"
-                                        " if there's some reason ps can't"
-					" print a field for any process, it's"
-                                        " removed from the output entirely)" },
-     {"reverse",   'r',       0,     0, "Reverse the order of any sort"},
-     {"gratuitously-long-reverse-option", 0, 0, OPTION_ALIAS},
-     {"session",   OPT_SESS,  "SID", OPTION_ARG_OPTIONAL,
-                                        "Add the processes from the session"
-					" SID (which defaults to the sid of"
-					" the current process)" },
-
-     {0,0,0,0, "Here are some more options:"},
-     {"foonly", 'f', "ZOT", 0, "Glork a foonly"},
-     {"zaza", 'z', 0, 0, "Snit a zar"},
-
-     {0}
-   }
-
-   Note that the last three options are automatically supplied by argp_parse,
-   unless you tell it not to with ARGP_NO_HELP.
-
-*/
-
-/* Returns true if CH occurs between BEG and END.  */
-static int
-find_char (char ch, char *beg, char *end)
-{
-  while (beg < end)
-    if (*beg == ch)
-      return 1;
-    else
-      beg++;
-  return 0;
-}
-
-struct hol_cluster;		/* fwd decl */
-
-struct hol_entry
-{
-  /* First option.  */
-  const struct argp_option *opt;
-  /* Number of options (including aliases).  */
-  unsigned num;
-
-  /* A pointers into the HOL's short_options field, to the first short option
-     letter for this entry.  The order of the characters following this point
-     corresponds to the order of options pointed to by OPT, and there are at
-     most NUM.  A short option recorded in a option following OPT is only
-     valid if it occurs in the right place in SHORT_OPTIONS (otherwise it's
-     probably been shadowed by some other entry).  */
-  char *short_options;
-
-  /* Entries are sorted by their group first, in the order:
-       1, 2, ..., n, 0, -m, ..., -2, -1
-     and then alphabetically within each group.  The default is 0.  */
-  int group;
-
-  /* The cluster of options this entry belongs to, or 0 if none.  */
-  struct hol_cluster *cluster;
-
-  /* The argp from which this option came.  */
-  const struct argp *argp;
-};
-
-/* A cluster of entries to reflect the argp tree structure.  */
-struct hol_cluster
-{
-  /* A descriptive header printed before options in this cluster.  */
-  const char *header;
-
-  /* Used to order clusters within the same group with the same parent,
-     according to the order in which they occurred in the parent argp's child
-     list.  */
-  int index;
-
-  /* How to sort this cluster with respect to options and other clusters at the
-     same depth (clusters always follow options in the same group).  */
-  int group;
-
-  /* The cluster to which this cluster belongs, or 0 if it's at the base
-     level.  */
-  struct hol_cluster *parent;
-
-  /* The argp from which this cluster is (eventually) derived.  */
-  const struct argp *argp;
-
-  /* The distance this cluster is from the root.  */
-  int depth;
-
-  /* Clusters in a given hol are kept in a linked list, to make freeing them
-     possible.  */
-  struct hol_cluster *next;
-};
-
-/* A list of options for help.  */
-struct hol
-{
-  /* An array of hol_entry's.  */
-  struct hol_entry *entries;
-  /* The number of entries in this hol.  If this field is zero, the others
-     are undefined.  */
-  unsigned num_entries;
-
-  /* A string containing all short options in this HOL.  Each entry contains
-     pointers into this string, so the order can't be messed with blindly.  */
-  char *short_options;
-
-  /* Clusters of entries in this hol.  */
-  struct hol_cluster *clusters;
-};
-
-/* Create a struct hol from the options in ARGP.  CLUSTER is the
-   hol_cluster in which these entries occur, or 0, if at the root.  */
-static struct hol *
-make_hol (const struct argp *argp, struct hol_cluster *cluster)
-{
-  char *so;
-  const struct argp_option *o;
-  const struct argp_option *opts = argp->options;
-  struct hol_entry *entry;
-  unsigned num_short_options = 0;
-  struct hol *hol = malloc (sizeof (struct hol));
-
-  assert (hol);
-
-  hol->num_entries = 0;
-  hol->clusters = 0;
-
-  if (opts)
-    {
-      int cur_group = 0;
-
-      /* The first option must not be an alias.  */
-      assert (! oalias (opts));
-
-      /* Calculate the space needed.  */
-      for (o = opts; ! oend (o); o++)
-	{
-	  if (! oalias (o))
-	    hol->num_entries++;
-	  if (oshort (o))
-	    num_short_options++;	/* This is an upper bound.  */
-	}
-
-      hol->entries = malloc (sizeof (struct hol_entry) * hol->num_entries);
-      hol->short_options = malloc (num_short_options + 1);
-
-      assert (hol->entries && hol->short_options);
-
-      /* Fill in the entries.  */
-      so = hol->short_options;
-      for (o = opts, entry = hol->entries; ! oend (o); entry++)
-	{
-	  entry->opt = o;
-	  entry->num = 0;
-	  entry->short_options = so;
-	  entry->group = cur_group =
-	    o->group
-	    ? o->group
-	    : ((!o->name && !o->key)
-	       ? cur_group + 1
-	       : cur_group);
-	  entry->cluster = cluster;
-	  entry->argp = argp;
-
-	  do
-	    {
-	      entry->num++;
-	      if (oshort (o) && ! find_char (o->key, hol->short_options, so))
-		/* O has a valid short option which hasn't already been used.*/
-		*so++ = o->key;
-	      o++;
-	    }
-	  while (! oend (o) && oalias (o));
-	}
-      *so = '\0';		/* null terminated so we can find the length */
-    }
-
-  return hol;
-}
-
-/* Add a new cluster to HOL, with the given GROUP and HEADER (taken from the
-   associated argp child list entry), INDEX, and PARENT, and return a pointer
-   to it.  ARGP is the argp that this cluster results from.  */
-static struct hol_cluster *
-hol_add_cluster (struct hol *hol, int group, const char *header, int index,
-		 struct hol_cluster *parent, const struct argp *argp)
-{
-  struct hol_cluster *cl = malloc (sizeof (struct hol_cluster));
-  if (cl)
-    {
-      cl->group = group;
-      cl->header = header;
-
-      cl->index = index;
-      cl->parent = parent;
-      cl->argp = argp;
-      cl->depth = parent ? parent->depth + 1 : 0;
-
-      cl->next = hol->clusters;
-      hol->clusters = cl;
-    }
-  return cl;
-}
-
-/* Free HOL and any resources it uses.  */
-static void
-hol_free (struct hol *hol)
-{
-  struct hol_cluster *cl = hol->clusters;
-
-  while (cl)
-    {
-      struct hol_cluster *next = cl->next;
-      free (cl);
-      cl = next;
-    }
-
-  if (hol->num_entries > 0)
-    {
-      free (hol->entries);
-      free (hol->short_options);
-    }
-
-  free (hol);
-}
-
-static inline int
-hol_entry_short_iterate (const struct hol_entry *entry,
-			 int (*func)(const struct argp_option *opt,
-				     const struct argp_option *real,
-				     const char *domain, void *cookie),
-			 const char *domain, void *cookie)
-{
-  unsigned nopts;
-  int val = 0;
-  const struct argp_option *opt, *real = entry->opt;
-  char *so = entry->short_options;
-
-  for (opt = real, nopts = entry->num; nopts > 0 && !val; opt++, nopts--)
-    if (oshort (opt) && *so == opt->key)
-      {
-	if (!oalias (opt))
-	  real = opt;
-	if (ovisible (opt))
-	  val = (*func)(opt, real, domain, cookie);
-	so++;
-      }
-
-  return val;
-}
-
-static inline int
-hol_entry_long_iterate (const struct hol_entry *entry,
-			int (*func)(const struct argp_option *opt,
-				    const struct argp_option *real,
-				    const char *domain, void *cookie),
-			const char *domain, void *cookie)
-{
-  unsigned nopts;
-  int val = 0;
-  const struct argp_option *opt, *real = entry->opt;
-
-  for (opt = real, nopts = entry->num; nopts > 0 && !val; opt++, nopts--)
-    if (opt->name)
-      {
-	if (!oalias (opt))
-	  real = opt;
-	if (ovisible (opt))
-	  val = (*func)(opt, real, domain, cookie);
-      }
-
-  return val;
-}
-
-/* Iterator that returns true for the first short option.  */
-static inline int
-until_short (const struct argp_option *opt, const struct argp_option *real UNUSED,
-	     const char *domain UNUSED, void *cookie UNUSED)
-{
-  return oshort (opt) ? opt->key : 0;
-}
-
-/* Returns the first valid short option in ENTRY, or 0 if there is none.  */
-static char
-hol_entry_first_short (const struct hol_entry *entry)
-{
-  return hol_entry_short_iterate (entry, until_short,
-				  entry->argp->argp_domain, 0);
-}
-
-/* Returns the first valid long option in ENTRY, or 0 if there is none.  */
-static const char *
-hol_entry_first_long (const struct hol_entry *entry)
-{
-  const struct argp_option *opt;
-  unsigned num;
-  for (opt = entry->opt, num = entry->num; num > 0; opt++, num--)
-    if (opt->name && ovisible (opt))
-      return opt->name;
-  return 0;
-}
-
-/* Returns the entry in HOL with the long option name NAME, or 0 if there is
-   none.  */
-static struct hol_entry *
-hol_find_entry (struct hol *hol, const char *name)
-{
-  struct hol_entry *entry = hol->entries;
-  unsigned num_entries = hol->num_entries;
-
-  while (num_entries-- > 0)
-    {
-      const struct argp_option *opt = entry->opt;
-      unsigned num_opts = entry->num;
-
-      while (num_opts-- > 0)
-	if (opt->name && ovisible (opt) && strcmp (opt->name, name) == 0)
-	  return entry;
-	else
-	  opt++;
-
-      entry++;
-    }
-
-  return 0;
-}
-
-/* If an entry with the long option NAME occurs in HOL, set it's special
-   sort position to GROUP.  */
-static void
-hol_set_group (struct hol *hol, const char *name, int group)
-{
-  struct hol_entry *entry = hol_find_entry (hol, name);
-  if (entry)
-    entry->group = group;
-}
-
-/* Order by group:  0, 1, 2, ..., n, -m, ..., -2, -1.
-   EQ is what to return if GROUP1 and GROUP2 are the same.  */
-static int
-group_cmp (int group1, int group2, int eq)
-{
-  if (group1 == group2)
-    return eq;
-  else if ((group1 < 0 && group2 < 0) || (group1 >= 0 && group2 >= 0))
-    return group1 - group2;
-  else
-    return group2 - group1;
-}
-
-/* Compare clusters CL1 & CL2 by the order that they should appear in
-   output.  */
-static int
-hol_cluster_cmp (const struct hol_cluster *cl1, const struct hol_cluster *cl2)
-{
-  /* If one cluster is deeper than the other, use its ancestor at the same
-     level, so that finding the common ancestor is straightforward.  */
-  while (cl1->depth < cl2->depth)
-    cl1 = cl1->parent;
-  while (cl2->depth < cl1->depth)
-    cl2 = cl2->parent;
-
-  /* Now reduce both clusters to their ancestors at the point where both have
-     a common parent; these can be directly compared.  */
-  while (cl1->parent != cl2->parent)
-    cl1 = cl1->parent, cl2 = cl2->parent;
-
-  return group_cmp (cl1->group, cl2->group, cl2->index - cl1->index);
-}
-
-/* Return the ancestor of CL that's just below the root (i.e., has a parent
-   of 0).  */
-static struct hol_cluster *
-hol_cluster_base (struct hol_cluster *cl)
-{
-  while (cl->parent)
-    cl = cl->parent;
-  return cl;
-}
-
-/* Return true if CL1 is a child of CL2.  */
-static int
-hol_cluster_is_child (const struct hol_cluster *cl1,
-		      const struct hol_cluster *cl2)
-{
-  while (cl1 && cl1 != cl2)
-    cl1 = cl1->parent;
-  return cl1 == cl2;
-}
-
-/* Given the name of a OPTION_DOC option, modifies NAME to start at the tail
-   that should be used for comparisons, and returns true iff it should be
-   treated as a non-option.  */
-
-/* FIXME: Can we use unsigned char * for the argument? */
-static int
-canon_doc_option (const char **name)
-{
-  int non_opt;
-  /* Skip initial whitespace.  */
-  while (isspace ( (unsigned char) **name))
-    (*name)++;
-  /* Decide whether this looks like an option (leading `-') or not.  */
-  non_opt = (**name != '-');
-  /* Skip until part of name used for sorting.  */
-  while (**name && !isalnum ( (unsigned char) **name))
-    (*name)++;
-  return non_opt;
-}
-
-/* Order ENTRY1 & ENTRY2 by the order which they should appear in a help
-   listing.  */
-static int
-hol_entry_cmp (const struct hol_entry *entry1,
-	       const struct hol_entry *entry2)
-{
-  /* The group numbers by which the entries should be ordered; if either is
-     in a cluster, then this is just the group within the cluster.  */
-  int group1 = entry1->group, group2 = entry2->group;
-
-  if (entry1->cluster != entry2->cluster)
-    {
-      /* The entries are not within the same cluster, so we can't compare them
-	 directly, we have to use the appropiate clustering level too.  */
-      if (! entry1->cluster)
-	/* ENTRY1 is at the `base level', not in a cluster, so we have to
-	   compare it's group number with that of the base cluster in which
-	   ENTRY2 resides.  Note that if they're in the same group, the
-	   clustered option always comes laster.  */
-	return group_cmp (group1, hol_cluster_base (entry2->cluster)->group, -1);
-      else if (! entry2->cluster)
-	/* Likewise, but ENTRY2's not in a cluster.  */
-	return group_cmp (hol_cluster_base (entry1->cluster)->group, group2, 1);
-      else
-	/* Both entries are in clusters, we can just compare the clusters.  */
-	return hol_cluster_cmp (entry1->cluster, entry2->cluster);
-    }
-  else if (group1 == group2)
-    /* The entries are both in the same cluster and group, so compare them
-       alphabetically.  */
-    {
-      int short1 = hol_entry_first_short (entry1);
-      int short2 = hol_entry_first_short (entry2);
-      int doc1 = odoc (entry1->opt);
-      int doc2 = odoc (entry2->opt);
-      /* FIXME: Can we use unsigned char * instead? */
-      const char *long1 = hol_entry_first_long (entry1);
-      const char *long2 = hol_entry_first_long (entry2);
-
-      if (doc1)
-	doc1 = canon_doc_option (&long1);
-      if (doc2)
-	doc2 = canon_doc_option (&long2);
-
-      if (doc1 != doc2)
-	/* `documentation' options always follow normal options (or
-	   documentation options that *look* like normal options).  */
-	return doc1 - doc2;
-      else if (!short1 && !short2 && long1 && long2)
-	/* Only long options.  */
-	return __strcasecmp (long1, long2);
-      else
-	/* Compare short/short, long/short, short/long, using the first
-	   character of long options.  Entries without *any* valid
-	   options (such as options with OPTION_HIDDEN set) will be put
-	   first, but as they're not displayed, it doesn't matter where
-	   they are.  */
-	{
-	  unsigned char first1 = short1 ? short1 : long1 ? *long1 : 0;
-	  unsigned char first2 = short2 ? short2 : long2 ? *long2 : 0;
-#ifdef _tolower
-	  int lower_cmp = _tolower (first1) - _tolower (first2);
-#else
-	  int lower_cmp = tolower (first1) - tolower (first2);
-#endif
-	  /* Compare ignoring case, except when the options are both the
-	     same letter, in which case lower-case always comes first.  */
-	  /* NOTE: The subtraction below does the right thing
-	     even with eight-bit chars: first1 and first2 are
-	     converted to int *before* the subtraction. */
-	  return lower_cmp ? lower_cmp : first2 - first1;
-	}
-    }
-  else
-    /* Within the same cluster, but not the same group, so just compare
-       groups.  */
-    return group_cmp (group1, group2, 0);
-}
-
-/* Version of hol_entry_cmp with correct signature for qsort.  */
-static int
-hol_entry_qcmp (const void *entry1_v, const void *entry2_v)
-{
-  return hol_entry_cmp (entry1_v, entry2_v);
-}
-
-/* Sort HOL by group and alphabetically by option name (with short options
-   taking precedence over long).  Since the sorting is for display purposes
-   only, the shadowing of options isn't effected.  */
-static void
-hol_sort (struct hol *hol)
-{
-  if (hol->num_entries > 0)
-    qsort (hol->entries, hol->num_entries, sizeof (struct hol_entry),
-	   hol_entry_qcmp);
-}
-
-/* Append MORE to HOL, destroying MORE in the process.  Options in HOL shadow
-   any in MORE with the same name.  */
-static void
-hol_append (struct hol *hol, struct hol *more)
-{
-  struct hol_cluster **cl_end = &hol->clusters;
-
-  /* Steal MORE's cluster list, and add it to the end of HOL's.  */
-  while (*cl_end)
-    cl_end = &(*cl_end)->next;
-  *cl_end = more->clusters;
-  more->clusters = 0;
-
-  /* Merge entries.  */
-  if (more->num_entries > 0)
-    {
-      if (hol->num_entries == 0)
-	{
-	  hol->num_entries = more->num_entries;
-	  hol->entries = more->entries;
-	  hol->short_options = more->short_options;
-	  more->num_entries = 0;	/* Mark MORE's fields as invalid.  */
-	}
-      else
-	/* Append the entries in MORE to those in HOL, taking care to only add
-	   non-shadowed SHORT_OPTIONS values.  */
-	{
-	  unsigned left;
-	  char *so, *more_so;
-	  struct hol_entry *e;
-	  unsigned num_entries = hol->num_entries + more->num_entries;
-	  struct hol_entry *entries =
-	    malloc (num_entries * sizeof (struct hol_entry));
-	  unsigned hol_so_len = strlen (hol->short_options);
-	  char *short_options =
-	    malloc (hol_so_len + strlen (more->short_options) + 1);
-
-	  __mempcpy (__mempcpy (entries, hol->entries,
-				hol->num_entries * sizeof (struct hol_entry)),
-		     more->entries,
-		     more->num_entries * sizeof (struct hol_entry));
-
-	  __mempcpy (short_options, hol->short_options, hol_so_len);
-
-	  /* Fix up the short options pointers from HOL.  */
-	  for (e = entries, left = hol->num_entries; left > 0; e++, left--)
-	    e->short_options += (short_options - hol->short_options);
-
-	  /* Now add the short options from MORE, fixing up its entries
-	     too.  */
-	  so = short_options + hol_so_len;
-	  more_so = more->short_options;
-	  for (left = more->num_entries; left > 0; e++, left--)
-	    {
-	      int opts_left;
-	      const struct argp_option *opt;
-
-	      e->short_options = so;
-
-	      for (opts_left = e->num, opt = e->opt; opts_left; opt++, opts_left--)
-		{
-		  int ch = *more_so;
-		  if (oshort (opt) && ch == opt->key)
-		    /* The next short option in MORE_SO, CH, is from OPT.  */
-		    {
-		      if (! find_char (ch, short_options,
-				       short_options + hol_so_len))
-			/* The short option CH isn't shadowed by HOL's options,
-			   so add it to the sum.  */
-			*so++ = ch;
-		      more_so++;
-		    }
-		}
-	    }
-
-	  *so = '\0';
-
-	  free (hol->entries);
-	  free (hol->short_options);
-
-	  hol->entries = entries;
-	  hol->num_entries = num_entries;
-	  hol->short_options = short_options;
-	}
-    }
-
-  hol_free (more);
-}
-
-/* Inserts enough spaces to make sure STREAM is at column COL.  */
-static void
-indent_to (argp_fmtstream_t stream, unsigned col)
-{
-  int needed = col - __argp_fmtstream_point (stream);
-  while (needed-- > 0)
-    __argp_fmtstream_putc (stream, ' ');
-}
-
-/* Output to STREAM either a space, or a newline if there isn't room for at
-   least ENSURE characters before the right margin.  */
-static void
-space (argp_fmtstream_t stream, size_t ensure)
-{
-  if (__argp_fmtstream_point (stream) + ensure
-      >= __argp_fmtstream_rmargin (stream))
-    __argp_fmtstream_putc (stream, '\n');
-  else
-    __argp_fmtstream_putc (stream, ' ');
-}
-
-/* If the option REAL has an argument, we print it in using the printf
-   format REQ_FMT or OPT_FMT depending on whether it's a required or
-   optional argument.  */
-static void
-arg (const struct argp_option *real, const char *req_fmt, const char *opt_fmt,
-     const char *domain UNUSED, argp_fmtstream_t stream)
-{
-  if (real->arg)
-    {
-      if (real->flags & OPTION_ARG_OPTIONAL)
-	__argp_fmtstream_printf (stream, opt_fmt,
-				 dgettext (domain, real->arg));
-      else
-	__argp_fmtstream_printf (stream, req_fmt,
-				 dgettext (domain, real->arg));
-    }
-}
-
-/* Helper functions for hol_entry_help.  */
-
-/* State used during the execution of hol_help.  */
-struct hol_help_state
-{
-  /* PREV_ENTRY should contain the previous entry printed, or 0.  */
-  struct hol_entry *prev_entry;
-
-  /* If an entry is in a different group from the previous one, and SEP_GROUPS
-     is true, then a blank line will be printed before any output. */
-  int sep_groups;
-
-  /* True if a duplicate option argument was suppressed (only ever set if
-     UPARAMS.dup_args is false).  */
-  int suppressed_dup_arg;
-};
-
-/* Some state used while printing a help entry (used to communicate with
-   helper functions).  See the doc for hol_entry_help for more info, as most
-   of the fields are copied from its arguments.  */
-struct pentry_state
-{
-  const struct hol_entry *entry;
-  argp_fmtstream_t stream;
-  struct hol_help_state *hhstate;
-
-  /* True if nothing's been printed so far.  */
-  int first;
-
-  /* If non-zero, the state that was used to print this help.  */
-  const struct argp_state *state;
-};
-
-/* If a user doc filter should be applied to DOC, do so.  */
-static const char *
-filter_doc (const char *doc, int key, const struct argp *argp,
-	    const struct argp_state *state)
-{
-  if (argp->help_filter)
-    /* We must apply a user filter to this output.  */
-    {
-      void *input = __argp_input (argp, state);
-      return (*argp->help_filter) (key, doc, input);
-    }
-  else
-    /* No filter.  */
-    return doc;
-}
-
-/* Prints STR as a header line, with the margin lines set appropiately, and
-   notes the fact that groups should be separated with a blank line.  ARGP is
-   the argp that should dictate any user doc filtering to take place.  Note
-   that the previous wrap margin isn't restored, but the left margin is reset
-   to 0.  */
-static void
-print_header (const char *str, const struct argp *argp,
-	      struct pentry_state *pest)
-{
-  const char *tstr = dgettext (argp->argp_domain, str);
-  const char *fstr = filter_doc (tstr, ARGP_KEY_HELP_HEADER, argp, pest->state);
-
-  if (fstr)
-    {
-      if (*fstr)
-	{
-	  if (pest->hhstate->prev_entry)
-	    /* Precede with a blank line.  */
-	    __argp_fmtstream_putc (pest->stream, '\n');
-	  indent_to (pest->stream, uparams.header_col);
-	  __argp_fmtstream_set_lmargin (pest->stream, uparams.header_col);
-	  __argp_fmtstream_set_wmargin (pest->stream, uparams.header_col);
-	  __argp_fmtstream_puts (pest->stream, fstr);
-	  __argp_fmtstream_set_lmargin (pest->stream, 0);
-	  __argp_fmtstream_putc (pest->stream, '\n');
-	}
-
-      pest->hhstate->sep_groups = 1; /* Separate subsequent groups. */
-    }
-
-  if (fstr != tstr)
-    free ((char *) fstr);
-}
-
-/* Inserts a comma if this isn't the first item on the line, and then makes
-   sure we're at least to column COL.  If this *is* the first item on a line,
-   prints any pending whitespace/headers that should precede this line. Also
-   clears FIRST.  */
-static void
-comma (unsigned col, struct pentry_state *pest)
-{
-  if (pest->first)
-    {
-      const struct hol_entry *pe = pest->hhstate->prev_entry;
-      const struct hol_cluster *cl = pest->entry->cluster;
-
-      if (pest->hhstate->sep_groups && pe && pest->entry->group != pe->group)
-	__argp_fmtstream_putc (pest->stream, '\n');
-
-      if (cl && cl->header && *cl->header
-	  && (!pe
-	      || (pe->cluster != cl
-		  && !hol_cluster_is_child (pe->cluster, cl))))
-	/* If we're changing clusters, then this must be the start of the
-	   ENTRY's cluster unless that is an ancestor of the previous one
-	   (in which case we had just popped into a sub-cluster for a bit).
-	   If so, then print the cluster's header line.  */
-	{
-	  int old_wm = __argp_fmtstream_wmargin (pest->stream);
-	  print_header (cl->header, cl->argp, pest);
-	  __argp_fmtstream_set_wmargin (pest->stream, old_wm);
-	}
-
-      pest->first = 0;
-    }
-  else
-    __argp_fmtstream_puts (pest->stream, ", ");
-
-  indent_to (pest->stream, col);
-}
-
-/* Print help for ENTRY to STREAM.  */
-static void
-hol_entry_help (struct hol_entry *entry, const struct argp_state *state,
-		argp_fmtstream_t stream, struct hol_help_state *hhstate)
-{
-  unsigned num;
-  const struct argp_option *real = entry->opt, *opt;
-  char *so = entry->short_options;
-  int have_long_opt = 0;	/* We have any long options.  */
-  /* Saved margins.  */
-  int old_lm = __argp_fmtstream_set_lmargin (stream, 0);
-  int old_wm = __argp_fmtstream_wmargin (stream);
-  /* PEST is a state block holding some of our variables that we'd like to
-     share with helper functions.  */
-
-  /* Decent initializers are a GNU extension, so don't use it here. */
-  struct pentry_state pest;
-  pest.entry = entry;
-  pest.stream = stream;
-  pest.hhstate = hhstate;
-  pest.first = 1;
-  pest.state = state;
-
-  if (! odoc (real))
-    for (opt = real, num = entry->num; num > 0; opt++, num--)
-      if (opt->name && ovisible (opt))
-	{
-	  have_long_opt = 1;
-	  break;
-	}
-
-  /* First emit short options.  */
-  __argp_fmtstream_set_wmargin (stream, uparams.short_opt_col); /* For truly bizarre cases. */
-  for (opt = real, num = entry->num; num > 0; opt++, num--)
-    if (oshort (opt) && opt->key == *so)
-      /* OPT has a valid (non shadowed) short option.  */
-      {
-	if (ovisible (opt))
-	  {
-	    comma (uparams.short_opt_col, &pest);
-	    __argp_fmtstream_putc (stream, '-');
-	    __argp_fmtstream_putc (stream, *so);
-	    if (!have_long_opt || uparams.dup_args)
-	      arg (real, " %s", "[%s]", state->root_argp->argp_domain, stream);
-	    else if (real->arg)
-	      hhstate->suppressed_dup_arg = 1;
-	  }
-	so++;
-      }
-
-  /* Now, long options.  */
-  if (odoc (real))
-    /* A `documentation' option.  */
-    {
-      __argp_fmtstream_set_wmargin (stream, uparams.doc_opt_col);
-      for (opt = real, num = entry->num; num > 0; opt++, num--)
-	if (opt->name && ovisible (opt))
-	  {
-	    comma (uparams.doc_opt_col, &pest);
-	    /* Calling gettext here isn't quite right, since sorting will
-	       have been done on the original; but documentation options
-	       should be pretty rare anyway...  */
-	    __argp_fmtstream_puts (stream,
-				   dgettext (state->root_argp->argp_domain,
-					     opt->name));
-	  }
-    }
-  else
-    /* A real long option.  */
-    {
-      int first_long_opt = 1;
-
-      __argp_fmtstream_set_wmargin (stream, uparams.long_opt_col);
-      for (opt = real, num = entry->num; num > 0; opt++, num--)
-	if (opt->name && ovisible (opt))
-	  {
-	    comma (uparams.long_opt_col, &pest);
-	    __argp_fmtstream_printf (stream, "--%s", opt->name);
-	    if (first_long_opt || uparams.dup_args)
-	      arg (real, "=%s", "[=%s]", state->root_argp->argp_domain,
-		   stream);
-	    else if (real->arg)
-	      hhstate->suppressed_dup_arg = 1;
-	  }
-    }
-
-  /* Next, documentation strings.  */
-  __argp_fmtstream_set_lmargin (stream, 0);
-
-  if (pest.first)
-    {
-      /* Didn't print any switches, what's up?  */
-      if (!oshort (real) && !real->name)
-	/* This is a group header, print it nicely.  */
-	print_header (real->doc, entry->argp, &pest);
-      else
-	/* Just a totally shadowed option or null header; print nothing.  */
-	goto cleanup;		/* Just return, after cleaning up.  */
-    }
-  else
-    {
-      const char *tstr = real->doc ? dgettext (state->root_argp->argp_domain,
-					       real->doc) : 0;
-      const char *fstr = filter_doc (tstr, real->key, entry->argp, state);
-      if (fstr && *fstr)
-	{
-	  unsigned int col = __argp_fmtstream_point (stream);
-
-	  __argp_fmtstream_set_lmargin (stream, uparams.opt_doc_col);
-	  __argp_fmtstream_set_wmargin (stream, uparams.opt_doc_col);
-
-	  if (col > (unsigned int) (uparams.opt_doc_col + 3))
-	    __argp_fmtstream_putc (stream, '\n');
-	  else if (col >= (unsigned int) uparams.opt_doc_col)
-	    __argp_fmtstream_puts (stream, "   ");
-	  else
-	    indent_to (stream, uparams.opt_doc_col);
-
-	  __argp_fmtstream_puts (stream, fstr);
-	}
-      if (fstr && fstr != tstr)
-	free ((char *) fstr);
-
-      /* Reset the left margin.  */
-      __argp_fmtstream_set_lmargin (stream, 0);
-      __argp_fmtstream_putc (stream, '\n');
-    }
-
-  hhstate->prev_entry = entry;
-
-cleanup:
-  __argp_fmtstream_set_lmargin (stream, old_lm);
-  __argp_fmtstream_set_wmargin (stream, old_wm);
-}
-
-/* Output a long help message about the options in HOL to STREAM.  */
-static void
-hol_help (struct hol *hol, const struct argp_state *state,
-	  argp_fmtstream_t stream)
-{
-  unsigned num;
-  struct hol_entry *entry;
-  struct hol_help_state hhstate = { 0, 0, 0 };
-
-  for (entry = hol->entries, num = hol->num_entries; num > 0; entry++, num--)
-    hol_entry_help (entry, state, stream, &hhstate);
-
-  if (hhstate.suppressed_dup_arg && uparams.dup_args_note)
-    {
-      const char *tstr = dgettext (state->root_argp->argp_domain, "\
-Mandatory or optional arguments to long options are also mandatory or \
-optional for any corresponding short options.");
-      const char *fstr = filter_doc (tstr, ARGP_KEY_HELP_DUP_ARGS_NOTE,
-				     state ? state->root_argp : 0, state);
-      if (fstr && *fstr)
-	{
-	  __argp_fmtstream_putc (stream, '\n');
-	  __argp_fmtstream_puts (stream, fstr);
-	  __argp_fmtstream_putc (stream, '\n');
-	}
-      if (fstr && fstr != tstr)
-	free ((char *) fstr);
-    }
-}
-
-/* Helper functions for hol_usage.  */
-
-/* If OPT is a short option without an arg, append its key to the string
-   pointer pointer to by COOKIE, and advance the pointer.  */
-static int
-add_argless_short_opt (const struct argp_option *opt,
-		       const struct argp_option *real,
-		       const char *domain UNUSED, void *cookie)
-{
-  char **snao_end = cookie;
-  if (!(opt->arg || real->arg)
-      && !((opt->flags | real->flags) & OPTION_NO_USAGE))
-    *(*snao_end)++ = opt->key;
-  return 0;
-}
-
-/* If OPT is a short option with an arg, output a usage entry for it to the
-   stream pointed at by COOKIE.  */
-static int
-usage_argful_short_opt (const struct argp_option *opt,
-			const struct argp_option *real,
-			const char *domain UNUSED, void *cookie)
-{
-  argp_fmtstream_t stream = cookie;
-  const char *arg = opt->arg;
-  int flags = opt->flags | real->flags;
-
-  if (! arg)
-    arg = real->arg;
-
-  if (arg && !(flags & OPTION_NO_USAGE))
-    {
-      arg = dgettext (domain, arg);
-
-      if (flags & OPTION_ARG_OPTIONAL)
-	__argp_fmtstream_printf (stream, " [-%c[%s]]", opt->key, arg);
-      else
-	{
-	  /* Manually do line wrapping so that it (probably) won't
-	     get wrapped at the embedded space.  */
-	  space (stream, 6 + strlen (arg));
-	  __argp_fmtstream_printf (stream, "[-%c %s]", opt->key, arg);
-	}
-    }
-
-  return 0;
-}
-
-/* Output a usage entry for the long option opt to the stream pointed at by
-   COOKIE.  */
-static int
-usage_long_opt (const struct argp_option *opt,
-		const struct argp_option *real,
-		const char *domain UNUSED, void *cookie)
-{
-  argp_fmtstream_t stream = cookie;
-  const char *arg = opt->arg;
-  int flags = opt->flags | real->flags;
-
-  if (! arg)
-    arg = real->arg;
-
-  if (! (flags & OPTION_NO_USAGE))
-    {
-      if (arg)
-	{
-	  arg = dgettext (domain, arg);
-	  if (flags & OPTION_ARG_OPTIONAL)
-	    __argp_fmtstream_printf (stream, " [--%s[=%s]]", opt->name, arg);
-	  else
-	    __argp_fmtstream_printf (stream, " [--%s=%s]", opt->name, arg);
-	}
-      else
-	__argp_fmtstream_printf (stream, " [--%s]", opt->name);
-    }
-
-  return 0;
-}
-
-/* Print a short usage description for the arguments in HOL to STREAM.  */
-static void
-hol_usage (struct hol *hol, argp_fmtstream_t stream)
-{
-  if (hol->num_entries > 0)
-    {
-      unsigned nentries;
-      struct hol_entry *entry;
-      char *short_no_arg_opts = alloca (strlen (hol->short_options) + 1);
-      char *snao_end = short_no_arg_opts;
-
-      /* First we put a list of short options without arguments.  */
-      for (entry = hol->entries, nentries = hol->num_entries
-	   ; nentries > 0
-	   ; entry++, nentries--)
-	hol_entry_short_iterate (entry, add_argless_short_opt,
-				 entry->argp->argp_domain, &snao_end);
-      if (snao_end > short_no_arg_opts)
-	{
-	  *snao_end++ = 0;
-	  __argp_fmtstream_printf (stream, " [-%s]", short_no_arg_opts);
-	}
-
-      /* Now a list of short options *with* arguments.  */
-      for (entry = hol->entries, nentries = hol->num_entries
-	   ; nentries > 0
-	   ; entry++, nentries--)
-	hol_entry_short_iterate (entry, usage_argful_short_opt,
-				 entry->argp->argp_domain, stream);
-
-      /* Finally, a list of long options (whew!).  */
-      for (entry = hol->entries, nentries = hol->num_entries
-	   ; nentries > 0
-	   ; entry++, nentries--)
-	hol_entry_long_iterate (entry, usage_long_opt,
-				entry->argp->argp_domain, stream);
-    }
-}
-
-/* Make a HOL containing all levels of options in ARGP.  CLUSTER is the
-   cluster in which ARGP's entries should be clustered, or 0.  */
-static struct hol *
-argp_hol (const struct argp *argp, struct hol_cluster *cluster)
-{
-  const struct argp_child *child = argp->children;
-  struct hol *hol = make_hol (argp, cluster);
-  if (child)
-    while (child->argp)
-      {
-	struct hol_cluster *child_cluster =
-	  ((child->group || child->header)
-	   /* Put CHILD->argp within its own cluster.  */
-	   ? hol_add_cluster (hol, child->group, child->header,
-			      child - argp->children, cluster, argp)
-	   /* Just merge it into the parent's cluster.  */
-	   : cluster);
-	hol_append (hol, argp_hol (child->argp, child_cluster)) ;
-	child++;
-      }
-  return hol;
-}
-
-/* Calculate how many different levels with alternative args strings exist in
-   ARGP.  */
-static size_t
-argp_args_levels (const struct argp *argp)
-{
-  size_t levels = 0;
-  const struct argp_child *child = argp->children;
-
-  if (argp->args_doc && strchr (argp->args_doc, '\n'))
-    levels++;
-
-  if (child)
-    while (child->argp)
-      levels += argp_args_levels ((child++)->argp);
-
-  return levels;
-}
-
-/* Print all the non-option args documented in ARGP to STREAM.  Any output is
-   preceded by a space.  LEVELS is a pointer to a byte vector the length
-   returned by argp_args_levels; it should be initialized to zero, and
-   updated by this routine for the next call if ADVANCE is true.  True is
-   returned as long as there are more patterns to output.  */
-static int
-argp_args_usage (const struct argp *argp, const struct argp_state *state,
-		 char **levels, int advance, argp_fmtstream_t stream)
-{
-  char *our_level = *levels;
-  int multiple = 0;
-  const struct argp_child *child = argp->children;
-  const char *tdoc = dgettext (argp->argp_domain, argp->args_doc), *nl = 0;
-  const char *fdoc = filter_doc (tdoc, ARGP_KEY_HELP_ARGS_DOC, argp, state);
-
-  if (fdoc)
-    {
-      const char *cp = fdoc;
-      nl = __strchrnul (cp, '\n');
-      if (*nl != '\0')
-	/* This is a `multi-level' args doc; advance to the correct position
-	   as determined by our state in LEVELS, and update LEVELS.  */
-	{
-	  int i;
-	  multiple = 1;
-	  for (i = 0; i < *our_level; i++)
-	    cp = nl + 1, nl = __strchrnul (cp, '\n');
-	  (*levels)++;
-	}
-
-      /* Manually do line wrapping so that it (probably) won't get wrapped at
-	 any embedded spaces.  */
-      space (stream, 1 + nl - cp);
-
-      __argp_fmtstream_write (stream, cp, nl - cp);
-    }
-  if (fdoc && fdoc != tdoc)
-    free ((char *)fdoc);	/* Free user's modified doc string.  */
-
-  if (child)
-    while (child->argp)
-      advance = !argp_args_usage ((child++)->argp, state, levels, advance, stream);
-
-  if (advance && multiple)
-    {
-      /* Need to increment our level.  */
-      if (*nl)
-	/* There's more we can do here.  */
-	{
-	  (*our_level)++;
-	  advance = 0;		/* Our parent shouldn't advance also. */
-	}
-      else if (*our_level > 0)
-	/* We had multiple levels, but used them up; reset to zero.  */
-	*our_level = 0;
-    }
-
-  return !advance;
-}
-
-/* Print the documentation for ARGP to STREAM; if POST is false, then
-   everything preceeding a `\v' character in the documentation strings (or
-   the whole string, for those with none) is printed, otherwise, everything
-   following the `\v' character (nothing for strings without).  Each separate
-   bit of documentation is separated a blank line, and if PRE_BLANK is true,
-   then the first is as well.  If FIRST_ONLY is true, only the first
-   occurrence is output.  Returns true if anything was output.  */
-static int
-argp_doc (const struct argp *argp, const struct argp_state *state,
-	  int post, int pre_blank, int first_only,
-	  argp_fmtstream_t stream)
-{
-  const char *text;
-  const char *inp_text;
-  void *input = 0;
-  int anything = 0;
-  size_t inp_text_limit = 0;
-  const char *doc = dgettext (argp->argp_domain, argp->doc);
-  const struct argp_child *child = argp->children;
-
-  if (doc)
-    {
-      char *vt = strchr (doc, '\v');
-      inp_text = post ? (vt ? vt + 1 : 0) : doc;
-      inp_text_limit = (!post && vt) ? (vt - doc) : 0;
-    }
-  else
-    inp_text = 0;
-
-  if (argp->help_filter)
-    /* We have to filter the doc strings.  */
-    {
-      if (inp_text_limit)
-	/* Copy INP_TEXT so that it's nul-terminated.  */
-	inp_text = STRNDUP (inp_text, inp_text_limit);
-      input = __argp_input (argp, state);
-      text =
-	(*argp->help_filter) (post
-			      ? ARGP_KEY_HELP_POST_DOC
-			      : ARGP_KEY_HELP_PRE_DOC,
-			      inp_text, input);
-    }
-  else
-    text = (const char *) inp_text;
-
-  if (text)
-    {
-      if (pre_blank)
-	__argp_fmtstream_putc (stream, '\n');
-
-      if (text == inp_text && inp_text_limit)
-	__argp_fmtstream_write (stream, inp_text, inp_text_limit);
-      else
-	__argp_fmtstream_puts (stream, text);
-
-      if (__argp_fmtstream_point (stream) > __argp_fmtstream_lmargin (stream))
-	__argp_fmtstream_putc (stream, '\n');
-
-      anything = 1;
-    }
-
-  if (text && text != inp_text)
-    free ((char *) text);	/* Free TEXT returned from the help filter.  */
-  if (inp_text && inp_text_limit && argp->help_filter)
-    free ((char *) inp_text);	/* We copied INP_TEXT, so free it now.  */
-
-  if (post && argp->help_filter)
-    /* Now see if we have to output a ARGP_KEY_HELP_EXTRA text.  */
-    {
-      text = (*argp->help_filter) (ARGP_KEY_HELP_EXTRA, 0, input);
-      if (text)
-	{
-	  if (anything || pre_blank)
-	    __argp_fmtstream_putc (stream, '\n');
-	  __argp_fmtstream_puts (stream, text);
-	  free ((char *) text);
-	  if (__argp_fmtstream_point (stream)
-	      > __argp_fmtstream_lmargin (stream))
-	    __argp_fmtstream_putc (stream, '\n');
-	  anything = 1;
-	}
-    }
-
-  if (child)
-    while (child->argp && !(first_only && anything))
-      anything |=
-	argp_doc ((child++)->argp, state,
-		  post, anything || pre_blank, first_only,
-		  stream);
-
-  return anything;
-}
-
-/* Output a usage message for ARGP to STREAM.  If called from
-   argp_state_help, STATE is the relevent parsing state.  FLAGS are from the
-   set ARGP_HELP_*.  NAME is what to use wherever a `program name' is
-   needed. */
-
-static void
-_help (const struct argp *argp, const struct argp_state *state, FILE *stream,
-       unsigned flags, const char *name)
-{
-  int anything = 0;		/* Whether we've output anything.  */
-  struct hol *hol = 0;
-  argp_fmtstream_t fs;
-
-  if (! stream)
-    return;
-
-  FLOCKFILE (stream);
-
-  if (! uparams.valid)
-    fill_in_uparams (state);
-
-  fs = __argp_make_fmtstream (stream, 0, uparams.rmargin, 0);
-  if (! fs)
-    {
-      FUNLOCKFILE (stream);
-      return;
-    }
-
-  if (flags & (ARGP_HELP_USAGE | ARGP_HELP_SHORT_USAGE | ARGP_HELP_LONG))
-    {
-      hol = argp_hol (argp, 0);
-
-      /* If present, these options always come last.  */
-      hol_set_group (hol, "help", -1);
-      hol_set_group (hol, "version", -1);
-
-      hol_sort (hol);
-    }
-
-  if (flags & (ARGP_HELP_USAGE | ARGP_HELP_SHORT_USAGE))
-    /* Print a short `Usage:' message.  */
-    {
-      int first_pattern = 1, more_patterns;
-      size_t num_pattern_levels = argp_args_levels (argp);
-      char *pattern_levels = alloca (num_pattern_levels);
-
-      memset (pattern_levels, 0, num_pattern_levels);
-
-      do
-	{
-	  int old_lm;
-	  int old_wm = __argp_fmtstream_set_wmargin (fs, uparams.usage_indent);
-	  char *levels = pattern_levels;
-
-	  if (first_pattern)
-	    __argp_fmtstream_printf (fs, "%s %s",
-				     dgettext (argp->argp_domain, "Usage:"),
-				     name);
-	  else
-	    __argp_fmtstream_printf (fs, "%s %s",
-				     dgettext (argp->argp_domain, "  or: "),
-				     name);
-
-	  /* We set the lmargin as well as the wmargin, because hol_usage
-	     manually wraps options with newline to avoid annoying breaks.  */
-	  old_lm = __argp_fmtstream_set_lmargin (fs, uparams.usage_indent);
-
-	  if (flags & ARGP_HELP_SHORT_USAGE)
-	    /* Just show where the options go.  */
-	    {
-	      if (hol->num_entries > 0)
-		__argp_fmtstream_puts (fs, dgettext (argp->argp_domain,
-						     " [OPTION...]"));
-	    }
-	  else
-	    /* Actually print the options.  */
-	    {
-	      hol_usage (hol, fs);
-	      flags |= ARGP_HELP_SHORT_USAGE; /* But only do so once.  */
-	    }
-
-	  more_patterns = argp_args_usage (argp, state, &levels, 1, fs);
-
-	  __argp_fmtstream_set_wmargin (fs, old_wm);
-	  __argp_fmtstream_set_lmargin (fs, old_lm);
-
-	  __argp_fmtstream_putc (fs, '\n');
-	  anything = 1;
-
-	  first_pattern = 0;
-	}
-      while (more_patterns);
-    }
-
-  if (flags & ARGP_HELP_PRE_DOC)
-    anything |= argp_doc (argp, state, 0, 0, 1, fs);
-
-  if (flags & ARGP_HELP_SEE)
-    {
-      __argp_fmtstream_printf (fs, dgettext (argp->argp_domain, "\
-Try `%s --help' or `%s --usage' for more information.\n"),
-			       name, name);
-      anything = 1;
-    }
-
-  if (flags & ARGP_HELP_LONG)
-    /* Print a long, detailed help message.  */
-    {
-      /* Print info about all the options.  */
-      if (hol->num_entries > 0)
-	{
-	  if (anything)
-	    __argp_fmtstream_putc (fs, '\n');
-	  hol_help (hol, state, fs);
-	  anything = 1;
-	}
-    }
-
-  if (flags & ARGP_HELP_POST_DOC)
-    /* Print any documentation strings at the end.  */
-    anything |= argp_doc (argp, state, 1, anything, 0, fs);
-
-  if ((flags & ARGP_HELP_BUG_ADDR) && argp_program_bug_address)
-    {
-      if (anything)
-	__argp_fmtstream_putc (fs, '\n');
-      __argp_fmtstream_printf (fs, dgettext (argp->argp_domain,
-					     "Report bugs to %s.\n"),
- 			       argp_program_bug_address);
-      anything = 1;
-    }
-
-  FUNLOCKFILE (stream);
-
-  if (hol)
-    hol_free (hol);
-
-  __argp_fmtstream_free (fs);
-}
-
-/* Output a usage message for ARGP to STREAM.  FLAGS are from the set
-   ARGP_HELP_*.  NAME is what to use wherever a `program name' is needed. */
-void __argp_help (const struct argp *argp, FILE *stream,
-		  unsigned flags, char *name)
-{
-  _help (argp, 0, stream, flags, name);
-}
-#ifdef weak_alias
-weak_alias (__argp_help, argp_help)
-#endif
-
-char *__argp_basename(char *name)
-{
-  char *short_name = strrchr(name, '/');
-  return short_name ? short_name + 1 : name;
-}
-
-char *
-__argp_short_program_name(const struct argp_state *state)
-{
-  if (state)
-    return state->name;
-#if HAVE_DECL_PROGRAM_INVOCATION_SHORT_NAME
-  return program_invocation_short_name;
-#elif HAVE_DECL_PROGRAM_INVOCATION_NAME
-  return __argp_basename(program_invocation_name);
-#else /* !HAVE_DECL_PROGRAM_INVOCATION_NAME */
-  /* FIXME: What now? Miles suggests that it is better to use NULL,
-     but currently the value is passed on directly to fputs_unlocked,
-     so that requires more changes. */
-# if __GNUC__
-  return "";
-# endif /* __GNUC__ */
-#endif /* !HAVE_DECL_PROGRAM_INVOCATION_NAME */
-}
-
-/* Output, if appropriate, a usage message for STATE to STREAM.  FLAGS are
-   from the set ARGP_HELP_*.  */
-void
-__argp_state_help (const struct argp_state *state, FILE *stream, unsigned flags)
-{
-  if ((!state || ! (state->flags & ARGP_NO_ERRS)) && stream)
-    {
-      if (state && (state->flags & ARGP_LONG_ONLY))
-	flags |= ARGP_HELP_LONG_ONLY;
-
-      _help (state ? state->root_argp : 0, state, stream, flags,
-	     __argp_short_program_name(state));
-
-      if (!state || ! (state->flags & ARGP_NO_EXIT))
-	{
-	  if (flags & ARGP_HELP_EXIT_ERR)
-	    exit (argp_err_exit_status);
-	  if (flags & ARGP_HELP_EXIT_OK)
-	    exit (0);
-	}
-  }
-}
-#ifdef weak_alias
-weak_alias (__argp_state_help, argp_state_help)
-#endif
-
-/* If appropriate, print the printf string FMT and following args, preceded
-   by the program name and `:', to stderr, and followed by a `Try ... --help'
-   message, then exit (1).  */
-void
-__argp_error (const struct argp_state *state, const char *fmt, ...)
-{
-  if (!state || !(state->flags & ARGP_NO_ERRS))
-    {
-      FILE *stream = state ? state->err_stream : stderr;
-
-      if (stream)
-	{
-	  va_list ap;
-
-	  FLOCKFILE (stream);
-
-	  FPUTS_UNLOCKED (__argp_short_program_name(state),
-			  stream);
-	  PUTC_UNLOCKED (':', stream);
-	  PUTC_UNLOCKED (' ', stream);
-
-	  va_start (ap, fmt);
-	  vfprintf (stream, fmt, ap);
-	  va_end (ap);
-
-	  PUTC_UNLOCKED ('\n', stream);
-
-	  __argp_state_help (state, stream, ARGP_HELP_STD_ERR);
-
-	  FUNLOCKFILE (stream);
-	}
-    }
-}
-#ifdef weak_alias
-weak_alias (__argp_error, argp_error)
-#endif
-
-/* Similar to the standard gnu error-reporting function error(), but will
-   respect the ARGP_NO_EXIT and ARGP_NO_ERRS flags in STATE, and will print
-   to STATE->err_stream.  This is useful for argument parsing code that is
-   shared between program startup (when exiting is desired) and runtime
-   option parsing (when typically an error code is returned instead).  The
-   difference between this function and argp_error is that the latter is for
-   *parsing errors*, and the former is for other problems that occur during
-   parsing but don't reflect a (syntactic) problem with the input.  */
-void
-__argp_failure (const struct argp_state *state, int status, int errnum,
-		const char *fmt, ...)
-{
-  if (!state || !(state->flags & ARGP_NO_ERRS))
-    {
-      FILE *stream = state ? state->err_stream : stderr;
-
-      if (stream)
-	{
-	  FLOCKFILE (stream);
-
-	  FPUTS_UNLOCKED (__argp_short_program_name(state),
-			  stream);
-
-	  if (fmt)
-	    {
-	      va_list ap;
-
-	      PUTC_UNLOCKED (':', stream);
-	      PUTC_UNLOCKED (' ', stream);
-
-	      va_start (ap, fmt);
-	      vfprintf (stream, fmt, ap);
-	      va_end (ap);
-	    }
-
-	  if (errnum)
-	    {
-	      PUTC_UNLOCKED (':', stream);
-	      PUTC_UNLOCKED (' ', stream);
-	      fputs (STRERROR (errnum), stream);
-	    }
-
-	  PUTC_UNLOCKED ('\n', stream);
-
-	  FUNLOCKFILE (stream);
-
-	  if (status && (!state || !(state->flags & ARGP_NO_EXIT)))
-	    exit (status);
-	}
-    }
-}
-#ifdef weak_alias
-weak_alias (__argp_failure, argp_failure)
-#endif
diff --git a/argp-standalone/argp-namefrob.h b/argp-standalone/argp-namefrob.h
deleted file mode 100644
index 0ce11481a7b..00000000000
--- a/argp-standalone/argp-namefrob.h
+++ /dev/null
@@ -1,96 +0,0 @@
-/* Name frobnication for compiling argp outside of glibc
-   Copyright (C) 1997 Free Software Foundation, Inc.
-   This file is part of the GNU C Library.
-   Written by Miles Bader <miles@gnu.ai.mit.edu>.
-
-   The GNU C Library is free software; you can redistribute it and/or
-   modify it under the terms of the GNU Library General Public License as
-   published by the Free Software Foundation; either version 2 of the
-   License, or (at your option) any later version.
-
-   The GNU C Library is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Library General Public License for more details.
-
-   You should have received a copy of the GNU Library General Public
-   License along with the GNU C Library; see the file COPYING.LIB.  If not,
-   write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330,
-   Boston, MA 02111-1307, USA.  */
-
-#if !_LIBC
-/* This code is written for inclusion in gnu-libc, and uses names in the
-   namespace reserved for libc.  If we're not compiling in libc, define those
-   names to be the normal ones instead.  */
-
-/* argp-parse functions */
-#undef __argp_parse
-#define __argp_parse argp_parse
-#undef __option_is_end
-#define __option_is_end _option_is_end
-#undef __option_is_short
-#define __option_is_short _option_is_short
-#undef __argp_input
-#define __argp_input _argp_input
-
-/* argp-help functions */
-#undef __argp_help
-#define __argp_help argp_help
-#undef __argp_error
-#define __argp_error argp_error
-#undef __argp_failure
-#define __argp_failure argp_failure
-#undef __argp_state_help
-#define __argp_state_help argp_state_help
-#undef __argp_usage
-#define __argp_usage argp_usage
-#undef __argp_basename
-#define __argp_basename _argp_basename
-#undef __argp_short_program_name
-#define __argp_short_program_name _argp_short_program_name
-
-/* argp-fmtstream functions */
-#undef __argp_make_fmtstream
-#define __argp_make_fmtstream argp_make_fmtstream
-#undef __argp_fmtstream_free
-#define __argp_fmtstream_free argp_fmtstream_free
-#undef __argp_fmtstream_putc
-#define __argp_fmtstream_putc argp_fmtstream_putc
-#undef __argp_fmtstream_puts
-#define __argp_fmtstream_puts argp_fmtstream_puts
-#undef __argp_fmtstream_write
-#define __argp_fmtstream_write argp_fmtstream_write
-#undef __argp_fmtstream_printf
-#define __argp_fmtstream_printf argp_fmtstream_printf
-#undef __argp_fmtstream_set_lmargin
-#define __argp_fmtstream_set_lmargin argp_fmtstream_set_lmargin
-#undef __argp_fmtstream_set_rmargin
-#define __argp_fmtstream_set_rmargin argp_fmtstream_set_rmargin
-#undef __argp_fmtstream_set_wmargin
-#define __argp_fmtstream_set_wmargin argp_fmtstream_set_wmargin
-#undef __argp_fmtstream_point
-#define __argp_fmtstream_point argp_fmtstream_point
-#undef __argp_fmtstream_update
-#define __argp_fmtstream_update _argp_fmtstream_update
-#undef __argp_fmtstream_ensure
-#define __argp_fmtstream_ensure _argp_fmtstream_ensure
-#undef __argp_fmtstream_lmargin
-#define __argp_fmtstream_lmargin argp_fmtstream_lmargin
-#undef __argp_fmtstream_rmargin
-#define __argp_fmtstream_rmargin argp_fmtstream_rmargin
-#undef __argp_fmtstream_wmargin
-#define __argp_fmtstream_wmargin argp_fmtstream_wmargin
-
-/* normal libc functions we call */
-#undef __sleep
-#define __sleep sleep
-#undef __strcasecmp
-#define __strcasecmp strcasecmp
-#undef __vsnprintf
-#define __vsnprintf vsnprintf
-
-#endif /* !_LIBC */
-
-#ifndef __set_errno
-#define __set_errno(e) (errno = (e))
-#endif
diff --git a/argp-standalone/argp-parse.c b/argp-standalone/argp-parse.c
deleted file mode 100644
index 78f7bf139b6..00000000000
--- a/argp-standalone/argp-parse.c
+++ /dev/null
@@ -1,1305 +0,0 @@
-/* Hierarchial argument parsing
-   Copyright (C) 1995, 96, 97, 98, 99, 2000,2003 Free Software Foundation, Inc.
-   This file is part of the GNU C Library.
-   Written by Miles Bader <miles@gnu.ai.mit.edu>.
-
-   The GNU C Library is free software; you can redistribute it and/or
-   modify it under the terms of the GNU Library General Public License as
-   published by the Free Software Foundation; either version 2 of the
-   License, or (at your option) any later version.
-
-   The GNU C Library is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Library General Public License for more details.
-
-   You should have received a copy of the GNU Library General Public
-   License along with the GNU C Library; see the file COPYING.LIB.  If not,
-   write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330,
-   Boston, MA 02111-1307, USA.  */
-
-#ifndef _GNU_SOURCE
-# define _GNU_SOURCE	1
-#endif
-
-#ifdef HAVE_CONFIG_H
-#include <config.h>
-#endif
-
-#if HAVE_ALLOCA_H
-#include <alloca.h>
-#endif
-
-#include <stdlib.h>
-#include <string.h>
-#if HAVE_UNISTD_H
-# include <unistd.h>
-#endif
-#include <limits.h>
-#include <assert.h>
-
-#if HAVE_MALLOC_H
-/* Needed, for alloca on windows */
-# include <malloc.h>
-#endif
-
-#ifndef _
-/* This is for other GNU distributions with internationalized messages.
-   When compiling libc, the _ macro is predefined.  */
-# if defined HAVE_LIBINTL_H || defined _LIBC
-#  include <libintl.h>
-#  ifdef _LIBC
-#   undef dgettext
-#   define dgettext(domain, msgid) __dcgettext (domain, msgid, LC_MESSAGES)
-#  endif
-# else
-#  define dgettext(domain, msgid) (msgid)
-#  define gettext(msgid) (msgid)
-# endif
-#endif
-#ifndef N_
-# define N_(msgid) (msgid)
-#endif
-
-#if _LIBC - 0
-#include <bits/libc-lock.h>
-#else
-#ifdef HAVE_CTHREADS_H
-#include <cthreads.h>
-#endif
-#endif /* _LIBC */
-
-#include "argp.h"
-#include "argp-namefrob.h"
-
-
-/* The meta-argument used to prevent any further arguments being interpreted
-   as options.  */
-#define QUOTE "--"
-
-/* EZ alias for ARGP_ERR_UNKNOWN.  */
-#define EBADKEY ARGP_ERR_UNKNOWN
-
-
-/* Default options.  */
-
-/* When argp is given the --HANG switch, _ARGP_HANG is set and argp will sleep
-   for one second intervals, decrementing _ARGP_HANG until it's zero.  Thus
-   you can force the program to continue by attaching a debugger and setting
-   it to 0 yourself.  */
-volatile int _argp_hang;
-
-#define OPT_PROGNAME	-2
-#define OPT_USAGE	-3
-#if HAVE_SLEEP && HAVE_GETPID
-#define OPT_HANG	-4
-#endif
-
-static const struct argp_option argp_default_options[] =
-{
-  {"help",	  '?',    	0, 0,  N_("Give this help list"), -1},
-  {"usage",	  OPT_USAGE,	0, 0,  N_("Give a short usage message"), 0 },
-  {"program-name",OPT_PROGNAME,"NAME", OPTION_HIDDEN,
-     N_("Set the program name"), 0},
-#if OPT_HANG
-  {"HANG",	  OPT_HANG,    "SECS", OPTION_ARG_OPTIONAL | OPTION_HIDDEN,
-     N_("Hang for SECS seconds (default 3600)"), 0 },
-#endif
-  {0, 0, 0, 0, 0, 0}
-};
-
-static error_t
-argp_default_parser (int key, char *arg, struct argp_state *state)
-{
-  switch (key)
-    {
-    case '?':
-      __argp_state_help (state, state->out_stream, ARGP_HELP_STD_HELP);
-      break;
-    case OPT_USAGE:
-      __argp_state_help (state, state->out_stream,
-		       ARGP_HELP_USAGE | ARGP_HELP_EXIT_OK);
-      break;
-
-    case OPT_PROGNAME:		/* Set the program name.  */
-#if HAVE_DECL_PROGRAM_INVOCATION_NAME
-      program_invocation_name = arg;
-#endif
-      /* [Note that some systems only have PROGRAM_INVOCATION_SHORT_NAME (aka
-	 __PROGNAME), in which case, PROGRAM_INVOCATION_NAME is just defined
-	 to be that, so we have to be a bit careful here.]  */
-
-      /* Update what we use for messages.  */
-
-      state->name = __argp_basename(arg);
-      
-#if HAVE_DECL_PROGRAM_INVOCATION_SHORT_NAME
-      program_invocation_short_name = state->name;
-#endif
-
-      if ((state->flags & (ARGP_PARSE_ARGV0 | ARGP_NO_ERRS))
-	  == ARGP_PARSE_ARGV0)
-	/* Update what getopt uses too.  */
-	state->argv[0] = arg;
-
-      break;
-      
-#if OPT_HANG
-    case OPT_HANG:
-      _argp_hang = atoi (arg ? arg : "3600");
-      fprintf(state->err_stream, "%s: pid = %ld\n",
-	      state->name, (long) getpid());
-      while (_argp_hang-- > 0)
-	__sleep (1);
-      break;
-#endif
-      
-    default:
-      return EBADKEY;
-    }
-  return 0;
-}
-
-static const struct argp argp_default_argp =
-  {argp_default_options, &argp_default_parser, NULL, NULL, NULL, NULL, "libc"};
-
-
-static const struct argp_option argp_version_options[] =
-{
-  {"version",	  'V',    	0, 0,  N_("Print program version"), -1},
-  {0, 0, 0, 0, 0, 0 }
-};
-
-static error_t
-argp_version_parser (int key, char *arg UNUSED, struct argp_state *state)
-{
-  switch (key)
-    {
-    case 'V':
-      if (argp_program_version_hook)
-	(*argp_program_version_hook) (state->out_stream, state);
-      else if (argp_program_version)
-	fprintf (state->out_stream, "%s\n", argp_program_version);
-      else
-	__argp_error (state, dgettext (state->root_argp->argp_domain,
-				       "(PROGRAM ERROR) No version known!?"));
-      if (! (state->flags & ARGP_NO_EXIT))
-	exit (0);
-      break;
-    default:
-      return EBADKEY;
-    }
-  return 0;
-}
-
-static const struct argp argp_version_argp =
-  {argp_version_options, &argp_version_parser, NULL, NULL, NULL, NULL, "libc"};
-
-
-
-/* The state of a `group' during parsing.  Each group corresponds to a
-   particular argp structure from the tree of such descending from the top
-   level argp passed to argp_parse.  */
-struct group
-{
-  /* This group's parsing function.  */
-  argp_parser_t parser;
-
-  /* Which argp this group is from.  */
-  const struct argp *argp;
-
-  /* The number of non-option args sucessfully handled by this parser.  */
-  unsigned args_processed;
-
-  /* This group's parser's parent's group.  */
-  struct group *parent;
-  unsigned parent_index;	/* And the our position in the parent.   */
-
-  /* These fields are swapped into and out of the state structure when
-     calling this group's parser.  */
-  void *input, **child_inputs;
-  void *hook;
-};
-
-/* Call GROUP's parser with KEY and ARG, swapping any group-specific info
-   from STATE before calling, and back into state afterwards.  If GROUP has
-   no parser, EBADKEY is returned.  */
-static error_t
-group_parse (struct group *group, struct argp_state *state, int key, char *arg)
-{
-  if (group->parser)
-    {
-      error_t err;
-      state->hook = group->hook;
-      state->input = group->input;
-      state->child_inputs = group->child_inputs;
-      state->arg_num = group->args_processed;
-      err = (*group->parser)(key, arg, state);
-      group->hook = state->hook;
-      return err;
-    }
-  else
-    return EBADKEY;
-}
-
-struct parser
-{
-  const struct argp *argp;
-
-  const char *posixly_correct;
-  
-  /* True if there are only no-option arguments left, which are just
-     passed verbatim with ARGP_KEY_ARG. This is set if we encounter a
-     quote, or the end of the proper options, but may be cleared again
-     if the user moves the next argument pointer backwards. */
-  int args_only;
-
-  /* Describe how to deal with options that follow non-option ARGV-elements.
-
-     If the caller did not specify anything, the default is
-     REQUIRE_ORDER if the environment variable POSIXLY_CORRECT is
-     defined, PERMUTE otherwise.
-
-     REQUIRE_ORDER means don't recognize them as options; stop option
-     processing when the first non-option is seen. This is what Unix
-     does. This mode of operation is selected by either setting the
-     environment variable POSIXLY_CORRECT, or using `+' as the first
-     character of the list of option characters.
-
-     PERMUTE is the default. We permute the contents of ARGV as we
-     scan, so that eventually all the non-options are at the end. This
-     allows options to be given in any order, even with programs that
-     were not written to expect this.
-
-     RETURN_IN_ORDER is an option available to programs that were
-     written to expect options and other ARGV-elements in any order
-     and that care about the ordering of the two. We describe each
-     non-option ARGV-element as if it were the argument of an option
-     with character code 1. Using `-' as the first character of the
-     list of option characters selects this mode of operation.
-
-  */
-  enum { REQUIRE_ORDER, PERMUTE, RETURN_IN_ORDER } ordering;
-
-  /* A segment of non-option arguments that have been skipped for
-     later processing, after all options. `first_nonopt' is the index
-     in ARGV of the first of them; `last_nonopt' is the index after
-     the last of them.
-
-     If quoted or args_only is non-zero, this segment should be empty. */
-
-  /* FIXME: I'd prefer to use unsigned, but it's more consistent to
-     use the same type as for state.next. */
-  int first_nonopt;
-  int last_nonopt;
-  
-  /* String of all recognized short options. Needed for ARGP_LONG_ONLY. */
-  /* FIXME: Perhaps change to a pointer to a suitable bitmap instead? */
-  char *short_opts;
-
-  /* For parsing combined short options. */
-  char *nextchar;
-  
-  /* States of the various parsing groups.  */
-  struct group *groups;
-  /* The end of the GROUPS array.  */
-  struct group *egroup;
-  /* An vector containing storage for the CHILD_INPUTS field in all groups.  */
-  void **child_inputs;
-  
-  /* State block supplied to parsing routines.  */
-  struct argp_state state;
-
-  /* Memory used by this parser.  */
-  void *storage;
-};
-
-/* Search for a group defining a short option. */
-static const struct argp_option *
-find_short_option(struct parser *parser, int key, struct group **p)
-{
-  struct group *group;
-  
-  assert(key >= 0);
-  assert(isascii(key));
-
-  for (group = parser->groups; group < parser->egroup; group++)
-    {
-      const struct argp_option *opts;
-
-      for (opts = group->argp->options; !__option_is_end(opts); opts++)
-	if (opts->key == key)
-	  {
-	    *p = group;
-	    return opts;
-	  }
-    }
-  return NULL;
-}
-
-enum match_result { MATCH_EXACT, MATCH_PARTIAL, MATCH_NO };
-
-/* If defined, allow complete.el-like abbreviations of long options. */
-#ifndef ARGP_COMPLETE
-#define ARGP_COMPLETE 0
-#endif
-
-/* Matches an encountern long-option argument ARG against an option NAME.
- * ARG is terminated by NUL or '='. */
-static enum match_result
-match_option(const char *arg, const char *name)
-{
-  unsigned i, j;
-  for (i = j = 0;; i++, j++)
-    {
-      switch(arg[i])
-	{
-	case '\0':
-	case '=':
-	  return name[j] ? MATCH_PARTIAL : MATCH_EXACT;
-#if ARGP_COMPLETE
-	case '-':
-	  while (name[j] != '-')
-	    if (!name[j++])
-	      return MATCH_NO;
-	  break;
-#endif
-	default:
-	  if (arg[i] != name[j])
-	    return MATCH_NO;
-	}
-    }
-}
-
-static const struct argp_option *
-find_long_option(struct parser *parser,
-		 const char *arg,
-		 struct group **p)
-{
-  struct group *group;
-
-  /* Partial match found so far. */
-  struct group *matched_group = NULL;
-  const struct argp_option *matched_option = NULL;
-
-  /* Number of partial matches. */
-  int num_partial = 0;
-
-  for (group = parser->groups; group < parser->egroup; group++)
-    {
-      const struct argp_option *opts;
-
-      for (opts = group->argp->options; !__option_is_end(opts); opts++)
-	{
-	  if (!opts->name)
-	    continue;
-	  switch (match_option(arg, opts->name))
-	    {
-	    case MATCH_NO:
-	      break;
-	    case MATCH_PARTIAL:
-	      num_partial++;
-
-	      matched_group = group;
-	      matched_option = opts;
-
-	      break;
-	    case MATCH_EXACT:
-	      /* Exact match. */
-	      *p = group;
-	      return opts;
-	    }
-	}
-    }
-  if (num_partial == 1)
-    {
-      *p = matched_group;
-      return matched_option;
-    }
-
-  return NULL;
-}
-
-
-/* The next usable entries in the various parser tables being filled in by
-   convert_options.  */
-struct parser_convert_state
-{
-  struct parser *parser;
-  char *short_end;
-  void **child_inputs_end;
-};
-
-/* Initialize GROUP from ARGP. If CVT->SHORT_END is non-NULL, short
-   options are recorded in the short options string. Returns the next
-   unused group entry. CVT holds state used during the conversion. */
-static struct group *
-convert_options (const struct argp *argp,
-		 struct group *parent, unsigned parent_index,
-		 struct group *group, struct parser_convert_state *cvt)
-{
-  const struct argp_option *opt = argp->options;
-  const struct argp_child *children = argp->children;
-
-  if (opt || argp->parser)
-    {
-      /* This parser needs a group. */
-      if (cvt->short_end)
-	{
-	  /* Record any short options. */
-	  for ( ; !__option_is_end (opt); opt++)
-	    if (__option_is_short(opt))
-	      *cvt->short_end++ = opt->key;
-	}
-      
-      group->parser = argp->parser;
-      group->argp = argp;
-      group->args_processed = 0;
-      group->parent = parent;
-      group->parent_index = parent_index;
-      group->input = 0;
-      group->hook = 0;
-      group->child_inputs = 0;
-      
-      if (children)
-	/* Assign GROUP's CHILD_INPUTS field some space from
-	   CVT->child_inputs_end.*/
-	{
-	  unsigned num_children = 0;
-	  while (children[num_children].argp)
-	    num_children++;
-	  group->child_inputs = cvt->child_inputs_end;
-	  cvt->child_inputs_end += num_children;
-	}
-      parent = group++;
-    }
-  else
-    parent = 0;
-
-  if (children)
-    {
-      unsigned index = 0;
-      while (children->argp)
-	group =
-	  convert_options (children++->argp, parent, index++, group, cvt);
-    }
-
-  return group;
-}
-/* Allocate and initialize the group structures, so that they are
-   ordered as if by traversing the corresponding argp parser tree in
-   pre-order. Also build the list of short options, if that is needed. */
-static void
-parser_convert (struct parser *parser, const struct argp *argp)
-{
-  struct parser_convert_state cvt;
-
-  cvt.parser = parser;
-  cvt.short_end = parser->short_opts;
-  cvt.child_inputs_end = parser->child_inputs;
-
-  parser->argp = argp;
-
-  if (argp)
-    parser->egroup = convert_options (argp, 0, 0, parser->groups, &cvt);
-  else
-    parser->egroup = parser->groups; /* No parsers at all! */
-
-  if (parser->short_opts)
-    *cvt.short_end ='\0';
-}
-
-/* Lengths of various parser fields which we will allocated.  */
-struct parser_sizes
-{
-  /* Needed only ARGP_LONG_ONLY */
-  size_t short_len;		/* Number of short options.  */
-
-  size_t num_groups;		/* Group structures we allocate.  */
-  size_t num_child_inputs;	/* Child input slots.  */
-};
-
-/* For ARGP, increments the NUM_GROUPS field in SZS by the total
-   number of argp structures descended from it, and the SHORT_LEN by
-   the total number of short options. */
-static void
-calc_sizes (const struct argp *argp,  struct parser_sizes *szs)
-{
-  const struct argp_child *child = argp->children;
-  const struct argp_option *opt = argp->options;
-
-  if (opt || argp->parser)
-    {
-      /* This parser needs a group. */
-      szs->num_groups++;
-      if (opt)
-	{
-	  while (__option_is_short (opt++))
-	    szs->short_len++;
-	}
-    }
-
-  if (child)
-    while (child->argp)
-      {
-	calc_sizes ((child++)->argp, szs);
-	szs->num_child_inputs++;
-      }
-}
-
-/* Initializes PARSER to parse ARGP in a manner described by FLAGS.  */
-static error_t
-parser_init (struct parser *parser, const struct argp *argp,
-	     int argc, char **argv, int flags, void *input)
-{
-  error_t err = 0;
-  struct group *group;
-  struct parser_sizes szs;
-
-  parser->posixly_correct = getenv ("POSIXLY_CORRECT");
-
-  if (flags & ARGP_IN_ORDER)
-    parser->ordering = RETURN_IN_ORDER;
-  else if (flags & ARGP_NO_ARGS)
-    parser->ordering = REQUIRE_ORDER;
-  else if (parser->posixly_correct)
-    parser->ordering = REQUIRE_ORDER;
-  else
-    parser->ordering = PERMUTE;
-  
-  szs.short_len = 0;
-  szs.num_groups = 0;
-  szs.num_child_inputs = 0;
-
-  if (argp)
-    calc_sizes (argp, &szs);
-
-  if (!(flags & ARGP_LONG_ONLY))
-    /* We have no use for the short option array. */
-    szs.short_len = 0;
-  
-  /* Lengths of the various bits of storage used by PARSER.  */
-#define GLEN (szs.num_groups + 1) * sizeof (struct group)
-#define CLEN (szs.num_child_inputs * sizeof (void *))
-#define SLEN (szs.short_len + 1)
-#define STORAGE(offset) ((void *) (((char *) parser->storage) + (offset)))
-
-  parser->storage = malloc (GLEN + CLEN + SLEN);
-  if (! parser->storage)
-    return ENOMEM;
-
-  parser->groups = parser->storage;
-
-  parser->child_inputs = STORAGE(GLEN);
-  memset (parser->child_inputs, 0, szs.num_child_inputs * sizeof (void *));
-
-  if (flags & ARGP_LONG_ONLY)
-    parser->short_opts = STORAGE(GLEN + CLEN);
-  else
-    parser->short_opts = NULL;
-
-  parser_convert (parser, argp);
-
-  memset (&parser->state, 0, sizeof (struct argp_state));
-  
-  parser->state.root_argp = parser->argp;
-  parser->state.argc = argc;
-  parser->state.argv = argv;
-  parser->state.flags = flags;
-  parser->state.err_stream = stderr;
-  parser->state.out_stream = stdout;
-  parser->state.pstate = parser;
-
-  parser->args_only = 0;
-  parser->nextchar = NULL;
-  parser->first_nonopt = parser->last_nonopt = 0;
-    
-  /* Call each parser for the first time, giving it a chance to propagate
-     values to child parsers.  */
-  if (parser->groups < parser->egroup)
-    parser->groups->input = input;
-  for (group = parser->groups;
-       group < parser->egroup && (!err || err == EBADKEY);
-       group++)
-    {
-      if (group->parent)
-	/* If a child parser, get the initial input value from the parent. */
-	group->input = group->parent->child_inputs[group->parent_index];
-
-      if (!group->parser
-	  && group->argp->children && group->argp->children->argp)
-	/* For the special case where no parsing function is supplied for an
-	   argp, propagate its input to its first child, if any (this just
-	   makes very simple wrapper argps more convenient).  */
-	group->child_inputs[0] = group->input;
-
-      err = group_parse (group, &parser->state, ARGP_KEY_INIT, 0);
-    }
-  if (err == EBADKEY)
-    err = 0;			/* Some parser didn't understand.  */
-
-  if (err)
-    return err;
-
-  if (argv[0] && !(parser->state.flags & ARGP_PARSE_ARGV0))
-    /* There's an argv[0]; use it for messages.  */
-    {
-      parser->state.name = __argp_basename(argv[0]);
-
-      /* Don't parse it as an argument. */
-      parser->state.next = 1;
-    }
-  else
-    parser->state.name = __argp_short_program_name(NULL);
-  
-  return 0;
-}
-
-/* Free any storage consumed by PARSER (but not PARSER itself).  */
-static error_t
-parser_finalize (struct parser *parser,
-		 error_t err, int arg_ebadkey, int *end_index)
-{
-  struct group *group;
-
-  if (err == EBADKEY && arg_ebadkey)
-    /* Suppress errors generated by unparsed arguments.  */
-    err = 0;
-
-  if (! err)
-    {
-      if (parser->state.next == parser->state.argc)
-	/* We successfully parsed all arguments!  Call all the parsers again,
-	   just a few more times... */
-	{
-	  for (group = parser->groups;
-	       group < parser->egroup && (!err || err==EBADKEY);
-	       group++)
-	    if (group->args_processed == 0)
-	      err = group_parse (group, &parser->state, ARGP_KEY_NO_ARGS, 0);
-	  for (group = parser->egroup - 1;
-	       group >= parser->groups && (!err || err==EBADKEY);
-	       group--)
-	    err = group_parse (group, &parser->state, ARGP_KEY_END, 0);
-
-	  if (err == EBADKEY)
-	    err = 0;		/* Some parser didn't understand.  */
-
-	  /* Tell the user that all arguments are parsed.  */
-	  if (end_index)
-	    *end_index = parser->state.next;
-	}
-      else if (end_index)
-	/* Return any remaining arguments to the user.  */
-	*end_index = parser->state.next;
-      else
-	/* No way to return the remaining arguments, they must be bogus. */
-	{
-	  if (!(parser->state.flags & ARGP_NO_ERRS)
-	      && parser->state.err_stream)
-	    fprintf (parser->state.err_stream,
-		     dgettext (parser->argp->argp_domain,
-			       "%s: Too many arguments\n"),
-		     parser->state.name);
-	  err = EBADKEY;
-	}
-    }
-
-  /* Okay, we're all done, with either an error or success; call the parsers
-     to indicate which one.  */
-
-  if (err)
-    {
-      /* Maybe print an error message.  */
-      if (err == EBADKEY)
-	/* An appropriate message describing what the error was should have
-	   been printed earlier.  */
-	__argp_state_help (&parser->state, parser->state.err_stream,
-			   ARGP_HELP_STD_ERR);
-
-      /* Since we didn't exit, give each parser an error indication.  */
-      for (group = parser->groups; group < parser->egroup; group++)
-	group_parse (group, &parser->state, ARGP_KEY_ERROR, 0);
-    }
-  else
-    /* Notify parsers of success, and propagate back values from parsers.  */
-    {
-      /* We pass over the groups in reverse order so that child groups are
-	 given a chance to do there processing before passing back a value to
-	 the parent.  */
-      for (group = parser->egroup - 1
-	   ; group >= parser->groups && (!err || err == EBADKEY)
-	   ; group--)
-	err = group_parse (group, &parser->state, ARGP_KEY_SUCCESS, 0);
-      if (err == EBADKEY)
-	err = 0;		/* Some parser didn't understand.  */
-    }
-
-  /* Call parsers once more, to do any final cleanup.  Errors are ignored.  */
-  for (group = parser->egroup - 1; group >= parser->groups; group--)
-    group_parse (group, &parser->state, ARGP_KEY_FINI, 0);
-
-  if (err == EBADKEY)
-    err = EINVAL;
-
-  free (parser->storage);
-
-  return err;
-}
-
-/* Call the user parsers to parse the non-option argument VAL, at the
-   current position, returning any error. The state NEXT pointer
-   should point to the argument; this function will adjust it
-   correctly to reflect however many args actually end up being
-   consumed. */
-static error_t
-parser_parse_arg (struct parser *parser, char *val)
-{
-  /* Save the starting value of NEXT */
-  int index = parser->state.next;
-  error_t err = EBADKEY;
-  struct group *group;
-  int key = 0;			/* Which of ARGP_KEY_ARG[S] we used.  */
-
-  /* Try to parse the argument in each parser.  */
-  for (group = parser->groups
-       ; group < parser->egroup && err == EBADKEY
-       ; group++)
-    {
-      parser->state.next++;	/* For ARGP_KEY_ARG, consume the arg.  */
-      key = ARGP_KEY_ARG;
-      err = group_parse (group, &parser->state, key, val);
-
-      if (err == EBADKEY)
-	/* This parser doesn't like ARGP_KEY_ARG; try ARGP_KEY_ARGS instead. */
-	{
-	  parser->state.next--;	/* For ARGP_KEY_ARGS, put back the arg.  */
-	  key = ARGP_KEY_ARGS;
-	  err = group_parse (group, &parser->state, key, 0);
-	}
-    }
-
-  if (! err)
-    {
-      if (key == ARGP_KEY_ARGS)
-	/* The default for ARGP_KEY_ARGS is to assume that if NEXT isn't
-	   changed by the user, *all* arguments should be considered
-	   consumed.  */
-	parser->state.next = parser->state.argc;
-
-      if (parser->state.next > index)
-	/* Remember that we successfully processed a non-option
-	   argument -- but only if the user hasn't gotten tricky and set
-	   the clock back.  */
-	(--group)->args_processed += (parser->state.next - index);
-      else
-	/* The user wants to reparse some args, so try looking for options again.  */
-	parser->args_only = 0;
-    }
-
-  return err;
-}
-
-/* Exchange two adjacent subsequences of ARGV.
-   One subsequence is elements [first_nonopt,last_nonopt)
-   which contains all the non-options that have been skipped so far.
-   The other is elements [last_nonopt,next), which contains all
-   the options processed since those non-options were skipped.
-
-   `first_nonopt' and `last_nonopt' are relocated so that they describe
-   the new indices of the non-options in ARGV after they are moved.  */
-
-static void
-exchange (struct parser *parser)
-{
-  int bottom = parser->first_nonopt;
-  int middle = parser->last_nonopt;
-  int top = parser->state.next;
-  char **argv = parser->state.argv;
-  
-  char *tem;
-
-  /* Exchange the shorter segment with the far end of the longer segment.
-     That puts the shorter segment into the right place.
-     It leaves the longer segment in the right place overall,
-     but it consists of two parts that need to be swapped next.  */
-
-  while (top > middle && middle > bottom)
-    {
-      if (top - middle > middle - bottom)
-	{
-	  /* Bottom segment is the short one.  */
-	  int len = middle - bottom;
-	  register int i;
-
-	  /* Swap it with the top part of the top segment.  */
-	  for (i = 0; i < len; i++)
-	    {
-	      tem = argv[bottom + i];
-	      argv[bottom + i] = argv[top - (middle - bottom) + i];
-	      argv[top - (middle - bottom) + i] = tem;
-	    }
-	  /* Exclude the moved bottom segment from further swapping.  */
-	  top -= len;
-	}
-      else
-	{
-	  /* Top segment is the short one.  */
-	  int len = top - middle;
-	  register int i;
-
-	  /* Swap it with the bottom part of the bottom segment.  */
-	  for (i = 0; i < len; i++)
-	    {
-	      tem = argv[bottom + i];
-	      argv[bottom + i] = argv[middle + i];
-	      argv[middle + i] = tem;
-	    }
-	  /* Exclude the moved top segment from further swapping.  */
-	  bottom += len;
-	}
-    }
-
-  /* Update records for the slots the non-options now occupy.  */
-
-  parser->first_nonopt += (parser->state.next - parser->last_nonopt);
-  parser->last_nonopt = parser->state.next;
-}
-
-
-
-enum arg_type { ARG_ARG, ARG_SHORT_OPTION,
-		ARG_LONG_OPTION, ARG_LONG_ONLY_OPTION,
-		ARG_QUOTE };
-
-static enum arg_type
-classify_arg(struct parser *parser, char *arg, char **opt)
-{
-  if (arg[0] == '-')
-    /* Looks like an option... */
-    switch (arg[1])
-      {
-      case '\0':
-	/* "-" is not an option. */
-	return ARG_ARG;
-      case '-':
-	/* Long option, or quote. */
-	if (!arg[2])
-	  return ARG_QUOTE;
-	  
-	/* A long option. */
-	if (opt)
-	  *opt = arg + 2;
-	return ARG_LONG_OPTION;
-
-      default:
-	/* Short option. But if ARGP_LONG_ONLY, it can also be a long option. */
-
-	if (opt)
-	  *opt = arg + 1;
-
-	if (parser->state.flags & ARGP_LONG_ONLY)
-	  {
-	    /* Rules from getopt.c:
-
-	       If long_only and the ARGV-element has the form "-f",
-	       where f is a valid short option, don't consider it an
-	       abbreviated form of a long option that starts with f.
-	       Otherwise there would be no way to give the -f short
-	       option.
-
-	       On the other hand, if there's a long option "fubar" and
-	       the ARGV-element is "-fu", do consider that an
-	       abbreviation of the long option, just like "--fu", and
-	       not "-f" with arg "u".
-
-	       This distinction seems to be the most useful approach. */
-
-	    assert(parser->short_opts);
-	    
-	    if (arg[2] || !strchr(parser->short_opts, arg[1]))
-	      return ARG_LONG_ONLY_OPTION;
-	  }
-
-	return ARG_SHORT_OPTION;
-      }
-  
-  else
-    return ARG_ARG;
-}
-
-/* Parse the next argument in PARSER (as indicated by PARSER->state.next).
-   Any error from the parsers is returned, and *ARGP_EBADKEY indicates
-   whether a value of EBADKEY is due to an unrecognized argument (which is
-   generally not fatal).  */
-static error_t
-parser_parse_next (struct parser *parser, int *arg_ebadkey)
-{
-  if (parser->state.quoted && parser->state.next < parser->state.quoted)
-    /* The next argument pointer has been moved to before the quoted
-       region, so pretend we never saw the quoting `--', and start
-       looking for options again. If the `--' is still there we'll just
-       process it one more time. */
-    parser->state.quoted = parser->args_only = 0;
-
-  /* Give FIRST_NONOPT & LAST_NONOPT rational values if NEXT has been
-     moved back by the user (who may also have changed the arguments).  */
-  if (parser->last_nonopt > parser->state.next)
-    parser->last_nonopt = parser->state.next;
-  if (parser->first_nonopt > parser->state.next)
-    parser->first_nonopt = parser->state.next;
-
-  if (parser->nextchar)
-    /* Deal with short options. */
-    {
-      struct group *group;
-      char c;
-      const struct argp_option *option;
-      char *value = NULL;;
-
-      assert(!parser->args_only);
-
-      c = *parser->nextchar++;
-      
-      option = find_short_option(parser, c, &group);
-      if (!option)
-	{
-	  if (parser->posixly_correct)
-	    /* 1003.2 specifies the format of this message.  */
-	    fprintf (parser->state.err_stream,
-		     dgettext(parser->state.root_argp->argp_domain,
-			      "%s: illegal option -- %c\n"),
-		     parser->state.name, c);
-	  else
-	    fprintf (parser->state.err_stream,
-		     dgettext(parser->state.root_argp->argp_domain,
-			      "%s: invalid option -- %c\n"),
-		     parser->state.name, c);
-
-	  *arg_ebadkey = 0;
-	  return EBADKEY;
-	}
-
-      if (!*parser->nextchar)
-	parser->nextchar = NULL;
-
-      if (option->arg)
-	{
-	  value = parser->nextchar;
-	  parser->nextchar = NULL;
-	      
-	  if (!value
-	      && !(option->flags & OPTION_ARG_OPTIONAL))
-	    /* We need an mandatory argument. */
-	    {
-	      if (parser->state.next == parser->state.argc)
-		/* Missing argument */
-		{
-		  /* 1003.2 specifies the format of this message.  */
-		  fprintf (parser->state.err_stream,
-			   dgettext(parser->state.root_argp->argp_domain,
-				    "%s: option requires an argument -- %c\n"),
-			   parser->state.name, c);
-
-		  *arg_ebadkey = 0;
-		  return EBADKEY;
-		}
-	      value = parser->state.argv[parser->state.next++];
-	    }
-	}
-      return group_parse(group, &parser->state,
-			 option->key, value);
-    }
-  else
-    /* Advance to the next ARGV-element.  */
-    {
-      if (parser->args_only)
-	{
-	  *arg_ebadkey = 1;
-	  if (parser->state.next >= parser->state.argc)
-	    /* We're done. */
-	    return EBADKEY;
-	  else
-	    return parser_parse_arg(parser,
-				    parser->state.argv[parser->state.next]);
-	}
-      
-      if (parser->state.next >= parser->state.argc)
-	/* Almost done. If there are non-options that we skipped
-	   previously, we should process them now. */
-	{
-	  *arg_ebadkey = 1;
-	  if (parser->first_nonopt != parser->last_nonopt)
-	    {
-	      exchange(parser);
-	      
-	      /* Start processing the arguments we skipped previously. */
-	      parser->state.next = parser->first_nonopt;
-	      
-	      parser->first_nonopt = parser->last_nonopt = 0;
-
-	      parser->args_only = 1;
-	      return 0;
-	    }
-	  else
-	    /* Indicate that we're really done. */
-	    return EBADKEY;
-	}
-      else
-	/* Look for options. */
-	{
-	  char *arg = parser->state.argv[parser->state.next];
-
-	  char *optstart;
-	  enum arg_type token = classify_arg(parser, arg, &optstart);
-	  
-	  switch (token)
-	    {
-	    case ARG_ARG:
-	      switch (parser->ordering)
-		{
-		case PERMUTE:
-		  if (parser->first_nonopt == parser->last_nonopt)
-		    /* Skipped sequence is empty; start a new one. */
-		    parser->first_nonopt = parser->last_nonopt = parser->state.next;
-
-		  else if (parser->last_nonopt != parser->state.next)
-		    /* We have a non-empty skipped sequence, and
-		       we're not at the end-point, so move it. */
-		    exchange(parser);
-
-		  assert(parser->last_nonopt == parser->state.next);
-		  
-		  /* Skip this argument for now. */
-		  parser->state.next++;
-		  parser->last_nonopt = parser->state.next; 
-		  
-		  return 0;
-
-		case REQUIRE_ORDER:
-		  /* Implicit quote before the first argument. */
-		   parser->args_only = 1;
-		   return 0;
-		   
-		case RETURN_IN_ORDER:
-		  *arg_ebadkey = 1;
-		  return parser_parse_arg(parser, arg);
-
-		default:
-		  abort();
-		}
-	    case ARG_QUOTE:
-	      /* Skip it, then exchange with any previous non-options. */
-	      parser->state.next++;
-	      assert (parser->last_nonopt != parser->state.next);
-
-	      if (parser->first_nonopt != parser->last_nonopt)
-		{
-		  exchange(parser);
-		  
-		  /* Start processing the skipped and the quoted
-		     arguments. */
-
-		  parser->state.quoted = parser->state.next = parser->first_nonopt;
-
-		  /* Also empty the skipped-list, to avoid confusion
-		     if the user resets the next pointer. */
-		  parser->first_nonopt = parser->last_nonopt = 0;
-		}
-	      else
-		parser->state.quoted = parser->state.next;
-
-	      parser->args_only = 1;	      
-	      return 0;
-
-	    case ARG_LONG_ONLY_OPTION:
-	    case ARG_LONG_OPTION:
-	      {
-		struct group *group;
-		const struct argp_option *option;
-		char *value;
-
-		parser->state.next++;
-		option = find_long_option(parser, optstart, &group);
-		
-		if (!option)
-		  {
-		    /* NOTE: This includes any "=something" in the output. */
-		    fprintf (parser->state.err_stream,
-			     dgettext(parser->state.root_argp->argp_domain,
-				      "%s: unrecognized option `%s'\n"),
-			     parser->state.name, arg);
-		    *arg_ebadkey = 0;
-		    return EBADKEY;
-		  }
-
-		value = strchr(optstart, '=');
-		if (value)
-		  value++;
-		
-		if (value && !option->arg)
-		  /* Unexpected argument. */
-		  {
-		    if (token == ARG_LONG_OPTION)
-		      /* --option */
-		      fprintf (parser->state.err_stream,
-			       dgettext(parser->state.root_argp->argp_domain,
-					"%s: option `--%s' doesn't allow an argument\n"),
-			       parser->state.name, option->name);
-		    else
-		      /* +option or -option */
-		      fprintf (parser->state.err_stream,
-			       dgettext(parser->state.root_argp->argp_domain,
-					"%s: option `%c%s' doesn't allow an argument\n"),
-			       parser->state.name, arg[0], option->name);
-
-		    *arg_ebadkey = 0;
-		    return EBADKEY;
-		  }
-		
-		if (option->arg && !value
-		    && !(option->flags & OPTION_ARG_OPTIONAL))
-		  /* We need an mandatory argument. */
-		  {
-		    if (parser->state.next == parser->state.argc)
-		      /* Missing argument */
-		      {
-			if (token == ARG_LONG_OPTION)
-			  /* --option */
-			  fprintf (parser->state.err_stream,
-				   dgettext(parser->state.root_argp->argp_domain,
-					    "%s: option `--%s' requires an argument\n"),
-				 parser->state.name, option->name);
-			else
-			  /* +option or -option */
-			  fprintf (parser->state.err_stream,
-				   dgettext(parser->state.root_argp->argp_domain,
-					    "%s: option `%c%s' requires an argument\n"),
-				   parser->state.name, arg[0], option->name);
-
-			*arg_ebadkey = 0;
-			return EBADKEY;
-		      }
-
-		    value = parser->state.argv[parser->state.next++];
-		  }
-		*arg_ebadkey = 0;
-		return group_parse(group, &parser->state,
-				   option->key, value);
-	      }
-	    case ARG_SHORT_OPTION:
-	      parser->state.next++;
-	      parser->nextchar = optstart;
-	      return 0;
-
-	    default:
-	      abort();
-	    }
-	}
-    }
-}
-
-/* Parse the options strings in ARGC & ARGV according to the argp in ARGP.
-   FLAGS is one of the ARGP_ flags above.  If END_INDEX is non-NULL, the
-   index in ARGV of the first unparsed option is returned in it.  If an
-   unknown option is present, EINVAL is returned; if some parser routine
-   returned a non-zero value, it is returned; otherwise 0 is returned.  */
-error_t
-__argp_parse (const struct argp *argp, int argc, char **argv, unsigned flags,
-	      int *end_index, void *input)
-{
-  error_t err;
-  struct parser parser;
-
-  /* If true, then err == EBADKEY is a result of a non-option argument failing
-     to be parsed (which in some cases isn't actually an error).  */
-  int arg_ebadkey = 0;
-
-  if (! (flags & ARGP_NO_HELP))
-    /* Add our own options.  */
-    {
-      struct argp_child *child = alloca (4 * sizeof (struct argp_child));
-      struct argp *top_argp = alloca (sizeof (struct argp));
-
-      /* TOP_ARGP has no options, it just serves to group the user & default
-	 argps.  */
-      memset (top_argp, 0, sizeof (*top_argp));
-      top_argp->children = child;
-
-      memset (child, 0, 4 * sizeof (struct argp_child));
-
-      if (argp)
-	(child++)->argp = argp;
-      (child++)->argp = &argp_default_argp;
-      if (argp_program_version || argp_program_version_hook)
-	(child++)->argp = &argp_version_argp;
-      child->argp = 0;
-
-      argp = top_argp;
-    }
-
-  /* Construct a parser for these arguments.  */
-  err = parser_init (&parser, argp, argc, argv, flags, input);
-
-  if (! err)
-    /* Parse! */
-    {
-      while (! err)
-	err = parser_parse_next (&parser, &arg_ebadkey);
-      err = parser_finalize (&parser, err, arg_ebadkey, end_index);
-    }
-
-  return err;
-}
-#ifdef weak_alias
-weak_alias (__argp_parse, argp_parse)
-#endif
-
-/* Return the input field for ARGP in the parser corresponding to STATE; used
-   by the help routines.  */
-void *
-__argp_input (const struct argp *argp, const struct argp_state *state)
-{
-  if (state)
-    {
-      struct group *group;
-      struct parser *parser = state->pstate;
-
-      for (group = parser->groups; group < parser->egroup; group++)
-	if (group->argp == argp)
-	  return group->input;
-    }
-
-  return 0;
-}
-#ifdef weak_alias
-weak_alias (__argp_input, _argp_input)
-#endif
-
-/* Defined here, in case a user is not inlining the definitions in
- * argp.h */
-void
-__argp_usage (__const struct argp_state *__state)
-{
-  __argp_state_help (__state, stderr, ARGP_HELP_STD_USAGE);
-}
-
-int
-__option_is_short (__const struct argp_option *__opt)
-{
-  if (__opt->flags & OPTION_DOC)
-    return 0;
-  else
-    {
-      int __key = __opt->key;
-      /* FIXME: whether or not a particular key implies a short option
-       * ought not to be locale dependent. */
-      return __key > 0 && isprint (__key);
-    }
-}
-
-int
-__option_is_end (__const struct argp_option *__opt)
-{
-  return !__opt->key && !__opt->name && !__opt->doc && !__opt->group;
-}
diff --git a/argp-standalone/argp-pv.c b/argp-standalone/argp-pv.c
deleted file mode 100644
index d7d374a66bd..00000000000
--- a/argp-standalone/argp-pv.c
+++ /dev/null
@@ -1,25 +0,0 @@
-/* Default definition for ARGP_PROGRAM_VERSION.
-   Copyright (C) 1996, 1997, 1999, 2004 Free Software Foundation, Inc.
-   This file is part of the GNU C Library.
-   Written by Miles Bader <miles@gnu.ai.mit.edu>.
-
-   The GNU C Library is free software; you can redistribute it and/or
-   modify it under the terms of the GNU Library General Public License as
-   published by the Free Software Foundation; either version 2 of the
-   License, or (at your option) any later version.
-
-   The GNU C Library is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Library General Public License for more details.
-
-   You should have received a copy of the GNU Library General Public
-   License along with the GNU C Library; see the file COPYING.LIB.  If not,
-   write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330,
-   Boston, MA 02111-1307, USA.  */
-
-/* If set by the user program to a non-zero value, then a default option
-   --version is added (unless the ARGP_NO_HELP flag is used), which will
-   print this this string followed by a newline and exit (unless the
-   ARGP_NO_EXIT flag is used).  Overridden by ARGP_PROGRAM_VERSION_HOOK.  */
-const char *argp_program_version = 0;
diff --git a/argp-standalone/argp-pvh.c b/argp-standalone/argp-pvh.c
deleted file mode 100644
index 829a1cda80d..00000000000
--- a/argp-standalone/argp-pvh.c
+++ /dev/null
@@ -1,32 +0,0 @@
-/* Default definition for ARGP_PROGRAM_VERSION_HOOK.
-   Copyright (C) 1996, 1997, 1999, 2004 Free Software Foundation, Inc.
-   This file is part of the GNU C Library.
-   Written by Miles Bader <miles@gnu.ai.mit.edu>.
-
-   The GNU C Library is free software; you can redistribute it and/or
-   modify it under the terms of the GNU Library General Public License as
-   published by the Free Software Foundation; either version 2 of the
-   License, or (at your option) any later version.
-
-   The GNU C Library is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Library General Public License for more details.
-
-   You should have received a copy of the GNU Library General Public
-   License along with the GNU C Library; see the file COPYING.LIB.  If not,
-   write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330,
-   Boston, MA 02111-1307, USA.  */
-
-#ifdef HAVE_CONFIG_H
-#include <config.h>
-#endif
-
-#include "argp.h"
-
-/* If set by the user program to a non-zero value, then a default option
-   --version is added (unless the ARGP_NO_HELP flag is used), which calls
-   this function with a stream to print the version to and a pointer to the
-   current parsing state, and then exits (unless the ARGP_NO_EXIT flag is
-   used).  This variable takes precedent over ARGP_PROGRAM_VERSION.  */
-void (*argp_program_version_hook) (FILE *stream, struct argp_state *state) = 0;
diff --git a/argp-standalone/argp.h b/argp-standalone/argp.h
deleted file mode 100644
index 29d3dfe9720..00000000000
--- a/argp-standalone/argp.h
+++ /dev/null
@@ -1,602 +0,0 @@
-/* Hierarchial argument parsing.
-   Copyright (C) 1995, 96, 97, 98, 99, 2003 Free Software Foundation, Inc.
-   This file is part of the GNU C Library.
-   Written by Miles Bader <miles@gnu.ai.mit.edu>.
-
-   The GNU C Library is free software; you can redistribute it and/or
-   modify it under the terms of the GNU Library General Public License as
-   published by the Free Software Foundation; either version 2 of the
-   License, or (at your option) any later version.
-
-   The GNU C Library is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Library General Public License for more details.
-
-   You should have received a copy of the GNU Library General Public
-   License along with the GNU C Library; see the file COPYING.LIB.  If not,
-   write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330,
-   Boston, MA 02111-1307, USA.  */
-
-#ifndef _ARGP_H
-#define _ARGP_H
-
-#include <stdio.h>
-#include <ctype.h>
-
-#define __need_error_t
-#include <errno.h>
-
-#ifndef __THROW
-# define __THROW
-#endif
-
-#ifndef __const
-# define __const const
-#endif
-
-#ifndef __error_t_defined
-typedef int error_t;
-# define __error_t_defined
-#endif
-
-/* FIXME: What's the right way to check for __restrict? Sun's cc seems
-   not to have it. Perhaps it's easiest to just delete the use of
-   __restrict from the prototypes. */
-#ifndef __restrict
-# ifndef __GNUC___
-#  define __restrict
-# endif
-#endif
-
-/* NOTE: We can't use the autoconf tests, since this is supposed to be
-   an installed header file and argp's config.h is of course not
-   installed. */
-#ifndef PRINTF_STYLE
-# if __GNUC__ >= 2
-#  define PRINTF_STYLE(f, a) __attribute__ ((__format__ (__printf__, f, a)))
-# else
-#  define PRINTF_STYLE(f, a)
-# endif
-#endif
-
-
-#ifdef  __cplusplus
-extern "C" {
-#endif
-
-/* A description of a particular option.  A pointer to an array of
-   these is passed in the OPTIONS field of an argp structure.  Each option
-   entry can correspond to one long option and/or one short option; more
-   names for the same option can be added by following an entry in an option
-   array with options having the OPTION_ALIAS flag set.  */
-struct argp_option
-{
-  /* The long option name.  For more than one name for the same option, you
-     can use following options with the OPTION_ALIAS flag set.  */
-  __const char *name;
-
-  /* What key is returned for this option.  If > 0 and printable, then it's
-     also accepted as a short option.  */
-  int key;
-
-  /* If non-NULL, this is the name of the argument associated with this
-     option, which is required unless the OPTION_ARG_OPTIONAL flag is set. */
-  __const char *arg;
-
-  /* OPTION_ flags.  */
-  int flags;
-
-  /* The doc string for this option.  If both NAME and KEY are 0, This string
-     will be printed outdented from the normal option column, making it
-     useful as a group header (it will be the first thing printed in its
-     group); in this usage, it's conventional to end the string with a `:'.  */
-  __const char *doc;
-
-  /* The group this option is in.  In a long help message, options are sorted
-     alphabetically within each group, and the groups presented in the order
-     0, 1, 2, ..., n, -m, ..., -2, -1.  Every entry in an options array with
-     if this field 0 will inherit the group number of the previous entry, or
-     zero if it's the first one, unless its a group header (NAME and KEY both
-     0), in which case, the previous entry + 1 is the default.  Automagic
-     options such as --help are put into group -1.  */
-  int group;
-};
-
-/* The argument associated with this option is optional.  */
-#define OPTION_ARG_OPTIONAL	0x1
-
-/* This option isn't displayed in any help messages.  */
-#define OPTION_HIDDEN	       	0x2
-
-/* This option is an alias for the closest previous non-alias option.  This
-   means that it will be displayed in the same help entry, and will inherit
-   fields other than NAME and KEY from the aliased option.  */
-#define OPTION_ALIAS		0x4
-
-/* This option isn't actually an option (and so should be ignored by the
-   actual option parser), but rather an arbitrary piece of documentation that
-   should be displayed in much the same manner as the options.  If this flag
-   is set, then the option NAME field is displayed unmodified (e.g., no `--'
-   prefix is added) at the left-margin (where a *short* option would normally
-   be displayed), and the documentation string in the normal place.  For
-   purposes of sorting, any leading whitespace and puncuation is ignored,
-   except that if the first non-whitespace character is not `-', this entry
-   is displayed after all options (and OPTION_DOC entries with a leading `-')
-   in the same group.  */
-#define OPTION_DOC		0x8
-
-/* This option shouldn't be included in `long' usage messages (but is still
-   included in help messages).  This is mainly intended for options that are
-   completely documented in an argp's ARGS_