nfs, nfs3: Base volume access on CHILD-UP-DOWN event

Overall, the aim of this patch is to change the result of an nfs
op depending on whether the underlying volume is up or down
as notified by CHILD_UP and CHILD_DOWN events.

This patch contains three intertwined changes:
o Only when the lookup on the root of a volume is successful does gnfs now
export the subvolume. Till now the result of the lookup was not used to
determine whether we should export that volume. Not accounting for root lookup
failure resulted in ESTALEs on first access because some children of distribute
were down at the time of the root lookup.

o Only when lookups on all the subvolumes have succeeded are
these exports enabled through NFS.

o When a child of say distribute goes down, on CHILD_DOWN event nfs will
ignore all incoming requests from the client because ignoring these
will prevent ESTALEs for those requests and in the hope that ignoring the
requests will make the client retransmit. There are risks in this approach
absent the DRC but we're willing to live with that for now.
When a child goes down, the mount exports list will continue to show it
but mount requests will be denied.

Signed-off-by: Shehjar Tikoo <shehjart@gluster.com>
Signed-off-by: Vijay Bellur <vijay@dev.gluster.com>

BUG: 1643 (Initial requests after mount ESTALE if DHT subvolumes connect after nfs startup)
URL: http://bugs.gluster.com/cgi-bin/bugzilla3/show_bug.cgi?id=1643

7 files changed, 418 insertions, 105 deletions

diff --git a/xlators/nfs/server/src/mount3.c b/xlators/nfs/server/src/mount3.c
index 3ea4bc8e893..84e1456fb64 100644
--- a/xlators/nfs/server/src/mount3.c
+++ b/xlators/nfs/server/src/mount3.c
@@ -766,6 +766,14 @@ mnt3svc_mnt (rpcsvc_request_t *req)
                 goto mnterr;
         }
 
+        if ((is_nfs_subvolume_disabled (nfs_state (ms->nfsx), exp->vol))) {
+                gf_log (GF_MNT, GF_LOG_ERROR, "Volume is disabled: %s",
+                        exp->vol->name);
+                ret = RPCSVC_ACTOR_ERROR;
+                mntstat = MNT3ERR_INVAL;
+                goto mnterr;
+        }
+
         ret = mnt3_check_client_net (ms, req, exp->vol);
         if (ret == -1) {
                 mntstat = MNT3ERR_ACCES;
diff --git a/xlators/nfs/server/src/nfs-common.c b/xlators/nfs/server/src/nfs-common.c
index 970eb41c571..369bb181663 100644
--- a/xlators/nfs/server/src/nfs-common.c
+++ b/xlators/nfs/server/src/nfs-common.c
@@ -33,6 +33,7 @@
 #include "nfs-mem-types.h"
 #include "rpcsvc.h"
 #include "iatt.h"
+#include "nfs.h"
 
 #include <libgen.h>
 
@@ -395,4 +396,42 @@ err:
 }
 
 
+int
+__is_nfs_subvolume_disabled (struct nfs_state *nfs, xlator_t *xl)
+{
+        int     x = 0;
+        int     notstarted = 0;
+
+        if ((!nfs) || (!xl))
+                return 1;
+
+        for (;x < nfs->allsubvols; ++x) {
+                if (!((&nfs->subvols[x])->subvol == xl))
+                        continue;
+
+                if (gf_nfs_subvolume_notstarted (&nfs->subvols[x])) {
+                        notstarted = 1;
+                        goto out;
+                }
+        }
+out:
+        return notstarted;
+}
 
+
+int
+is_nfs_subvolume_disabled (struct nfs_state *nfs, xlator_t *xl)
+{
+        int     notstarted = 0;
+
+        if ((!nfs) || (!xl))
+                return 1;
+
+        LOCK (&nfs->svinitlock);
+        {
+                notstarted = __is_nfs_subvolume_disabled (nfs, xl);
+        }
+        UNLOCK (&nfs->svinitlock);
+
+        return notstarted;
+}
diff --git a/xlators/nfs/server/src/nfs-common.h b/xlators/nfs/server/src/nfs-common.h
index 25fc1cf405b..3e791a90549 100644
--- a/xlators/nfs/server/src/nfs-common.h
+++ b/xlators/nfs/server/src/nfs-common.h
@@ -31,6 +31,7 @@
 #include "rpcsvc.h"
 #include "iatt.h"
 #include "uuid.h"
+#include "nfs.h"
 
 #define NFS_PATH_MAX    PATH_MAX
 #define NFS_NAME_MAX    NAME_MAX
@@ -76,4 +77,7 @@ nfs_entry_loc_fill (inode_table_t *itable, uuid_t pargfid, char *entry,
 
 extern int
 nfs_root_loc_fill (inode_table_t *itable, loc_t *loc);
+
+extern int
+is_nfs_subvolume_disabled (struct nfs_state *nfs, xlator_t *xl);
 #endif
diff --git a/xlators/nfs/server/src/nfs-mem-types.h b/xlators/nfs/server/src/nfs-mem-types.h
index 2198a43015b..74499fb0851 100644
--- a/xlators/nfs/server/src/nfs-mem-types.h
+++ b/xlators/nfs/server/src/nfs-mem-types.h
@@ -43,6 +43,7 @@ enum gf_nfs_mem_types_ {
         gf_nfs_mt_mnt3_resolve,
         gf_nfs_mt_mnt3_export,
         gf_nfs_mt_inode_q,
+        gf_nfs_mt_subvolumes,
         gf_nfs_mt_end
 };
 #endif
diff --git a/xlators/nfs/server/src/nfs.c b/xlators/nfs/server/src/nfs.c
index 431765332ca..3fa46662f2a 100644
--- a/xlators/nfs/server/src/nfs.c
+++ b/xlators/nfs/server/src/nfs.c
@@ -169,7 +169,7 @@ ret:
 
 
 int
-nfs_subvolume_started (struct nfs_state *nfs, xlator_t *xl)
+__is_nfs_subvolume_lookup_sent (struct nfs_state *nfs, xlator_t *xl)
 {
         int     x = 0;
         int     started = 0;
@@ -177,16 +177,32 @@ nfs_subvolume_started (struct nfs_state *nfs, xlator_t *xl)
         if ((!nfs) || (!xl))
                 return 1;
 
+        for (;x < nfs->allsubvols; ++x) {
+                if (!((&nfs->subvols[x])->subvol == xl))
+                        continue;
+
+                if (gf_nfs_subvolume_lookupsent (&nfs->subvols[x])) {
+                        started = 1;
+                        goto out;
+                }
+        }
+out:
+        return started;
+}
+
+
+int
+is_nfs_subvolume_lookup_sent (struct nfs_state *nfs, xlator_t *xl)
+{
+        int     started = 0;
+
+        if ((!nfs) || (!xl))
+                return 1;
+
         LOCK (&nfs->svinitlock);
         {
-                for (;x < nfs->allsubvols; ++x) {
-                        if (nfs->initedxl[x] == xl) {
-                                started = 1;
-                                goto unlock;
-                       }
-               }
+                started = __is_nfs_subvolume_lookup_sent (nfs, xl);
         }
-unlock:
         UNLOCK (&nfs->svinitlock);
 
         return started;
@@ -194,27 +210,187 @@ unlock:
 
 
 int
-nfs_subvolume_set_started (struct nfs_state *nfs, xlator_t *xl)
+__is_nfs_subvolume_started (struct nfs_state *nfs, xlator_t *xl)
 {
         int     x = 0;
+        int     started = 0;
+
+        if ((!nfs) || (!xl))
+                return 1;
+
+        for (;x < nfs->allsubvols; ++x) {
+                if (!((&nfs->subvols[x])->subvol == xl))
+                        continue;
+
+                if (gf_nfs_subvolume_started (&nfs->subvols[x])) {
+                        started = 1;
+                        goto out;
+                }
+        }
+out:
+        return started;
+}
+
+
+int
+is_nfs_subvolume_started (struct nfs_state *nfs, xlator_t *xl)
+{
+        int     started = 0;
 
         if ((!nfs) || (!xl))
                 return 1;
 
         LOCK (&nfs->svinitlock);
         {
-                for (;x < nfs->allsubvols; ++x) {
-                        if (nfs->initedxl[x] == NULL) {
-                                nfs->initedxl[x] = xl;
-                                ++nfs->upsubvols;
-                                gf_log (GF_NFS, GF_LOG_DEBUG, "Starting up: %s "
-                                        ", vols started till now: %d", xl->name,
-                                        nfs->upsubvols);
-                                goto unlock;
-                        }
-               }
+                started = __is_nfs_subvolume_started (nfs, xl);
+        }
+        UNLOCK (&nfs->svinitlock);
+
+        return started;
+}
+
+
+int
+__nfs_subvolume_lookup_again_later (struct nfs_state *nfs, xlator_t *xl)
+{
+        int                     x = 0;
+        struct nfs_subvolume    *sv = NULL;
+
+        if ((!nfs) || (!xl))
+                return -1;
+
+        for (;x < nfs->allsubvols; ++x) {
+                if (((&nfs->subvols[x])->subvol == xl)) {
+                        sv = &nfs->subvols[x];
+                        break;
+                }
+        }
+
+        if (!sv)
+                goto err;
+        gf_log (GF_NFS, GF_LOG_TRACE, "Will lookup subvol later: %s",
+                xl->name);
+        sv->status = GF_NFS_SUBVOLUME_NOTSTARTED;
+err:
+        return 0;
+}
+
+
+int
+__nfs_subvolume_start (struct nfs_state *nfs, xlator_t *xl)
+{
+        int                     x = 0;
+        int                     ret = 0;
+        struct nfs_subvolume    *sv = NULL;
+
+        if ((!nfs) || (!xl))
+                return -1;
+
+        for (;x < nfs->allsubvols; ++x) {
+                if (((&nfs->subvols[x])->subvol == xl)) {
+                        sv = &nfs->subvols[x];
+                        break;
+                }
+        }
+
+        if (!sv) {
+                gf_log (GF_NFS, GF_LOG_ERROR, "Could not find subvol slot");
+                goto err;
+        }
+
+        /* If the subvolume got disconnected or sent a CHILD-DOWN between the
+         * lookup fop and the lookup callback.
+         */
+        if (gf_nfs_subvolume_notstarted (sv)) {
+                goto err;
+        }
+
+        sv->status = GF_NFS_SUBVOLUME_STARTED;
+        ++nfs->upsubvols;
+        gf_log (GF_NFS, GF_LOG_DEBUG, "Starting up: %s, vols started till now: "
+                "%d", xl->name, nfs->upsubvols);
+
+        if (gf_nfs_all_subvolumes_started (nfs)) {
+                nfs->subvols_started = 1;
+                gf_log (GF_NFS, GF_LOG_INFO, "All exports up");
+                ret = nfs_init_versions (nfs, nfs->nfsx);
+                if (ret == -1)
+                        gf_log (GF_NFS, GF_LOG_CRITICAL, "Failed to initialize "
+                                "protocols");
+        }
+err:
+        return 0;
+}
+
+
+int
+nfs_subvolume_start (struct nfs_state *nfs, xlator_t *xl)
+{
+        if ((!nfs) || (!xl))
+                return 1;
+
+        LOCK (&nfs->svinitlock);
+        {
+                __nfs_subvolume_start (nfs, xl);
+        }
+        UNLOCK (&nfs->svinitlock);
+
+        return 0;
+}
+
+
+int
+__nfs_subvolume_lookup_sent (struct nfs_state *nfs, xlator_t *xl)
+{
+        struct nfs_subvolume    *sv = NULL;
+        int                     x = 0;
+
+        if ((!nfs) || (!xl))
+                return -1;
+
+        for (;x < nfs->allsubvols; ++x) {
+                if (((&nfs->subvols[x])->subvol == xl)) {
+                        sv = &nfs->subvols[x];
+                        break;
+                }
+        }
+
+        if (sv) {
+                sv->status = GF_NFS_SUBVOLUME_LOOKUPSENT;
+        } else
+                gf_log (GF_NFS, GF_LOG_ERROR, "Could not find subvol slot: %s",
+                        xl->name);
+
+        return 0;
+}
+
+
+int
+nfs_subvolume_lookup_sent (struct nfs_state *nfs, xlator_t *xl)
+{
+        if ((!nfs) || (!xl))
+                return 1;
+
+        LOCK (&nfs->svinitlock);
+        {
+                __nfs_subvolume_lookup_sent (nfs, xl);
+        }
+        UNLOCK (&nfs->svinitlock);
+
+        return 0;
+}
+
+
+int
+nfs_subvolume_lookup_again_later (struct nfs_state *nfs, xlator_t *xl)
+{
+        if ((!nfs) || (!xl))
+                return 1;
+
+        LOCK (&nfs->svinitlock);
+        {
+                __nfs_subvolume_lookup_again_later (nfs, xl);
         }
-unlock:
         UNLOCK (&nfs->svinitlock);
 
         return 0;
@@ -227,36 +403,71 @@ nfs_start_subvol_lookup_cbk (call_frame_t *frame, void *cookie,
                              inode_t *inode, struct iatt *buf, dict_t *xattr,
                              struct iatt *postparent)
 {
+        struct nfs_state	*nfs = frame->local;
+
         if (op_ret == -1) {
                 gf_log (GF_NFS, GF_LOG_CRITICAL, "Failed to lookup root: %s",
                         strerror (op_errno));
+                nfs_subvolume_lookup_again_later (nfs, (xlator_t *)cookie);
                 goto err;
         }
 
-        gf_log (GF_NFS, GF_LOG_TRACE, "Started %s", ((xlator_t *)cookie)->name);
+        nfs_subvolume_start (nfs, (xlator_t *)cookie);
 err:
         return 0;
 }
 
 
 int
+__nfs_startup_subvolume_check (struct nfs_state *nfs, xlator_t *xl)
+{
+        int     ret = 0;
+
+        if (__is_nfs_subvolume_started (nfs, xl)) {
+                gf_log (GF_NFS,GF_LOG_TRACE, "Subvolume already started: %s",
+                        xl->name);
+                ret = 1;
+                goto out;
+        }
+
+        if (__is_nfs_subvolume_lookup_sent (nfs, xl)) {
+                gf_log (GF_NFS,GF_LOG_TRACE,"Subvolume lookup already sent: %s",
+                        xl->name);
+                ret = 1;
+                goto out;
+        }
+
+out:
+        return ret;
+}
+
+
+int
 nfs_startup_subvolume (xlator_t *nfsx, xlator_t *xl)
 {
-        int             ret = -1;
-        loc_t           rootloc = {0, };
-        nfs_user_t      nfu = {0, };
+        int                     ret = -1;
+        loc_t                   rootloc = {0, };
+        nfs_user_t              nfu = {0, };
+        struct nfs_state        *nfs = NULL;
 
         if ((!nfsx) || (!xl))
                 return -1;
 
-        if (nfs_subvolume_started (nfsx->private, xl)) {
-                gf_log (GF_NFS,GF_LOG_TRACE, "Subvolume already started: %s",
-                        xl->name);
-                ret = 0;
-                goto err;
+        nfs = nfsx->private;
+        LOCK (&nfs->svinitlock);
+        {
+                ret = __nfs_startup_subvolume_check (nfsx->private, xl);
+                if (ret)
+                        goto unlock;
+
+                __nfs_subvolume_lookup_sent (nfs, xl);
         }
+unlock:
+        UNLOCK (&nfs->svinitlock);
+
+        if (ret)
+                goto err;
 
-        nfs_subvolume_set_started (nfsx->private, xl);
         ret = nfs_inode_loc_fill (xl->itable->root, &rootloc);
         if (ret == -1) {
                 gf_log (GF_NFS, GF_LOG_CRITICAL, "Failed to init root loc");
@@ -279,35 +490,6 @@ err:
         return ret;
 }
 
-int
-nfs_startup_subvolumes (xlator_t *nfsx)
-{
-        int                     ret = -1;
-        xlator_list_t           *cl = NULL;
-        struct nfs_state        *nfs = NULL;
-
-        if (!nfsx)
-                return -1;
-
-        nfs = nfsx->private;
-        cl = nfs->subvols;
-        while (cl) {
-                gf_log (GF_NFS, GF_LOG_DEBUG, "Starting subvolume: %s",
-                        cl->xlator->name);
-                ret = nfs_startup_subvolume (nfsx, cl->xlator);
-                if (ret == -1) {
-                        gf_log (GF_NFS, GF_LOG_CRITICAL, "Failed to start-up "
-                                "xlator: %s", cl->xlator->name);
-                        goto err;
-                }
-                cl = cl->next;
-        }
-
-        ret = 0;
-err:
-        return ret;
-}
-
 
 int
 nfs_init_subvolume (struct nfs_state *nfs, xlator_t *xl)
@@ -333,17 +515,19 @@ err:
 int
 nfs_init_subvolumes (struct nfs_state *nfs, xlator_list_t *cl)
 {
-        int             ret = -1;
-        unsigned int    lrusize = 0;
-        int             svcount = 0;
+        int                     ret = -1;
+        unsigned int            lrusize = 0;
+        int                     svcount = 0;
+        int                     x = 0;
+        xlator_list_t           *tmpcl = NULL;
+        struct nfs_subvolume    *sv = NULL;
 
         if ((!nfs) || (!cl))
                 return -1;
 
         lrusize = nfs->memfactor * GF_NFS_INODE_LRU_MULT;
-        nfs->subvols = cl;
         gf_log (GF_NFS, GF_LOG_TRACE, "inode table lru: %d", lrusize);
-
+        tmpcl = cl;
         while (cl) {
                 gf_log (GF_NFS, GF_LOG_DEBUG, "Initing subvolume: %s",
                         cl->xlator->name);
@@ -358,16 +542,23 @@ nfs_init_subvolumes (struct nfs_state *nfs, xlator_list_t *cl)
         }
 
         LOCK_INIT (&nfs->svinitlock);
-        nfs->initedxl = GF_CALLOC (svcount, sizeof (xlator_t *), 
-                                   gf_nfs_mt_xlator_t );
-        if (!nfs->initedxl) {
-                gf_log (GF_NFS, GF_LOG_ERROR, "Failed to allocated inited xls");
+        nfs->subvols = GF_CALLOC (svcount, sizeof (struct nfs_subvolume),
+                                  gf_nfs_mt_subvolumes);
+        if (!nfs->subvols) {
+                gf_log (GF_NFS, GF_LOG_ERROR, "Failed to allocate subvol list");
                 ret = -1;
                 goto err;
         }
 
-        gf_log (GF_NFS, GF_LOG_TRACE, "Inited volumes: %d", svcount);
+        cl = tmpcl;
         nfs->allsubvols = svcount;
+        for (;x < nfs->allsubvols; x++) {
+                sv = &nfs->subvols[x];
+                sv->subvol = cl->xlator;
+                sv->status = GF_NFS_SUBVOLUME_NOTSTARTED;
+        }
+
+        gf_log (GF_NFS, GF_LOG_TRACE, "inited volumes: %d", svcount);
         ret = 0;
 err:
         return ret;
@@ -482,6 +673,7 @@ nfs_init_state (xlator_t *this)
                 return NULL;
         }
 
+        nfs->nfsx = this;
         /* RPC service needs to be started before NFS versions can be
          * inited. */
         nfs->rpcsvc =  nfs_rpcsvc_init (this->ctx, this->options);
@@ -580,33 +772,48 @@ err:
 
 
 int
+nfs_disable_subvolume (xlator_t *nfsx, xlator_t *xl)
+{
+        struct nfs_state *nfs = NULL;
+
+        if ((!nfsx) || (!xl))
+                return -1;
+
+        nfs = nfsx->private;
+        gf_log (GF_NFS, GF_LOG_TRACE, "Disabling subvolume: %s", xl->name);
+        LOCK (&nfs->svinitlock);
+        {
+                if (!__is_nfs_subvolume_started (nfsx->private, xl)) {
+                        gf_log (GF_NFS,GF_LOG_TRACE, "Subvolume not started, "
+                                "no point disabling: %s", xl->name);
+                        goto unlock;
+                }
+                --nfs->upsubvols;
+                __nfs_subvolume_lookup_again_later (nfs, xl);
+        }
+unlock:
+        UNLOCK (&nfs->svinitlock);
+
+        return 0;
+}
+
+
+int
 notify (xlator_t *this, int32_t event, void *data, ...)
 {
         struct nfs_state        *nfs = NULL;
         xlator_t                *subvol = NULL;
-        int                     ret = -1;
 
         nfs = (struct nfs_state *)this->private;
         subvol = (xlator_t *)data;
 
-        gf_log (GF_NFS, GF_LOG_TRACE, "Notification received: %d",
-                event);
+        gf_log (GF_NFS, GF_LOG_TRACE, "Notification received: %s",
+                glusterfs_strevent (event));
         switch (event)
         {
                 case GF_EVENT_CHILD_UP:
                 {
                         nfs_startup_subvolume (this, subvol);
-                        if ((nfs->upsubvols == nfs->allsubvols) &&
-                            (!nfs->subvols_started)) {
-                                nfs->subvols_started = 1;
-                                gf_log (GF_NFS, GF_LOG_TRACE, "All children up,"
-                                " starting RPC");
-                                ret = nfs_init_versions (nfs, this);
-                                if (ret == -1)
-                                        gf_log (GF_NFS, GF_LOG_CRITICAL,
-                                                "Failed to initialize "
-                                                "protocols");
-                        }
                         break;
                 }
 
@@ -615,6 +822,12 @@ notify (xlator_t *this, int32_t event, void *data, ...)
                         default_notify (this, GF_EVENT_PARENT_UP, data);
                         break;
                 }
+
+                case GF_EVENT_CHILD_DOWN:
+                {
+                        nfs_disable_subvolume (this, subvol);
+                        break;
+                }
         }
 
         return 0;
diff --git a/xlators/nfs/server/src/nfs.h b/xlators/nfs/server/src/nfs.h
index baee0ce0f6a..6e6b433ce66 100644
--- a/xlators/nfs/server/src/nfs.h
+++ b/xlators/nfs/server/src/nfs.h
@@ -58,20 +58,37 @@ struct nfs_initer_list {
         rpcsvc_program_t        *program;
 };
 
+typedef enum {
+        GF_NFS_SUBVOLUME_NOTSTARTED = 1,
+        GF_NFS_SUBVOLUME_LOOKUPSENT,
+        GF_NFS_SUBVOLUME_STARTED,
+} nfs_subvolume_status_t;
+
+struct nfs_subvolume {
+        xlator_t                *subvol;
+        nfs_subvolume_status_t  status;
+};
+
+#define gf_nfs_subvolume_started(svl)   ((svl)->status == GF_NFS_SUBVOLUME_STARTED)
+#define gf_nfs_subvolume_lookupsent(svl)   ((svl)->status == GF_NFS_SUBVOLUME_LOOKUPSENT)
+#define gf_nfs_subvolume_notstarted(svl) ((svl)->status == GF_NFS_SUBVOLUME_NOTSTARTED)
+
+#define gf_nfs_subvolume_unused_slot(svl) ((svl)->subvol == NULL)
+#define gf_nfs_all_subvolumes_started(nf) ((nf->upsubvols == nf->allsubvols) && (!nf->subvols_started))
 
 struct nfs_state {
         rpcsvc_t                *rpcsvc;
         struct list_head        versions;
         struct mem_pool         *foppool;
         unsigned int            memfactor;
-        xlator_list_t           *subvols;
 
         gf_lock_t               svinitlock;
         int                     allsubvols;
         int                     upsubvols;
-        xlator_t                **initedxl;
+        struct nfs_subvolume    *subvols;
         int                     subvols_started;
         int                     dynamicvolumes;
+        xlator_t                *nfsx;
 };
 
 #define gf_nfs_dvm_on(nfsstt)   (((struct nfs_state *)nfsstt)->dynamicvolumes == GF_NFS_DVM_ON)
diff --git a/xlators/nfs/server/src/nfs3.c b/xlators/nfs/server/src/nfs3.c
index 2d5df536937..ad2b6581c3e 100644
--- a/xlators/nfs/server/src/nfs3.c
+++ b/xlators/nfs/server/src/nfs3.c
@@ -37,6 +37,7 @@
 #include "nfs-generics.h"
 #include "nfs3-helpers.h"
 #include "nfs-mem-types.h"
+#include "nfs-common.h"
 
 
 #include <sys/socket.h>
@@ -67,6 +68,16 @@
                 }                                                       \
         } while (0);                                                    \
 
+#define nfs3_volume_disabled_check(nf3stt, vlm, rtval, erlbl)           \
+        do {                                                            \
+              if ((is_nfs_subvolume_disabled (nfs_state (nf3stt->nfsx), vlm))){\
+                      gf_log (GF_NFS3, GF_LOG_ERROR, "Volume is disabled: %s",\
+                              vlm->name);                               \
+                      rtval = RPCSVC_ACTOR_SUCCESS;                     \
+                      goto erlbl;                                       \
+              }                                                         \
+        } while (0)                                                     \
+
 
 struct nfs3_export *
 __nfs3_get_export_by_index (struct nfs3_state *nfs3, uuid_t exportid)
@@ -671,6 +682,7 @@ nfs3_getattr (rpcsvc_request_t *req, struct nfs3_fh *fh)
         nfs3_validate_gluster_fh (fh, stat, nfs3err);
         nfs3_validate_nfs3_state (req, nfs3, stat, nfs3err, ret);
         nfs3_map_fh_to_volume (nfs3, fh, req, vol, stat, nfs3err);
+        nfs3_volume_disabled_check (nfs3, vol, ret, out);
         nfs3_handle_call_state_init (nfs3, cstate, req, vol, stat, nfs3err);
 
         ret = nfs3_fh_resolve_and_resume (cstate, fh, NULL,nfs3_getattr_resume);
@@ -685,7 +697,7 @@ nfs3err:
                 ret = 0;
                 nfs3_call_state_wipe (cstate);
         }
-
+out:
         return ret;
 }
 
@@ -927,6 +939,7 @@ nfs3_setattr (rpcsvc_request_t *req, struct nfs3_fh *fh, sattr3 *sattr,
         nfs3_validate_gluster_fh (fh, stat, nfs3err);
         nfs3_validate_nfs3_state (req, nfs3, stat, nfs3err, ret);
         nfs3_map_fh_to_volume (nfs3, fh, req, vol, stat, nfs3err);
+        nfs3_volume_disabled_check (nfs3, vol, ret, out);
         nfs3_check_rw_volaccess (nfs3, fh->exportid, stat, nfs3err);
         nfs3_handle_call_state_init (nfs3, cs, req, vol, stat, nfs3err);
 
@@ -962,7 +975,7 @@ nfs3err:
                  */
                 ret = 0;
         }
-
+out:
         return ret;
 }
 
@@ -1209,6 +1222,7 @@ nfs3_lookup (rpcsvc_request_t *req, struct nfs3_fh *fh, int fhlen, char *name)
                 nfs3_validate_gluster_fh (fh, stat, nfs3err);
         nfs3_validate_strlen_or_goto (name, NFS_NAME_MAX, nfs3err, stat, ret);
         nfs3_map_fh_to_volume (nfs3, fh, req, vol, stat, nfs3err);
+        nfs3_volume_disabled_check (nfs3, vol, ret, out);
         nfs3_handle_call_state_init (nfs3, cs, req, vol, stat, nfs3err);
 
         if (!nfs3_is_parentdir_entry (name))
@@ -1233,7 +1247,7 @@ nfs3err:
                  */
                 ret = 0;
         }
-
+out:
         return ret;
 }
 
@@ -1354,6 +1368,7 @@ nfs3_access (rpcsvc_request_t *req, struct nfs3_fh *fh, uint32_t accbits)
         nfs3_validate_gluster_fh (fh, stat, nfs3err);
         nfs3_validate_nfs3_state (req, nfs3, stat, nfs3err, ret);
         nfs3_map_fh_to_volume (nfs3, fh, req, vol, stat, nfs3err);
+        nfs3_volume_disabled_check (nfs3, vol, ret, out);
         nfs3_handle_call_state_init (nfs3, cs, req, vol, stat, nfs3err);
         cs->accessbits = accbits;
 
@@ -1369,7 +1384,7 @@ nfs3err:
                 nfs3_call_state_wipe (cs);
                 ret = 0;
         }
-
+out:
         return ret;
 }
 
@@ -1494,6 +1509,7 @@ nfs3_readlink (rpcsvc_request_t *req, struct nfs3_fh *fh)
         nfs3_validate_gluster_fh (fh, stat, nfs3err);
         nfs3_validate_nfs3_state (req, nfs3, stat, nfs3err, ret);
         nfs3_map_fh_to_volume (nfs3, fh, req, vol, stat, nfs3err);
+        nfs3_volume_disabled_check (nfs3, vol, ret, out);
         nfs3_handle_call_state_init (nfs3, cs, req, vol, stat, nfs3err);
 
         ret = nfs3_fh_resolve_and_resume (cs, fh, NULL, nfs3_readlink_resume);
@@ -1511,7 +1527,7 @@ nfs3err:
                  */
                 ret = 0;
         }
-
+out:
         return ret;
 }
 
@@ -1685,6 +1701,7 @@ nfs3_read (rpcsvc_request_t *req, struct nfs3_fh *fh, offset3 offset,
         nfs3_validate_gluster_fh (fh, stat, nfs3err);
         nfs3_validate_nfs3_state (req, nfs3, stat, nfs3err, ret);
         nfs3_map_fh_to_volume (nfs3, fh, req, vol, stat, nfs3err);
+        nfs3_volume_disabled_check (nfs3, vol, ret, out);
         nfs3_handle_call_state_init (nfs3, cs, req, vol, stat, nfs3err);
 
         cs->datacount = count;
@@ -1701,7 +1718,7 @@ nfs3err:
                 nfs3_call_state_wipe (cs);
                 ret = 0;
         }
-
+out:
         return ret;
 }
 
@@ -2008,6 +2025,7 @@ nfs3_write (rpcsvc_request_t *req, struct nfs3_fh *fh, offset3 offset,
         nfs3_validate_gluster_fh (fh, stat, nfs3err);
         nfs3_validate_nfs3_state (req, nfs3, stat, nfs3err, ret);
         nfs3_map_fh_to_volume (nfs3, fh, req, vol, stat, nfs3err);
+        nfs3_volume_disabled_check (nfs3, vol, ret, out);
         nfs3_check_rw_volaccess (nfs3, fh->exportid, stat, nfs3err);
         nfs3_handle_call_state_init (nfs3, cs, req, vol, stat, nfs3err);
         cs->datacount = count;
@@ -2029,7 +2047,7 @@ nfs3err:
                 nfs3_call_state_wipe (cs);
                 ret = 0;
         }
-
+out:
         return ret;
 }
 
@@ -2399,6 +2417,7 @@ nfs3_create (rpcsvc_request_t *req, struct nfs3_fh *dirfh, char *name,
         nfs3_validate_nfs3_state (req, nfs3, stat, nfs3err, ret);
         nfs3_validate_strlen_or_goto (name, NFS_NAME_MAX, nfs3err, stat, ret);
         nfs3_map_fh_to_volume (nfs3, dirfh, req, vol, stat, nfs3err);
+        nfs3_volume_disabled_check (nfs3, vol, ret, out);
         nfs3_check_rw_volaccess (nfs3, dirfh->exportid, stat, nfs3err);
         nfs3_handle_call_state_init (nfs3, cs, req, vol, stat, nfs3err);
 
@@ -2420,7 +2439,7 @@ nfs3err:
                 nfs3_call_state_wipe (cs);
                 ret = 0;
         }
-
+out:
         return ret;
 }
 
@@ -2606,6 +2625,7 @@ nfs3_mkdir (rpcsvc_request_t *req, struct nfs3_fh *dirfh, char *name,
         nfs3_validate_nfs3_state (req, nfs3, stat, nfs3err, ret);
         nfs3_validate_strlen_or_goto (name, NFS_NAME_MAX, nfs3err, stat, ret);
         nfs3_map_fh_to_volume (nfs3, dirfh, req, vol, stat, nfs3err);
+        nfs3_volume_disabled_check (nfs3, vol, ret, out);
         nfs3_check_rw_volaccess (nfs3, dirfh->exportid, stat, nfs3err);
         nfs3_handle_call_state_init (nfs3, cs, req, vol, stat, nfs3err);
 
@@ -2624,7 +2644,7 @@ nfs3err:
                 nfs3_call_state_wipe (cs);
                 ret = 0;
         }
-
+out:
         return ret;
 }
 
@@ -2756,6 +2776,7 @@ nfs3_symlink (rpcsvc_request_t *req, struct nfs3_fh *dirfh, char *name,
         nfs3_validate_nfs3_state (req, nfs3, stat, nfs3err, ret);
         nfs3_validate_strlen_or_goto (name, NFS_NAME_MAX, nfs3err, stat, ret);
         nfs3_map_fh_to_volume (nfs3, dirfh, req, vol, stat, nfs3err);
+        nfs3_volume_disabled_check (nfs3, vol, ret, out);
         nfs3_check_rw_volaccess (nfs3, dirfh->exportid, stat, nfs3err);
         nfs3_handle_call_state_init (nfs3, cs, req, vol, stat, nfs3err);
 
@@ -2782,7 +2803,7 @@ nfs3err:
                  */
                 ret = 0;
         }
-
+out:
         return ret;
 }
 
@@ -3028,6 +3049,7 @@ nfs3_mknod (rpcsvc_request_t *req, struct nfs3_fh *fh, char *name,
         nfs3_validate_nfs3_state (req, nfs3, stat, nfs3err, ret);
         nfs3_validate_strlen_or_goto (name, NFS_NAME_MAX, nfs3err, stat, ret);
         nfs3_map_fh_to_volume (nfs3, fh, req, vol, stat, nfs3err);
+        nfs3_volume_disabled_check (nfs3, vol, ret, out);
         nfs3_check_rw_volaccess (nfs3, fh->exportid, stat, nfs3err);
         nfs3_handle_call_state_init (nfs3, cs, req, vol, stat, nfs3err);
 
@@ -3067,7 +3089,7 @@ nfs3err:
                 nfs3_call_state_wipe (cs);
                 ret = 0;
         }
-
+out:
         return ret;
 }
 
@@ -3227,6 +3249,7 @@ nfs3_remove (rpcsvc_request_t *req, struct nfs3_fh *fh, char *name)
         nfs3_validate_nfs3_state (req, nfs3, stat, nfs3err, ret);
         nfs3_validate_strlen_or_goto (name, NFS_NAME_MAX, nfs3err, stat, ret);
         nfs3_map_fh_to_volume (nfs3, fh, req, vol, stat, nfs3err);
+        nfs3_volume_disabled_check (nfs3, vol, ret, out);
         nfs3_check_rw_volaccess (nfs3, fh->exportid, stat, nfs3err);
         nfs3_handle_call_state_init (nfs3, cs, req, vol, stat, nfs3err);
 
@@ -3245,7 +3268,7 @@ nfs3err:
                  */
                 ret = 0;
         }
-
+out:
         return ret;
 }
 
@@ -3370,6 +3393,7 @@ nfs3_rmdir (rpcsvc_request_t *req, struct nfs3_fh *fh, char *name)
         nfs3_validate_nfs3_state (req, nfs3, stat, nfs3err, ret);
         nfs3_validate_strlen_or_goto (name, NFS_NAME_MAX, nfs3err, stat, ret);
         nfs3_map_fh_to_volume (nfs3, fh, req, vol, stat, nfs3err);
+        nfs3_volume_disabled_check (nfs3, vol, ret, out);
         nfs3_check_rw_volaccess (nfs3, fh->exportid, stat, nfs3err);
         nfs3_handle_call_state_init (nfs3, cs, req, vol, stat, nfs3err);
 
@@ -3388,7 +3412,7 @@ nfs3err:
                  */
                 ret = 0;
         }
-
+out:
         return ret;
 }
 
@@ -3573,6 +3597,7 @@ nfs3_rename (rpcsvc_request_t *req, struct nfs3_fh *olddirfh, char *oldname,
         nfs3_validate_strlen_or_goto(oldname, NFS_NAME_MAX, nfs3err, stat, ret);
         nfs3_validate_strlen_or_goto(newname, NFS_NAME_MAX, nfs3err, stat, ret);
         nfs3_map_fh_to_volume (nfs3, olddirfh, req, vol, stat, nfs3err);
+        nfs3_volume_disabled_check (nfs3, vol, ret, out);
         nfs3_check_rw_volaccess (nfs3, olddirfh->exportid, stat, nfs3err);
         nfs3_handle_call_state_init (nfs3, cs, req, vol, stat, nfs3err);
 
@@ -3603,7 +3628,7 @@ nfs3err:
                  */
                 ret = 0;
         }
-
+out:
         return ret;
 }
 
@@ -3762,6 +3787,7 @@ nfs3_link (rpcsvc_request_t *req, struct nfs3_fh *targetfh,
         nfs3_validate_nfs3_state (req, nfs3, stat, nfs3err, ret);
         nfs3_validate_strlen_or_goto(newname, NFS_NAME_MAX, nfs3err, stat, ret);
         nfs3_map_fh_to_volume (nfs3, dirfh, req, vol, stat, nfs3err);
+        nfs3_volume_disabled_check (nfs3, vol, ret, out);
         nfs3_check_rw_volaccess (nfs3, dirfh->exportid, stat, nfs3err);
         nfs3_handle_call_state_init (nfs3, cs, req, vol, stat, nfs3err);
 
@@ -3789,7 +3815,7 @@ nfs3err:
                  */
                 ret = 0;
         }
-
+out:
         return ret;
 }
 
@@ -4085,6 +4111,7 @@ nfs3_readdir (rpcsvc_request_t *req, struct nfs3_fh *fh, cookie3 cookie,
         nfs3_validate_gluster_fh (fh, stat, nfs3err);
         nfs3_validate_nfs3_state (req, nfs3, stat, nfs3err, ret);
         nfs3_map_fh_to_volume (nfs3, fh, req, vol, stat, nfs3err);
+        nfs3_volume_disabled_check (nfs3, vol, ret, out);
         nfs3_handle_call_state_init (nfs3, cs, req, vol, stat, nfs3err);
 
         cs->cookieverf = cverf;
@@ -4116,7 +4143,7 @@ nfs3err:
                 ret = 0;
                 nfs3_call_state_wipe (cs);
         }
-
+out:
         return ret;
 }
 
@@ -4307,6 +4334,7 @@ nfs3_fsstat (rpcsvc_request_t *req, struct nfs3_fh *fh)
         nfs3_validate_gluster_fh (fh, stat, nfs3err);
         nfs3_validate_nfs3_state (req, nfs3, stat, nfs3err, ret);
         nfs3_map_fh_to_volume (nfs3, fh, req, vol, stat, nfs3err);
+        nfs3_volume_disabled_check (nfs3, vol, ret, out);
         nfs3_handle_call_state_init (nfs3, cs, req, vol, stat, nfs3err);
 
         ret = nfs3_fh_resolve_and_resume (cs, fh, NULL, nfs3_fsstat_resume);
@@ -4324,7 +4352,7 @@ nfs3err:
                  */
                 ret = 0;
         }
-
+out:
         return ret;
 }
 
@@ -4449,6 +4477,7 @@ nfs3_fsinfo (rpcsvc_request_t *req, struct nfs3_fh *fh)
         nfs3_validate_gluster_fh (fh, stat, nfs3err);
         nfs3_validate_nfs3_state (req, nfs3, stat, nfs3err, ret);
         nfs3_map_fh_to_volume (nfs3, fh, req, vol, stat, nfs3err);
+        nfs3_volume_disabled_check (nfs3, vol, ret, out);
         nfs3_handle_call_state_init (nfs3, cs, req, vol, stat, nfs3err);
 
         ret = nfs3_fh_resolve_and_resume (cs, fh, NULL, nfs3_fsinfo_resume);
@@ -4463,7 +4492,7 @@ nfs3err:
                 nfs3_call_state_wipe (cs);
                 ret = 0;
         }
-
+out:
         return ret;
 }
 
@@ -4586,6 +4615,7 @@ nfs3_pathconf (rpcsvc_request_t *req, struct nfs3_fh *fh)
         nfs3_validate_gluster_fh (fh, stat, nfs3err);
         nfs3_validate_nfs3_state (req, nfs3, stat, nfs3err, ret);
         nfs3_map_fh_to_volume (nfs3, fh, req, vol, stat, nfs3err);
+        nfs3_volume_disabled_check (nfs3, vol, ret, out);
         nfs3_handle_call_state_init (nfs3, cs, req, vol, stat, nfs3err);
 
         ret = nfs3_fh_resolve_and_resume (cs, fh, NULL, nfs3_pathconf_resume);
@@ -4603,7 +4633,7 @@ nfs3err:
                  */
                 ret = 0;
         }
-
+out:
         return ret;
 }
 
@@ -4764,6 +4794,7 @@ nfs3_commit (rpcsvc_request_t *req, struct nfs3_fh *fh, offset3 offset,
         nfs3_validate_gluster_fh (fh, stat, nfs3err);
         nfs3_validate_nfs3_state (req, nfs3, stat, nfs3err, ret);
         nfs3_map_fh_to_volume (nfs3, fh, req, vol, stat, nfs3err);
+        nfs3_volume_disabled_check (nfs3, vol, ret, out);
         nfs3_check_rw_volaccess (nfs3, fh->exportid, stat, nfs3err);
         nfs3_handle_call_state_init (nfs3, cs, req, vol, stat, nfs3err);
 
@@ -4782,7 +4813,7 @@ nfs3err:
                 nfs3_call_state_wipe (cs);
                 ret = 0;
         }
-
+out:
         return ret;
 }