nfs3: Unref & unbind dir fd with inode lock on EOF

..so that when EOF is reached on this fd, any further
requests on the same inode do not get handled through this
fd but result in a new fd being opened.

Unbinding results in the fd getting deleted from the inode's fd list.

Signed-off-by: Shehjar Tikoo <shehjart@gluster.com>
Signed-off-by: Vijay Bellur <vijay@dev.gluster.com>

BUG: 1619 (glusterfs nfs server crashed on dht+replica(2x2))
URL: http://bugs.gluster.com/cgi-bin/bugzilla3/show_bug.cgi?id=1619

3 files changed, 38 insertions, 6 deletions

diff --git a/libglusterfs/src/fd.c b/libglusterfs/src/fd.c
index 9e91cdb83..5e25b59cf 100644
--- a/libglusterfs/src/fd.c
+++ b/libglusterfs/src/fd.c
@@ -499,6 +499,30 @@ fd_bind (fd_t *fd)
         return fd;
 }
 
+
+void
+fd_unref_unbind (fd_t *fd)
+{
+        assert (fd->refcount);
+
+        LOCK (&fd->inode->lock);
+        {
+	        --fd->refcount;
+                /* Better know what you're doing with this function
+                 * because it does not do what fd_destroy does when
+                 * refcount goes to 0.
+                 * Make sure you only call this when you know there are
+                 * pending refs on the fd.
+                 */
+                assert (fd->refcount);
+		list_del_init (&fd->inode_list);
+        }
+        UNLOCK (&fd->inode->lock);
+
+        return;
+}
+
+
 fd_t *
 fd_create (inode_t *inode, pid_t pid)
 {
diff --git a/libglusterfs/src/fd.h b/libglusterfs/src/fd.h
index 4ea7fc165..48037b722 100644
--- a/libglusterfs/src/fd.h
+++ b/libglusterfs/src/fd.h
@@ -170,4 +170,6 @@ _fd_ref (fd_t *fd);
 void
 fd_ctx_dump (fd_t *fd, char *prefix);
 
+extern void
+fd_unref_unbind (fd_t *fd);
 #endif /* _FD_H */
diff --git a/xlators/nfs/server/src/nfs3.c b/xlators/nfs/server/src/nfs3.c
index 0a55e55c9..2d5df5369 100644
--- a/xlators/nfs/server/src/nfs3.c
+++ b/xlators/nfs/server/src/nfs3.c
@@ -3885,6 +3885,17 @@ nfs3svc_readdir_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
         stat = NFS3_OK;
 nfs3err:
+
+        /* On end-of-directory, unref the fd to have it removed from the cache
+         * and also unbind it from the fd so that any subsequent request on the
+         * on the directory do not get this fd when fd_lookup is called in
+         * dir open resume path.
+         */
+        if (is_eof) {
+                gf_log (GF_NFS3, GF_LOG_TRACE, "EOF REF: %d", cs->fd->refcount);
+                fd_unref_unbind (cs->fd);
+        }
+
         if (cs->maxcount == 0) {
                 nfs3_log_readdir_res (nfs_rpcsvc_request_xid (cs->req), stat,
                                       op_errno, (uintptr_t)cs->fd,
@@ -3902,11 +3913,6 @@ nfs3err:
                                      cs->maxcount, is_eof);
         }
 
-        if (is_eof) {
-                gf_log (GF_NFS3, GF_LOG_TRACE, "EOF REF: %d", cs->fd->refcount);
-                fd_unref (cs->fd);
-        }
-
         gf_log (GF_NFS3, GF_LOG_TRACE, "CS WIPE REF: %d", cs->fd->refcount);
         nfs3_call_state_wipe (cs);
         return 0;
@@ -3958,7 +3964,7 @@ err:
          * so that next time the dir is read, we'll get any changed directory
          * entries.
          */
-        fd_unref (cs->fd);
+        fd_unref_unbind (cs->fd);
         nfs3_call_state_wipe (cs);
 ret:
         return 0;