storage/posix: Add active-fd-count option in gluster

Problem:
when dd happens on sharded replicate volume all the writes on shards happen
through anon-fd. When the writes don't come quick enough, old anon-fd closes
and new fd gets created to serve the new writes. open-fd-count is decremented
only after the fd is closed as part of fd_destroy(). So even when one fd is on
the way to be closed a new fd will be created and during this short period it
appears as though there are multiple fds opened on the file. AFR thinks another
application opened the same file and switches off eager-lock leading to
extra latency.

Fix:
Have a different option called active-fd whose life cycle starts at
fd_bind() and ends just before fd_destroy()

BUG: 1557932
Change-Id: I2e221f6030feeedf29fbb3bd6554673b8a5b9c94
Signed-off-by: Pranith Kumar K <pkarampu@redhat.com>

7 files changed, 51 insertions, 33 deletions

diff --git a/libglusterfs/src/fd.c b/libglusterfs/src/fd.c
index c6b1a4de611..d31b106aa8b 100644
--- a/libglusterfs/src/fd.c
+++ b/libglusterfs/src/fd.c
@@ -540,6 +540,7 @@ fd_unref (fd_t *fd)
                 if (refcount == 0) {
                         if (!list_empty (&fd->inode_list)) {
                                 list_del_init (&fd->inode_list);
+                                fd->inode->active_fd_count--;
                                 bound = _gf_true;
                         }
                 }
@@ -561,6 +562,7 @@ __fd_bind (fd_t *fd)
         list_del_init (&fd->inode_list);
         list_add (&fd->inode_list, &fd->inode->fd_list);
         fd->inode->fd_count++;
+        fd->inode->active_fd_count++;
 
         return fd;
 }
diff --git a/libglusterfs/src/glusterfs.h b/libglusterfs/src/glusterfs.h
index 250d8e69a4f..28666de3da4 100644
--- a/libglusterfs/src/glusterfs.h
+++ b/libglusterfs/src/glusterfs.h
@@ -181,6 +181,7 @@
 #define GLUSTERFS_WRITE_IS_APPEND "glusterfs.write-is-append"
 #define GLUSTERFS_WRITE_UPDATE_ATOMIC "glusterfs.write-update-atomic"
 #define GLUSTERFS_OPEN_FD_COUNT "glusterfs.open-fd-count"
+#define GLUSTERFS_ACTIVE_FD_COUNT "glusterfs.open-active-fd-count"
 #define GLUSTERFS_INODELK_COUNT "glusterfs.inodelk-count"
 #define GLUSTERFS_ENTRYLK_COUNT "glusterfs.entrylk-count"
 #define GLUSTERFS_POSIXLK_COUNT "glusterfs.posixlk-count"
diff --git a/libglusterfs/src/inode.c b/libglusterfs/src/inode.c
index 1c8d1badf4a..2100ea3cad2 100644
--- a/libglusterfs/src/inode.c
+++ b/libglusterfs/src/inode.c
@@ -2343,6 +2343,8 @@ inode_dump (inode_t *inode, char *prefix)
                 gf_proc_dump_write("gfid", "%s", uuid_utoa (inode->gfid));
                 gf_proc_dump_write("nlookup", "%ld", inode->nlookup);
                 gf_proc_dump_write("fd-count", "%u", inode->fd_count);
+                gf_proc_dump_write("active-fd-count", "%u",
+                                   inode->active_fd_count);
                 gf_proc_dump_write("ref", "%u", inode->ref);
                 gf_proc_dump_write("ia_type", "%d", inode->ia_type);
                 if (inode->_ctx) {
diff --git a/libglusterfs/src/inode.h b/libglusterfs/src/inode.h
index a1b238041b4..f77778ff3ad 100644
--- a/libglusterfs/src/inode.h
+++ b/libglusterfs/src/inode.h
@@ -92,6 +92,7 @@ struct _inode {
         gf_lock_t            lock;
         uint64_t             nlookup;
         uint32_t             fd_count;      /* Open fd count */
+        uint32_t             active_fd_count;      /* Active open fd count */
         uint32_t             ref;           /* reference count on this inode */
         ia_type_t            ia_type;       /* what kind of file */
         struct list_head     fd_list;       /* list of open files on this inode */
diff --git a/tests/volume.rc b/tests/volume.rc
index add812d4779..44428606711 100644
--- a/tests/volume.rc
+++ b/tests/volume.rc
@@ -808,7 +808,19 @@ function get_fd_count {
         local fname=$4
         local gfid_str=$(gf_gfid_xattr_to_str $(gf_get_gfid_xattr $brick/$fname))
         local statedump=$(generate_brick_statedump $vol $host $brick)
-        local count=$(grep "gfid=$gfid_str" $statedump -A2 -B1 | grep $brick -A3 | grep fd-count | cut -f2 -d'=' | tail -1)
+        local count=$(grep "gfid=$gfid_str" $statedump -A2 -B1 | grep $brick -A3 | grep -w fd-count | cut -f2 -d'=' | tail -1)
+        rm -f $statedump
+        echo $count
+}
+
+function get_active_fd_count {
+        local vol=$1
+        local host=$2
+        local brick=$3
+        local fname=$4
+        local gfid_str=$(gf_gfid_xattr_to_str $(gf_get_gfid_xattr $brick/$fname))
+        local statedump=$(generate_brick_statedump $vol $host $brick)
+        local count=$(grep "gfid=$gfid_str" $statedump -A2 -B1 | grep $brick -A3 | grep -w active-fd-count | cut -f2 -d'=' | tail -1)
         rm -f $statedump
         echo $count
 }
diff --git a/xlators/storage/posix/src/posix-helpers.c b/xlators/storage/posix/src/posix-helpers.c
index 377389d22a3..816fb3587d2 100644
--- a/xlators/storage/posix/src/posix-helpers.c
+++ b/xlators/storage/posix/src/posix-helpers.c
@@ -392,27 +392,6 @@ _get_filler_inode (posix_xattr_filler_t *filler)
 }
 
 static int
-_posix_filler_get_openfd_count (posix_xattr_filler_t *filler, char *key)
-{
-        inode_t  *inode            = NULL;
-        int      ret               = -1;
-
-        inode = _get_filler_inode (filler);
-        if (!inode || gf_uuid_is_null (inode->gfid))
-                        goto out;
-
-        ret = dict_set_uint32 (filler->xattr, key, inode->fd_count);
-        if (ret < 0) {
-                gf_msg (filler->this->name, GF_LOG_WARNING, 0,
-                        P_MSG_DICT_SET_FAILED,
-                        "Failed to set dictionary value for %s", key);
-                goto out;
-        }
-out:
-        return ret;
-}
-
-static int
 _posix_xattr_get_set (dict_t *xattr_req, char *key, data_t *data,
                       void *xattrargs)
 {
@@ -420,11 +399,11 @@ _posix_xattr_get_set (dict_t *xattr_req, char *key, data_t *data,
         int       ret      = -1;
         char     *databuf  = NULL;
         int       _fd      = -1;
-        loc_t    *loc      = NULL;
         ssize_t  req_size  = 0;
         int32_t  list_offset = 0;
         ssize_t  remaining_size = 0;
         char     *xattr    = NULL;
+        inode_t  *inode    = NULL;
 
         if (posix_xattr_ignorable (key))
                 goto out;
@@ -500,16 +479,25 @@ _posix_xattr_get_set (dict_t *xattr_req, char *key, data_t *data,
                         GF_FREE (databuf);
                 }
         } else if (!strcmp (key, GLUSTERFS_OPEN_FD_COUNT)) {
-                ret = _posix_filler_get_openfd_count (filler, key);
-                loc = filler->loc;
-                if (loc) {
-                        ret = dict_set_uint32 (filler->xattr, key,
-                                               loc->inode->fd_count);
-                        if (ret < 0)
-                                gf_msg (filler->this->name, GF_LOG_WARNING, 0,
-                                        P_MSG_XDATA_GETXATTR,
-                                        "Failed to set dictionary value for %s",
-                                        key);
+                inode = _get_filler_inode (filler);
+                if (!inode || gf_uuid_is_null (inode->gfid))
+                                goto out;
+                ret = dict_set_uint32 (filler->xattr, key, inode->fd_count);
+                if (ret < 0) {
+                        gf_msg (filler->this->name, GF_LOG_WARNING, 0,
+                                P_MSG_DICT_SET_FAILED,
+                                "Failed to set dictionary value for %s", key);
+                }
+        } else if (!strcmp (key, GLUSTERFS_ACTIVE_FD_COUNT)) {
+                inode = _get_filler_inode (filler);
+                if (!inode || gf_uuid_is_null (inode->gfid))
+                                goto out;
+                ret = dict_set_uint32 (filler->xattr, key,
+                                       inode->active_fd_count);
+                if (ret < 0) {
+                        gf_msg (filler->this->name, GF_LOG_WARNING, 0,
+                                P_MSG_DICT_SET_FAILED,
+                                "Failed to set dictionary value for %s", key);
                 }
         } else if (!strcmp (key, GET_ANCESTRY_PATH_KEY)) {
                 /* As of now, the only consumers of POSIX_ANCESTRY_PATH attempt
diff --git a/xlators/storage/posix/src/posix-inode-fd-ops.c b/xlators/storage/posix/src/posix-inode-fd-ops.c
index bca7419eee0..dafac59fe5b 100644
--- a/xlators/storage/posix/src/posix-inode-fd-ops.c
+++ b/xlators/storage/posix/src/posix-inode-fd-ops.c
@@ -1526,6 +1526,18 @@ _fill_writev_xdata (fd_t *fd, dict_t *xdata, xlator_t *this, int is_append)
                 }
         }
 
+        if (dict_get (xdata, GLUSTERFS_ACTIVE_FD_COUNT)) {
+                ret = dict_set_uint32 (rsp_xdata, GLUSTERFS_ACTIVE_FD_COUNT,
+                                       fd->inode->active_fd_count);
+                if (ret < 0) {
+                        gf_msg (this->name, GF_LOG_WARNING, 0,
+                                P_MSG_DICT_SET_FAILED, "%s: Failed to set "
+                                "dictionary value for %s",
+                                uuid_utoa (fd->inode->gfid),
+                                GLUSTERFS_ACTIVE_FD_COUNT);
+                }
+        }
+
         if (dict_get (xdata, GLUSTERFS_WRITE_IS_APPEND)) {
                 ret = dict_set_uint32 (rsp_xdata, GLUSTERFS_WRITE_IS_APPEND,
                                        is_append);