gfapi: async fops should unref in callbacks

If fd is unref'd at the end of async call then the unref in cbks would
lead to double unref and possible crash. Removing duplicate unrefs.

Added unref only in failure cases.

A simple test case has been added to test async write case. Need to
extend the same for other async APIs too.

Details:
All glfd based calls in libgfapi, except for glfs_open and glfs_close,
behave in the same way. At the start of the operation, they take a ref
on glfd and fd. At the end of the operation, they unref it. Async calls
are a little different as they unref in the cbk function. A successfull
open call does not unref either the glfd or fd, thereby functioning as a
reference for a OPEN file object. glfs_close makes a syncop_flush call
sandwiched between a fd ref and unref(this can be removed, more on this
below), followed by a call to glfs_mark_glfd_for_deletion which unrefs
glfd and also calls glfs_fd_destroy as a release function thereby doing
a unref on fd too.

Functionally, there is no problem with how everything works when as
described above. However, it is a little non-intuitive that we need to
perform a fd_unref as a consequence of a implicit fd_ref that happens
within glfs_resolve_fd. As we perform a GF_REF_GET(glfd) at the start of
every operation, it would be worthwhile to remove the fd_ref that
glfs_resovle_fd takes and do away with explicit fd_unref()s at the end
of every operation. This is the same reason why we don't need the fd_ref
in glfs_close. This is however not in the scope of this patch.

Change-Id: I86b1d3b2ad846b16ea527d541dc82b5e90b0ba85
BUG: 1391086
Signed-off-by: Raghavendra Talur <rtalur@redhat.com>
Reviewed-on: http://review.gluster.org/15768
Smoke: Gluster Build System <jenkins@build.gluster.org>
NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org>
Reviewed-by: Rajesh Joseph <rjoseph@redhat.com>
Reviewed-by: Xavier Hernandez <xhernandez@datalab.es>
Reviewed-by: soumya k <skoduri@redhat.com>
Reviewed-by: Prasanna Kumar Kalever <pkalever@redhat.com>
CentOS-regression: Gluster Build System <jenkins@build.gluster.org>
Reviewed-by: Shyamsundar Ranganathan <srangana@redhat.com>

3 files changed, 215 insertions, 18 deletions

diff --git a/api/src/glfs-fops.c b/api/src/glfs-fops.c
index 0a59a8dcf1c..ea61c8b7a43 100644
--- a/api/src/glfs-fops.c
+++ b/api/src/glfs-fops.c
@@ -934,10 +934,9 @@ pub_glfs_preadv_async (struct glfs_fd *glfd, const struct iovec *iovec,
 			   offset, flags, NULL);
 
 out:
-        if (fd)
-                fd_unref (fd);
-
         if (ret) {
+                if (fd)
+                        fd_unref (fd);
                 if (glfd)
                         GF_REF_PUT (glfd);
                 if (gio) {
@@ -1252,10 +1251,9 @@ pub_glfs_pwritev_async (struct glfs_fd *glfd, const struct iovec *iovec,
 
         ret = 0;
 out:
-        if (fd)
-                fd_unref (fd);
-
         if (ret) {
+                if (fd)
+                        fd_unref (fd);
                 if (glfd)
                         GF_REF_PUT (glfd);
                 GF_FREE (gio);
@@ -1439,10 +1437,9 @@ glfs_fsync_async_common (struct glfs_fd *glfd, glfs_io_cbk fn, void *data,
                            subvol->fops->fsync, fd, dataonly, NULL);
 
 out:
-        if (fd)
-                fd_unref (fd);
-
         if (ret) {
+                if (fd)
+                        fd_unref (fd);
                 if (glfd)
                         GF_REF_PUT (glfd);
                 GF_FREE (gio);
@@ -1693,10 +1690,9 @@ pub_glfs_ftruncate_async (struct glfs_fd *glfd, off_t offset, glfs_io_cbk fn,
         ret = 0;
 
 out:
-        if (fd)
-                fd_unref (fd);
-
         if (ret) {
+                if (fd)
+                        fd_unref (fd);
                 if (glfd)
                         GF_REF_PUT (glfd);
                 GF_FREE (gio);
@@ -2552,10 +2548,9 @@ pub_glfs_discard_async (struct glfs_fd *glfd, off_t offset, size_t len,
 
         ret = 0;
 out:
-        if (fd)
-                fd_unref (fd);
-
         if (ret) {
+                if (fd)
+                        fd_unref (fd);
                 if (glfd)
                         GF_REF_PUT (glfd);
                 GF_FREE (gio);
@@ -2640,10 +2635,9 @@ pub_glfs_zerofill_async (struct glfs_fd *glfd, off_t offset, off_t len,
                            subvol->fops->zerofill, fd, offset, len, NULL);
         ret = 0;
 out:
-        if (fd)
-                fd_unref (fd);
-
         if (ret) {
+                if (fd)
+                        fd_unref (fd);
                 if (glfd)
                         GF_REF_PUT (glfd);
                 GF_FREE (gio);
diff --git a/tests/basic/gfapi/gfapi-async-calls-test.c b/tests/basic/gfapi/gfapi-async-calls-test.c
new file mode 100644
index 00000000000..277067bee2c
--- /dev/null
+++ b/tests/basic/gfapi/gfapi-async-calls-test.c
@@ -0,0 +1,177 @@
+#include <fcntl.h>
+#include <unistd.h>
+#include <time.h>
+#include <limits.h>
+#include <string.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <errno.h>
+#include <glusterfs/api/glfs.h>
+#include <glusterfs/api/glfs-handles.h>
+
+#define LOG_ERR(msg) do { \
+        fprintf (stderr, "%s : Error (%s)\n", msg, strerror (errno)); \
+        } while (0)
+
+int cbk_complete = 0;
+int cbk_ret_val = -1;
+
+int
+fill_iov (struct iovec *iov, char fillchar, int count)
+{
+        int ret = -1;
+
+        iov->iov_base = calloc (count + 1, sizeof(fillchar));
+        if (iov->iov_base == NULL) {
+                return ret;
+        } else {
+                iov->iov_len = count;
+                ret = 0;
+        }
+        memset (iov->iov_base, fillchar, count);
+        memset (iov->iov_base + count, '\0', 1);
+
+        return ret;
+}
+
+glfs_t *
+init_glfs (const char *hostname, const char *volname,
+           const char *logfile)
+{
+        int     ret     = -1;
+        glfs_t *fs      = NULL;
+
+        fs = glfs_new (volname);
+        if (!fs) {
+                LOG_ERR ("glfs_new failed");
+                return NULL;
+        }
+
+        ret = glfs_set_volfile_server (fs, "tcp", hostname, 24007);
+        if (ret < 0) {
+                LOG_ERR ("glfs_set_volfile_server failed");
+                goto out;
+        }
+
+        ret = glfs_set_logging (fs, logfile, 7);
+        if (ret < 0) {
+                LOG_ERR ("glfs_set_logging failed");
+                goto out;
+        }
+
+        ret = glfs_init (fs);
+        if (ret < 0) {
+                LOG_ERR ("glfs_init failed");
+                goto out;
+        }
+
+        ret = 0;
+out:
+        if (ret) {
+                glfs_fini (fs);
+                fs = NULL;
+        }
+
+        return fs;
+}
+
+void
+write_async_cbk (glfs_fd_t *fd, ssize_t ret, void *cookie)
+{
+
+        if (ret < 0) {
+                LOG_ERR ("glfs_write failed");
+        }
+        cbk_ret_val = ret;
+        cbk_complete = 1;
+}
+
+int
+write_async (glfs_t *fs, glfs_fd_t *glfd, int char_count)
+{
+        ssize_t         ret          = -1;
+        int             flags        = O_RDWR;
+        const char     *buff         = "This is from my prog\n";
+        struct iovec    iov          = {0};
+        void           *write_cookie = NULL;
+        void           *read_cookie  = NULL;
+
+
+
+        ret = fill_iov (&iov, 'a', char_count);
+        if (ret) {
+                LOG_ERR ("failed to create iov");
+                goto out;
+        }
+
+        write_cookie = strdup ("write_cookie");
+        ret = glfs_pwritev_async (glfd, &iov, 1, 0, flags, write_async_cbk,
+                                  &write_cookie);
+out:
+        if (ret < 0) {
+                LOG_ERR ("glfs_pwritev async failed");
+        }
+        return ret;
+
+}
+
+int
+main (int argc, char *argv[])
+{
+        int         ret      = 0;
+        char       *hostname = NULL;
+        char       *volname  = NULL;
+        char       *logfile  = NULL;
+        glfs_t     *fs       = NULL;
+        const char *filename = "glfs_test.txt";
+        int         flags    = (O_RDWR|O_CREAT);
+        glfs_fd_t  *glfd     = NULL;
+        int         count    = 200;
+
+        if (argc != 4) {
+                fprintf (stderr, "Invalid argument\n");
+                exit(1);
+        }
+
+        hostname = argv[1];
+        volname = argv[2];
+        logfile = argv[3];
+
+        fs = init_glfs (hostname, volname, logfile);
+        if (fs == NULL) {
+                LOG_ERR ("init_glfs failed");
+                return -1;
+        }
+
+        glfd = glfs_creat (fs, filename, flags, 0644);
+        if (glfd == NULL) {
+                LOG_ERR ("glfs_creat failed");
+                exit(1);
+        }
+
+        ret = write_async (fs, glfd, count);
+        if (ret) {
+                LOG_ERR ("glfs_test_function failed");
+                exit(1);
+        }
+
+        while (cbk_complete != 1) {
+                sleep(1);
+        }
+
+        ret = glfs_close (glfd);
+        if (ret < 0) {
+                LOG_ERR ("glfs close  failed");
+        }
+
+        /*
+         * skipping fini
+         */
+
+        if (cbk_ret_val == count)
+                return 0;
+        else
+                return -1;
+}
+
+
diff --git a/tests/basic/gfapi/gfapi-async-calls-test.t b/tests/basic/gfapi/gfapi-async-calls-test.t
new file mode 100644
index 00000000000..9e5cd7c92cc
--- /dev/null
+++ b/tests/basic/gfapi/gfapi-async-calls-test.t
@@ -0,0 +1,26 @@
+#!/bin/bash
+
+. $(dirname $0)/../../include.rc
+. $(dirname $0)/../../volume.rc
+
+cleanup;
+EXIT_EARLY=1;
+
+TEST glusterd
+
+TEST $CLI volume create $V0 ${H0}:$B0/brick1;
+EXPECT 'Created' volinfo_field $V0 'Status';
+
+TEST $CLI volume start $V0;
+EXPECT 'Started' volinfo_field $V0 'Status';
+
+logdir=`gluster --print-logdir`
+
+TEST build_tester $(dirname $0)/gfapi-async-calls-test.c -lgfapi
+
+
+TEST ./$(dirname $0)/gfapi-async-calls-test ${H0} $V0 $logdir/gfapi-async-calls-test.log
+
+cleanup_tester $(dirname $0)/gfapi-async-calls-test
+
+cleanup;