gfapi : RECALL_LEASE implementation

Right now there are two types of upcalls
* poll method
* registering callback

But callback can be registered per fs and same callback fn shall be used
for any lease recall with object handle as argument as done for cache
invalidation.

TODO: RECALL LEASE for each glfd (for future reference)
 (may be needed fo Samba as they do not deal with
  object handles.

    In case of RECALL_LEASE, we could associate separate
     cbk function for each glfd either by
       - extending pub_glfs_lease to accept new args (recall_cbk_fn, cookie)
       - or by defining new API "glfs_register_recall_cbk_fn (glfd, recall_cbk_fn, cookie)
       . In such cases, flag it and instead of calling below upcall functions, define
        a new one to go through the glfd list and invoke each of theirs recall_cbk_fn.

Plus added following as well
* passed lease id to dict in required arguments
* added flag check in pub_glfs_open

Updates: #350
Change-Id: I07a971f0f26ec6aae0b9f9a5613504317dee153b
Signed-off-by: Soumya Koduri <skoduri@redhat.com>
Signed-off-by: Poornima G <pgurusid@redhat.com>
Signed-off-by: Jiffin Tony Thottan <jthottan@redhat.com>

8 files changed, 532 insertions, 28 deletions

diff --git a/api/src/gfapi.aliases b/api/src/gfapi.aliases
index 42256726f63..a05b8dd1f55 100644
--- a/api/src/gfapi.aliases
+++ b/api/src/gfapi.aliases
@@ -164,6 +164,7 @@ _pub_glfs_setfsleaseid _glfs_setfsleaseid$GFAPI_4.0.0
 _pub_glfs_file_lock _glfs_file_lock$GFAPI_4.0.0
 _pub_glfs_lease _glfs_lease$GFAPI_4.0.0
 _pub_glfs_h_lease _glfs_h_lease$GFAPI_4.0.0
+_pub_glfs_recall_lease _glfs_recall_lease$GFAPI_4.0.0
 _pub_glfs_read_async _glfs_read_async$GFAPI_4.0.0
 _pub_glfs_write_async _glfs_write_async$GFAPI_4.0.0
 _pub_glfs_readv_async _glfs_readv_async$GFAPI_4.0.0
diff --git a/api/src/gfapi.map b/api/src/gfapi.map
index e9b5f88037f..6fa0326d78a 100644
--- a/api/src/gfapi.map
+++ b/api/src/gfapi.map
@@ -207,6 +207,7 @@ GFAPI_4.0.0 {
                 glfs_file_lock;
                 glfs_lease;
                 glfs_h_lease;
+                glfs_recall_lease;
                 glfs_read_async;
                 glfs_write_async;
                 glfs_readv_async;
diff --git a/api/src/glfs-fops.c b/api/src/glfs-fops.c
index 726574714e2..66b764dd0da 100644
--- a/api/src/glfs-fops.c
+++ b/api/src/glfs-fops.c
@@ -135,6 +135,41 @@ out:
 }
 
 int
+glfs_get_upcall_lease (struct gf_upcall *to_up_data,
+                             struct gf_upcall *from_up_data)
+{
+
+        struct gf_upcall_recall_lease *ca_data = NULL;
+        struct gf_upcall_recall_lease *f_ca_data = NULL;
+        int                                 ret      = -1;
+
+        GF_VALIDATE_OR_GOTO (THIS->name, to_up_data, out);
+        GF_VALIDATE_OR_GOTO (THIS->name, from_up_data, out);
+
+        f_ca_data = from_up_data->data;
+        GF_VALIDATE_OR_GOTO (THIS->name, f_ca_data, out);
+
+        ca_data = GF_CALLOC (1, sizeof(*ca_data),
+                            glfs_mt_upcall_entry_t);
+
+        if (!ca_data) {
+                gf_msg (THIS->name, GF_LOG_ERROR, errno,
+                        API_MSG_ALLOC_FAILED,
+                        "Upcall entry allocation failed.");
+                goto out;
+        }
+
+        to_up_data->data = ca_data;
+
+        ca_data->lease_type   = f_ca_data->lease_type;
+        gf_uuid_copy (ca_data->tid, f_ca_data->tid);
+        ca_data->dict       = f_ca_data->dict;
+
+        ret = 0;
+out:
+        return ret;
+}
+int
 glfs_loc_link (loc_t *loc, struct iatt *iatt)
 {
 	int ret = -1;
@@ -192,6 +227,7 @@ pub_glfs_open (struct glfs *fs, const char *path, int flags)
 	loc_t            loc = {0, };
 	struct iatt      iatt = {0, };
 	int              reval = 0;
+        dict_t          *fop_attr = NULL;
 
         DECLARE_OLD_THIS;
         __GLFS_ENTRY_VALIDATE_FS (fs, invalid_fs);
@@ -243,13 +279,20 @@ retry:
 	}
         glfd->fd->flags = flags;
 
-	ret = syncop_open (subvol, &loc, flags, glfd->fd, NULL, NULL);
+        ret = get_fop_attr_thrd_key (&fop_attr);
+        if (ret)
+                gf_msg_debug ("gfapi", 0, "Getting leaseid from thread failed");
+
+	ret = syncop_open (subvol, &loc, flags, glfd->fd, fop_attr, NULL);
         DECODE_SYNCOP_ERR (ret);
 
 	ESTALE_RETRY (ret, errno, reval, &loc, retry);
 out:
 	loc_wipe (&loc);
 
+        if (fop_attr)
+                dict_unref (fop_attr);
+
 	if (ret && glfd) {
                 GF_REF_PUT (glfd);
 		glfd = NULL;
@@ -274,6 +317,7 @@ pub_glfs_close (struct glfs_fd *glfd)
 	int        ret = -1;
 	fd_t      *fd = NULL;
 	struct glfs *fs = NULL;
+        dict_t    *fop_attr = NULL;
 
         DECLARE_OLD_THIS;
 	__GLFS_ENTRY_VALIDATE_FD (glfd, invalid_fs);
@@ -297,14 +341,20 @@ pub_glfs_close (struct glfs_fd *glfd)
                 if (ret)
                         goto out;
         }
+        ret = get_fop_attr_thrd_key (&fop_attr);
+        if (ret)
+                gf_msg_debug ("gfapi", 0, "Getting leaseid from thread failed");
 
-	ret = syncop_flush (subvol, fd, NULL, NULL);
+	ret = syncop_flush (subvol, fd, fop_attr, NULL);
         DECODE_SYNCOP_ERR (ret);
 out:
 	fs = glfd->fs;
 
         if (fd)
                 fd_unref (fd);
+        if (fop_attr)
+                dict_unref (fop_attr);
+
 
         glfs_mark_glfd_for_deletion (glfd);
 	glfs_subvol_done (fs, subvol);
@@ -551,8 +601,10 @@ retry:
 	}
         glfd->fd->flags = flags;
 
+        if (get_fop_attr_thrd_key (&xattr_req))
+                gf_msg_debug ("gfapi", 0, "Getting leaseid from thread failed");
 	if (ret == 0) {
-		ret = syncop_open (subvol, &loc, flags, glfd->fd, NULL, NULL);
+		ret = syncop_open (subvol, &loc, flags, glfd->fd, xattr_req, NULL);
                 DECODE_SYNCOP_ERR (ret);
 	} else {
 		ret = syncop_create (subvol, &loc, flags, mode, glfd->fd,
@@ -708,6 +760,7 @@ glfs_preadv_common (struct glfs_fd *glfd, const struct iovec *iovec,
 	struct iobref  *iobref = NULL;
 	fd_t           *fd = NULL;
         struct iatt     iatt = {0, };
+        dict_t         *fop_attr = NULL;
 
         DECLARE_OLD_THIS;
 	__GLFS_ENTRY_VALIDATE_FD (glfd, invalid_fs);
@@ -730,8 +783,12 @@ glfs_preadv_common (struct glfs_fd *glfd, const struct iovec *iovec,
 
 	size = iov_length (iovec, iovcnt);
 
+        ret = get_fop_attr_thrd_key (&fop_attr);
+        if (ret)
+                gf_msg_debug ("gfapi", 0, "Getting leaseid from thread failed");
+
 	ret = syncop_readv (subvol, fd, size, offset, 0, &iov, &cnt, &iobref,
-                            &iatt, NULL, NULL);
+                            &iatt, fop_attr, NULL);
         DECODE_SYNCOP_ERR (ret);
 
         if (ret >= 0 && poststat)
@@ -755,6 +812,8 @@ out:
 		fd_unref (fd);
         if (glfd)
                 GF_REF_PUT (glfd);
+        if (fop_attr)
+                dict_unref (fop_attr);
 
 	glfs_subvol_done (glfd->fs, subvol);
 
@@ -956,6 +1015,7 @@ glfs_preadv_async_common (struct glfs_fd *glfd, const struct iovec *iovec,
 	xlator_t       *subvol = NULL;
 	glfs_t         *fs = NULL;
 	fd_t           *fd = NULL;
+        dict_t         *fop_attr = NULL;
 
         DECLARE_OLD_THIS;
 	__GLFS_ENTRY_VALIDATE_FD (glfd, invalid_fs);
@@ -1010,9 +1070,13 @@ glfs_preadv_async_common (struct glfs_fd *glfd, const struct iovec *iovec,
 
 	frame->local = gio;
 
+        ret = get_fop_attr_thrd_key (&fop_attr);
+        if (ret)
+                gf_msg_debug ("gfapi", 0, "Getting leaseid from thread failed");
+
 	STACK_WIND_COOKIE (frame, glfs_preadv_async_cbk, subvol, subvol,
 			   subvol->fops->readv, fd, iov_length (iovec, count),
-			   offset, flags, NULL);
+			   offset, flags, fop_attr);
 
 out:
         if (ret) {
@@ -1029,6 +1093,8 @@ out:
                 }
 		glfs_subvol_done (fs, subvol);
 	}
+        if (fop_attr)
+                dict_unref (fop_attr);
 
         __GLFS_EXIT_FS;
 
@@ -1222,6 +1288,7 @@ glfs_pwritev_common (struct glfs_fd *glfd, const struct iovec *iovec,
 	struct iovec    iov = {0, };
 	fd_t           *fd = NULL;
         struct iatt     preiatt = {0, }, postiatt = {0, };
+        dict_t         *fop_attr = NULL;
 
         DECLARE_OLD_THIS;
 	__GLFS_ENTRY_VALIDATE_FD (glfd, invalid_fs);
@@ -1246,8 +1313,12 @@ glfs_pwritev_common (struct glfs_fd *glfd, const struct iovec *iovec,
         if (ret)
                 goto out;
 
+        ret = get_fop_attr_thrd_key (&fop_attr);
+        if (ret)
+                gf_msg_debug ("gfapi", 0, "Getting leaseid from thread failed");
+
         ret = syncop_writev (subvol, fd, &iov, 1, offset, iobref, flags,
-                             &preiatt, &postiatt, NULL, NULL);
+                             &preiatt, &postiatt, fop_attr, NULL);
         DECODE_SYNCOP_ERR (ret);
 
         if (ret >= 0) {
@@ -1270,6 +1341,8 @@ out:
 		fd_unref (fd);
         if (glfd)
                 GF_REF_PUT (glfd);
+        if (fop_attr)
+                dict_unref (fop_attr);
 
 	glfs_subvol_done (glfd->fs, subvol);
 
@@ -1383,6 +1456,7 @@ glfs_pwritev_async_common (struct glfs_fd *glfd, const struct iovec *iovec,
         fd_t           *fd = NULL;
         struct iobref  *iobref = NULL;
         struct iobuf   *iobuf = NULL;
+        dict_t         *fop_attr = NULL;
 
         DECLARE_OLD_THIS;
         __GLFS_ENTRY_VALIDATE_FD (glfd, invalid_fs);
@@ -1438,9 +1512,13 @@ glfs_pwritev_async_common (struct glfs_fd *glfd, const struct iovec *iovec,
 
         frame->local = gio;
 
+        ret = get_fop_attr_thrd_key (&fop_attr);
+        if (ret)
+                gf_msg_debug ("gfapi", 0, "Getting leaseid from thread failed");
+
         STACK_WIND_COOKIE (frame, glfs_pwritev_async_cbk, subvol, subvol,
                            subvol->fops->writev, fd, gio->iov,
-                           gio->count, offset, flags, iobref, NULL);
+                           gio->count, offset, flags, iobref, fop_attr);
 
         ret = 0;
 out:
@@ -1456,6 +1534,8 @@ out:
                  */
                 glfs_subvol_done (glfd->fs, subvol);
         }
+        if (fop_attr)
+                dict_unref (fop_attr);
 
         if (iobuf)
                 iobuf_unref (iobuf);
@@ -1604,6 +1684,7 @@ glfs_fsync_common (struct glfs_fd *glfd, struct stat *prestat,
 	xlator_t        *subvol = NULL;
 	fd_t            *fd = NULL;
         struct iatt      preiatt = {0, }, postiatt = {0, };
+        dict_t          *fop_attr = NULL;
 
         DECLARE_OLD_THIS;
 	__GLFS_ENTRY_VALIDATE_FD (glfd, invalid_fs);
@@ -1624,7 +1705,11 @@ glfs_fsync_common (struct glfs_fd *glfd, struct stat *prestat,
 		goto out;
 	}
 
-	ret = syncop_fsync (subvol, fd, 0, &preiatt, &postiatt, NULL, NULL);
+        ret = get_fop_attr_thrd_key (&fop_attr);
+        if (ret)
+                gf_msg_debug ("gfapi", 0, "Getting leaseid from thread failed");
+
+	ret = syncop_fsync (subvol, fd, 0, &preiatt, &postiatt, fop_attr, NULL);
         DECODE_SYNCOP_ERR (ret);
 
         if (ret >= 0) {
@@ -1638,6 +1723,8 @@ out:
 		fd_unref (fd);
         if (glfd)
                 GF_REF_PUT (glfd);
+        if (fop_attr)
+                dict_unref (fop_attr);
 
 	glfs_subvol_done (glfd->fs, subvol);
 
@@ -1794,6 +1881,7 @@ glfs_fdatasync_common (struct glfs_fd *glfd, struct stat *prestat,
 	xlator_t        *subvol = NULL;
 	fd_t            *fd = NULL;
         struct iatt      preiatt = {0, }, postiatt = {0, };
+        dict_t          *fop_attr = NULL;
 
         DECLARE_OLD_THIS;
 	__GLFS_ENTRY_VALIDATE_FD (glfd, invalid_fs);
@@ -1814,7 +1902,11 @@ glfs_fdatasync_common (struct glfs_fd *glfd, struct stat *prestat,
 		goto out;
 	}
 
-	ret = syncop_fsync (subvol, fd, 1, &preiatt, &postiatt, NULL, NULL);
+        ret = get_fop_attr_thrd_key (&fop_attr);
+        if (ret)
+                gf_msg_debug ("gfapi", 0, "Getting leaseid from thread failed");
+
+	ret = syncop_fsync (subvol, fd, 1, &preiatt, &postiatt, fop_attr, NULL);
         DECODE_SYNCOP_ERR (ret);
 
         if (ret >= 0) {
@@ -1828,6 +1920,8 @@ out:
 		fd_unref (fd);
         if (glfd)
                 GF_REF_PUT (glfd);
+        if (fop_attr)
+                dict_unref (fop_attr);
 
 	glfs_subvol_done (glfd->fs, subvol);
 
@@ -1902,6 +1996,7 @@ glfs_ftruncate_common (struct glfs_fd *glfd, off_t offset,
 	xlator_t        *subvol = NULL;
 	fd_t            *fd = NULL;
         struct iatt      preiatt = {0, }, postiatt = {0, };
+        dict_t         *fop_attr = NULL;
 
         DECLARE_OLD_THIS;
         __GLFS_ENTRY_VALIDATE_FD (glfd, invalid_fs);
@@ -1922,8 +2017,12 @@ glfs_ftruncate_common (struct glfs_fd *glfd, off_t offset,
 		goto out;
 	}
 
+        ret = get_fop_attr_thrd_key (&fop_attr);
+        if (ret)
+                gf_msg_debug ("gfapi", 0, "Getting leaseid from thread failed");
+
 	ret = syncop_ftruncate (subvol, fd, offset, &preiatt, &postiatt,
-                                NULL, NULL);
+                                fop_attr, NULL);
         DECODE_SYNCOP_ERR (ret);
 
         if (ret >= 0) {
@@ -1937,6 +2036,8 @@ out:
 		fd_unref (fd);
         if (glfd)
                 GF_REF_PUT (glfd);
+        if (fop_attr)
+                dict_unref (fop_attr);
 
 	glfs_subvol_done (glfd->fs, subvol);
 
@@ -2030,6 +2131,7 @@ glfs_ftruncate_async_common (struct glfs_fd *glfd, off_t offset,
         call_frame_t   *frame = NULL;
         xlator_t       *subvol = NULL;
         fd_t           *fd = NULL;
+        dict_t         *fop_attr = NULL;
 
         DECLARE_OLD_THIS;
         __GLFS_ENTRY_VALIDATE_FD (glfd, invalid_fs);
@@ -2072,8 +2174,12 @@ glfs_ftruncate_async_common (struct glfs_fd *glfd, off_t offset,
 
         frame->local = gio;
 
+        ret = get_fop_attr_thrd_key (&fop_attr);
+        if (ret)
+                gf_msg_debug ("gfapi", 0, "Getting leaseid from thread failed");
+
         STACK_WIND_COOKIE (frame, glfs_ftruncate_async_cbk, subvol, subvol,
-                           subvol->fops->ftruncate, fd, offset, NULL);
+                           subvol->fops->ftruncate, fd, offset, fop_attr);
 
         ret = 0;
 
@@ -2088,6 +2194,8 @@ out:
                         STACK_DESTROY (frame->root);
                 glfs_subvol_done (glfd->fs, subvol);
         }
+        if (fop_attr)
+                dict_unref (fop_attr);
 
         __GLFS_EXIT_FS;
 
@@ -2511,6 +2619,7 @@ retry:
 		goto out;
 	}
 
+        /* TODO: Add leaseid */
 	ret = syncop_unlink (subvol, &loc, NULL, NULL);
         DECODE_SYNCOP_ERR (ret);
 
@@ -2632,7 +2741,8 @@ retrynew:
                 }
         }
 
-	/* TODO: check if new or old is a prefix of the other, and fail EINVAL */
+	/* TODO: - check if new or old is a prefix of the other, and fail EINVAL
+         *       - Add leaseid */
 
 	ret = syncop_rename (subvol, &oldloc, &newloc, NULL, NULL);
         DECODE_SYNCOP_ERR (ret);
@@ -2906,6 +3016,7 @@ glfs_discard_async_common (struct glfs_fd *glfd, off_t offset, size_t len,
         call_frame_t   *frame = NULL;
         xlator_t       *subvol = NULL;
         fd_t           *fd = NULL;
+        dict_t         *fop_attr = NULL;
 
         DECLARE_OLD_THIS;
         __GLFS_ENTRY_VALIDATE_FD (glfd, invalid_fs);
@@ -2948,9 +3059,12 @@ glfs_discard_async_common (struct glfs_fd *glfd, off_t offset, size_t len,
 	gio->data   = data;
 
         frame->local = gio;
+        ret = get_fop_attr_thrd_key (&fop_attr);
+        if (ret)
+                gf_msg_debug ("gfapi", 0, "Getting leaseid from thread failed");
 
         STACK_WIND_COOKIE (frame, glfs_discard_async_cbk, subvol, subvol,
-                           subvol->fops->discard, fd, offset, len, NULL);
+                           subvol->fops->discard, fd, offset, len, fop_attr);
 
         ret = 0;
 out:
@@ -3015,6 +3129,7 @@ glfs_zerofill_async_common (struct glfs_fd *glfd, off_t offset, off_t len,
         call_frame_t   *frame = NULL;
         xlator_t       *subvol = NULL;
         fd_t           *fd = NULL;
+        dict_t         *fop_attr = NULL;
 
         DECLARE_OLD_THIS;
         __GLFS_ENTRY_VALIDATE_FD (glfd, invalid_fs);
@@ -3058,8 +3173,12 @@ glfs_zerofill_async_common (struct glfs_fd *glfd, off_t offset, off_t len,
 
         frame->local = gio;
 
+        ret = get_fop_attr_thrd_key (&fop_attr);
+        if (ret)
+                gf_msg_debug ("gfapi", 0, "Getting leaseid from thread failed");
+
         STACK_WIND_COOKIE (frame, glfs_zerofill_async_cbk, subvol, subvol,
-                           subvol->fops->zerofill, fd, offset, len, NULL);
+                           subvol->fops->zerofill, fd, offset, len, fop_attr);
         ret = 0;
 out:
         if (ret) {
@@ -3072,6 +3191,8 @@ out:
                         STACK_DESTROY (frame->root);
                 glfs_subvol_done (glfd->fs, subvol);
         }
+        if (fop_attr)
+                dict_unref (fop_attr);
 
         __GLFS_EXIT_FS;
 
@@ -3427,6 +3548,7 @@ retry:
 	if (ret)
 		goto out;
 
+        /* TODO : Add leaseid */
 	ret = syncop_setattr (subvol, &loc, iatt, valid, 0, 0, NULL, NULL);
         DECODE_SYNCOP_ERR (ret);
 
@@ -3469,6 +3591,7 @@ glfs_fsetattr (struct glfs_fd *glfd, struct iatt *iatt, int valid)
 		goto out;
 	}
 
+        /* TODO : Add leaseid */
 	ret = syncop_fsetattr (subvol, fd, iatt, valid, 0, 0, NULL, NULL);
         DECODE_SYNCOP_ERR (ret);
 out:
@@ -4286,6 +4409,7 @@ pub_glfs_fallocate (struct glfs_fd *glfd, int keep_size, off_t offset, size_t le
 	int              ret = -1;
 	xlator_t        *subvol = NULL;
 	fd_t		*fd = NULL;
+        dict_t          *fop_attr = NULL;
 
         DECLARE_OLD_THIS;
 	__GLFS_ENTRY_VALIDATE_FD (glfd, invalid_fs);
@@ -4306,13 +4430,19 @@ pub_glfs_fallocate (struct glfs_fd *glfd, int keep_size, off_t offset, size_t le
 		goto out;
 	}
 
-	ret = syncop_fallocate (subvol, fd, keep_size, offset, len, NULL, NULL);
+        ret = get_fop_attr_thrd_key (&fop_attr);
+        if (ret)
+                gf_msg_debug ("gfapi", 0, "Getting leaseid from thread failed");
+
+	ret = syncop_fallocate (subvol, fd, keep_size, offset, len, fop_attr, NULL);
         DECODE_SYNCOP_ERR (ret);
 out:
 	if (fd)
 		fd_unref(fd);
         if (glfd)
                 GF_REF_PUT (glfd);
+        if (fop_attr)
+                dict_unref (fop_attr);
 
 	glfs_subvol_done (glfd->fs, subvol);
 
@@ -4331,6 +4461,7 @@ pub_glfs_discard (struct glfs_fd *glfd, off_t offset, size_t len)
 	int              ret = -1;
 	xlator_t        *subvol = NULL;
 	fd_t		*fd = NULL;
+        dict_t          *fop_attr = NULL;
 
         DECLARE_OLD_THIS;
 	__GLFS_ENTRY_VALIDATE_FD (glfd, invalid_fs);
@@ -4351,13 +4482,19 @@ pub_glfs_discard (struct glfs_fd *glfd, off_t offset, size_t len)
 		goto out;
 	}
 
-	ret = syncop_discard (subvol, fd, offset, len, NULL, NULL);
+        ret = get_fop_attr_thrd_key (&fop_attr);
+        if (ret)
+                gf_msg_debug ("gfapi", 0, "Getting leaseid from thread failed");
+
+	ret = syncop_discard (subvol, fd, offset, len, fop_attr, NULL);
         DECODE_SYNCOP_ERR (ret);
 out:
 	if (fd)
 		fd_unref(fd);
         if (glfd)
                 GF_REF_PUT (glfd);
+        if (fop_attr)
+                dict_unref (fop_attr);
 
 	glfs_subvol_done (glfd->fs, subvol);
 
@@ -4376,6 +4513,7 @@ pub_glfs_zerofill (struct glfs_fd *glfd, off_t offset, off_t len)
         int               ret             = -1;
         xlator_t         *subvol          = NULL;
         fd_t             *fd              = NULL;
+        dict_t           *fop_attr         = NULL;
 
         DECLARE_OLD_THIS;
         __GLFS_ENTRY_VALIDATE_FD (glfd, invalid_fs);
@@ -4394,13 +4532,19 @@ pub_glfs_zerofill (struct glfs_fd *glfd, off_t offset, off_t len)
                 goto out;
         }
 
-        ret = syncop_zerofill (subvol, fd, offset, len, NULL, NULL);
+        ret = get_fop_attr_thrd_key (&fop_attr);
+        if (ret)
+                gf_msg_debug ("gfapi", 0, "Getting leaseid from thread failed");
+
+        ret = syncop_zerofill (subvol, fd, offset, len, fop_attr, NULL);
         DECODE_SYNCOP_ERR (ret);
 out:
         if (fd)
                 fd_unref(fd);
         if (glfd)
                 GF_REF_PUT (glfd);
+        if (fop_attr)
+                dict_unref (fop_attr);
 
         glfs_subvol_done (glfd->fs, subvol);
 
@@ -4730,6 +4874,10 @@ glfs_lock_common (struct glfs_fd *glfd, int cmd, struct flock *flock,
                         goto out;
         }
 
+        ret = get_fop_attr_thrd_key (&xdata);
+        if (ret)
+                gf_msg_debug ("gfapi", 0, "Getting leaseid from thread failed");
+
         ret = syncop_lk (subvol, fd, cmd, &gf_flock, xdata, NULL);
         DECODE_SYNCOP_ERR (ret);
 
@@ -4926,6 +5074,10 @@ glfs_enqueue_upcall_data (struct glfs *fs, struct gf_upcall *upcall_data)
                 ret = glfs_get_upcall_cache_invalidation (&u_list->upcall_data,
                                                           upcall_data);
                 break;
+        case GF_UPCALL_RECALL_LEASE:
+                ret = glfs_get_upcall_lease (&u_list->upcall_data,
+                                                   upcall_data);
+                break;
         default:
                 break;
         }
@@ -4954,6 +5106,172 @@ out:
 }
 
 static void
+glfs_free_upcall_lease (void *to_free)
+{
+        struct glfs_upcall_lease *arg = to_free;
+
+        if (!arg)
+                return;
+
+        if (arg->object)
+                glfs_h_close (arg->object);
+
+        GF_FREE (arg);
+}
+
+int
+glfs_recall_lease_fd (struct glfs *fs,
+                      struct gf_upcall *up_data)
+{
+        struct gf_upcall_recall_lease *recall_lease = NULL;
+        xlator_t                      *subvol       = NULL;
+        int                            ret          = 0;
+        inode_t                       *inode        = NULL;
+        struct glfs_fd                *glfd         = NULL;
+        struct glfs_fd                *tmp          = NULL;
+        struct list_head               glfd_list;
+        fd_t                          *fd           = NULL;
+        uint64_t                       value        = 0;
+        struct glfs_lease              lease        = {0, };
+
+        GF_VALIDATE_OR_GOTO ("gfapi", up_data, out);
+        GF_VALIDATE_OR_GOTO ("gfapi", fs, out);
+
+        recall_lease = up_data->data;
+        GF_VALIDATE_OR_GOTO ("gfapi", recall_lease, out);
+
+        subvol = glfs_active_subvol (fs);
+        if (!subvol) {
+                ret = -1;
+                errno = EIO;
+                goto out;
+        }
+
+        gf_msg_debug (THIS->name, 0,
+                      "Recall lease received for gfid:%s",
+                      uuid_utoa(up_data->gfid));
+
+
+        inode = inode_find (subvol->itable, up_data->gfid);
+        if (!inode) {
+                ret = -1;
+                gf_msg (THIS->name, GF_LOG_ERROR, errno,
+                        API_MSG_INODE_FIND_FAILED,
+                        "Unable to find inode entry for gfid:%s graph id:%d",
+                        uuid_utoa(up_data->gfid), subvol->graph->id);
+                goto out;
+        }
+
+        LOCK (&inode->lock);
+        {
+                list_for_each_entry (fd, &inode->fd_list, inode_list) {
+                        ret = fd_ctx_get (fd, subvol, &value);
+                        glfd = (void *) value;
+                        if (glfd) {
+                                gf_msg_trace (THIS->name, 0,
+                                              "glfd (%p) has held lease", glfd);
+                                GF_REF_GET (glfd);
+                                list_add_tail (&glfd->list, &glfd_list);
+                        }
+                }
+        }
+        UNLOCK (&inode->lock);
+
+        list_for_each_entry_safe (glfd, tmp, &glfd_list, list) {
+                LOCK (&glfd->lock);
+                {
+                        if (glfd->state != GLFD_CLOSE)
+                                gf_msg_trace (THIS->name, 0,
+                                              "glfd (%p) has held lease, "
+                                              "calling recall cbk", glfd);
+                                glfd->cbk (lease, glfd->cookie);
+                }
+                UNLOCK (&glfd->lock);
+
+                list_del_init (&glfd->list);
+                GF_REF_PUT (glfd);
+        }
+
+out:
+        return ret;
+}
+
+int
+glfs_recall_lease_upcall (struct glfs *fs,
+                          struct glfs_upcall *up_arg,
+                          struct gf_upcall *up_data)
+{
+        struct gf_upcall_recall_lease *recall_lease = NULL;
+        struct glfs_object                  *object       = NULL;
+        xlator_t                      *subvol       = NULL;
+        int                            ret          = 0;
+        struct glfs_upcall_lease      *up_lease_arg = NULL;
+
+        GF_VALIDATE_OR_GOTO ("gfapi", up_data, out);
+        GF_VALIDATE_OR_GOTO ("gfapi", fs, out);
+
+        recall_lease = up_data->data;
+        GF_VALIDATE_OR_GOTO ("gfapi", recall_lease, out);
+
+        subvol = glfs_active_subvol (fs);
+        if (!subvol) {
+                ret = -1;
+                errno = EIO;
+                goto out;
+        }
+
+        gf_msg_debug (THIS->name, 0,
+                      "Recall lease received for gfid:%s",
+                      uuid_utoa(up_data->gfid));
+
+        object = glfs_h_find_handle (fs, up_data->gfid,
+                                     GFAPI_HANDLE_LENGTH);
+        if (!object) {
+                /* The reason handle creation will fail is because we
+                 * couldn't find the inode in the gfapi inode table.
+                 *
+                 * But since application would have taken inode_ref, the
+                 * only case when this can happen is when it has closed
+                 * the handle and hence will no more be interested in
+                 * the upcall for this particular gfid.
+                 */
+                gf_msg (THIS->name, GF_LOG_DEBUG, errno,
+                        API_MSG_CREATE_HANDLE_FAILED,
+                        "handle creation of %s failed",
+                         uuid_utoa (up_data->gfid));
+                errno = ESTALE;
+                goto out;
+        }
+
+        up_lease_arg = GF_CALLOC (1, sizeof (struct glfs_upcall_lease),
+                                  glfs_mt_upcall_inode_t);
+        up_lease_arg->object = object;
+
+        GF_VALIDATE_OR_GOTO ("glfs_recall_lease",
+                             up_lease_arg, out);
+
+
+        up_lease_arg->lease_type = recall_lease->lease_type;
+
+        up_arg->reason = GF_UPCALL_RECALL_LEASE;
+        up_arg->event = up_lease_arg;
+        up_arg->free_event = glfs_free_upcall_lease;
+
+        ret = 0;
+
+out:
+        if (ret) {
+                /* Close p_object and oldp_object as well if being referenced.*/
+                if (object)
+                        glfs_h_close (object);
+
+                /* Set reason to prevent applications from using ->event */
+                up_arg->reason = GF_UPCALL_EVENT_NULL;
+        }
+        return ret;
+}
+
+static void
 glfs_cbk_upcall_data (struct glfs *fs, struct gf_upcall *upcall_data)
 {
         int ret = -1;
@@ -4980,6 +5298,9 @@ glfs_cbk_upcall_data (struct glfs *fs, struct gf_upcall *upcall_data)
         case GF_UPCALL_CACHE_INVALIDATION:
                 ret = glfs_h_poll_cache_invalidation (fs, up_arg, upcall_data);
                 break;
+        case GF_UPCALL_RECALL_LEASE:
+                ret = glfs_recall_lease_upcall (fs, up_arg, upcall_data);
+                break;
         default:
                 errno = EINVAL;
         }
@@ -5069,9 +5390,14 @@ priv_glfs_process_upcall_event (struct glfs *fs, void *data)
                       (char *)(upcall_data->gfid));
 
         /* *
-         * TODO: RECALL LEASE
-         * Refer issue #350
-         * Proposed patch https://review.gluster.org/#/c/14019/
+         * TODO: RECALL LEASE for each glfd
+         *
+         * In case of RECALL_LEASE, we could associate separate
+         * cbk function for each glfd either by
+         * - extending pub_glfs_lease to accept new args (recall_cbk_fn, cookie)
+         * - or by defining new API "glfs_register_recall_cbk_fn (glfd, recall_cbk_fn, cookie)
+         * . In such cases, flag it and instead of calling below upcall functions, define
+         * a new one to go through the glfd list and invoke each of theirs recall_cbk_fn.
          * */
 
         if (fs->up_cbk) { /* upcall cbk registered */
@@ -5165,6 +5491,7 @@ glfs_anonymous_pwritev (struct glfs *fs, struct glfs_object *object,
         iov.iov_base = iobuf_ptr (iobuf);
         iov.iov_len = size;
 
+        /* TODO : set leaseid */
         ret = syncop_writev (subvol, fd, &iov, 1, offset, iobref, flags,
                              NULL, NULL, NULL, NULL);
         DECODE_SYNCOP_ERR (ret);
@@ -5234,6 +5561,7 @@ glfs_anonymous_preadv (struct glfs *fs,  struct glfs_object *object,
 
         size = iov_length (iovec, iovcnt);
 
+        /* TODO : set leaseid */
 	ret = syncop_readv (subvol, fd, size, offset, flags, &iov, &cnt,
                             &iobref, NULL, NULL, NULL);
         DECODE_SYNCOP_ERR (ret);
@@ -5427,7 +5755,8 @@ glfs_lease_to_gf_lease (struct glfs_lease *lease, struct gf_lease *gf_lease)
 }
 
 int
-pub_glfs_lease (struct glfs_fd *glfd, struct glfs_lease *lease)
+pub_glfs_lease (struct glfs_fd *glfd, struct glfs_lease *lease,
+                glfs_recall_cbk fn, void *data)
 {
         int              ret = -1;
         loc_t            loc = {0, };
@@ -5460,6 +5789,30 @@ pub_glfs_lease (struct glfs_fd *glfd, struct glfs_lease *lease)
                 goto out;
         }
 
+        switch (lease->lease_type) {
+        case GLFS_RD_LEASE:
+                if ((fd->flags != O_RDONLY) && !(fd->flags & O_RDWR)) {
+                        ret = -1;
+                        errno = EINVAL;
+                        goto out;
+                }
+                break;
+        case GLFS_RW_LEASE:
+                if (!((fd->flags & O_WRONLY) || (fd->flags & O_RDWR))) {
+                        ret = -1;
+                        errno = EINVAL;
+                        goto out;
+                }
+                break;
+        default:
+                if (lease->cmd != GLFS_GET_LEASE) {
+                        ret = -1;
+                        errno = EINVAL;
+                        goto out;
+                }
+                break;
+        }
+
         /* populate loc */
         GLFS_LOC_FILL_INODE (fd->inode, loc, out);
 
@@ -5474,15 +5827,21 @@ pub_glfs_lease (struct glfs_fd *glfd, struct glfs_lease *lease)
         if (ret == 0 && (cmd == F_SETLK || cmd == F_SETLKW))
                 fd_lk_insert_and_merge (fd, cmd, &saved_flock);
         */
+        if (ret == 0) {
+                   ret = fd_ctx_set (glfd->fd, subvol, (uint64_t)(long)glfd);
+                   if (ret) {
+                           gf_msg (subvol->name, GF_LOG_ERROR, ENOMEM, API_MSG_FDCTX_SET_FAILED,
+                                           "Setting fd ctx failed for fd(%p)", glfd->fd);
+                           goto out;
+                   }
+                   glfd->cbk = fn;
+                   glfd->cookie = data;
+        }
 
 out:
-        if (fd)
-                fd_unref (fd);
 
-        if (glfd) {
+        if (glfd)
                 GF_REF_PUT (glfd);
-                glfd = NULL;
-        }
 
         if (subvol)
                 glfs_subvol_done (glfd->fs, subvol);
diff --git a/api/src/glfs-handleops.c b/api/src/glfs-handleops.c
index f8193408ec0..cb37de30e4a 100644
--- a/api/src/glfs-handleops.c
+++ b/api/src/glfs-handleops.c
@@ -622,6 +622,7 @@ pub_glfs_h_open (struct glfs *fs, struct glfs_object *object, int flags)
         xlator_t        *subvol = NULL;
         inode_t         *inode = NULL;
         loc_t            loc = {0, };
+        dict_t          *fop_attr = NULL;
 
         /* validate in args */
         if ((fs == NULL) || (object == NULL)) {
@@ -678,6 +679,10 @@ pub_glfs_h_open (struct glfs *fs, struct glfs_object *object, int flags)
         GLFS_LOC_FILL_INODE (inode, loc, out);
 
         /* fop/op */
+        ret = get_fop_attr_thrd_key (&fop_attr);
+        if (ret)
+                gf_msg_debug ("gfapi", 0, "Getting leaseid from thread failed");
+
         ret = syncop_open (subvol, &loc, flags, glfd->fd, NULL, NULL);
         DECODE_SYNCOP_ERR (ret);
 
@@ -688,6 +693,8 @@ out:
 
         if (inode)
                 inode_unref (inode);
+        if (fop_attr)
+                dict_unref (fop_attr);
 
         if (ret && glfd) {
                 GF_REF_PUT (glfd);
diff --git a/api/src/glfs-handles.h b/api/src/glfs-handles.h
index 97c058a11c7..b89d09c3404 100644
--- a/api/src/glfs-handles.h
+++ b/api/src/glfs-handles.h
@@ -332,6 +332,10 @@ int glfs_h_lease (struct glfs *fs, struct glfs_object *object,
                   struct glfs_lease *lease) __THROW
         GFAPI_PUBLIC(glfs_h_lease, 4.0.0);
 
+struct glfs_object *
+glfs_h_find_handle (struct glfs *fs, unsigned char *handle,
+                    int len) __THROW
+        GFAPI_PUBLIC(glfs_h_lease, 4.0.0);
 __END_DECLS
 
 #endif /* !_GLFS_HANDLES_H */
diff --git a/api/src/glfs-internal.h b/api/src/glfs-internal.h
index 145509c3caf..788c6b00545 100644
--- a/api/src/glfs-internal.h
+++ b/api/src/glfs-internal.h
@@ -213,6 +213,7 @@ enum glfs_fd_state {
 
 struct glfs_fd {
 	struct list_head   openfds;
+        struct list_head   list;
         GF_REF_DECL;
 	struct glfs       *fs;
         enum glfs_fd_state state;
@@ -224,6 +225,8 @@ struct glfs_fd {
         gf_lkowner_t       lk_owner;
         glfs_leaseid_t     lease_id; /* Stores lease_id of client in glfd */
         gf_lock_t          lock; /* lock taken before updating fd state */
+        glfs_recall_cbk    cbk;
+        void              *cookie;
 };
 
 /* glfs object handle introduced for the alternate gfapi implementation based
@@ -254,6 +257,16 @@ struct glfs_upcall_inode {
         struct stat           oldp_buf; /* Latest stat of old parent dir handle */
 };
 
+struct glfs_upcall_lease {
+        struct glfs_object   *object;  /* Object which need to be acted upon */
+        uint32_t  lease_type; /* Lease type to which client can downgrade to*/
+};
+
+struct glfs_upcall_lease_fd {
+        uint32_t  lease_type; /* Lease type to which client can downgrade to*/
+        void   *fd_cookie;  /* Object which need to be acted upon */
+};
+
 struct glfs_xreaddirp_stat {
         struct stat st; /* Stat for that dirent - corresponds to GFAPI_XREADDIRP_STAT */
         struct glfs_object *object; /* handled for GFAPI_XREADDIRP_HANDLE */
@@ -614,4 +627,16 @@ void
 glfs_lease_to_gf_lease (struct glfs_lease *lease, struct gf_lease *gf_lease);
 
 void glfs_release_upcall (void *ptr);
+
+int
+get_fop_attr_glfd (dict_t **fop_attr, struct glfs_fd *glfd);
+
+int
+set_fop_attr_glfd (struct glfs_fd *glfd);
+
+int
+get_fop_attr_thrd_key (dict_t **fop_attr);
+
+void
+unset_fop_attr (dict_t **fop_attr);
 #endif /* !_GLFS_INTERNAL_H */
diff --git a/api/src/glfs.c b/api/src/glfs.c
index 6724534fb31..da6bc3a07ba 100644
--- a/api/src/glfs.c
+++ b/api/src/glfs.c
@@ -578,7 +578,7 @@ pub_glfs_setfsleaseid (glfs_leaseid_t leaseid)
 
         GF_VALIDATE_OR_GOTO (THIS->name, leaseid, out);
 
-        gleaseid = glusterfs_leaseid_buf_get();
+        gleaseid = gf_leaseid_get();
         if (gleaseid) {
                 memcpy (gleaseid, leaseid, LEASE_ID_SIZE);
                 ret = 0;
@@ -589,6 +589,87 @@ out:
 
 GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_setfsleaseid, 4.0.0);
 
+int
+get_fop_attr_glfd (dict_t **fop_attr, struct glfs_fd *glfd)
+{
+        char *leaseid            = NULL;
+        int   ret                = 0;
+        gf_boolean_t dict_create = _gf_false;
+
+        leaseid = GF_CALLOC (1, LEASE_ID_SIZE, gf_common_mt_char);
+        GF_CHECK_ALLOC_AND_LOG("gfapi", leaseid, ret, "lease id alloc failed", out);
+        memcpy (leaseid, glfd->lease_id, LEASE_ID_SIZE);
+        if (*fop_attr == NULL) {
+                *fop_attr = dict_new ();
+                dict_create = _gf_true;
+        }
+        GF_CHECK_ALLOC_AND_LOG("gfapi", *fop_attr, ret, "dict_new failed", out);
+        ret = dict_set_static_bin (*fop_attr, "lease-id", leaseid, LEASE_ID_SIZE);
+out:
+        if (ret) {
+                GF_FREE (leaseid);
+                if (dict_create) {
+                        if (*fop_attr)
+                                dict_unref (*fop_attr);
+                        *fop_attr = NULL;
+                }
+        }
+        return ret;
+}
+
+int
+set_fop_attr_glfd (struct glfs_fd *glfd)
+{
+        char     *lease_id = NULL;
+        int       ret      = -1;
+
+        lease_id = gf_existing_leaseid ();
+        if (lease_id) {
+                memcpy (glfd->lease_id, lease_id, LEASE_ID_SIZE);
+                ret = 0;
+        }
+        return ret;
+}
+
+int
+get_fop_attr_thrd_key (dict_t **fop_attr)
+{
+        char     *lease_id       = NULL;
+        int       ret            = 0;
+        gf_boolean_t dict_create = _gf_false;
+
+        lease_id = gf_existing_leaseid ();
+        if (lease_id) {
+                if (*fop_attr == NULL) {
+                        *fop_attr = dict_new ();
+                        dict_create = _gf_true;
+                }
+                GF_CHECK_ALLOC_AND_LOG("gfapi", *fop_attr, ret, "dict_new failed", out);
+                ret = dict_set_bin (*fop_attr, "lease-id", gf_strdup (lease_id),
+                                    LEASE_ID_SIZE);
+        }
+
+out:
+        if (ret && dict_create) {
+                if (*fop_attr)
+                        dict_unref (*fop_attr);
+                *fop_attr = NULL;
+        }
+        return ret;
+}
+
+void
+unset_fop_attr (dict_t **fop_attr)
+{
+        char     *lease_id = NULL;
+        lease_id = gf_existing_leaseid ();
+        if (lease_id)
+                memset (lease_id, 0, LEASE_ID_SIZE);
+        if (*fop_attr) {
+                dict_unref (*fop_attr);
+                *fop_attr = NULL;
+        }
+}
 struct glfs *
 pub_glfs_from_glfd (struct glfs_fd *glfd)
 {
@@ -1486,6 +1567,22 @@ pub_glfs_upcall_inode_get_oldpstat (struct glfs_upcall_inode *arg)
 GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_inode_get_oldpstat, 3.7.16);
 
 
+/*struct glfs_object*
+pub_glfs_upcall_lease_get_object (struct glfs_upcall_recall_inode *arg)
+{
+        return arg->object;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_lease_get_object, 4.0.0);
+*/
+
+uint32_t
+pub_glfs_upcall_lease_get_lease_type (struct glfs_upcall_lease *arg)
+{
+        return arg->lease_type;
+}
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_lease_get_lease_type, 4.0.0);
+
 /* definitions of the GLFS_SYSRQ_* chars are in glfs.h */
 static struct glfs_sysrq_help {
         char  sysrq;
@@ -1578,8 +1675,10 @@ glfs_upcall_register (struct glfs *fs, uint32_t event_list,
                          */
                         fs->upcall_events |=  GF_UPCALL_CACHE_INVALIDATION;
                         ret |= GF_UPCALL_CACHE_INVALIDATION;
+                } else if (event_list & GLFS_EVENT_INODE_INVALIDATE) {
+                        fs->upcall_events |=  GF_UPCALL_RECALL_LEASE;
+                        ret |= GF_UPCALL_RECALL_LEASE;
                 }
-
                 /* Override cbk function if existing */
                 fs->up_cbk = cbk;
                 fs->up_data = data;
diff --git a/api/src/glfs.h b/api/src/glfs.h
index 59e5784f91f..f9f8d4f36a2 100644
--- a/api/src/glfs.h
+++ b/api/src/glfs.h
@@ -1151,10 +1151,18 @@ typedef void (*glfs_recall_cbk) (struct glfs_lease lease, void *data);
   0:  Successfull completion
   <0: Failure. @errno will be set with the type of failure
 */
+struct gf_upcall;
 
 int glfs_lease (struct glfs_fd *glfd, struct glfs_lease *lease,
                 glfs_recall_cbk fn, void *data) __THROW
         GFAPI_PUBLIC(glfs_lease, 4.0.0);
+int
+glfs_recall_lease_fd (struct glfs *fs, struct gf_upcall *up_data) __THROW
+        GFAPI_PUBLIC(glfs_recall_lease_fd, 4.0.0);
 
+int
+glfs_recall_lease_upcall (struct glfs *fs, struct glfs_upcall *up_arg,
+                          struct gf_upcall *up_data) __THROW
+        GFAPI_PUBLIC(glfs_recall_lease_upcall, 4.0.0);
 __END_DECLS
 #endif /* !_GLFS_H */