storage/posix: Implement .unlink directory

Problem: For EC volume, If a file descriptor is open and
file has been unlinked, any further write on that fd will
fail. When a write request comes, EC internally reads some
blocks using anonymous fd. This read will fail as the file
has already been unlinked.

Solution: To solve this issue, we are using .unlink directory
to keep track of unlinked file. If a file is to be unlinked
while its fd is open, move this to .unlink directory and unlink
it from .glusterfs and real path. Once all the fd will be closed,
remove this entry form .unlink directory.

Change-Id: I8344edb0d340bdb883dc46458c16edbc336916b9
BUG: 1286029
Signed-off-by: Ashish Pandey <aspandey@redhat.com>
Reviewed-on: http://review.gluster.org/12816
Reviewed-by: Krutika Dhananjay <kdhananj@redhat.com>
Reviewed-by: Xavier Hernandez <xhernandez@datalab.es>
Tested-by: NetBSD Build System <jenkins@build.gluster.org>
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Pranith Kumar Karampuri <pkarampu@redhat.com>

9 files changed, 322 insertions, 24 deletions

diff --git a/libglusterfs/src/common-utils.h b/libglusterfs/src/common-utils.h
index e541df6240a..0e7aa016cbd 100644
--- a/libglusterfs/src/common-utils.h
+++ b/libglusterfs/src/common-utils.h
@@ -359,6 +359,8 @@ union gf_sock_union {
 };
 
 #define GF_HIDDEN_PATH ".glusterfs"
+#define GF_UNLINK_PATH GF_HIDDEN_PATH"/unlink"
+#define GF_LANDFILL_PATH GF_HIDDEN_PATH"/landfill"
 
 #define IOV_MIN(n) min(IOV_MAX,n)
 
diff --git a/libglusterfs/src/inode.h b/libglusterfs/src/inode.h
index 46fe8f6ac80..37083fae05b 100644
--- a/libglusterfs/src/inode.h
+++ b/libglusterfs/src/inode.h
@@ -93,7 +93,7 @@ struct _inode {
         struct list_head     hash;          /* hash table pointers */
         struct list_head     list;          /* active/lru/purge */
 
-	struct _inode_ctx   *_ctx;    /* replacement for dict_t *(inode->ctx) */
+        struct _inode_ctx   *_ctx;    /* replacement for dict_t *(inode->ctx) */
 };
 
 
diff --git a/tests/basic/ec/ec-anonymous-fd.t b/tests/basic/ec/ec-anonymous-fd.t
new file mode 100644
index 00000000000..a61628182ef
--- /dev/null
+++ b/tests/basic/ec/ec-anonymous-fd.t
@@ -0,0 +1,42 @@
+#!/bin/bash
+
+. $(dirname $0)/../../include.rc
+. $(dirname $0)/../../volume.rc
+. $(dirname $0)/../../fileio.rc
+
+cleanup
+function num_entries {
+        ls -l $1 | wc -l
+}
+
+TEST glusterd
+TEST pidof glusterd
+TEST $CLI volume create $V0 disperse 6 redundancy 2 $H0:$B0/${V0}{0..5}
+TEST $CLI volume start $V0
+TEST glusterfs --entry-timeout=0 --attribute-timeout=0 -s $H0 --volfile-id $V0 $M0
+EXPECT_WITHIN $CHILD_UP_TIMEOUT "6" ec_child_up_count $V0 0
+
+fd1=`fd_available`
+TEST fd_open $fd1 'w' $M0/file1
+TEST fd_write $fd1 testing
+TEST cat  $M0/file1
+TEST rm -rf $M0/file1
+TEST fd_write $fd1 testing
+TEST fd_write $fd1 testing
+TEST fd_write $fd1 testing
+
+EXPECT "^2$" num_entries $B0/${V0}0/.glusterfs/unlink/
+EXPECT "^2$" num_entries $B0/${V0}1/.glusterfs/unlink/
+EXPECT "^2$" num_entries $B0/${V0}2/.glusterfs/unlink/
+EXPECT "^2$" num_entries $B0/${V0}3/.glusterfs/unlink/
+EXPECT "^2$" num_entries $B0/${V0}4/.glusterfs/unlink/
+EXPECT "^2$" num_entries $B0/${V0}5/.glusterfs/unlink/
+TEST fd_close $fd1;
+EXPECT_WITHIN $UNLINK_TIMEOUT "^1$" num_entries $B0/${V0}0/.glusterfs/unlink/
+EXPECT_WITHIN $UNLINK_TIMEOUT "^1$" num_entries $B0/${V0}1/.glusterfs/unlink/
+EXPECT_WITHIN $UNLINK_TIMEOUT "^1$" num_entries $B0/${V0}2/.glusterfs/unlink/
+EXPECT_WITHIN $UNLINK_TIMEOUT "^1$" num_entries $B0/${V0}3/.glusterfs/unlink/
+EXPECT_WITHIN $UNLINK_TIMEOUT "^1$" num_entries $B0/${V0}4/.glusterfs/unlink/
+EXPECT_WITHIN $UNLINK_TIMEOUT "^1$" num_entries $B0/${V0}5/.glusterfs/unlink/
+
+cleanup;
diff --git a/tests/include.rc b/tests/include.rc
index 79fbd6c4b0b..4bd4c49a33b 100644
--- a/tests/include.rc
+++ b/tests/include.rc
@@ -72,6 +72,7 @@ UMOUNT_TIMEOUT=5
 CONFIG_UPDATE_TIMEOUT=5
 AUTH_REFRESH_INTERVAL=10
 GRAPH_SWITCH_TIMEOUT=10
+UNLINK_TIMEOUT=5
 
 LOGDIR=$(gluster --print-logdir)
 
diff --git a/xlators/storage/posix/src/posix-handle.c b/xlators/storage/posix/src/posix-handle.c
index 9a521d019a4..5a8f180b9f4 100644
--- a/xlators/storage/posix/src/posix-handle.c
+++ b/xlators/storage/posix/src/posix-handle.c
@@ -16,6 +16,8 @@
 #include <alloca.h>
 #endif
 
+#include "common-utils.h"
+
 #include "posix-handle.h"
 #include "posix.h"
 #include "xlator.h"
@@ -447,7 +449,6 @@ out:
         return len;
 }
 
-
 int
 posix_handle_init (xlator_t *this)
 {
diff --git a/xlators/storage/posix/src/posix-helpers.c b/xlators/storage/posix/src/posix-helpers.c
index 09552a30ec7..d6e1736bfd5 100644
--- a/xlators/storage/posix/src/posix-helpers.c
+++ b/xlators/storage/posix/src/posix-helpers.c
@@ -1604,7 +1604,6 @@ out:
         return ret;
 }
 
-
 static int
 __posix_fd_ctx_get (fd_t *fd, xlator_t *this, struct posix_fd **pfd_p)
 {
@@ -1612,34 +1611,37 @@ __posix_fd_ctx_get (fd_t *fd, xlator_t *this, struct posix_fd **pfd_p)
         struct posix_fd  *pfd = NULL;
         int               ret = -1;
         char             *real_path = NULL;
+        char             *unlink_path = NULL;
         int               _fd = -1;
         DIR              *dir = NULL;
 
+        struct posix_private    *priv      = NULL;
+
+        priv = this->private;
+
         ret = __fd_ctx_get (fd, this, &tmp_pfd);
         if (ret == 0) {
                 pfd = (void *)(long) tmp_pfd;
-                ret = 0;
+                goto out;
+        }
+        if (!fd_is_anonymous(fd)) {
+                gf_msg (this->name, GF_LOG_ERROR, 0,
+                        P_MSG_READ_FAILED,
+                        "Failed to get fd context for a non-anonymous fd, "
+                        "file: %s, gfid: %s", real_path,
+                        uuid_utoa (fd->inode->gfid));
                 goto out;
         }
 
         MAKE_HANDLE_PATH (real_path, this, fd->inode->gfid, NULL);
         if (!real_path) {
                 gf_msg (this->name, GF_LOG_ERROR, 0,
-                        P_MSG_HANDLE_PATH_CREATE_FAILED,
+                        P_MSG_READ_FAILED,
                         "Failed to create handle path (%s)",
                         uuid_utoa (fd->inode->gfid));
                 ret = -1;
                 goto out;
         }
-
-        if (!fd_is_anonymous(fd)) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to get fd context for a non-anonymous fd, "
-                        "file: %s, gfid: %s", real_path,
-                        uuid_utoa (fd->inode->gfid));
-                goto out;
-        }
-
         pfd = GF_CALLOC (1, sizeof (*pfd), gf_posix_mt_posix_fd);
         if (!pfd) {
                 goto out;
@@ -1663,6 +1665,16 @@ __posix_fd_ctx_get (fd_t *fd, xlator_t *this, struct posix_fd **pfd_p)
         if (fd->inode->ia_type == IA_IFREG) {
                 _fd = open (real_path, fd->flags);
                 if (_fd == -1) {
+                        POSIX_GET_FILE_UNLINK_PATH (priv->base_path,
+                                                    fd->inode->gfid,
+                                                    unlink_path);
+                        _fd = open (unlink_path, fd->flags);
+                }
+                if (_fd == -1) {
+                        gf_msg (this->name, GF_LOG_ERROR, errno,
+                                P_MSG_READ_FAILED,
+                                "Failed to get anonymous "
+                                "real_path: %s _fd = %d", real_path, _fd);
                         GF_FREE (pfd);
                         pfd = NULL;
                         goto out;
@@ -2164,3 +2176,40 @@ posix_fdget_objectsignature (int fd, dict_t *xattr)
  error_return:
         return -EINVAL;
 }
+
+
+int
+posix_inode_ctx_get (inode_t *inode, xlator_t *this, uint64_t *ctx)
+{
+        int             ret     = -1;
+        uint64_t        ctx_int = 0;
+
+        GF_VALIDATE_OR_GOTO (this->name, this, out);
+        GF_VALIDATE_OR_GOTO (this->name, inode, out);
+
+        ret = inode_ctx_get (inode, this, &ctx_int);
+
+        if (ret)
+                return ret;
+
+        if (ctx)
+                *ctx = ctx_int;
+
+out:
+        return ret;
+}
+
+
+int
+posix_inode_ctx_set (inode_t *inode, xlator_t *this, uint64_t ctx)
+{
+        int             ret = -1;
+
+        GF_VALIDATE_OR_GOTO (this->name, this, out);
+        GF_VALIDATE_OR_GOTO (this->name, inode, out);
+        GF_VALIDATE_OR_GOTO (this->name, ctx, out);
+
+        ret = inode_ctx_set (inode, this, &ctx);
+out:
+        return ret;
+}
diff --git a/xlators/storage/posix/src/posix-mem-types.h b/xlators/storage/posix/src/posix-mem-types.h
index 81752c17e78..b463c086be5 100644
--- a/xlators/storage/posix/src/posix-mem-types.h
+++ b/xlators/storage/posix/src/posix-mem-types.h
@@ -21,6 +21,7 @@ enum gf_posix_mem_types_ {
         gf_posix_mt_posix_dev_t,
         gf_posix_mt_trash_path,
 	gf_posix_mt_paiocb,
+        gf_posix_mt_inode_ctx_t,
         gf_posix_mt_end
 };
 #endif
diff --git a/xlators/storage/posix/src/posix.c b/xlators/storage/posix/src/posix.c
index efb5f03bef1..86ac65a897d 100644
--- a/xlators/storage/posix/src/posix.c
+++ b/xlators/storage/posix/src/posix.c
@@ -101,10 +101,32 @@ int
 posix_forget (xlator_t *this, inode_t *inode)
 {
         uint64_t tmp_cache = 0;
-        if (!inode_ctx_del (inode, this, &tmp_cache))
-                dict_destroy ((dict_t *)(long)tmp_cache);
+        int ret = 0;
+        char *unlink_path = NULL;
+        struct posix_private    *priv_posix = NULL;
 
-        return 0;
+        priv_posix = (struct posix_private *) this->private;
+
+        ret = inode_ctx_del (inode, this, &tmp_cache);
+        if (ret < 0) {
+                ret = 0;
+                goto out;
+        }
+        if (tmp_cache == GF_UNLINK_TRUE) {
+                POSIX_GET_FILE_UNLINK_PATH(priv_posix->base_path,
+                                           inode->gfid, unlink_path);
+                if (!unlink_path) {
+                        gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
+                                P_MSG_UNLINK_FAILED,
+                                "Failed to remove gfid :%s",
+                                uuid_utoa (inode->gfid));
+                        ret = -1;
+                        goto out;
+                }
+                ret = sys_unlink(unlink_path);
+        }
+out:
+        return ret;
 }
 
 /* Regular fops */
@@ -1446,16 +1468,93 @@ out:
         return 0;
 }
 
+int
+posix_add_unlink_to_ctx (inode_t *inode, xlator_t *this, char *unlink_path)
+{
+        uint64_t ctx = GF_UNLINK_FALSE;
+        int ret = 0;
+
+        if (!unlink_path) {
+                gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
+                        P_MSG_UNLINK_FAILED,
+                        "Creation of unlink entry failed for gfid: %s",
+                        unlink_path);
+                ret = -1;
+                goto out;
+        }
+
+        ctx = GF_UNLINK_TRUE;
+        ret = posix_inode_ctx_set (inode, this, ctx);
+        if (ret < 0) {
+                goto out;
+        }
+
+out:
+        return ret;
+}
+
+int32_t
+posix_move_gfid_to_unlink (xlator_t *this, uuid_t gfid, loc_t *loc)
+{
+        char *unlink_path = NULL;
+        char *gfid_path = NULL;
+        struct stat  stbuf = {0, };
+        int ret = 0;
+        struct posix_private    *priv_posix = NULL;
+
+        priv_posix = (struct posix_private *) this->private;
+
+        MAKE_HANDLE_GFID_PATH (gfid_path, this, gfid, NULL);
+
+        POSIX_GET_FILE_UNLINK_PATH (priv_posix->base_path,
+                                    loc->inode->gfid, unlink_path);
+        if (!unlink_path) {
+                ret = -1;
+                goto out;
+        }
+        gf_msg_debug (this->name, 0,
+                      "Moving gfid: %s to unlink_path : %s",
+                      gfid_path, unlink_path);
+        ret = sys_rename (gfid_path, unlink_path);
+        if (ret < 0) {
+                gf_msg (this->name, GF_LOG_ERROR, errno,
+                        P_MSG_UNLINK_FAILED,
+                        "Creation of unlink entry failed for gfid: %s",
+                        unlink_path);
+                goto out;
+        }
+        ret = posix_add_unlink_to_ctx (loc->inode, this, unlink_path);
+        if (ret < 0)
+                goto out;
+
+out:
+        return ret;
+}
+
 int32_t
 posix_unlink_gfid_handle_and_entry (xlator_t *this, const char *real_path,
-                                    struct iatt *stbuf, int32_t *op_errno)
+                                    struct iatt *stbuf, int32_t *op_errno,
+                                    loc_t *loc)
 {
-        int32_t             ret      =   0;
+        int32_t                 ret    = 0;
+        struct posix_private    *priv  = NULL;
+        int fd_count = 0;
 
-        /*  Unlink the gfid_handle_first */
+        priv = this->private;
 
+        /*  Unlink the gfid_handle_first */
         if (stbuf && stbuf->ia_nlink == 1) {
-                ret = posix_handle_unset (this, stbuf->ia_gfid, NULL);
+
+                LOCK (&loc->inode->lock);
+
+                if (loc->inode->fd_count == 0) {
+                        UNLOCK (&loc->inode->lock);
+                        ret = posix_handle_unset (this, stbuf->ia_gfid, NULL);
+                } else {
+                        UNLOCK (&loc->inode->lock);
+                        ret = posix_move_gfid_to_unlink (this, stbuf->ia_gfid,
+                                                         loc);
+                }
                 if (ret) {
                         gf_msg (this->name, GF_LOG_ERROR, errno,
                                 P_MSG_UNLINK_FAILED, "unlink of gfid handle "
@@ -1469,7 +1568,6 @@ posix_unlink_gfid_handle_and_entry (xlator_t *this, const char *real_path,
         if (ret == -1) {
                 if (op_errno)
                         *op_errno = errno;
-
                 gf_msg (this->name, GF_LOG_ERROR, errno, P_MSG_UNLINK_FAILED,
                         "unlink of %s failed", real_path);
                 goto err;
@@ -1623,7 +1721,7 @@ posix_unlink (call_frame_t *frame, xlator_t *this,
         }
 
         op_ret =  posix_unlink_gfid_handle_and_entry (this, real_path, &stbuf,
-                                                      &op_errno);
+                                                      &op_errno, loc);
         if (op_ret == -1) {
                 goto out;
         }
@@ -6176,6 +6274,77 @@ out:
 	return ret;
 }
 
+int32_t
+posix_create_unlink_dir (xlator_t *this) {
+
+        char  *unlink_path = NULL;
+        char  *landfill_path = NULL;
+        struct posix_private *priv = NULL;
+        struct stat           stbuf;
+        int ret = -1;
+        uuid_t                gfid = {0};
+        char                  gfid_str[64] = {0};
+
+        priv = this->private;
+
+        unlink_path = alloca (strlen (priv->base_path) + 1 +
+                              strlen (GF_UNLINK_PATH) + 1);
+        sprintf (unlink_path, "%s/%s", priv->base_path,
+                 GF_UNLINK_PATH);
+
+        gf_uuid_generate (gfid);
+        uuid_utoa_r (gfid, gfid_str);
+
+        landfill_path = alloca (strlen (priv->base_path) + 1 +
+                                strlen (GF_LANDFILL_PATH) + 1 +
+                                strlen (gfid_str) + 1);
+        sprintf (landfill_path, "%s/%s/%s", priv->base_path,
+                 GF_LANDFILL_PATH, gfid_str);
+
+        ret = sys_stat (unlink_path, &stbuf);
+        switch (ret) {
+        case -1:
+                if (errno != ENOENT) {
+                        gf_msg (this->name, GF_LOG_ERROR, errno,
+                                P_MSG_HANDLE_CREATE,
+                                "Checking for %s failed",
+                                unlink_path);
+                        return -1;
+                }
+                break;
+        case 0:
+                if (!S_ISDIR (stbuf.st_mode)) {
+                        gf_msg (this->name, GF_LOG_ERROR, 0,
+                                P_MSG_HANDLE_CREATE,
+                                "Not a directory: %s",
+                                unlink_path);
+                        return -1;
+                }
+                ret = sys_rename (unlink_path, landfill_path);
+                if (ret) {
+                        gf_msg (this->name, GF_LOG_ERROR, errno,
+                                P_MSG_HANDLE_CREATE,
+                                "Can not delete directory %s ",
+                                unlink_path);
+                        return -1;
+                }
+                break;
+        default:
+                break;
+        }
+        ret = sys_mkdir (unlink_path, 0600);
+        if (ret) {
+                gf_msg (this->name, GF_LOG_ERROR, errno,
+                        P_MSG_HANDLE_CREATE,
+                        "Creating directory %s failed",
+                        unlink_path);
+                return -1;
+        }
+
+        return 0;
+}
+
+
 
 /**
  * init -
@@ -6587,6 +6756,15 @@ init (xlator_t *this)
                 goto out;
 	}
 
+        op_ret = posix_create_unlink_dir (this);
+        if (op_ret == -1) {
+                gf_msg (this->name, GF_LOG_ERROR, 0,
+                        P_MSG_HANDLE_CREATE,
+                        "Creation of unlink directory failed");
+                ret = -1;
+                goto out;
+        }
+
 	_private->aio_init_done = _gf_false;
 	_private->aio_capable = _gf_false;
 
diff --git a/xlators/storage/posix/src/posix.h b/xlators/storage/posix/src/posix.h
index 27006ff3beb..c48a77b4f0a 100644
--- a/xlators/storage/posix/src/posix.h
+++ b/xlators/storage/posix/src/posix.h
@@ -56,6 +56,8 @@
                                                + SLEN(GF_HIDDEN_PATH) + SLEN("/") \
                                                + SLEN("00/")            \
                                                + SLEN("00/") + SLEN(UUID0_STR) + 1) /* '\0' */;
+#define GF_UNLINK_TRUE 0x0000000000000001
+#define GF_UNLINK_FALSE 0x0000000000000000
 
 /**
  * posix_fd - internal structure common to file and directory fd's
@@ -184,14 +186,36 @@ typedef struct {
         int32_t     op_errno;
 } posix_xattr_filler_t;
 
-
 #define POSIX_BASE_PATH(this) (((struct posix_private *)this->private)->base_path)
 
 #define POSIX_BASE_PATH_LEN(this) (((struct posix_private *)this->private)->base_path_length)
 
 #define POSIX_PATH_MAX(this) (((struct posix_private *)this->private)->path_max)
 
+#define POSIX_GET_FILE_UNLINK_PATH(base_path, gfid, unlink_path)                           \
+        do {                                                                               \
+                int  path_len = 0;                                                         \
+                char gfid_str[64] = {0};                                                   \
+                uuid_utoa_r (gfid, gfid_str);                                              \
+                path_len = strlen (base_path) + 1 +                                        \
+                          strlen (GF_UNLINK_PATH) + 1 +                                    \
+                          strlen (gfid_str) + 1;                                           \
+                unlink_path = alloca (path_len);                                           \
+                if (!unlink_path) {                                                        \
+                        gf_msg ("posix", GF_LOG_ERROR, ENOMEM,                             \
+                                P_MSG_UNLINK_FAILED,                                       \
+                                "Failed to get unlink_path");                              \
+                        break;                                                             \
+                }                                                                          \
+                sprintf (unlink_path, "%s/%s/%s",                                          \
+                         base_path, GF_UNLINK_PATH, gfid_str);                             \
+         } while (0)
+
+
 /* Helper functions */
+int posix_inode_ctx_get (inode_t *inode, xlator_t *this, uint64_t *ctx);
+int posix_inode_ctx_set (inode_t *inode, xlator_t *this, uint64_t ctx);
+
 int posix_gfid_set (xlator_t *this, const char *path, loc_t *loc,
                     dict_t *xattr_req);
 int posix_fdstat (xlator_t *this, int fd, struct iatt *stbuf_p);