protocol/client: cluster/afr: Support lock recovery and self heal.

Signed-off-by: Pavan Vilas Sondur <pavan@gluster.com>
Signed-off-by: Vijay Bellur <vijay@dev.gluster.com>

BUG: 865 (Add locks recovery support in GlusterFS)
URL: http://bugs.gluster.com/cgi-bin/bugzilla3/show_bug.cgi?id=865

5 files changed, 150 insertions, 3 deletions

diff --git a/xlators/cluster/afr/src/afr-common.c b/xlators/cluster/afr/src/afr-common.c
index 64e22c322..9d9f5d041 100644
--- a/xlators/cluster/afr/src/afr-common.c
+++ b/xlators/cluster/afr/src/afr-common.c
@@ -1197,6 +1197,16 @@ afr_fd_ctx_set (xlator_t *this, fd_t *fd)
                 fd_ctx->up_count   = priv->up_count;
                 fd_ctx->down_count = priv->down_count;
 
+                fd_ctx->locked_on = GF_CALLOC (sizeof (*fd_ctx->locked_on),
+                                               priv->child_count,
+                                               gf_afr_mt_char);
+                if (!fd_ctx->locked_on) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Out of memory");
+                        ret = -ENOMEM;
+                        goto unlock;
+                }
+
                 ret = __fd_ctx_set (fd, this, (uint64_t)(long) fd_ctx);
 
                 INIT_LIST_HEAD (&fd_ctx->entries);
@@ -1426,6 +1436,9 @@ afr_cleanup_fd_ctx (xlator_t *this, fd_t *fd)
                 if (fd_ctx->opened_on)
                         GF_FREE (fd_ctx->opened_on);
 
+                if (fd_ctx->locked_on)
+                        GF_FREE (fd_ctx->locked_on);
+
                 GF_FREE (fd_ctx);
         }
 
@@ -2298,8 +2311,9 @@ int32_t
 afr_lk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
             int32_t op_ret, int32_t op_errno, struct flock *lock)
 {
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
+	afr_local_t *local = NULL;
+	afr_private_t *priv = NULL;
+        int            ret  = 0;
 
         int child_index = -1;
 
@@ -2339,7 +2353,18 @@ afr_lk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         } else {
                 /* locking has succeeded on all nodes that are up */
 
-                AFR_STACK_UNWIND (lk, frame, local->op_ret, local->op_errno,
+                ret = afr_mark_locked_nodes (this, local->fd,
+                                             local->cont.lk.locked_nodes);
+                if (ret)
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "Could not save locked nodes info in fdctx");
+
+                ret = afr_save_locked_fd (this, local->fd);
+                if (ret)
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "Could not save locked fd");
+
+		AFR_STACK_UNWIND (lk, frame, local->op_ret, local->op_errno,
                                   &local->cont.lk.ret_flock);
         }
 
diff --git a/xlators/cluster/afr/src/afr-lk-common.c b/xlators/cluster/afr/src/afr-lk-common.c
index ee53d1d7b..de95a6c76 100644
--- a/xlators/cluster/afr/src/afr-lk-common.c
+++ b/xlators/cluster/afr/src/afr-lk-common.c
@@ -1678,3 +1678,104 @@ afr_unlock (call_frame_t *frame, xlator_t *this)
 
         return 0;
 }
+
+int
+afr_mark_locked_nodes (xlator_t *this, fd_t *fd,
+                       unsigned char *locked_nodes)
+{
+        afr_private_t *priv  = NULL;
+        afr_fd_ctx_t  *fdctx = NULL;
+        uint64_t       tmp   = 0;
+        int            ret   = 0;
+
+        priv = this->private;
+
+        afr_fd_ctx_set (this, fd);
+        if (ret < 0)
+                goto out;
+
+        ret = fd_ctx_get (fd, this, &tmp);
+        fdctx = (afr_fd_ctx_t *) (long) tmp;
+
+        GF_ASSERT (fdctx->locked_on);
+
+        memcpy (fdctx->locked_on, locked_nodes,
+                priv->child_count);
+
+out:
+        return ret;
+}
+
+static int
+__is_fd_saved (xlator_t *this, fd_t *fd)
+{
+        afr_locked_fd_t *locked_fd = NULL;
+        afr_private_t   *priv      = NULL;
+        int              found     = 0;
+
+        priv = this->private;
+
+        list_for_each_entry (locked_fd, &priv->saved_fds, list) {
+                if (locked_fd->fd == fd) {
+                        found = 1;
+                        break;
+                }
+        }
+
+        return found;
+}
+
+static int
+__afr_save_locked_fd (xlator_t *this, fd_t *fd)
+{
+        afr_private_t   *priv      = NULL;
+        afr_locked_fd_t *locked_fd = NULL;
+        int              ret       = 0;
+
+        priv = this->private;
+
+        locked_fd = GF_CALLOC (1, sizeof (*locked_fd),
+                               gf_afr_mt_locked_fd);
+        if (!locked_fd) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Out of memory");
+                ret = -1;
+                goto out;
+        }
+
+        INIT_LIST_HEAD (&locked_fd->list);
+
+        list_add_tail (&locked_fd->list, &priv->saved_fds);
+
+out:
+        return ret;
+}
+
+int
+afr_save_locked_fd (xlator_t *this, fd_t *fd)
+{
+        afr_private_t   *priv      = NULL;
+        int              ret       = 0;
+
+        priv = this->private;
+
+        pthread_mutex_lock (&priv->mutex);
+        {
+                if (__is_fd_saved (this, fd)) {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "fd=%p already saved", fd);
+                        goto unlock;
+                }
+
+                ret = __afr_save_locked_fd (this, fd);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "fd=%p could not be saved");
+                        goto unlock;
+                }
+        }
+unlock:
+        pthread_mutex_unlock (&priv->mutex);
+
+        return ret;
+}
diff --git a/xlators/cluster/afr/src/afr-mem-types.h b/xlators/cluster/afr/src/afr-mem-types.h
index c1a86b827..31a80e8f5 100644
--- a/xlators/cluster/afr/src/afr-mem-types.h
+++ b/xlators/cluster/afr/src/afr-mem-types.h
@@ -41,6 +41,7 @@ enum gf_afr_mem_types_ {
         gf_afr_mt_loc_t,
         gf_afr_mt_entry_name,
         gf_afr_mt_pump_priv,
+        gf_afr_mt_locked_fd,
         gf_afr_mt_end
 };
 #endif
diff --git a/xlators/cluster/afr/src/afr.c b/xlators/cluster/afr/src/afr.c
index 09094bdbe..69b281d97 100644
--- a/xlators/cluster/afr/src/afr.c
+++ b/xlators/cluster/afr/src/afr.c
@@ -641,6 +641,9 @@ init (xlator_t *this)
         priv->first_lookup = 1;
         priv->root_inode = NULL;
 
+        pthread_mutex_init (&priv->mutex, NULL);
+        INIT_LIST_HEAD (&priv->saved_fds);
+
 	ret = 0;
 out:
 	return ret;
diff --git a/xlators/cluster/afr/src/afr.h b/xlators/cluster/afr/src/afr.h
index 26f2c989f..68b4a1e30 100644
--- a/xlators/cluster/afr/src/afr.h
+++ b/xlators/cluster/afr/src/afr.h
@@ -85,6 +85,9 @@ typedef struct _afr_private {
 
 	struct _pump_private *pump_private; /* Set if we are loaded as pump */
         int                   use_afr_in_pump;
+
+        pthread_mutex_t  mutex;
+        struct list_head saved_fds;   /* list of fds on which locks have succeeded */
 } afr_private_t;
 
 typedef struct {
@@ -262,6 +265,11 @@ typedef struct {
 
 } afr_internal_lock_t;
 
+typedef struct _afr_locked_fd {
+        fd_t  *fd;
+        struct list_head list;
+} afr_locked_fd_t;
+
 typedef struct _afr_local {
 	unsigned int call_count;
 	unsigned int success_count;
@@ -619,6 +627,8 @@ typedef struct {
         int  hit, miss;
         gf_boolean_t failed_over;
         struct list_head entries; /* needed for readdir failover */
+
+        unsigned char *locked_on; /* which subvolumes locks have been successful */
 } afr_fd_ctx_t;
 
 
@@ -655,6 +665,13 @@ int32_t
 afr_notify (xlator_t *this, int32_t event,
             void *data, ...);
 
+int
+afr_save_locked_fd (xlator_t *this, fd_t *fd);
+
+int
+afr_mark_locked_nodes (xlator_t *this, fd_t *fd,
+                       unsigned char *locked_nodes);
+
 void
 afr_set_lk_owner (call_frame_t *frame, xlator_t *this);