cluster/afr: Use 2 phase locking for transactions and self heal.

Signed-off-by: Pavan Vilas Sondur <pavan@gluster.com>
Signed-off-by: Anand V. Avati <avati@dev.gluster.com>

BUG: 960 ()
URL: http://bugs.gluster.com/cgi-bin/bugzilla3/show_bug.cgi?id=960

1 files changed, 73 insertions, 130 deletions

diff --git a/xlators/cluster/afr/src/afr-self-heal-data.c b/xlators/cluster/afr/src/afr-self-heal-data.c
index 366cac817..8e87bc555 100644
--- a/xlators/cluster/afr/src/afr-self-heal-data.c
+++ b/xlators/cluster/afr/src/afr-self-heal-data.c
@@ -56,7 +56,6 @@ afr_sh_data_done (call_frame_t *frame, xlator_t *this)
 	afr_local_t     *local = NULL;
 	afr_self_heal_t *sh = NULL;
 	afr_private_t   *priv = NULL;
-        int              i = 0;
 
 	local = frame->local;
 	sh = &local->self_heal;
@@ -73,8 +72,8 @@ afr_sh_data_done (call_frame_t *frame, xlator_t *this)
 		sh->healing_fd = NULL;
         }
 
-        for (i = 0; i < priv->child_count; i++)
-                sh->locked_nodes[i] = 0;
+/*         for (i = 0; i < priv->child_count; i++) */
+/*                 sh->locked_nodes[i] = 0; */
 
 	gf_log (this->name, GF_LOG_TRACE,
 		"self heal of %s completed",
@@ -268,58 +267,18 @@ afr_sh_data_unlck_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 int
 afr_sh_data_unlock (call_frame_t *frame, xlator_t *this)
 {
-	struct flock flock;			
-	int i = 0;				
-	int call_count = 0;		     
+        afr_local_t         *local    = NULL;
+        afr_internal_lock_t *int_lock = NULL;
+	afr_self_heal_t     *sh       = NULL;
 
-	afr_local_t *   local = NULL;
-	afr_private_t * priv  = NULL;
-	afr_self_heal_t * sh  = NULL;
-
-
-	local = frame->local;
-	sh = &local->self_heal;
-	priv = this->private;
-
-        if (sh->data_lock_held) {
-                /* not our job to unlock, proceed to close */
-
-                afr_sh_data_close (frame, this);
-                return 0;
-        }
+        local    = frame->local;
+        int_lock = &local->internal_lock;
+	sh       = &local->self_heal;
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (sh->locked_nodes[i])
-                        call_count++;
-        }
+        GF_ASSERT (!sh->data_lock_held);
 
-        if (call_count == 0) {
-                afr_sh_data_close (frame, this);
-                return 0;
-        }
-
-	local->call_count = call_count;		
-
-	flock.l_start = 0;
-	flock.l_len   = 0;
-	flock.l_type  = F_UNLCK;
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (sh->locked_nodes[i]) {
-			gf_log (this->name, GF_LOG_TRACE,
-				"unlocking %s on subvolume %s",
-				local->loc.path, priv->children[i]->name);
-
-			STACK_WIND_COOKIE (frame, afr_sh_data_unlck_cbk,
-					   (void *) (long) i,
-					   priv->children[i], 
-					   priv->children[i]->fops->inodelk,
-                                           this->name,
-					   &local->loc, F_SETLK, &flock); 
-			if (!--call_count)
-				break;
-		}
-	}
+        int_lock->lock_cbk = afr_sh_data_close;
+        afr_unlock (frame, this);
 
 	return 0;
 }
@@ -329,13 +288,18 @@ int
 afr_sh_data_finish (call_frame_t *frame, xlator_t *this)
 {
 	afr_local_t   *local = NULL;
+	afr_self_heal_t *sh = NULL;
 
 	local = frame->local;
+	sh = &local->self_heal;
 
 	gf_log (this->name, GF_LOG_TRACE,
 		"finishing data selfheal of %s", local->loc.path);
 
-	afr_sh_data_unlock (frame, this);
+        if (!sh->data_lock_held)
+                afr_sh_data_unlock (frame, this);
+        else
+                afr_sh_data_close (frame, this);
 
 	return 0;
 }
@@ -388,7 +352,7 @@ afr_sh_data_erase_pending (call_frame_t *frame, xlator_t *this)
                                  priv->child_count, AFR_DATA_TRANSACTION);
 
 	erase_xattr = GF_CALLOC (sizeof (*erase_xattr), priv->child_count,
-                                 gf_afr_mt_dict_t);
+                              gf_afr_mt_dict_t);
 
 	for (i = 0; i < priv->child_count; i++) {
 		if (sh->xattr[i]) {
@@ -772,8 +736,8 @@ afr_self_heal_get_source (xlator_t *this, afr_local_t *local, dict_t **xattr)
                                         gf_afr_mt_int32_t);
 	for (i = 0; i < priv->child_count; i++) {
 		sh->pending_matrix[i] = GF_CALLOC (sizeof (int32_t),
-					           priv->child_count,
-                                                   gf_afr_mt_int32_t);
+						priv->child_count,
+                                                gf_afr_mt_int32_t);
 	}
 
 	sh->sources = GF_CALLOC (priv->child_count, sizeof (*sh->sources),
@@ -958,96 +922,79 @@ afr_sh_data_fxattrop (call_frame_t *frame, xlator_t *this)
 
 
 int
-afr_sh_data_lock_rec (call_frame_t *frame, xlator_t *this, int child_index);
+afr_sh_data_lock_rec (call_frame_t *frame, xlator_t *this);
 
 int
-afr_sh_data_lock_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		      int32_t op_ret, int32_t op_errno)
+afr_sh_data_post_blocking_inodelk_cbk (call_frame_t *frame, xlator_t *this)
 {
-	afr_local_t     *local = NULL;
-	afr_self_heal_t *sh = NULL;
-	int              child_index = (long) cookie;
-
-	/* TODO: what if lock fails? */
-	
-	local = frame->local;
-	sh = &local->self_heal;
+        afr_internal_lock_t *int_lock = NULL;
+        afr_local_t         *local    = NULL;
 
-	LOCK (&frame->lock);
-	{
-		if (op_ret == -1) {
-                        sh->locked_nodes[child_index] = 0;
+        local    = frame->local;
+        int_lock = &local->internal_lock;
 
-			gf_log (this->name, GF_LOG_DEBUG,
-				"locking of %s on child %d failed: %s",
-				local->loc.path, child_index,
-				strerror (op_errno));
-		} else {
-                        sh->locked_nodes[child_index] = 1;
-                        sh->lock_count++;
-
-			gf_log (this->name, GF_LOG_TRACE,
-				"inode of %s on child %d locked",
-				local->loc.path, child_index);
-		}
-	}
-	UNLOCK (&frame->lock);
+        if (int_lock->lock_op_ret < 0) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "Blocking inodelks failed.");
+                afr_sh_data_done (frame, this);
+        } else {
 
-        afr_sh_data_lock_rec (frame, this, child_index + 1);
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "Blocking inodelks done. Proceeding to FOP");
+                afr_sh_data_fxattrop (frame, this);
+        }
 
-	return 0;
+        return 0;
 }
 
-
 int
-afr_sh_data_lock_rec (call_frame_t *frame, xlator_t *this, int child_index)
+afr_sh_data_post_nonblocking_inodelk_cbk (call_frame_t *frame, xlator_t *this)
 {
-	struct flock flock;			
-	int i = 0;				
+        afr_internal_lock_t *int_lock = NULL;
+        afr_local_t         *local    = NULL;
 
-	afr_local_t *   local = NULL;
-	afr_private_t * priv  = NULL;
-	afr_self_heal_t * sh  = NULL;
+        local    = frame->local;
+        int_lock = &local->internal_lock;
 
-	local = frame->local;
-	sh = &local->self_heal;
-	priv = this->private;
+        /* Initiate blocking locks if non-blocking has failed */
+        if (int_lock->lock_op_ret < 0) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "Non blocking inodelks failed. Proceeding to blocking");
+                int_lock->lock_cbk = afr_sh_data_post_blocking_inodelk_cbk;
+                afr_blocking_lock (frame, this);
+        } else {
 
-	flock.l_start = 0;
-	flock.l_len   = 0;
-	flock.l_type  = F_WRLCK;			
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "Non blocking inodelks done. Proceeding to FOP");
+                afr_sh_data_fxattrop (frame, this);
+        }
 
-	/* skip over children that are down */
-	while ((child_index < priv->child_count)
-	       && !local->child_up[child_index])
-		child_index++;
+        return 0;
+}
 
-	if ((child_index == priv->child_count) &&
-	    sh->lock_count == 0) {
+int
+afr_sh_data_lock_rec (call_frame_t *frame, xlator_t *this)
+{
+        afr_internal_lock_t *int_lock = NULL;
+        afr_local_t         *local    = NULL;
+        afr_self_heal_t     *sh       = NULL;
 
-		gf_log (this->name, GF_LOG_DEBUG,
-			"unable to lock on even one child");
+        local    = frame->local;
+        int_lock = &local->internal_lock;
+        sh       = &local->self_heal;
 
-                afr_sh_data_done (frame, this);
-		return 0;
-	}
+        int_lock->transaction_lk_type = AFR_SELFHEAL_LK;
+        int_lock->selfheal_lk_type    = AFR_DATA_SELF_HEAL_LK;
 
-        if ((child_index == priv->child_count)
-            || (sh->lock_count == afr_lock_server_count (priv, AFR_DATA_TRANSACTION))) {
-                afr_sh_data_fxattrop (frame, this);
-                return 0;
-        }
+        afr_set_lock_number (frame, this);
+
+        int_lock->lk_flock.l_start = 0;
+        int_lock->lk_flock.l_len   = 0;
+        int_lock->lk_flock.l_type  = F_WRLCK;
+        int_lock->lock_cbk         = afr_sh_data_post_nonblocking_inodelk_cbk;
 
-        gf_log (this->name, GF_LOG_TRACE,
-                "locking %s on subvolume %s",
-                local->loc.path, priv->children[i]->name);
+        afr_nonblocking_inodelk (frame, this);
 
-        STACK_WIND_COOKIE (frame, afr_sh_data_lock_cbk,
-                           (void *) (long) child_index,
-                           priv->children[i],
-                           priv->children[i]->fops->inodelk,
-                           this->name,
-                           &local->loc, F_SETLKW, &flock);
 
 	return 0;
 }
@@ -1060,7 +1007,6 @@ afr_sh_data_lock (call_frame_t *frame, xlator_t *this)
 	afr_private_t * priv  = NULL;
 	afr_self_heal_t * sh  = NULL;
 
-        int i = 0;
 
 	local = frame->local;
 	sh    = &local->self_heal;
@@ -1074,10 +1020,7 @@ afr_sh_data_lock (call_frame_t *frame, xlator_t *this)
                 return 0;
         }
 
-        for (i = 0; i < priv->child_count; i++)
-                sh->locked_nodes[i] = 0;
-
-        return afr_sh_data_lock_rec (frame, this, 0);
+        return afr_sh_data_lock_rec (frame, this);
 }