prevent spurious unlocks from afr selfheal

afr selfheal now remembers all the nodes on which locks were successfully
held and sends unlocks only to those nodes

Signed-off-by: Anand V. Avati <avati@dev.gluster.com>

BUG: 112 (parallel deletion of files mounted by different clients on the same back-end hangs and/or does not completely delete)
URL: http://bugs.gluster.com/cgi-bin/bugzilla3/show_bug.cgi?id=112

6 files changed, 90 insertions, 23 deletions

diff --git a/xlators/cluster/afr/src/afr-self-heal-common.c b/xlators/cluster/afr/src/afr-self-heal-common.c
index adc3f282d1a..36c80f1c337 100644
--- a/xlators/cluster/afr/src/afr-self-heal-common.c
+++ b/xlators/cluster/afr/src/afr-self-heal-common.c
@@ -713,7 +713,11 @@ afr_sh_missing_entries_done (call_frame_t *frame, xlator_t *this)
 
 //	memset (sh->child_errno, 0, sizeof (int) * priv->child_count);
 	memset (sh->buf, 0, sizeof (struct stat) * priv->child_count);
-	
+
+        for (i = 0; i < priv->child_count; i++) {
+                sh->locked_nodes[i] = 0;
+        }
+
 	for (i = 0; i < priv->child_count; i++) {
 		if (sh->xattr[i])
 			dict_unref (sh->xattr[i]);
@@ -780,12 +784,20 @@ sh_missing_entries_finish (call_frame_t *frame, xlator_t *this)
 	sh = &local->self_heal;
 	priv = this->private;
 
-	call_count = local->child_count;
+        for (i = 0; i < priv->child_count; i++) {
+                if (sh->locked_nodes[i])
+                        call_count++;
+        }
+
+        if (call_count == 0) {
+                afr_sh_missing_entries_done (frame, this);
+                return 0;
+        }
 
 	local->call_count = call_count;
 
 	for (i = 0; i < priv->child_count; i++) {
-		if (local->child_up[i]) {
+		if (sh->locked_nodes[i]) {
 			gf_log (this->name, GF_LOG_TRACE,
 				"unlocking %"PRId64"/%s on subvolume %s",
 				sh->parent_loc.inode->ino, local->loc.name,
@@ -1280,11 +1292,13 @@ sh_missing_entries_lk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 		if (op_ret == -1) {
 			sh->op_failed = 1;
 
+                        sh->locked_nodes[child_index] = 0;
 			gf_log (this->name, GF_LOG_DEBUG,
 				"locking inode of %s on child %d failed: %s",
 				local->loc.path, child_index,
 				strerror (op_errno));
 		} else {
+                        sh->locked_nodes[child_index] = 1;
 			gf_log (this->name, GF_LOG_TRACE,
 				"inode of %s on child %d locked",
 				local->loc.path, child_index);
@@ -1333,12 +1347,13 @@ afr_self_heal_missing_entries (call_frame_t *frame, xlator_t *this)
 
 	for (i = 0; i < priv->child_count; i++) {
 		if (local->child_up[i]) {
-			STACK_WIND (frame, sh_missing_entries_lk_cbk,
-				    priv->children[i],
-				    priv->children[i]->fops->entrylk,
-                                    this->name,
-				    &sh->parent_loc, local->loc.name,
-				    ENTRYLK_LOCK_NB, ENTRYLK_WRLCK);
+			STACK_WIND_COOKIE (frame, sh_missing_entries_lk_cbk,
+                                           (void *) (long) i,
+                                           priv->children[i],
+                                           priv->children[i]->fops->entrylk,
+                                           this->name,
+                                           &sh->parent_loc, local->loc.name,
+                                           ENTRYLK_LOCK_NB, ENTRYLK_WRLCK);
 			if (!--call_count)
 				break;
 		}
@@ -1376,6 +1391,7 @@ afr_self_heal (call_frame_t *frame, xlator_t *this,
 	sh->success = CALLOC (priv->child_count, sizeof (int));
 	sh->xattr = CALLOC (priv->child_count, sizeof (dict_t *));
 	sh->sources = CALLOC (sizeof (*sh->sources), priv->child_count);
+	sh->locked_nodes = CALLOC (sizeof (*sh->locked_nodes), priv->child_count);
 
 	sh->pending_matrix = CALLOC (sizeof (int32_t *), priv->child_count);
 	for (i = 0; i < priv->child_count; i++) {
diff --git a/xlators/cluster/afr/src/afr-self-heal-data.c b/xlators/cluster/afr/src/afr-self-heal-data.c
index bbcda721adc..d24aa57a1d0 100644
--- a/xlators/cluster/afr/src/afr-self-heal-data.c
+++ b/xlators/cluster/afr/src/afr-self-heal-data.c
@@ -56,6 +56,7 @@ afr_sh_data_done (call_frame_t *frame, xlator_t *this)
 	afr_local_t     *local = NULL;
 	afr_self_heal_t *sh = NULL;
 	afr_private_t   *priv = NULL;
+        int              i = 0;
 
 	local = frame->local;
 	sh = &local->self_heal;
@@ -65,6 +66,14 @@ afr_sh_data_done (call_frame_t *frame, xlator_t *this)
 	   TODO: cleanup sh->* 
 	 */
 
+        if (sh->healing_fd) {
+		fd_unref (sh->healing_fd);
+		sh->healing_fd = NULL;
+        }
+
+        for (i = 0; i < priv->child_count; i++)
+                sh->locked_nodes[i] = 0;
+
 	gf_log (this->name, GF_LOG_TRACE,
 		"self heal of %s completed",
 		local->loc.path);
@@ -96,8 +105,6 @@ afr_sh_data_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 	call_count = afr_frame_return (frame);
 
 	if (call_count == 0) {
-		fd_unref (sh->healing_fd);
-		sh->healing_fd = NULL;
 		afr_sh_data_done (frame, this);
 	}
 
@@ -258,7 +265,15 @@ afr_sh_data_unlock (call_frame_t *frame, xlator_t *this)
 	sh = &local->self_heal;
 	priv = this->private;
 
-	call_count = local->child_count;
+        for (i = 0; i < priv->child_count; i++) {
+                if (sh->locked_nodes[i])
+                        call_count++;
+        }
+
+        if (call_count == 0) {
+                afr_sh_data_close (frame, this);
+                return 0;
+        }
 
 	local->call_count = call_count;		
 
@@ -267,7 +282,7 @@ afr_sh_data_unlock (call_frame_t *frame, xlator_t *this)
 	flock.l_type  = F_UNLCK;
 
 	for (i = 0; i < priv->child_count; i++) {
-		if (local->child_up[i]) {
+		if (sh->locked_nodes[i]) {
 			gf_log (this->name, GF_LOG_TRACE,
 				"unlocking %s on subvolume %s",
 				local->loc.path, priv->children[i]->name);
@@ -992,12 +1007,13 @@ afr_sh_data_lock_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 		if (op_ret == -1) {
 			sh->op_failed = 1;
 
-			gf_log (this->name,
-                                GF_LOG_DEBUG,
+                        sh->locked_nodes[child_index] = 0;
+			gf_log (this->name, GF_LOG_DEBUG,
 				"locking of %s on child %d failed: %s",
 				local->loc.path, child_index,
 				strerror (op_errno));
 		} else {
+                        sh->locked_nodes[child_index] = 1;
 			gf_log (this->name, GF_LOG_TRACE,
 				"inode of %s on child %d locked",
 				local->loc.path, child_index);
diff --git a/xlators/cluster/afr/src/afr-self-heal-entry.c b/xlators/cluster/afr/src/afr-self-heal-entry.c
index f3759b113a4..676f2ef1b31 100644
--- a/xlators/cluster/afr/src/afr-self-heal-entry.c
+++ b/xlators/cluster/afr/src/afr-self-heal-entry.c
@@ -56,6 +56,7 @@ afr_sh_entry_done (call_frame_t *frame, xlator_t *this)
 	afr_local_t     *local = NULL;
 	afr_self_heal_t *sh = NULL;
 	afr_private_t   *priv = NULL;
+        int              i = 0;
 
 	local = frame->local;
 	sh = &local->self_heal;
@@ -65,6 +66,14 @@ afr_sh_entry_done (call_frame_t *frame, xlator_t *this)
 	   TODO: cleanup sh->* 
 	*/
 
+        if (sh->healing_fd)
+                fd_unref (sh->healing_fd);
+        sh->healing_fd = NULL;
+
+        for (i = 0; i < priv->child_count; i++) {
+                sh->locked_nodes[i] = 0;
+        }
+
 	gf_log (this->name, GF_LOG_TRACE,
 		"self heal of %s completed",
 		local->loc.path);
@@ -107,9 +116,6 @@ afr_sh_entry_unlck_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 	call_count = afr_frame_return (frame);
 
 	if (call_count == 0) {
-		if (sh->healing_fd)
-			fd_unref (sh->healing_fd);
-		sh->healing_fd = NULL;
 		afr_sh_entry_done (frame, this);
 	}
 
@@ -132,12 +138,20 @@ afr_sh_entry_unlock (call_frame_t *frame, xlator_t *this)
 	sh = &local->self_heal;
 	priv = this->private;
 
-	call_count = local->child_count;
+        for (i = 0; i < priv->child_count; i++) {
+                if (sh->locked_nodes[i])
+                        call_count++;
+        }
+
+        if (call_count == 0) {
+                afr_sh_entry_done (frame, this);
+                return 0;
+        }
 
 	local->call_count = call_count;		
 
 	for (i = 0; i < priv->child_count; i++) {
-		if (local->child_up[i]) {
+		if (sh->locked_nodes[i]) {
 			gf_log (this->name, GF_LOG_TRACE,
 				"unlocking %s on subvolume %s",
 				local->loc.path, priv->children[i]->name);
@@ -2013,11 +2027,13 @@ afr_sh_entry_lock_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 		if (op_ret == -1) {
 			sh->op_failed = 1;
 
+                        sh->locked_nodes[child_index] = 0;
 			gf_log (this->name, GF_LOG_DEBUG,
 				"locking inode of %s on child %d failed: %s",
 				local->loc.path, child_index,
 				strerror (op_errno));
 		} else {
+                        sh->locked_nodes[child_index] = 1;
 			gf_log (this->name, GF_LOG_TRACE,
 				"inode of %s on child %d locked",
 				local->loc.path, child_index);
diff --git a/xlators/cluster/afr/src/afr-self-heal-metadata.c b/xlators/cluster/afr/src/afr-self-heal-metadata.c
index 69a5f8878f8..0bf72aec103 100644
--- a/xlators/cluster/afr/src/afr-self-heal-metadata.c
+++ b/xlators/cluster/afr/src/afr-self-heal-metadata.c
@@ -64,7 +64,11 @@ afr_sh_metadata_done (call_frame_t *frame, xlator_t *this)
 //	memset (sh->child_errno, 0, sizeof (int) * priv->child_count);
 	memset (sh->buf, 0, sizeof (struct stat) * priv->child_count);
 	memset (sh->success, 0, sizeof (int) * priv->child_count);
-	
+
+        for (i = 0; i < priv->child_count; i++) {
+                sh->locked_nodes[i] = 1;
+        }
+
 	for (i = 0; i < priv->child_count; i++) {
 		if (sh->xattr[i])
 			dict_unref (sh->xattr[i]);
@@ -137,7 +141,16 @@ afr_sh_metadata_finish (call_frame_t *frame, xlator_t *this)
 	sh = &local->self_heal;
 	priv = this->private;
 
-	call_count = local->child_count;
+        for (i = 0; i < priv->child_count; i++) {
+                if (sh->locked_nodes[i])
+                        call_count++;
+        }
+
+        if (call_count == 0) {
+                afr_sh_metadata_done (frame, this);
+                return 0;
+        }
+
 	local->call_count = call_count;
 
 	for (i = 0; i < priv->child_count; i++) {
@@ -145,7 +158,7 @@ afr_sh_metadata_finish (call_frame_t *frame, xlator_t *this)
 		flock.l_len     = 0;
 		flock.l_type    = F_UNLCK;
 
-		if (local->child_up[i]) {
+		if (sh->locked_nodes[i]) {
 			gf_log (this->name, GF_LOG_TRACE,
 				"unlocking %s on subvolume %s",
 				local->loc.path, priv->children[i]->name);
@@ -713,11 +726,13 @@ afr_sh_metadata_lk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 		if (op_ret == -1) {
 			sh->op_failed = 1;
 
+                        sh->locked_nodes[child_index] = 0;
 			gf_log (this->name, GF_LOG_DEBUG,
 				"locking of %s on child %d failed: %s",
 				local->loc.path, child_index,
 				strerror (op_errno));
 		} else {
+                        sh->locked_nodes[child_index] = 1;
 			gf_log (this->name, GF_LOG_TRACE,
 				"inode of %s on child %d locked",
 				local->loc.path, child_index);
diff --git a/xlators/cluster/afr/src/afr.c b/xlators/cluster/afr/src/afr.c
index df227ceae4d..3e1808721a1 100644
--- a/xlators/cluster/afr/src/afr.c
+++ b/xlators/cluster/afr/src/afr.c
@@ -200,6 +200,9 @@ afr_local_sh_cleanup (afr_local_t *local, xlator_t *this)
 	if (sh->success)
 		FREE (sh->success);
 
+	if (sh->locked_nodes)
+		FREE (sh->locked_nodes);
+
 	if (sh->healing_fd) {
 		fd_unref (sh->healing_fd);
 		sh->healing_fd = NULL;
diff --git a/xlators/cluster/afr/src/afr.h b/xlators/cluster/afr/src/afr.h
index ea291f83905..fff53e56988 100644
--- a/xlators/cluster/afr/src/afr.h
+++ b/xlators/cluster/afr/src/afr.h
@@ -82,6 +82,7 @@ typedef struct {
 	int active_source;
 	int active_sinks;
 	int *success;
+	int *locked_nodes;
 
 	fd_t *healing_fd;
 	int   op_failed;