ec: Fix return errors when not enough bricks

Changes introduced by this patch:

* Fix an incorrect error propagation when the state of the life
  cycle of a fop returns an error.

* Fix incorrect unlocking of failed locks.

* Return ENOTCONN if there aren't enough bricks online.

* In readdir(p) check that the fd has been successfully open by
  a previous opendir.

Change-Id: Ib44f25a1297849ebcbab839332f3b6359f275ebe
BUG: 1162805
Signed-off-by: Xavier Hernandez <xhernandez@datalab.es>
Reviewed-on: http://review.gluster.org/9098
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Vijay Bellur <vbellur@redhat.com>

8 files changed, 81 insertions, 14 deletions

diff --git a/xlators/cluster/ec/src/ec-common.c b/xlators/cluster/ec/src/ec-common.c
index fbdad86950a..89c78c69bae 100644
--- a/xlators/cluster/ec/src/ec-common.c
+++ b/xlators/cluster/ec/src/ec-common.c
@@ -313,6 +313,7 @@ void ec_resume_parent(ec_fop_data_t * fop, int32_t error)
     parent = fop->parent;
     if (parent != NULL)
     {
+        ec_trace("RESUME_PARENT", fop, "error=%u", error);
         fop->parent = NULL;
         ec_resume(parent, error);
     }
@@ -1148,7 +1149,7 @@ int32_t ec_unlocked(call_frame_t *frame, void *cookie, xlator_t *this,
 
 void ec_unlock_lock(ec_fop_data_t *fop, ec_lock_t *lock)
 {
-    if (lock->mask != 0) {
+    if ((lock->mask != 0) && lock->acquired) {
         ec_owner_set(fop->frame, lock);
 
         switch (lock->kind) {
@@ -1327,7 +1328,10 @@ void ec_unlock_timer_add(ec_lock_link_t *link)
         lock->refs--;
 
         UNLOCK(&lock->loc.inode->lock);
-    } else {
+    } else if (lock->acquired) {
+        delay.tv_sec = 1;
+        delay.tv_nsec = 0;
+
         LOCK(&fop->lock);
 
         fop->jobs++;
@@ -1361,6 +1365,12 @@ void ec_unlock_timer_add(ec_lock_link_t *link)
         if (refs == 0) {
             ec_unlock_now(fop, lock);
         }
+    } else {
+        *lock->plock = NULL;
+
+        UNLOCK(&lock->loc.inode->lock);
+
+        ec_lock_destroy(lock);
     }
 }
 
@@ -1461,23 +1471,25 @@ void ec_lock_reuse(ec_fop_data_t *fop)
 
 void __ec_manager(ec_fop_data_t * fop, int32_t error)
 {
-    do
-    {
-        ec_trace("MANAGER", fop, "error=%d", error);
+    ec_t *ec = fop->xl->private;
 
-        if (fop->state == EC_STATE_END)
-        {
-            ec_fop_data_release(fop);
+    do {
+        ec_trace("MANAGER", fop, "error=%d", error);
 
-            break;
+        if (ec->xl_up_count < ec->fragments) {
+            error = ENOTCONN;
         }
-
-        if (error != 0)
-        {
+        if (error != 0) {
             fop->error = error;
             fop->state = -fop->state;
         }
 
+        if ((fop->state == EC_STATE_END) || (fop->state == -EC_STATE_END)) {
+            ec_fop_data_release(fop);
+
+            break;
+        }
+
         fop->state = fop->handler(fop, fop->state);
 
         error = ec_check_complete(fop, __ec_manager);
diff --git a/xlators/cluster/ec/src/ec-dir-read.c b/xlators/cluster/ec/src/ec-dir-read.c
index 95d80efdf8b..c705b80fe82 100644
--- a/xlators/cluster/ec/src/ec-dir-read.c
+++ b/xlators/cluster/ec/src/ec-dir-read.c
@@ -155,8 +155,21 @@ int32_t ec_manager_opendir(ec_fop_data_t * fop, int32_t state)
                         cbk->op_errno = EIO;
                     }
                 }
-                if (cbk->op_ret < 0)
-                {
+                if (cbk->op_ret >= 0) {
+                    /* Save which subvolumes successfully opened the directory.
+                     * If ctx->open is 0, it means that readdir cannot be
+                     * processed in this directory.
+                     */
+                    LOCK(&fop->fd->lock);
+
+                    ctx = __ec_fd_get(fop->fd, fop->xl);
+                    if (ctx != NULL) {
+                        ctx->open |= cbk->mask;
+                    }
+
+                    UNLOCK(&fop->fd->lock);
+                }
+                if (cbk->op_ret < 0) {
                     ec_fop_set_error(fop, cbk->op_errno);
                 }
             }
@@ -180,6 +193,7 @@ int32_t ec_manager_opendir(ec_fop_data_t * fop, int32_t state)
 
             return EC_STATE_LOCK_REUSE;
 
+        case -EC_STATE_INIT:
         case -EC_STATE_LOCK:
         case -EC_STATE_GET_SIZE_AND_VERSION:
         case -EC_STATE_DISPATCH:
@@ -360,9 +374,20 @@ void ec_wind_readdir(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
 
 int32_t ec_manager_readdir(ec_fop_data_t * fop, int32_t state)
 {
+    ec_fd_t *ctx;
+
     switch (state)
     {
         case EC_STATE_INIT:
+            /* Return error if opendir has not been successfully called on
+             * any subvolume. */
+            ctx = ec_fd_get(fop->fd, fop->xl);
+            if ((ctx == NULL) || (ctx->open == 0)) {
+                fop->error = EINVAL;
+
+                return EC_STATE_REPORT;
+            }
+
             if (fop->xdata == NULL)
             {
                 fop->xdata = dict_new();
@@ -402,6 +427,7 @@ int32_t ec_manager_readdir(ec_fop_data_t * fop, int32_t state)
 
             return EC_STATE_REPORT;
 
+        case -EC_STATE_INIT:
         case -EC_STATE_REPORT:
             if (fop->id == GF_FOP_READDIR)
             {
diff --git a/xlators/cluster/ec/src/ec-dir-write.c b/xlators/cluster/ec/src/ec-dir-write.c
index 2b1064a98f0..91467ae26e7 100644
--- a/xlators/cluster/ec/src/ec-dir-write.c
+++ b/xlators/cluster/ec/src/ec-dir-write.c
@@ -278,6 +278,7 @@ int32_t ec_manager_create(ec_fop_data_t * fop, int32_t state)
 
             return EC_STATE_LOCK_REUSE;
 
+        case -EC_STATE_INIT:
         case -EC_STATE_LOCK:
         case -EC_STATE_DISPATCH:
         case -EC_STATE_PREPARE_ANSWER:
@@ -560,6 +561,7 @@ int32_t ec_manager_link(ec_fop_data_t * fop, int32_t state)
 
             return EC_STATE_LOCK_REUSE;
 
+        case -EC_STATE_INIT:
         case -EC_STATE_LOCK:
         case -EC_STATE_GET_SIZE_AND_VERSION:
         case -EC_STATE_DISPATCH:
@@ -839,6 +841,7 @@ int32_t ec_manager_mkdir(ec_fop_data_t * fop, int32_t state)
 
             return EC_STATE_LOCK_REUSE;
 
+        case -EC_STATE_INIT:
         case -EC_STATE_LOCK:
         case -EC_STATE_DISPATCH:
         case -EC_STATE_PREPARE_ANSWER:
@@ -1139,6 +1142,7 @@ int32_t ec_manager_mknod(ec_fop_data_t * fop, int32_t state)
 
             return EC_STATE_LOCK_REUSE;
 
+        case -EC_STATE_INIT:
         case -EC_STATE_LOCK:
         case -EC_STATE_DISPATCH:
         case -EC_STATE_PREPARE_ANSWER:
@@ -1403,6 +1407,7 @@ int32_t ec_manager_rename(ec_fop_data_t * fop, int32_t state)
 
             return EC_STATE_LOCK_REUSE;
 
+        case -EC_STATE_INIT:
         case -EC_STATE_LOCK:
         case -EC_STATE_GET_SIZE_AND_VERSION:
         case -EC_STATE_DISPATCH:
@@ -1642,6 +1647,7 @@ int32_t ec_manager_rmdir(ec_fop_data_t * fop, int32_t state)
 
             return EC_STATE_LOCK_REUSE;
 
+        case -EC_STATE_INIT:
         case -EC_STATE_LOCK:
         case -EC_STATE_DISPATCH:
         case -EC_STATE_PREPARE_ANSWER:
@@ -1898,6 +1904,7 @@ int32_t ec_manager_symlink(ec_fop_data_t * fop, int32_t state)
 
             return EC_STATE_LOCK_REUSE;
 
+        case -EC_STATE_INIT:
         case -EC_STATE_LOCK:
         case -EC_STATE_DISPATCH:
         case -EC_STATE_PREPARE_ANSWER:
@@ -2146,6 +2153,7 @@ int32_t ec_manager_unlink(ec_fop_data_t * fop, int32_t state)
 
             return EC_STATE_LOCK_REUSE;
 
+        case -EC_STATE_INIT:
         case -EC_STATE_LOCK:
         case -EC_STATE_GET_SIZE_AND_VERSION:
         case -EC_STATE_DISPATCH:
diff --git a/xlators/cluster/ec/src/ec-generic.c b/xlators/cluster/ec/src/ec-generic.c
index ffc40f01de1..bccbc009457 100644
--- a/xlators/cluster/ec/src/ec-generic.c
+++ b/xlators/cluster/ec/src/ec-generic.c
@@ -133,6 +133,7 @@ int32_t ec_manager_flush(ec_fop_data_t * fop, int32_t state)
 
             return EC_STATE_LOCK_REUSE;
 
+        case -EC_STATE_INIT:
         case -EC_STATE_LOCK:
         case -EC_STATE_DISPATCH:
         case -EC_STATE_PREPARE_ANSWER:
@@ -383,6 +384,7 @@ int32_t ec_manager_fsync(ec_fop_data_t * fop, int32_t state)
 
             return EC_STATE_LOCK_REUSE;
 
+        case -EC_STATE_INIT:
         case -EC_STATE_LOCK:
         case -EC_STATE_GET_SIZE_AND_VERSION:
         case -EC_STATE_DISPATCH:
@@ -596,6 +598,7 @@ int32_t ec_manager_fsyncdir(ec_fop_data_t * fop, int32_t state)
 
             return EC_STATE_LOCK_REUSE;
 
+        case -EC_STATE_INIT:
         case -EC_STATE_LOCK:
         case -EC_STATE_DISPATCH:
         case -EC_STATE_PREPARE_ANSWER:
@@ -997,6 +1000,7 @@ int32_t ec_manager_lookup(ec_fop_data_t * fop, int32_t state)
 
             return EC_STATE_END;
 
+        case -EC_STATE_INIT:
         case -EC_STATE_DISPATCH:
         case -EC_STATE_PREPARE_ANSWER:
         case -EC_STATE_REPORT:
@@ -1203,6 +1207,7 @@ int32_t ec_manager_statfs(ec_fop_data_t * fop, int32_t state)
 
             return EC_STATE_END;
 
+        case -EC_STATE_INIT:
         case -EC_STATE_DISPATCH:
         case -EC_STATE_PREPARE_ANSWER:
         case -EC_STATE_REPORT:
@@ -1442,6 +1447,7 @@ int32_t ec_manager_xattrop(ec_fop_data_t * fop, int32_t state)
 
             return EC_STATE_LOCK_REUSE;
 
+        case -EC_STATE_INIT:
         case -EC_STATE_LOCK:
         case -EC_STATE_DISPATCH:
         case -EC_STATE_PREPARE_ANSWER:
diff --git a/xlators/cluster/ec/src/ec-heal.c b/xlators/cluster/ec/src/ec-heal.c
index da5f5947de3..ad6417bc420 100644
--- a/xlators/cluster/ec/src/ec-heal.c
+++ b/xlators/cluster/ec/src/ec-heal.c
@@ -1455,6 +1455,7 @@ int32_t ec_manager_heal(ec_fop_data_t * fop, int32_t state)
 
             return EC_STATE_END;
 
+        case -EC_STATE_INIT:
         case -EC_STATE_DISPATCH:
         case -EC_STATE_PREPARE_ANSWER:
         case -EC_STATE_REPORT:
diff --git a/xlators/cluster/ec/src/ec-inode-read.c b/xlators/cluster/ec/src/ec-inode-read.c
index f9d1bcb052c..3b5315b187a 100644
--- a/xlators/cluster/ec/src/ec-inode-read.c
+++ b/xlators/cluster/ec/src/ec-inode-read.c
@@ -72,6 +72,7 @@ int32_t ec_manager_access(ec_fop_data_t * fop, int32_t state)
 
             return EC_STATE_REPORT;
 
+        case -EC_STATE_INIT:
         case -EC_STATE_REPORT:
             if (fop->cbks.access != NULL)
             {
@@ -310,6 +311,7 @@ int32_t ec_manager_getxattr(ec_fop_data_t * fop, int32_t state)
 
             return EC_STATE_LOCK_REUSE;
 
+        case -EC_STATE_INIT:
         case -EC_STATE_LOCK:
         case -EC_STATE_DISPATCH:
         case -EC_STATE_PREPARE_ANSWER:
@@ -792,6 +794,7 @@ int32_t ec_manager_open(ec_fop_data_t * fop, int32_t state)
 
             return EC_STATE_END;
 
+        case -EC_STATE_INIT:
         case -EC_STATE_DISPATCH:
         case -EC_STATE_PREPARE_ANSWER:
         case -EC_STATE_REPORT:
@@ -957,6 +960,7 @@ int32_t ec_manager_readlink(ec_fop_data_t * fop, int32_t state)
 
             return EC_STATE_REPORT;
 
+        case -EC_STATE_INIT:
         case -EC_STATE_REPORT:
             if (fop->cbks.readlink != NULL)
             {
@@ -1344,6 +1348,7 @@ int32_t ec_manager_readv(ec_fop_data_t * fop, int32_t state)
 
             return EC_STATE_LOCK_REUSE;
 
+        case -EC_STATE_INIT:
         case -EC_STATE_LOCK:
         case -EC_STATE_GET_SIZE_AND_VERSION:
         case -EC_STATE_DISPATCH:
@@ -1606,6 +1611,7 @@ int32_t ec_manager_stat(ec_fop_data_t * fop, int32_t state)
 
             return EC_STATE_LOCK_REUSE;
 
+        case -EC_STATE_INIT:
         case -EC_STATE_LOCK:
         case -EC_STATE_GET_SIZE_AND_VERSION:
         case -EC_STATE_DISPATCH:
diff --git a/xlators/cluster/ec/src/ec-inode-write.c b/xlators/cluster/ec/src/ec-inode-write.c
index 169ba1ae091..c120f067b6a 100644
--- a/xlators/cluster/ec/src/ec-inode-write.c
+++ b/xlators/cluster/ec/src/ec-inode-write.c
@@ -149,6 +149,7 @@ int32_t ec_manager_removexattr(ec_fop_data_t * fop, int32_t state)
 
             return EC_STATE_LOCK_REUSE;
 
+        case -EC_STATE_INIT:
         case -EC_STATE_LOCK:
         case -EC_STATE_DISPATCH:
         case -EC_STATE_PREPARE_ANSWER:
@@ -562,6 +563,7 @@ int32_t ec_manager_setattr(ec_fop_data_t * fop, int32_t state)
 
             return EC_STATE_LOCK_REUSE;
 
+        case -EC_STATE_INIT:
         case -EC_STATE_LOCK:
         case -EC_STATE_GET_SIZE_AND_VERSION:
         case -EC_STATE_DISPATCH:
@@ -938,6 +940,7 @@ int32_t ec_manager_setxattr(ec_fop_data_t * fop, int32_t state)
 
             return EC_STATE_LOCK_REUSE;
 
+        case -EC_STATE_INIT:
         case -EC_STATE_LOCK:
         case -EC_STATE_DISPATCH:
         case -EC_STATE_PREPARE_ANSWER:
@@ -1465,6 +1468,7 @@ int32_t ec_manager_truncate(ec_fop_data_t * fop, int32_t state)
 
             return EC_STATE_LOCK_REUSE;
 
+        case -EC_STATE_INIT:
         case -EC_STATE_LOCK:
         case -EC_STATE_GET_SIZE_AND_VERSION:
         case -EC_STATE_DISPATCH:
@@ -2106,6 +2110,7 @@ int32_t ec_manager_writev(ec_fop_data_t * fop, int32_t state)
 
             return EC_STATE_LOCK_REUSE;
 
+        case -EC_STATE_INIT:
         case -EC_STATE_LOCK:
         case -EC_STATE_GET_SIZE_AND_VERSION:
         case -EC_STATE_DISPATCH:
diff --git a/xlators/cluster/ec/src/ec-locks.c b/xlators/cluster/ec/src/ec-locks.c
index c1fbcd3c11e..2e301631b3f 100644
--- a/xlators/cluster/ec/src/ec-locks.c
+++ b/xlators/cluster/ec/src/ec-locks.c
@@ -283,6 +283,7 @@ int32_t ec_manager_entrylk(ec_fop_data_t * fop, int32_t state)
 
             return EC_STATE_END;
 
+        case -EC_STATE_INIT:
         case -EC_STATE_DISPATCH:
         case -EC_STATE_PREPARE_ANSWER:
         case -EC_STATE_REPORT:
@@ -726,6 +727,7 @@ int32_t ec_manager_inodelk(ec_fop_data_t * fop, int32_t state)
 
             return EC_STATE_END;
 
+        case -EC_STATE_INIT:
         case -EC_STATE_DISPATCH:
         case -EC_STATE_PREPARE_ANSWER:
         case -EC_STATE_REPORT:
@@ -1194,6 +1196,7 @@ int32_t ec_manager_lk(ec_fop_data_t * fop, int32_t state)
 
             return EC_STATE_END;
 
+        case -EC_STATE_INIT:
         case -EC_STATE_DISPATCH:
         case -EC_STATE_PREPARE_ANSWER:
         case -EC_STATE_REPORT: