cluster/ec: Prevent double unwind

Problem:
1) ec_access/ec_readlink_/ec_readdir[p] _cbks are trying to recover only from
ENOTCONN.
2) When the fop succeeds it unwinds right away. But when its
ec_fop_manager resumes, if the number of bricks that are up is less than
ec->fragments, the the state machine will resume with -EC_STATE_REPORT which
unwinds again. This will lead to crashes.

Fix:
- If fop fails retry on other subvols, as ESTALE/ENOENT/EBADFD etc are also
recoverable.
- unwind success/failure in _cbks

Change-Id: I2cac3c2f9669a4e6160f1ff4abc39f0299303222
BUG: 1228952
Signed-off-by: Pranith Kumar K <pkarampu@redhat.com>
Reviewed-on: http://review.gluster.org/11111
Reviewed-by: Xavier Hernandez <xhernandez@datalab.es>
Tested-by: Gluster Build System <jenkins@build.gluster.com>

4 files changed, 12 insertions, 13 deletions

diff --git a/xlators/cluster/ec/src/ec-common.c b/xlators/cluster/ec/src/ec-common.c
index c4bc4707257..f214e7e065f 100644
--- a/xlators/cluster/ec/src/ec-common.c
+++ b/xlators/cluster/ec/src/ec-common.c
@@ -535,11 +535,9 @@ void ec_dispatch_one(ec_fop_data_t * fop)
     }
 }
 
-int32_t ec_dispatch_one_retry(ec_fop_data_t * fop, int32_t idx, int32_t op_ret,
-                              int32_t op_errno)
+int32_t ec_dispatch_one_retry(ec_fop_data_t *fop, int32_t idx, int32_t op_ret)
 {
-    if ((op_ret < 0) && (op_errno == ENOTCONN))
-    {
+    if (op_ret < 0) {
         return (ec_dispatch_next(fop, idx) >= 0);
     }
 
diff --git a/xlators/cluster/ec/src/ec-common.h b/xlators/cluster/ec/src/ec-common.h
index c0db0218699..e3f01cac26a 100644
--- a/xlators/cluster/ec/src/ec-common.h
+++ b/xlators/cluster/ec/src/ec-common.h
@@ -75,8 +75,7 @@ typedef enum {
 #define EC_STATE_HEAL_POST_INODELK_UNLOCK   217
 #define EC_STATE_HEAL_DISPATCH              218
 
-int32_t ec_dispatch_one_retry(ec_fop_data_t * fop, int32_t idx, int32_t op_ret,
-                              int32_t op_errno);
+int32_t ec_dispatch_one_retry(ec_fop_data_t *fop, int32_t idx, int32_t op_ret);
 int32_t ec_dispatch_next(ec_fop_data_t * fop, int32_t idx);
 
 void ec_complete(ec_fop_data_t * fop);
diff --git a/xlators/cluster/ec/src/ec-dir-read.c b/xlators/cluster/ec/src/ec-dir-read.c
index 354c63d3683..782187856fc 100644
--- a/xlators/cluster/ec/src/ec-dir-read.c
+++ b/xlators/cluster/ec/src/ec-dir-read.c
@@ -361,7 +361,7 @@ int32_t ec_readdir_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
         ec_adjust_readdir(fop->xl->private, idx, entries);
     }
 
-    if (!ec_dispatch_one_retry(fop, idx, op_ret, op_errno))
+    if (!ec_dispatch_one_retry(fop, idx, op_ret))
     {
         if (fop->cbks.readdir != NULL)
         {
@@ -429,6 +429,8 @@ int32_t ec_manager_readdir(ec_fop_data_t * fop, int32_t state)
 
             if (fop->offset != 0)
             {
+            /* Non-zero offset is irrecoverable error as the offset may not be
+             * valid on other bricks*/
                 int32_t idx = -1;
 
                 idx = ec_deitransform (fop->xl, fop->offset);
@@ -448,7 +450,6 @@ int32_t ec_manager_readdir(ec_fop_data_t * fop, int32_t state)
             return EC_STATE_REPORT;
 
         case -EC_STATE_INIT:
-        case -EC_STATE_REPORT:
             if (fop->id == GF_FOP_READDIR)
             {
                 if (fop->cbks.readdir != NULL)
@@ -467,6 +468,7 @@ int32_t ec_manager_readdir(ec_fop_data_t * fop, int32_t state)
             }
 
         case EC_STATE_REPORT:
+        case -EC_STATE_REPORT:
             return EC_STATE_END;
 
         default:
@@ -564,7 +566,7 @@ int32_t ec_readdirp_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
         ec_adjust_readdir(fop->xl->private, idx, entries);
     }
 
-    if (!ec_dispatch_one_retry(fop, idx, op_ret, op_errno))
+    if (!ec_dispatch_one_retry(fop, idx, op_ret))
     {
         if (fop->cbks.readdirp != NULL)
         {
diff --git a/xlators/cluster/ec/src/ec-inode-read.c b/xlators/cluster/ec/src/ec-inode-read.c
index ae02e964c3d..ef2170f8924 100644
--- a/xlators/cluster/ec/src/ec-inode-read.c
+++ b/xlators/cluster/ec/src/ec-inode-read.c
@@ -35,7 +35,7 @@ int32_t ec_access_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
     ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx,
              frame, op_ret, op_errno);
 
-    if (!ec_dispatch_one_retry(fop, idx, op_ret, op_errno))
+    if (!ec_dispatch_one_retry(fop, idx, op_ret))
     {
         if (fop->cbks.access != NULL)
         {
@@ -73,13 +73,13 @@ int32_t ec_manager_access(ec_fop_data_t * fop, int32_t state)
             return EC_STATE_REPORT;
 
         case -EC_STATE_INIT:
-        case -EC_STATE_REPORT:
             if (fop->cbks.access != NULL)
             {
                 fop->cbks.access(fop->req_frame, fop, fop->xl, -1, fop->error,
                                  NULL);
             }
 
+        case -EC_STATE_REPORT:
         case EC_STATE_REPORT:
             return EC_STATE_END;
 
@@ -968,7 +968,7 @@ int32_t ec_readlink_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
         ec_iatt_rebuild(fop->xl->private, buf, 1, 1);
     }
 
-    if (!ec_dispatch_one_retry(fop, idx, op_ret, op_errno))
+    if (!ec_dispatch_one_retry(fop, idx, op_ret))
     {
         if (fop->cbks.readlink != NULL)
         {
@@ -1006,7 +1006,6 @@ int32_t ec_manager_readlink(ec_fop_data_t * fop, int32_t state)
             return EC_STATE_REPORT;
 
         case -EC_STATE_INIT:
-        case -EC_STATE_REPORT:
             if (fop->cbks.readlink != NULL)
             {
                 fop->cbks.readlink(fop->req_frame, fop, fop->xl, -1,
@@ -1014,6 +1013,7 @@ int32_t ec_manager_readlink(ec_fop_data_t * fop, int32_t state)
             }
 
         case EC_STATE_REPORT:
+        case -EC_STATE_REPORT:
             return EC_STATE_END;
 
         default: