dht: Ongoing IO is failing on non-distribute volumes after just add-brick

Problem: On a non-distributed volumes linux kernel untar is failed
         after running add-brick operation

Solution:
1) Save hashed subvol as a MDS in case while MDS has not been
   populated

Fixes: #1328
Change-Id: I9967e136da008c6367973a7346637617dfa8f934
Signed-off-by: Mohit Agrawal <moagrawa@redhat.com>

3 files changed, 41 insertions, 8 deletions

diff --git a/xlators/cluster/dht/src/dht-common.c b/xlators/cluster/dht/src/dht-common.c
index d874415699c..42e143e3a5a 100644
--- a/xlators/cluster/dht/src/dht-common.c
+++ b/xlators/cluster/dht/src/dht-common.c
@@ -382,7 +382,7 @@ out:
 
 /* Code to save hashed subvol on inode ctx as a mds subvol
  */
-static int
+int
 dht_inode_ctx_mdsvol_set(inode_t *inode, xlator_t *this, xlator_t *mds_subvol)
 {
     dht_inode_ctx_t *ctx = NULL;
diff --git a/xlators/cluster/dht/src/dht-common.h b/xlators/cluster/dht/src/dht-common.h
index 07bbe204997..adb362f7784 100644
--- a/xlators/cluster/dht/src/dht-common.h
+++ b/xlators/cluster/dht/src/dht-common.h
@@ -1482,4 +1482,6 @@ dht_set_parent_layout_in_dict(loc_t *loc, xlator_t *this, dht_local_t *local);
 int
 dht_dir_layout_error_check(xlator_t *this, inode_t *inode);
 
+int
+dht_inode_ctx_mdsvol_set(inode_t *inode, xlator_t *this, xlator_t *mds_subvol);
 #endif /* _DHT_H */
diff --git a/xlators/cluster/dht/src/dht-selfheal.c b/xlators/cluster/dht/src/dht-selfheal.c
index 1b6571cd43c..8efa3b85b22 100644
--- a/xlators/cluster/dht/src/dht-selfheal.c
+++ b/xlators/cluster/dht/src/dht-selfheal.c
@@ -1330,9 +1330,11 @@ dht_selfheal_dir_mkdir(call_frame_t *frame, loc_t *loc, dht_layout_t *layout,
     int ret = -1;
     dht_local_t *local = NULL;
     xlator_t *this = NULL;
+    dht_conf_t *conf = NULL;
 
     local = frame->local;
     this = frame->this;
+    conf = this->private;
 
     local->selfheal.force_mkdir = force;
     local->selfheal.hole_cnt = 0;
@@ -1372,15 +1374,44 @@ dht_selfheal_dir_mkdir(call_frame_t *frame, loc_t *loc, dht_layout_t *layout,
         return 0;
     }
 
-    if (local->hashed_subvol == NULL)
-        local->hashed_subvol = dht_subvol_get_hashed(this, loc);
+    /* MDS xattr is populated only while DHT is having more than one
+     subvol.In case of graph switch while adding more dht subvols need to
+     consider hash subvol as a MDS to avoid MDS check failure at the time
+     of running fop on directory
+    */
+    if (!dict_get(local->xattr, conf->mds_xattr_key) &&
+        (conf->subvolume_cnt > 1)) {
+        if (local->hashed_subvol == NULL) {
+            local->hashed_subvol = dht_subvol_get_hashed(this, loc);
+            if (local->hashed_subvol == NULL) {
+                local->op_errno = EINVAL;
+                gf_smsg(this->name, GF_LOG_WARNING, local->op_errno,
+                        DHT_MSG_HASHED_SUBVOL_GET_FAILED, "gfid=%s",
+                        loc->pargfid, "name=%s", loc->name, "path=%s",
+                        loc->path, NULL);
+                goto err;
+            }
+        }
+        ret = dht_inode_ctx_mdsvol_set(local->inode, this,
+                                       local->hashed_subvol);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_SET_INODE_CTX_FAILED,
+                   "Failed to set hashed subvol for %s on inode vol is %s",
+                   local->loc.path,
+                   local->hashed_subvol ? local->hashed_subvol->name : "NULL");
+            goto err;
+        }
+    }
 
     if (local->hashed_subvol == NULL) {
-        local->op_errno = EINVAL;
-        gf_smsg(this->name, GF_LOG_WARNING, local->op_errno,
-                DHT_MSG_HASHED_SUBVOL_GET_FAILED, "gfid=%s", loc->pargfid,
-                "name=%s", loc->name, "path=%s", loc->path, NULL);
-        goto err;
+        local->hashed_subvol = dht_subvol_get_hashed(this, loc);
+        if (local->hashed_subvol == NULL) {
+            local->op_errno = EINVAL;
+            gf_smsg(this->name, GF_LOG_WARNING, local->op_errno,
+                    DHT_MSG_HASHED_SUBVOL_GET_FAILED, "gfid=%s", loc->pargfid,
+                    "name=%s", loc->name, "path=%s", loc->path, NULL);
+            goto err;
+        }
     }
 
     local->current = &local->lock[0];