dht: utilize the framework to pass-through xlator tasks

Also fixes the issue caused due to not converting back the fn function
to after getting its address. We wanted the value of the field, not
the address of the pt_fop field.

With this patch, DHT will always be started in pass-through
mode if the number of subvols is just 1.

Fixes some tests to make sure DHT is in full config (ie, subvols > 1).
  - increased timeout of brick-mux test as it was bordering on 300 seconds.
  - Also change the volume type to supported 'replica 3' from 'replica 2'.
  - also no DHT tests should assume presence of DHT when there is
    just 1 brick in volume

Credits: Nithya B <nbalacha@redhat.com>

fixes: #405
Change-Id: I8e55239ce58d6ac6ae1901e2e384be1ecbd33d6e
Signed-off-by: Amar Tumballi <amarts@redhat.com>

7 files changed, 231 insertions, 27 deletions

diff --git a/xlators/cluster/dht/src/dht-common.c b/xlators/cluster/dht/src/dht-common.c
index f43a10bec2f..4ca54a1f5c0 100644
--- a/xlators/cluster/dht/src/dht-common.c
+++ b/xlators/cluster/dht/src/dht-common.c
@@ -11030,3 +11030,135 @@ dht_remove_stale_linkto_cbk(int ret, call_frame_t *sync_frame, void *data)
     DHT_STACK_DESTROY(sync_frame);
     return 0;
 }
+
+int
+dht_pt_mkdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                 int op_errno, inode_t *inode, struct iatt *stbuf,
+                 struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+
+    local = frame->local;
+
+    if (!op_ret) {
+        dht_layout_set(this, inode, local->layout);
+    }
+
+    DHT_STACK_UNWIND(mkdir, frame, op_ret, op_errno, inode, stbuf, preparent,
+                     postparent, NULL);
+
+    return 0;
+}
+
+int32_t
+dht_pt_mkdir(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+             mode_t umask, dict_t *xdata)
+{
+    dht_layout_t *layout = NULL;
+    dht_conf_t *conf = NULL;
+    dht_local_t *local = NULL;
+    bool free_xdata = false;
+    int ret = 0;
+    int op_errno = 0;
+    int32_t *disk_layout_p = NULL;
+
+    conf = this->private;
+
+    local = dht_local_init(frame, loc, NULL, GF_FOP_MKDIR);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    layout = dht_layout_new(this, conf->subvolume_cnt);
+    if (!layout)
+        goto wind;
+
+    local->layout = layout;
+
+    if (!xdata) {
+        xdata = dict_new();
+        if (!xdata)
+            goto wind;
+        free_xdata = true;
+    }
+
+    /*Set the xlator or the following will crash*/
+    layout->list[0].xlator = conf->subvolumes[0];
+
+    dht_selfheal_layout_new_directory(frame, loc, layout);
+
+    dht_disk_layout_extract(this, layout, 0, &disk_layout_p);
+
+    ret = dict_set_bin(xdata, conf->xattr_name, disk_layout_p, 4 * 4);
+    if (ret) {
+        gf_msg("dht", GF_LOG_DEBUG, EINVAL, DHT_MSG_DICT_SET_FAILED,
+               "dht layout dict set failed");
+    }
+wind:
+    STACK_WIND(frame, dht_pt_mkdir_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->mkdir, loc, mode, umask, xdata);
+    if (free_xdata)
+        dict_unref(xdata);
+    return 0;
+
+err:
+    op_errno = local ? local->op_errno : op_errno;
+    DHT_STACK_UNWIND(mkdir, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL);
+
+    return 0;
+}
+
+static int
+dht_pt_getxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int op_ret, int op_errno, dict_t *xattr, dict_t *xdata)
+{
+    dht_conf_t *conf = NULL;
+
+    conf = this->private;
+    dict_del(xattr, conf->xattr_name);
+
+    if (frame->root->pid >= 0) {
+        GF_REMOVE_INTERNAL_XATTR("trusted.glusterfs.quota*", xattr);
+        GF_REMOVE_INTERNAL_XATTR("trusted.pgfid*", xattr);
+    }
+
+    DHT_STACK_UNWIND(getxattr, frame, op_ret, op_errno, xattr, xdata);
+    return 0;
+}
+
+int
+dht_pt_getxattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                const char *key, dict_t *xdata)
+{
+    STACK_WIND(frame, dht_pt_getxattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->getxattr, loc, key, xdata);
+    return 0;
+}
+
+static int
+dht_pt_fgetxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int op_ret, int op_errno, dict_t *xattr, dict_t *xdata)
+{
+    dht_conf_t *conf = NULL;
+
+    conf = this->private;
+    dict_del(xattr, conf->xattr_name);
+
+    if (frame->root->pid >= 0) {
+        GF_REMOVE_INTERNAL_XATTR("trusted.glusterfs.quota*", xattr);
+        GF_REMOVE_INTERNAL_XATTR("trusted.pgfid*", xattr);
+    }
+
+    DHT_STACK_UNWIND(fgetxattr, frame, op_ret, op_errno, xattr, xdata);
+    return 0;
+}
+
+int
+dht_pt_fgetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, const char *key,
+                 dict_t *xdata)
+{
+    STACK_WIND(frame, dht_pt_fgetxattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fgetxattr, fd, key, xdata);
+    return 0;
+}
diff --git a/xlators/cluster/dht/src/dht-common.h b/xlators/cluster/dht/src/dht-common.h
index bd30fd84b88..cc1e1f02912 100644
--- a/xlators/cluster/dht/src/dht-common.h
+++ b/xlators/cluster/dht/src/dht-common.h
@@ -1502,4 +1502,24 @@ dht_read_iatt_from_xdata(xlator_t *this, dict_t *xdata, struct iatt *stbuf);
 
 int
 is_permission_different(ia_prot_t *prot1, ia_prot_t *prot2);
+void
+dht_selfheal_layout_new_directory(call_frame_t *frame, loc_t *loc,
+                                  dht_layout_t *new_layout);
+
+int
+dht_pt_fgetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, const char *key,
+                 dict_t *xdata);
+
+int
+dht_pt_getxattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                const char *key, dict_t *xdata);
+
+int32_t
+dht_pt_mkdir(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+             mode_t umask, dict_t *xdata);
+
+int
+dht_pt_rename(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+              dict_t *xdata);
+
 #endif /* _DHT_H */
diff --git a/xlators/cluster/dht/src/dht-helper.c b/xlators/cluster/dht/src/dht-helper.c
index 6d6ec24729d..e463aad3a0c 100644
--- a/xlators/cluster/dht/src/dht-helper.c
+++ b/xlators/cluster/dht/src/dht-helper.c
@@ -1162,6 +1162,10 @@ dht_init_subvolumes(xlator_t *this, dht_conf_t *conf)
         return -1;
     }
     conf->subvolume_cnt = cnt;
+    /* Doesn't make sense to do any dht layer tasks
+       if the subvol count is 1. Set it as pass_through */
+    if (cnt == 1)
+        this->pass_through = _gf_true;
 
     conf->local_subvols_cnt = 0;
 
diff --git a/xlators/cluster/dht/src/dht-rename.c b/xlators/cluster/dht/src/dht-rename.c
index 45808a2bfa6..57a1a9f8b0c 100644
--- a/xlators/cluster/dht/src/dht-rename.c
+++ b/xlators/cluster/dht/src/dht-rename.c
@@ -1828,33 +1828,14 @@ dht_rename(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
     int op_errno = -1;
     int ret = -1;
     dht_local_t *local = NULL;
-    dht_conf_t *conf = NULL;
     char gfid[GF_UUID_BUF_SIZE] = {0};
     char newgfid[GF_UUID_BUF_SIZE] = {0};
-    gf_boolean_t free_xdata = _gf_false;
 
     VALIDATE_OR_GOTO(frame, err);
     VALIDATE_OR_GOTO(this, err);
     VALIDATE_OR_GOTO(oldloc, err);
     VALIDATE_OR_GOTO(newloc, err);
 
-    conf = this->private;
-
-    if (conf->subvolume_cnt == 1) {
-        if (!IA_ISDIR(oldloc->inode->ia_type)) {
-            if (!xdata) {
-                free_xdata = _gf_true;
-            }
-            DHT_CHANGELOG_TRACK_AS_RENAME(xdata, oldloc, newloc);
-        }
-        default_rename(frame, this, oldloc, newloc, xdata);
-        if (free_xdata && xdata) {
-            dict_unref(xdata);
-            xdata = NULL;
-        }
-        return 0;
-    }
-
     gf_uuid_unparse(oldloc->inode->gfid, gfid);
 
     src_hashed = dht_subvol_get_hashed(this, oldloc);
@@ -1941,3 +1922,24 @@ err:
 
     return 0;
 }
+
+int
+dht_pt_rename(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+              dict_t *xdata)
+{
+    gf_boolean_t free_xdata = _gf_false;
+
+    /* Just a pass through */
+    if (!IA_ISDIR(oldloc->inode->ia_type)) {
+        if (!xdata) {
+            free_xdata = _gf_true;
+        }
+        DHT_CHANGELOG_TRACK_AS_RENAME(xdata, oldloc, newloc);
+    }
+    default_rename(frame, this, oldloc, newloc, xdata);
+    if (free_xdata && xdata) {
+        dict_unref(xdata);
+        xdata = NULL;
+    }
+    return 0;
+}
diff --git a/xlators/cluster/dht/src/dht-shared.c b/xlators/cluster/dht/src/dht-shared.c
index 5fd97130d22..afb5880b459 100644
--- a/xlators/cluster/dht/src/dht-shared.c
+++ b/xlators/cluster/dht/src/dht-shared.c
@@ -41,7 +41,8 @@
    - handle all cases in self heal layout reconstruction
    - complete linkfile selfheal
 */
-struct volume_options options[];
+/* FIXME: Not sure why someone defined this here */
+/* struct volume_options options[]; */
 
 extern dht_methods_t dht_methods;
 
diff --git a/xlators/cluster/dht/src/dht.c b/xlators/cluster/dht/src/dht.c
index 677905f236e..6dfaa3b23ed 100644
--- a/xlators/cluster/dht/src/dht.c
+++ b/xlators/cluster/dht/src/dht.c
@@ -11,10 +11,32 @@
 #include "statedump.h"
 #include "dht-common.h"
 
-class_methods_t class_methods = {.init = dht_init,
-                                 .fini = dht_fini,
-                                 .reconfigure = dht_reconfigure,
-                                 .notify = dht_notify};
+struct xlator_fops dht_pt_fops = {
+    /* we need to keep mkdir to make sure we
+       have layout on new directory */
+    .mkdir = dht_pt_mkdir,
+    .getxattr = dht_pt_getxattr,
+    .fgetxattr = dht_pt_fgetxattr,
+
+    /* required to trace fop properly in changelog */
+    .rename = dht_pt_rename,
+
+    /* FIXME: commenting the '.lookup()' below made some of
+       the failing tests to pass. I would remove the below
+       line, but keeping it here as a reminder for people
+       to check for issues if they find concerns with DHT
+       pass-through logic  */
+    /*
+      .lookup = dht_lookup,
+      .readdir = dht_readdir,
+      .readdirp = dht_readdirp,
+    */
+    /* Keeping above as commented, mainly to support the
+       usecase of a gluster volume getting to 1x(anytype),
+       due to remove-brick (shrinking) exercise. In that case,
+       we would need above fops to be available, so we can
+       handle the case of dangling linkto files (if any) */
+};
 
 struct xlator_fops fops = {
     .ipc = dht_ipc,
@@ -74,6 +96,28 @@ struct xlator_dumpops dumpops = {
     .inodectx = dht_inodectx_dump,
 };
 
-struct xlator_cbks cbks = {.release = dht_release,
-                           //      .releasedir = dht_releasedir,
-                           .forget = dht_forget};
+struct xlator_cbks cbks = {
+    .release = dht_release,
+    //      .releasedir = dht_releasedir,
+    .forget = dht_forget,
+};
+
+extern int32_t
+mem_acct_init(xlator_t *this);
+extern struct volume_options options[];
+
+xlator_api_t xlator_api = {
+    .init = dht_init,
+    .fini = dht_fini,
+    .notify = dht_notify,
+    .reconfigure = dht_reconfigure,
+    .mem_acct_init = mem_acct_init,
+    .op_version = {1}, /* Present from the initial version */
+    .dumpops = &dumpops,
+    .fops = &fops,
+    .cbks = &cbks,
+    .options = options,
+    .identifier = "distribute",
+    .pass_through_fops = &dht_pt_fops,
+    .category = GF_MAINTAINED,
+};
diff --git a/xlators/cluster/dht/src/dht.sym b/xlators/cluster/dht/src/dht.sym
index 780b5fc0387..24241a91baf 100644
--- a/xlators/cluster/dht/src/dht.sym
+++ b/xlators/cluster/dht/src/dht.sym
@@ -1,3 +1,4 @@
+xlator_api
 fops
 cbks
 class_methods