io-threads: use slow/normal/fast classification of fops

Signed-off-by: Anand Avati <avati@gluster.com>
Signed-off-by: Anand V. Avati <avati@dev.gluster.com>

BUG: 2241 (GlusterFs Stat Actions Degrade During I/O)
URL: http://bugs.gluster.com/cgi-bin/bugzilla3/show_bug.cgi?id=2241

1 files changed, 75 insertions, 168 deletions

diff --git a/xlators/performance/io-threads/src/io-threads.c b/xlators/performance/io-threads/src/io-threads.c
index ace065c6d..09b240e88 100644
--- a/xlators/performance/io-threads/src/io-threads.c
+++ b/xlators/performance/io-threads/src/io-threads.c
@@ -228,8 +228,7 @@ iot_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xattr_req)
                 goto out;
         }
 
-        ret = iot_schedule_unordered ((iot_conf_t *)this->private, loc->inode,
-                                      stub);
+        ret = iot_schedule_fast (this->private, stub);
 
 out:
         if (ret < 0) {
@@ -281,8 +280,7 @@ iot_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
                 goto out;
         }
 
-        ret = iot_schedule_unordered ((iot_conf_t *)this->private,
-                                      loc->inode, stub);
+        ret = iot_schedule (this->private, stub);
 
 out:
         if (ret < 0) {
@@ -333,8 +331,7 @@ iot_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
                 goto out;
         }
 
-        ret = iot_schedule_ordered ((iot_conf_t *)this->private, fd->inode,
-                                    stub);
+        ret = iot_schedule (this->private, stub);
 
 out:
         if (ret < 0) {
@@ -380,8 +377,7 @@ iot_access (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t mask)
                 goto out;
         }
 
-        ret = iot_schedule_unordered ((iot_conf_t *)this->private, loc->inode,
-                                      stub);
+        ret = iot_schedule_fast (this->private, stub);
 out:
         if (ret < 0) {
                 STACK_UNWIND_STRICT (access, frame, -1, -ret);
@@ -430,8 +426,7 @@ iot_readlink (call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size)
                 goto out;
         }
 
-        ret = iot_schedule_unordered ((iot_conf_t *)this->private, loc->inode,
-                                      stub);
+        ret = iot_schedule_fast (this->private, stub);
 
 out:
         if (ret < 0) {
@@ -484,8 +479,7 @@ iot_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
                 goto out;
         }
 
-        ret = iot_schedule_unordered ((iot_conf_t *)this->private, loc->inode,
-                                      stub);
+        ret = iot_schedule (this->private, stub);
 
 out:
         if (ret < 0) {
@@ -537,8 +531,7 @@ iot_mkdir (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
                 goto out;
         }
 
-        ret = iot_schedule_unordered ((iot_conf_t *)this->private, loc->inode,
-                                      stub);
+        ret = iot_schedule (this->private, stub);
 
 out:
         if (ret < 0) {
@@ -587,8 +580,7 @@ iot_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags)
                 goto out;
         }
 
-        ret = iot_schedule_unordered ((iot_conf_t *)this->private, loc->inode,
-                                      stub);
+        ret = iot_schedule (this->private, stub);
 out:
         if (ret < 0) {
                 STACK_UNWIND_STRICT (rmdir, frame, -1, -ret, NULL, NULL);
@@ -639,8 +631,7 @@ iot_symlink (call_frame_t *frame, xlator_t *this, const char *linkname,
                 goto out;
         }
 
-        ret = iot_schedule_unordered ((iot_conf_t *)this->private, loc->inode,
-                                      stub);
+        ret = iot_schedule (this->private, stub);
 
 out:
         if (ret < 0) {
@@ -691,8 +682,7 @@ iot_rename (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc)
                 goto out;
         }
 
-        ret = iot_schedule_unordered ((iot_conf_t *)this->private,
-                                      oldloc->inode, stub);
+        ret = iot_schedule (this->private, stub);
 
 out:
         if (ret < 0) {
@@ -742,8 +732,7 @@ iot_open (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
                 goto out;
         }
 
-	ret = iot_schedule_unordered ((iot_conf_t *)this->private, loc->inode,
-                                      stub);
+	ret = iot_schedule_fast (this->private, stub);
 
 out:
         if (ret < 0) {
@@ -799,8 +788,7 @@ iot_create (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
                 goto out;
         }
 
-        ret = iot_schedule_unordered ((iot_conf_t *)this->private, loc->inode,
-                                      stub);
+        ret = iot_schedule (this->private, stub);
 
 out:
         if (ret < 0) {
@@ -856,8 +844,7 @@ iot_readv (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
                 goto out;
 	}
 
-        ret = iot_schedule_ordered ((iot_conf_t *)this->private, fd->inode,
-                                    stub);
+        ret = iot_schedule_slow (this->private, stub);
 
 out:
         if (ret < 0) {
@@ -906,8 +893,7 @@ iot_flush (call_frame_t *frame, xlator_t *this, fd_t *fd)
                 goto out;
 	}
 
-        ret = iot_schedule_ordered ((iot_conf_t *)this->private, fd->inode,
-                                    stub);
+        ret = iot_schedule (this->private, stub);
 out:
         if (ret < 0) {
 		STACK_UNWIND_STRICT (flush, frame, -1, -ret);
@@ -957,8 +943,7 @@ iot_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t datasync)
                 goto out;
 	}
 
-        ret = iot_schedule_ordered ((iot_conf_t *)this->private, fd->inode,
-                                    stub);
+        ret = iot_schedule_slow (this->private, stub);
 
 out:
         if (ret < 0) {
@@ -1014,8 +999,7 @@ iot_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
                 goto out;
 	}
 
-        ret = iot_schedule_ordered ((iot_conf_t *)this->private, fd->inode,
-                                    stub);
+        ret = iot_schedule_slow (this->private, stub);
 out:
         if (ret < 0) {
 		STACK_UNWIND_STRICT (writev, frame, -1, -ret, NULL, NULL);
@@ -1067,8 +1051,7 @@ iot_lk (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t cmd,
                 goto out;
 	}
 
-        ret = iot_schedule_ordered ((iot_conf_t *)this->private, fd->inode,
-                                    stub);
+        ret = iot_schedule (this->private, stub);
 out:
         if (ret < 0) {
 		STACK_UNWIND_STRICT (lk, frame, -1, -ret, NULL);
@@ -1105,7 +1088,6 @@ int
 iot_stat (call_frame_t *frame, xlator_t *this, loc_t *loc)
 {
 	call_stub_t *stub = NULL;
-	fd_t        *fd = NULL;
         int         ret = -1;
 
         stub = fop_stat_stub (frame, iot_stat_wrapper, loc);
@@ -1117,17 +1099,7 @@ iot_stat (call_frame_t *frame, xlator_t *this, loc_t *loc)
                 goto out;
 	}
 
-	fd = fd_lookup (loc->inode, frame->root->pid);
-        /* File is not open, so we can send it through unordered pool.
-         */
-	if (fd == NULL)
-                ret = iot_schedule_unordered ((iot_conf_t *)this->private,
-                                              loc->inode, stub);
-        else {
-                ret = iot_schedule_ordered ((iot_conf_t *)this->private,
-                                            loc->inode, stub);
-	        fd_unref (fd);
-        }
+        ret = iot_schedule_fast (this->private, stub);
 
 out:
         if (ret < 0) {
@@ -1176,8 +1148,7 @@ iot_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd)
                 goto out;
 	}
 
-        ret = iot_schedule_ordered ((iot_conf_t *)this->private, fd->inode,
-                                    stub);
+        ret = iot_schedule_fast (this->private, stub);
 out:
         if (ret < 0) {
 		STACK_UNWIND_STRICT (fstat, frame, -1, -ret, NULL);
@@ -1217,7 +1188,6 @@ int
 iot_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset)
 {
 	call_stub_t *stub;
-	fd_t        *fd = NULL;
         int         ret = -1;
 
         stub = fop_truncate_stub (frame, iot_truncate_wrapper, loc, offset);
@@ -1230,15 +1200,7 @@ iot_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset)
                 goto out;
 	}
 
-	fd = fd_lookup (loc->inode, frame->root->pid);
-	if (fd == NULL)
-                ret = iot_schedule_unordered ((iot_conf_t *)this->private,
-                                              loc->inode, stub);
-        else {
-                ret = iot_schedule_ordered ((iot_conf_t *)this->private,
-                                            loc->inode, stub);
-	        fd_unref (fd);
-        }
+        ret = iot_schedule_slow (this->private, stub);
 
 out:
         if (ret < 0) {
@@ -1290,8 +1252,8 @@ iot_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset)
                 ret = -ENOMEM;
                 goto out;
 	}
-        ret = iot_schedule_ordered ((iot_conf_t *)this->private, fd->inode,
-                                    stub);
+
+        ret = iot_schedule_slow (this->private, stub);
 out:
         if (ret < 0) {
 		STACK_UNWIND_STRICT (ftruncate, frame, -1, -ret, NULL, NULL);
@@ -1342,8 +1304,7 @@ iot_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc)
                 goto out;
 	}
 
-        ret = iot_schedule_unordered((iot_conf_t *)this->private, loc->inode,
-                                     stub);
+        ret = iot_schedule (this->private, stub);
 
 out:
         if (ret < 0) {
@@ -1393,8 +1354,7 @@ iot_link (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc)
                 goto out;
         }
 
-        ret = iot_schedule_unordered ((iot_conf_t *)this->private,
-                                      oldloc->inode, stub);
+        ret = iot_schedule (this->private, stub);
 out:
         if (ret < 0) {
                 STACK_UNWIND_STRICT (link, frame, -1, -ret, NULL, NULL, NULL,
@@ -1440,8 +1400,7 @@ iot_opendir (call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd)
                 goto out;
         }
 
-        ret = iot_schedule_unordered ((iot_conf_t *)this->private, loc->inode,
-                                      stub);
+        ret = iot_schedule_fast (this->private, stub);
 out:
         if (ret < 0) {
                 STACK_UNWIND_STRICT (opendir, frame, -1, -ret, NULL);
@@ -1487,8 +1446,7 @@ iot_fsyncdir (call_frame_t *frame, xlator_t *this, fd_t *fd, int datasync)
                 goto out;
         }
 
-        ret = iot_schedule_ordered ((iot_conf_t *)this->private, fd->inode,
-                                    stub);
+        ret = iot_schedule_slow (this->private, stub);
 out:
         if (ret < 0) {
                 STACK_UNWIND_STRICT (fsyncdir, frame, -1, -ret);
@@ -1533,8 +1491,7 @@ iot_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc)
                 goto out;
         }
 
-        ret = iot_schedule_unordered ((iot_conf_t *)this->private, loc->inode,
-                                      stub);
+        ret = iot_schedule_fast (this->private, stub);
 out:
         if (ret < 0) {
                 STACK_UNWIND_STRICT (statfs, frame, -1, -ret, NULL);
@@ -1571,7 +1528,6 @@ iot_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
               int32_t flags)
 {
         call_stub_t     *stub = NULL;
-        fd_t            *fd = NULL;
         int              ret = -1;
 
         stub = fop_setxattr_stub (frame, iot_setxattr_wrapper, loc, dict,
@@ -1583,15 +1539,7 @@ iot_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
                 goto out;
         }
 
-        fd = fd_lookup (loc->inode, frame->root->pid);
-        if (fd == NULL)
-                ret = iot_schedule_unordered ((iot_conf_t *)this->private,
-                                              loc->inode, stub);
-        else {
-                ret = iot_schedule_ordered ((iot_conf_t *)this->private,
-                                            loc->inode, stub);
-                fd_unref (fd);
-        }
+        ret = iot_schedule (this->private, stub);
 
 out:
         if (ret < 0) {
@@ -1629,8 +1577,7 @@ iot_getxattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
               const char *name)
 {
         call_stub_t     *stub = NULL;
-        fd_t            *fd = NULL;
-        int             ret = -1;
+        int              ret = -1;
 
         stub = fop_getxattr_stub (frame, iot_getxattr_wrapper, loc, name);
         if (!stub) {
@@ -1640,15 +1587,7 @@ iot_getxattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
                 goto out;
         }
 
-        fd = fd_lookup (loc->inode, frame->root->pid);
-        if (!fd)
-                ret = iot_schedule_unordered ((iot_conf_t *)this->private,
-                                              loc->inode, stub);
-        else {
-                ret = iot_schedule_ordered ((iot_conf_t *)this->private,
-                                            loc->inode, stub);
-                fd_unref (fd);
-        }
+        ret = iot_schedule (this->private, stub);
 
 out:
         if (ret < 0) {
@@ -1696,8 +1635,7 @@ iot_fgetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
                 goto out;
         }
 
-        ret = iot_schedule_ordered ((iot_conf_t *)this->private, fd->inode,
-                                    stub);
+        ret = iot_schedule (this->private, stub);
 out:
         if (ret < 0) {
                 STACK_UNWIND_STRICT (fgetxattr, frame, -1, -ret, NULL);
@@ -1745,8 +1683,7 @@ iot_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict,
                 goto out;
         }
 
-        ret = iot_schedule_ordered ((iot_conf_t *)this->private, fd->inode,
-                                    stub);
+        ret = iot_schedule (this->private, stub);
 out:
         if (ret < 0) {
                 STACK_UNWIND_STRICT (fsetxattr, frame, -1, -ret);
@@ -1783,7 +1720,6 @@ iot_removexattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
                  const char *name)
 {
         call_stub_t     *stub = NULL;
-        fd_t            *fd = NULL;
         int             ret = -1;
 
         stub = fop_removexattr_stub (frame, iot_removexattr_wrapper, loc,
@@ -1795,16 +1731,7 @@ iot_removexattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
                 goto out;
         }
 
-        fd = fd_lookup (loc->inode, frame->root->pid);
-        if (!fd)
-                ret = iot_schedule_unordered ((iot_conf_t *)this->private,
-                                              loc->inode, stub);
-        else {
-                ret = iot_schedule_ordered ((iot_conf_t *)this->private,
-                                            loc->inode, stub);
-                fd_unref (fd);
-        }
-
+        ret = iot_schedule (this->private, stub);
 out:
         if (ret < 0) {
                 STACK_UNWIND_STRICT (removexattr, frame, -1, -ret);
@@ -1852,8 +1779,7 @@ iot_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
                 goto out;
         }
 
-        ret = iot_schedule_ordered ((iot_conf_t *)this->private, fd->inode,
-                                    stub);
+        ret = iot_schedule_fast (this->private, stub);
 out:
         if (ret < 0) {
                 STACK_UNWIND_STRICT (readdirp, frame, -1, -ret, NULL);
@@ -1900,8 +1826,7 @@ iot_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
                 goto out;
         }
 
-        ret = iot_schedule_ordered ((iot_conf_t *)this->private, fd->inode,
-                                    stub);
+        ret = iot_schedule_fast (this->private, stub);
 out:
         if (ret < 0) {
                 STACK_UNWIND_STRICT (readdir, frame, -1, -ret, NULL);
@@ -1938,7 +1863,6 @@ iot_xattrop (call_frame_t *frame, xlator_t *this, loc_t *loc,
              gf_xattrop_flags_t optype, dict_t *xattr)
 {
         call_stub_t     *stub = NULL;
-        fd_t            *fd = NULL;
         int             ret = -1;
 
         stub = fop_xattrop_stub (frame, iot_xattrop_wrapper, loc, optype,
@@ -1950,16 +1874,7 @@ iot_xattrop (call_frame_t *frame, xlator_t *this, loc_t *loc,
                 goto out;
         }
 
-        fd = fd_lookup (loc->inode, frame->root->pid);
-        if (!fd)
-                ret = iot_schedule_unordered ((iot_conf_t *)this->private,
-                                              loc->inode, stub);
-        else {
-                ret = iot_schedule_ordered ((iot_conf_t *)this->private,
-                                            loc->inode, stub);
-                fd_unref (fd);
-        }
-
+        ret = iot_schedule_slow (this->private, stub);
 out:
         if (ret < 0) {
                 STACK_UNWIND_STRICT (xattrop, frame, -1, -ret, NULL);
@@ -2006,8 +1921,7 @@ iot_fxattrop (call_frame_t *frame, xlator_t *this, fd_t *fd,
                 goto out;
         }
 
-        ret = iot_schedule_ordered ((iot_conf_t *)this->private, fd->inode,
-                                    stub);
+        ret = iot_schedule_slow (this->private, stub);
 out:
         if (ret < 0) {
                 STACK_UNWIND_STRICT (fxattrop, frame, -1, -ret, NULL);
@@ -2056,8 +1970,7 @@ iot_rchecksum (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
                 goto out;
         }
 
-        ret = iot_schedule_ordered ((iot_conf_t *)this->private, fd->inode,
-                                    stub);
+        ret = iot_schedule_slow (this->private, stub);
 out:
         if (ret < 0) {
                 STACK_UNWIND_STRICT (rchecksum, frame, -1, -ret, -1, NULL);
@@ -2338,50 +2251,44 @@ fini (xlator_t *this)
 	return;
 }
 
-/*
- * O - Goes to ordered threadpool.
- * U - Goes to un-ordered threadpool.
- * V - Variable, depends on whether the file is open.
- *     If it is, then goes to ordered, otherwise to
- *     un-ordered.
- */
+
 struct xlator_fops fops = {
-	.open        = iot_open,        /* U */
-	.create      = iot_create,      /* U */
-	.readv       = iot_readv,       /* O */
-	.writev      = iot_writev,      /* O */
-	.flush       = iot_flush,       /* O */
-	.fsync       = iot_fsync,       /* O */
-	.lk          = iot_lk,          /* O */
-	.stat        = iot_stat,        /* V */
-	.fstat       = iot_fstat,       /* O */
-	.truncate    = iot_truncate,    /* V */
-	.ftruncate   = iot_ftruncate,   /* O */
-	.unlink      = iot_unlink,      /* U */
-        .lookup      = iot_lookup,      /* U */
-        .setattr     = iot_setattr,     /* U */
-        .fsetattr    = iot_fsetattr,    /* O */
-        .access      = iot_access,      /* U */
-        .readlink    = iot_readlink,    /* U */
-        .mknod       = iot_mknod,       /* U */
-        .mkdir       = iot_mkdir,       /* U */
-        .rmdir       = iot_rmdir,       /* U */
-        .symlink     = iot_symlink,     /* U */
-        .rename      = iot_rename,      /* U */
-        .link        = iot_link,        /* U */
-        .opendir     = iot_opendir,     /* U */
-        .fsyncdir    = iot_fsyncdir,    /* O */
-        .statfs      = iot_statfs,      /* U */
-        .setxattr    = iot_setxattr,    /* U */
-        .getxattr    = iot_getxattr,    /* U */
-        .fgetxattr   = iot_fgetxattr,   /* O */
-        .fsetxattr   = iot_fsetxattr,   /* O */
-        .removexattr = iot_removexattr, /* U */
-        .readdir     = iot_readdir,     /* O */
-        .readdirp    = iot_readdirp,    /* O */
-        .xattrop     = iot_xattrop,     /* U */
-	.fxattrop    = iot_fxattrop,    /* O */
-        .rchecksum   = iot_rchecksum,   /* O */
+	.open        = iot_open,
+	.create      = iot_create,
+	.readv       = iot_readv,
+	.writev      = iot_writev,
+	.flush       = iot_flush,
+	.fsync       = iot_fsync,
+	.lk          = iot_lk,
+	.stat        = iot_stat,
+	.fstat       = iot_fstat,
+	.truncate    = iot_truncate,
+	.ftruncate   = iot_ftruncate,
+	.unlink      = iot_unlink,
+        .lookup      = iot_lookup,
+        .setattr     = iot_setattr,
+        .fsetattr    = iot_fsetattr,
+        .access      = iot_access,
+        .readlink    = iot_readlink,
+        .mknod       = iot_mknod,
+        .mkdir       = iot_mkdir,
+        .rmdir       = iot_rmdir,
+        .symlink     = iot_symlink,
+        .rename      = iot_rename,
+        .link        = iot_link,
+        .opendir     = iot_opendir,
+        .fsyncdir    = iot_fsyncdir,
+        .statfs      = iot_statfs,
+        .setxattr    = iot_setxattr,
+        .getxattr    = iot_getxattr,
+        .fgetxattr   = iot_fgetxattr,
+        .fsetxattr   = iot_fsetxattr,
+        .removexattr = iot_removexattr,
+        .readdir     = iot_readdir,
+        .readdirp    = iot_readdirp,
+        .xattrop     = iot_xattrop,
+	.fxattrop    = iot_fxattrop,
+        .rchecksum   = iot_rchecksum,
 };
 
 struct xlator_cbks cbks = {