debug/delay-gen: Implement delay-generation feature

Background:
I was working on a customer issue where the disks were responding some times
after seconds. It was becoming very difficult to recreate the issues in our
labs, so had to come up with this feature.

Requirements:
We need an xlator which can delay x% of ops for y micro seconds.
We should be able to enable delays for specific fops.

This feature is modeled after error-gen. Most of the logic
is borrowed from that xlator. This is a minimum implementation
of the feature which satisfied the requirements I had. May be
in future with more requirements and understanding of the problem
further we can improve upon this implementation.

Here are the commands and what they do:
Enable delay-gen: (This is similar to how err-gen is enabled on the brick side)
- gluster volume set <volname> delay-gen posix

Set the percentage of fops that need to be delayed
- gluster volume set <volname> delay-gen.delay-percentage 50
Default is 10%

Set the delay in micro seconds
- gluster volume set <volname> delay-gen.delay-duration 500000
Default is 100000

Set comma separated fops to be delayed
- gluster v set r2 delay-gen.enable read,write
Default is all fops.

Fixes #257
Change-Id: Ib547bd39cc024c9cdb63754d21e3aa62fc9d6473
Signed-off-by: Pranith Kumar K <pkarampu@redhat.com>
Reviewed-on: https://review.gluster.org/17591
Smoke: Gluster Build System <jenkins@build.gluster.org>
CentOS-regression: Gluster Build System <jenkins@build.gluster.org>
Reviewed-by: Jeff Darcy <jeff@pl.atyp.us>

13 files changed, 902 insertions, 27 deletions

diff --git a/configure.ac b/configure.ac
index 413d071c1db..1c3a7ee18a5 100644
--- a/configure.ac
+++ b/configure.ac
@@ -112,6 +112,8 @@ AC_CONFIG_FILES([Makefile
                 xlators/debug/trace/src/Makefile
                 xlators/debug/error-gen/Makefile
                 xlators/debug/error-gen/src/Makefile
+                xlators/debug/delay-gen/Makefile
+                xlators/debug/delay-gen/src/Makefile
                 xlators/debug/io-stats/Makefile
                 xlators/debug/io-stats/src/Makefile
                 xlators/protocol/Makefile
diff --git a/glusterfs.spec.in b/glusterfs.spec.in
index 5c3567ead49..f00f52c1ed4 100644
--- a/glusterfs.spec.in
+++ b/glusterfs.spec.in
@@ -998,6 +998,7 @@ exit 0
 %dir %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator
 %dir %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/debug
      %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/debug/error-gen.so
+     %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/debug/delay-gen.so
      %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/debug/io-stats.so
      %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/debug/sink.so
      %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/debug/trace.so
diff --git a/tests/features/delay-gen.t b/tests/features/delay-gen.t
new file mode 100755
index 00000000000..38b2a499dc0
--- /dev/null
+++ b/tests/features/delay-gen.t
@@ -0,0 +1,39 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../volume.rc
+
+cleanup;
+
+TEST glusterd
+TEST pidof glusterd
+
+TEST $CLI volume create $V0 $H0:$B0/${V0}1
+
+EXPECT "$V0" volinfo_field $V0 'Volume Name'
+EXPECT 'Created' volinfo_field $V0 'Status'
+
+TEST $CLI volume set $V0 delay-gen posix
+TEST $CLI volume set $V0 delay-gen.delay-duration 1000000
+TEST $CLI volume set $V0 delay-gen.delay-percentage 100
+TEST $CLI volume set $V0 delay-gen.enable read,write
+
+TEST $CLI volume start $V0
+EXPECT 'Started' volinfo_field $V0 'Status'
+
+TEST $CLI volume profile $V0 start
+## Mount FUSE with caching disabled (read-write)
+TEST $GFS -s $H0 --volfile-id $V0 $M0
+
+TEST dd if=/dev/zero of=$M0/1 count=1 bs=128k oflag=sync
+
+#Write should take at least a second
+write_max_latency=$($CLI volume profile $V0 info | grep WRITE | awk 'BEGIN {max = 0} {if ($6 > max) max=$6;} END {print max}' | cut -d. -f 1 | egrep "[0-9]{7,}")
+
+#Create should not take a second
+create_max_latency=$($CLI volume profile $V0 info | grep CREATE | awk 'BEGIN {max = 0} {if ($6 > max) max=$6;} END {print max}' | cut -d. -f 1 | egrep "[0-9]{7,}")
+
+TEST [ ! -z $write_max_latency ];
+TEST [ -z $create_max_latency ];
+
+cleanup;
diff --git a/xlators/debug/Makefile.am b/xlators/debug/Makefile.am
index 6e476152ddc..88fac1c6d9e 100644
--- a/xlators/debug/Makefile.am
+++ b/xlators/debug/Makefile.am
@@ -1,3 +1,3 @@
-SUBDIRS = error-gen io-stats sink trace
+SUBDIRS = error-gen io-stats sink trace delay-gen
 
 CLEANFILES = 
diff --git a/xlators/debug/delay-gen/Makefile.am b/xlators/debug/delay-gen/Makefile.am
new file mode 100644
index 00000000000..a985f42a877
--- /dev/null
+++ b/xlators/debug/delay-gen/Makefile.am
@@ -0,0 +1,3 @@
+SUBDIRS = src
+
+CLEANFILES =
diff --git a/xlators/debug/delay-gen/src/Makefile.am b/xlators/debug/delay-gen/src/Makefile.am
new file mode 100644
index 00000000000..8f758dec199
--- /dev/null
+++ b/xlators/debug/delay-gen/src/Makefile.am
@@ -0,0 +1,11 @@
+
+xlator_LTLIBRARIES = delay-gen.la
+xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/debug
+delay_gen_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
+delay_gen_la_SOURCES = delay-gen.c
+delay_gen_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
+noinst_HEADERS = delay-gen.h delay-gen-mem-types.h delay-gen-messages.h
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
+	   -I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src
+AM_CFLAGS = -Wall -fno-strict-aliasing $(GF_CFLAGS)
+CLEANFILES =
diff --git a/xlators/debug/delay-gen/src/delay-gen-mem-types.h b/xlators/debug/delay-gen/src/delay-gen-mem-types.h
new file mode 100644
index 00000000000..8c54f5f922e
--- /dev/null
+++ b/xlators/debug/delay-gen/src/delay-gen-mem-types.h
@@ -0,0 +1,22 @@
+/*
+ *   Copyright (c) 2017 Red Hat, Inc. <http://www.redhat.com>
+ *   This file is part of GlusterFS.
+ *
+ *   This file is licensed to you under your choice of the GNU Lesser
+ *   General Public License, version 3 or any later version (LGPLv3 or
+ *   later), or the GNU General Public License, version 2 (GPLv2), in all
+ *   cases as published by the Free Software Foundation.
+ */
+
+
+#ifndef __DELAY_GEN_MEM_TYPES_H__
+#define __DELAY_GEN_MEM_TYPES_H__
+
+#include "mem-types.h"
+
+enum gf_delay_gen_mem_types_ {
+        gf_delay_gen_mt_dg_t = gf_common_mt_end + 1,
+        gf_delay_gen_mt_end
+};
+
+#endif /* __DELAY_GEN_MEM_TYPES_H__ */
diff --git a/xlators/debug/delay-gen/src/delay-gen-messages.h b/xlators/debug/delay-gen/src/delay-gen-messages.h
new file mode 100644
index 00000000000..db828722ca2
--- /dev/null
+++ b/xlators/debug/delay-gen/src/delay-gen-messages.h
@@ -0,0 +1,17 @@
+/*
+ *   Copyright (c) 2017 Red Hat, Inc. <http://www.redhat.com>
+ *   This file is part of GlusterFS.
+ *
+ *   This file is licensed to you under your choice of the GNU Lesser
+ *   General Public License, version 3 or any later version (LGPLv3 or
+ *   later), or the GNU General Public License, version 2 (GPLv2), in all
+ *   cases as published by the Free Software Foundation.
+ */
+
+
+#ifndef __DELAY_GEN_MESSAGES_H__
+#define __DELAY_GEN_MESSAGES_H__
+
+
+
+#endif /* __DELAY_GEN_MESSAGES_H__ */
diff --git a/xlators/debug/delay-gen/src/delay-gen.c b/xlators/debug/delay-gen/src/delay-gen.c
new file mode 100644
index 00000000000..9dc82338bfc
--- /dev/null
+++ b/xlators/debug/delay-gen/src/delay-gen.c
@@ -0,0 +1,736 @@
+/*
+ *  Copyright (c) 2017 Red Hat, Inc. <http://www.redhat.com>
+ *  This file is part of GlusterFS.
+ *
+ *  This file is licensed to you under your choice of the GNU Lesser
+ *  General Public License, version 3 or any later version (LGPLv3 or
+ *  later), or the GNU General Public License, version 2 (GPLv2), in all
+ *  cases as published by the Free Software Foundation.
+ */
+
+
+#include "delay-gen.h"
+
+#define DELAY_GRANULARITY     (1 << 20)
+
+#define DG_FOP(fop, name, frame, this, args...)                                \
+        do {                                                                   \
+                delay_gen (this, fop);                                         \
+                default_##name (frame, this, args);                            \
+        } while (0)
+
+int
+delay_gen (xlator_t *this, int fop)
+{
+        dg_t             *dg = this->private;
+
+        if (!dg->enable[fop] || !dg->delay_ppm)
+                return 0;
+
+        if ((rand () % DELAY_GRANULARITY) < dg->delay_ppm)
+                usleep (dg->delay_duration);
+
+        return 0;
+}
+
+int32_t
+dg_rename (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+           dict_t *xdata)
+{
+        DG_FOP (GF_FOP_RENAME, rename, frame, this,  oldloc,  newloc, xdata);
+        return 0;
+}
+
+
+int32_t
+dg_ipc (call_frame_t *frame, xlator_t *this, int32_t op, dict_t *xdata)
+{
+        DG_FOP (GF_FOP_IPC, ipc, frame, this, op, xdata);
+        return 0;
+}
+
+int32_t
+dg_setactivelk (call_frame_t *frame, xlator_t *this, loc_t *loc,
+                lock_migration_info_t *locklist, dict_t *xdata)
+{
+        DG_FOP (GF_FOP_SETACTIVELK, setactivelk, frame, this, loc,
+                locklist, xdata);
+        return 0;
+}
+
+int32_t
+dg_flush (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
+{
+        DG_FOP (GF_FOP_FLUSH, flush, frame, this, fd, xdata);
+        return 0;
+}
+
+int32_t
+dg_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+            off_t off, dict_t *xdata)
+{
+        DG_FOP (GF_FOP_READDIR, readdir, frame, this,  fd, size, off, xdata);
+        return 0;
+}
+
+int32_t
+dg_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
+             int32_t flags, dict_t *xdata)
+{
+        DG_FOP (GF_FOP_SETXATTR, setxattr, frame, this, loc, dict, flags,
+                xdata);
+        return 0;
+}
+
+int32_t
+dg_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+          dev_t rdev, mode_t umask, dict_t *xdata)
+{
+        DG_FOP (GF_FOP_MKNOD, mknod, frame, this, loc, mode, rdev, umask,
+                xdata);
+        return 0;
+}
+
+int32_t
+dg_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict,
+              int32_t flags, dict_t *xdata)
+{
+        DG_FOP (GF_FOP_FSETXATTR, fsetxattr, frame, this, fd, dict, flags,
+                xdata);
+        return 0;
+}
+
+int32_t
+dg_readv (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+          off_t offset, uint32_t flags, dict_t *xdata)
+{
+        DG_FOP (GF_FOP_READ, readv, frame, this, fd, size, offset, flags,
+                xdata);
+        return 0;
+}
+
+int32_t
+dg_inodelk (call_frame_t *frame, xlator_t *this, const char *volume,
+            loc_t *loc, int32_t cmd, struct gf_flock *lock, dict_t *xdata)
+{
+        DG_FOP (GF_FOP_INODELK, inodelk, frame, this, volume, loc, cmd, lock,
+                xdata);
+        return 0;
+}
+
+int32_t
+dg_fremovexattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
+                 const char *name, dict_t *xdata)
+{
+        DG_FOP (GF_FOP_FREMOVEXATTR, fremovexattr, frame, this, fd, name,
+                xdata);
+        return 0;
+}
+
+int32_t
+dg_open (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+         fd_t *fd, dict_t *xdata)
+{
+        DG_FOP (GF_FOP_OPEN, open, frame, this, loc, flags, fd, xdata);
+        return 0;
+}
+
+int32_t
+dg_xattrop (call_frame_t *frame, xlator_t *this, loc_t *loc,
+            gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
+{
+        DG_FOP (GF_FOP_XATTROP, xattrop, frame, this, loc, flags, dict, xdata);
+        return 0;
+}
+
+int32_t
+dg_entrylk (call_frame_t *frame, xlator_t *this, const char *volume,
+            loc_t *loc, const char *basename, entrylk_cmd cmd,
+            entrylk_type type, dict_t *xdata)
+{
+        DG_FOP (GF_FOP_ENTRYLK, entrylk, frame, this, volume,
+                loc, basename, cmd, type, xdata);
+        return 0;
+}
+
+int32_t
+dg_getactivelk (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+{
+        DG_FOP (GF_FOP_GETACTIVELK, getactivelk, frame, this, loc, xdata);
+        return 0;
+}
+
+int32_t
+dg_finodelk (call_frame_t *frame, xlator_t *this, const char *volume,
+             fd_t *fd, int32_t cmd, struct gf_flock *lock, dict_t *xdata)
+{
+        DG_FOP (GF_FOP_FINODELK, finodelk, frame, this, volume, fd, cmd, lock,
+                xdata);
+        return 0;
+}
+
+int32_t
+dg_create (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+           mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
+{
+        DG_FOP (GF_FOP_CREATE, create, frame, this, loc, flags, mode, umask, fd,
+                xdata);
+        return 0;
+}
+
+int32_t
+dg_discard (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+            size_t len, dict_t *xdata)
+{
+        DG_FOP (GF_FOP_DISCARD, discard, frame, this, fd, offset, len, xdata);
+        return 0;
+}
+
+int32_t
+dg_mkdir (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+          mode_t umask, dict_t *xdata)
+{
+        DG_FOP (GF_FOP_MKDIR, mkdir, frame, this, loc, mode, umask, xdata);
+        return 0;
+}
+
+int32_t
+dg_lk (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t cmd,
+       struct gf_flock *lock, dict_t *xdata)
+{
+        DG_FOP (GF_FOP_LK, lk, frame, this, fd, cmd, lock, xdata);
+        return 0;
+}
+
+int32_t
+dg_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
+           struct iovec *vector, int32_t count, off_t off, uint32_t flags,
+           struct iobref *iobref, dict_t *xdata)
+{
+        DG_FOP (GF_FOP_WRITE, writev, frame, this, fd,
+                vector, count, off, flags, iobref, xdata);
+        return 0;
+}
+
+int32_t
+dg_access (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t mask,
+           dict_t *xdata)
+{
+        DG_FOP (GF_FOP_ACCESS, access, frame, this, loc, mask, xdata);
+        return 0;
+}
+
+int32_t
+dg_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+{
+        DG_FOP (GF_FOP_LOOKUP, lookup, frame, this, loc, xdata);
+        return 0;
+}
+
+
+
+int32_t
+dg_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc,
+          int32_t flags, dict_t *xdata)
+{
+        DG_FOP (GF_FOP_RMDIR, rmdir, frame, this, loc, flags, xdata);
+        return 0;
+}
+
+
+
+int32_t
+dg_fallocate (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t keep_size,
+              off_t offset, size_t len, dict_t *xdata)
+{
+        DG_FOP (GF_FOP_FALLOCATE, fallocate, frame, this, fd, keep_size, offset,
+                len, xdata);
+        return 0;
+}
+
+
+
+int32_t
+dg_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
+{
+        DG_FOP (GF_FOP_FSTAT, fstat, frame, this, fd, xdata);
+        return 0;
+}
+
+
+
+int32_t
+dg_lease (call_frame_t *frame, xlator_t *this, loc_t *loc,
+          struct gf_lease *lease, dict_t *xdata)
+{
+        DG_FOP (GF_FOP_LEASE, lease, frame, this, loc, lease, xdata);
+        return 0;
+}
+
+
+
+int32_t
+dg_stat (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+{
+        DG_FOP (GF_FOP_STAT, stat, frame, this, loc, xdata);
+        return 0;
+}
+
+
+
+int32_t
+dg_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
+             dict_t *xdata)
+{
+        DG_FOP (GF_FOP_TRUNCATE, truncate, frame, this, loc, offset, xdata);
+        return 0;
+}
+
+
+
+int32_t
+dg_getxattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
+             const char *name, dict_t *xdata)
+{
+        DG_FOP (GF_FOP_GETXATTR, getxattr, frame, this, loc, name, xdata);
+        return 0;
+}
+
+
+
+int32_t
+dg_symlink (call_frame_t *frame, xlator_t *this, const char *linkpath,
+            loc_t *loc, mode_t umask, dict_t *xdata)
+{
+        DG_FOP (GF_FOP_SYMLINK, symlink, frame, this, linkpath, loc, umask,
+                xdata);
+        return 0;
+}
+
+
+
+int32_t
+dg_zerofill (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+             off_t len, dict_t *xdata)
+{
+        DG_FOP (GF_FOP_ZEROFILL, zerofill, frame, this, fd, offset, len, xdata);
+        return 0;
+}
+
+
+
+int32_t
+dg_fsyncdir (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags,
+             dict_t *xdata)
+{
+        DG_FOP (GF_FOP_FSYNCDIR, fsyncdir, frame, this, fd, flags, xdata);
+        return 0;
+}
+
+
+
+int32_t
+dg_fgetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd, const char *name,
+              dict_t *xdata)
+{
+        DG_FOP (GF_FOP_FGETXATTR, fgetxattr, frame, this, fd, name, xdata);
+        return 0;
+}
+
+
+
+int32_t
+dg_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+             off_t off, dict_t *xdata)
+{
+        DG_FOP (GF_FOP_READDIRP, readdirp, frame, this, fd, size, off,  xdata);
+        return 0;
+}
+
+
+
+int32_t
+dg_link (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+         dict_t *xdata)
+{
+        DG_FOP (GF_FOP_LINK, link, frame, this, oldloc, newloc, xdata);
+        return 0;
+}
+
+
+
+int32_t
+dg_fxattrop (call_frame_t *frame, xlator_t *this, fd_t *fd,
+             gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
+{
+        DG_FOP (GF_FOP_FXATTROP, fxattrop, frame, this, fd, flags, dict, xdata);
+        return 0;
+}
+
+
+
+int32_t
+dg_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+              dict_t *xdata)
+{
+        DG_FOP (GF_FOP_FTRUNCATE, ftruncate, frame, this, fd, offset,  xdata);
+        return 0;
+}
+
+
+
+int32_t
+dg_rchecksum (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+              int32_t len, dict_t *xdata)
+{
+        DG_FOP (GF_FOP_RCHECKSUM, rchecksum, frame, this, fd, offset, len,
+                xdata);
+        return 0;
+}
+
+
+
+int32_t
+dg_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+           dict_t *xdata)
+{
+        DG_FOP (GF_FOP_UNLINK, unlink, frame, this, loc, flags, xdata);
+        return 0;
+}
+
+
+
+int32_t
+dg_fentrylk (call_frame_t *frame, xlator_t *this, const char *volume,
+             fd_t *fd, const char *basename, entrylk_cmd cmd,
+             entrylk_type type, dict_t *xdata)
+{
+        DG_FOP (GF_FOP_FENTRYLK, fentrylk, frame, this, volume,  fd, basename,
+                cmd, type, xdata);
+        return 0;
+}
+
+
+
+int32_t
+dg_getspec (call_frame_t *frame, xlator_t *this, const char *key,
+            int32_t flags)
+{
+        DG_FOP (GF_FOP_GETSPEC, getspec, frame, this, key, flags);
+        return 0;
+}
+
+
+
+int32_t
+dg_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
+            struct iatt *stbuf, int32_t valid, dict_t *xdata)
+{
+        DG_FOP (GF_FOP_SETATTR, setattr, frame, this, loc, stbuf, valid, xdata);
+        return 0;
+}
+
+
+
+int32_t
+dg_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags,
+          dict_t *xdata)
+{
+        DG_FOP (GF_FOP_FSYNC, fsync, frame, this,  fd, flags, xdata);
+        return 0;
+}
+
+
+
+int32_t
+dg_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+{
+        DG_FOP (GF_FOP_STATFS, statfs, frame, this, loc, xdata);
+        return 0;
+}
+
+
+
+int32_t
+dg_seek (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+         gf_seek_what_t what, dict_t *xdata)
+{
+        DG_FOP (GF_FOP_SEEK, seek, frame, this, fd, offset, what, xdata);
+        return 0;
+}
+
+
+
+int32_t
+dg_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
+             struct iatt *stbuf, int32_t valid, dict_t *xdata)
+{
+        DG_FOP (GF_FOP_FSETATTR, fsetattr, frame, this, fd,
+                stbuf, valid, xdata);
+        return 0;
+}
+
+
+
+int32_t
+dg_opendir (call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd,
+            dict_t *xdata)
+{
+        DG_FOP (GF_FOP_OPENDIR, opendir, frame, this, loc, fd, xdata);
+        return 0;
+}
+
+
+
+int32_t
+dg_readlink (call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size,
+             dict_t *xdata)
+{
+        DG_FOP (GF_FOP_READLINK, readlink, frame, this, loc, size, xdata);
+        return 0;
+}
+
+
+
+int32_t
+dg_removexattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
+                const char *name, dict_t *xdata)
+{
+        DG_FOP (GF_FOP_REMOVEXATTR, removexattr, frame, this, loc, name, xdata);
+        return 0;
+}
+
+int32_t
+dg_forget (xlator_t *this, inode_t *inode)
+{
+        return 0;
+}
+
+int32_t
+dg_release (xlator_t *this, fd_t *fd)
+{
+        return 0;
+}
+
+int32_t
+dg_releasedir (xlator_t *this, fd_t *fd)
+{
+        return 0;
+}
+
+static int
+delay_gen_parse_fill_fops (dg_t *dg, char *enable_fops)
+{
+        char            *op_no_str = NULL;
+        int              op_no = -1;
+        int              i = 0;
+        int              ret = 0;
+        xlator_t        *this = THIS;
+        char            *saveptr = NULL;
+        char            *dup_enable_fops = NULL;
+
+        if (!enable_fops) {
+                for (i = GF_FOP_NULL + 1; i < GF_FOP_MAXVALUE; i++)
+                        dg->enable[i] = 1;
+        } else {
+                dup_enable_fops = gf_strdup (enable_fops);
+                if (!dup_enable_fops) {
+                        ret = -1;
+                        goto out;
+                }
+                op_no_str = strtok_r (dup_enable_fops, ",", &saveptr);
+                while (op_no_str) {
+                        op_no = gf_fop_int (op_no_str);
+                        if (op_no == -1) {
+                                gf_log (this->name, GF_LOG_WARNING,
+                                        "Wrong option value %s", op_no_str);
+                                ret = -1;
+                                goto out;
+                        } else {
+                                dg->enable[op_no] = 1;
+                        }
+
+                        op_no_str = strtok_r (NULL, ",", &saveptr);
+                }
+        }
+out:
+        GF_FREE (dup_enable_fops);
+        return ret;
+}
+
+void
+delay_gen_set_delay_ppm (dg_t *dg, double percent)
+{
+        double ppm;
+
+        ppm = (percent / 100.0) * (double) DELAY_GRANULARITY;
+        dg->delay_ppm = ppm;
+}
+
+int32_t
+init (xlator_t *this)
+{
+        dg_t            *dg = NULL;
+        int32_t          ret = 0;
+        double          delay_percent = 0;
+        char            *delay_enable_fops = NULL;
+
+        if (!this->children || this->children->next) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "delay-gen not configured with one subvolume");
+                ret = -1;
+                goto out;
+        }
+
+        if (!this->parents) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "dangling volume. check volfile ");
+        }
+
+        dg = GF_CALLOC (1, sizeof (*dg), gf_delay_gen_mt_dg_t);
+
+        if (!dg) {
+                ret = -1;
+                goto out;
+        }
+
+        ret = -1;
+
+        GF_OPTION_INIT ("delay-percentage", delay_percent, percent, out);
+        GF_OPTION_INIT ("enable", delay_enable_fops, str, out);
+        GF_OPTION_INIT ("delay-duration", dg->delay_duration, int32, out);
+
+        delay_gen_set_delay_ppm (dg, delay_percent);
+
+        ret = delay_gen_parse_fill_fops (dg, delay_enable_fops);
+        if (ret)
+                goto out;
+
+        this->private = dg;
+
+        ret = 0;
+out:
+        if (ret)
+                GF_FREE (dg);
+        return ret;
+}
+
+void
+fini (xlator_t *this)
+{
+        GF_FREE (this->private);
+}
+
+int32_t
+mem_acct_init (xlator_t *this)
+{
+        int     ret = -1;
+
+        if (!this)
+                return ret;
+
+        ret = xlator_mem_acct_init (this, gf_delay_gen_mt_end + 1);
+
+        if (ret != 0) {
+                gf_log (this->name, GF_LOG_ERROR, "Memory accounting init"
+                        " failed");
+                return ret;
+        }
+
+        return ret;
+}
+
+int32_t
+reconfigure (xlator_t *this, dict_t *dict)
+{
+        /*At the moment I don't see any need to implement this. In future
+         *if this is needed we can add code here.
+         */
+        return 0;
+}
+
+int
+notify (xlator_t *this, int event, void *data, ...)
+{
+        return default_notify (this, event, data);
+}
+
+
+struct xlator_fops fops = {
+        .rename               = dg_rename,
+        .ipc                  = dg_ipc,
+        .setactivelk          = dg_setactivelk,
+        .flush                = dg_flush,
+        .readdir              = dg_readdir,
+        .setxattr             = dg_setxattr,
+        .mknod                = dg_mknod,
+        .fsetxattr            = dg_fsetxattr,
+        .readv                = dg_readv,
+        .inodelk              = dg_inodelk,
+        .fremovexattr         = dg_fremovexattr,
+        .open                 = dg_open,
+        .xattrop              = dg_xattrop,
+        .entrylk              = dg_entrylk,
+        .getactivelk          = dg_getactivelk,
+        .finodelk             = dg_finodelk,
+        .create               = dg_create,
+        .discard              = dg_discard,
+        .mkdir                = dg_mkdir,
+        .lk                   = dg_lk,
+        .writev               = dg_writev,
+        .access               = dg_access,
+        .lookup               = dg_lookup,
+        .rmdir                = dg_rmdir,
+        .fallocate            = dg_fallocate,
+        .fstat                = dg_fstat,
+        .lease                = dg_lease,
+        .stat                 = dg_stat,
+        .truncate             = dg_truncate,
+        .getxattr             = dg_getxattr,
+        .symlink              = dg_symlink,
+        .zerofill             = dg_zerofill,
+        .fsyncdir             = dg_fsyncdir,
+        .fgetxattr            = dg_fgetxattr,
+        .readdirp             = dg_readdirp,
+        .link                 = dg_link,
+        .fxattrop             = dg_fxattrop,
+        .ftruncate            = dg_ftruncate,
+        .rchecksum            = dg_rchecksum,
+        .unlink               = dg_unlink,
+        .fentrylk             = dg_fentrylk,
+        .getspec              = dg_getspec,
+        .setattr              = dg_setattr,
+        .fsync                = dg_fsync,
+        .statfs               = dg_statfs,
+        .seek                 = dg_seek,
+        .fsetattr             = dg_fsetattr,
+        .opendir              = dg_opendir,
+        .readlink             = dg_readlink,
+        .removexattr          = dg_removexattr,
+};
+
+struct xlator_cbks cbks = {
+        .forget               = dg_forget,
+        .release              = dg_release,
+        .releasedir           = dg_releasedir,
+};
+
+struct volume_options options[] = {
+        { .key  = {"delay-percentage"},
+          .type = GF_OPTION_TYPE_PERCENT,
+          .default_value = "10%",
+          .description = "Percentage delay of operations when enabled.",
+        },
+
+        { .key  = {"delay-duration"},
+          .type = GF_OPTION_TYPE_INT,
+          .description = "Delay duration in micro seconds",
+          .default_value = "100000",
+        },
+
+        { .key  = {"enable"},
+          .type = GF_OPTION_TYPE_STR,
+        },
+
+        { .key  = {NULL} }
+};
diff --git a/xlators/debug/delay-gen/src/delay-gen.h b/xlators/debug/delay-gen/src/delay-gen.h
new file mode 100644
index 00000000000..e1111c782e2
--- /dev/null
+++ b/xlators/debug/delay-gen/src/delay-gen.h
@@ -0,0 +1,28 @@
+/*
+ *   Copyright (c) 2017 Red Hat, Inc. <http://www.redhat.com>
+ *   This file is part of GlusterFS.
+ *
+ *   This file is licensed to you under your choice of the GNU Lesser
+ *   General Public License, version 3 or any later version (LGPLv3 or
+ *   later), or the GNU General Public License, version 2 (GPLv2), in all
+ *   cases as published by the Free Software Foundation.
+ */
+
+
+#ifndef __DELAY_GEN_H__
+#define __DELAY_GEN_H__
+
+#include "delay-gen-mem-types.h"
+#include "delay-gen-messages.h"
+#include "glusterfs.h"
+#include "xlator.h"
+#include "defaults.h"
+
+typedef struct {
+        int enable[GF_FOP_MAXVALUE];
+        int op_count;
+        int delay_ppm;
+        int delay_duration;
+} dg_t;
+
+#endif /* __DELAY_GEN_H__ */
diff --git a/xlators/mgmt/glusterd/src/glusterd-volgen.c b/xlators/mgmt/glusterd/src/glusterd-volgen.c
index e14e3133679..f65ad94ace6 100644
--- a/xlators/mgmt/glusterd/src/glusterd-volgen.c
+++ b/xlators/mgmt/glusterd/src/glusterd-volgen.c
@@ -2548,16 +2548,15 @@ debugxl_option_handler (volgen_graph_t *graph, struct volopt_map_entry *vme,
                 return 0;
 
         if (!strcmp (vme->key , "debug.trace") ||
-            !strcmp (vme->key, "debug.error-gen")) {
+            !strcmp (vme->key, "debug.error-gen") ||
+            !strcmp (vme->key, "debug.delay-gen")) {
                 if (get_server_xlator (vme->value) == GF_XLATOR_NONE &&
                     get_client_xlator (vme->value) == GF_CLNT_XLATOR_NONE)
                         return 0;
-                else
-                        goto add_graph;
         }
 
         if (gf_string2boolean (vme->value, &enabled) == -1)
-                return -1;
+                goto add_graph;
         if (!enabled)
                 return 0;
 
@@ -2574,34 +2573,28 @@ int
 check_and_add_debug_xl (volgen_graph_t *graph, dict_t *set_dict, char *volname,
                         char *xlname)
 {
+        int       i         = 0;
         int       ret       = 0;
         char     *value_str = NULL;
+        static char     *xls[] = {"debug.trace", "debug.error-gen",
+                           "debug.delay-gen", NULL};
 
         if (!xlname)
                 goto out;
 
-        ret = dict_get_str (set_dict, "debug.trace", &value_str);
-        if (!ret) {
-                if (strcmp (xlname, value_str) == 0) {
-                        ret = volgen_graph_set_options_generic (graph,
-                                                set_dict, volname,
-                                                &debugxl_option_handler);
-                        if (ret)
-                                goto out;
-                }
-        }
-
-        ret = dict_get_str (set_dict, "debug.error-gen", &value_str);
-        if (!ret) {
-                if (strcmp (xlname, value_str) == 0) {
-                        ret = volgen_graph_set_options_generic (graph,
-                                                set_dict, volname,
-                                                &debugxl_option_handler);
-                        if (ret)
-                        goto out;
+        while (xls[i]) {
+                ret = dict_get_str (set_dict, xls[i], &value_str);
+                if (!ret) {
+                        if (strcmp (xlname, value_str) == 0) {
+                                ret = volgen_graph_set_options_generic (graph,
+                                                        set_dict, volname,
+                                                       &debugxl_option_handler);
+                                if (ret)
+                                        goto out;
+                        }
                 }
+                i++;
         }
-
         ret = 0;
 
 out:
diff --git a/xlators/mgmt/glusterd/src/glusterd-volgen.h b/xlators/mgmt/glusterd/src/glusterd-volgen.h
index 6c79c776436..aa4359c2589 100644
--- a/xlators/mgmt/glusterd/src/glusterd-volgen.h
+++ b/xlators/mgmt/glusterd/src/glusterd-volgen.h
@@ -163,8 +163,8 @@ struct volgen_brick_xlator {
         /* function that builds a xlator */
         brick_xlator_builder builder;
         /* debug key for a xlator that
-         * gets used for adding debug translators like trace, error-gen
-         * before this xlator */
+         * gets used for adding debug translators like trace, error-gen,
+         * delay-gen before this xlator */
         char *dbg_key;
 };
 typedef struct volgen_brick_xlator volgen_brick_xlator_t;
diff --git a/xlators/mgmt/glusterd/src/glusterd-volume-set.c b/xlators/mgmt/glusterd/src/glusterd-volume-set.c
index 3dcfafb4a3e..2c4dc413b0d 100644
--- a/xlators/mgmt/glusterd/src/glusterd-volume-set.c
+++ b/xlators/mgmt/glusterd/src/glusterd-volume-set.c
@@ -3526,6 +3526,29 @@ struct volopt_map_entry glusterd_volopt_map[] = {
           .op_version  = GD_OP_VERSION_3_12_0,
           .validate_fn = validate_boolean
         },
+        { .key        = "debug.delay-gen",
+          .voltype    = "debug/delay-gen",
+          .option     = "!debug",
+          .value      = "off",
+          .type       = NO_DOC,
+          .op_version = GD_OP_VERSION_4_0_0,
+          .flags      = VOLOPT_FLAG_XLATOR_OPT
+        },
+        { .key        = "delay-gen.delay-percentage",
+          .voltype    = "debug/delay-gen",
+          .type       = NO_DOC,
+          .op_version = GD_OP_VERSION_4_0_0,
+        },
+        { .key        = "delay-gen.delay-duration",
+          .voltype    = "debug/delay-gen",
+          .type       = NO_DOC,
+          .op_version = GD_OP_VERSION_4_0_0,
+        },
+        { .key        = "delay-gen.enable",
+          .voltype    = "debug/delay-gen",
+          .type       = NO_DOC,
+          .op_version = GD_OP_VERSION_4_0_0,
+        },
         { .key         = NULL
         }
 };