diff options
author | Xavi Hernandez <xhernandez@redhat.com> | 2020-05-12 23:54:54 +0200 |
---|---|---|
committer | Rinku Kothiya <rkothiya@redhat.com> | 2020-06-15 10:53:24 +0000 |
commit | dcbf54581717d10cf82cdce9bc08415ba1e1d19b (patch) | |
tree | a2e7d9d50bf60cd7e5c890898131ca682d4449d6 /tests | |
parent | 928b56eb0d54f08ee292f582d07101c89d6fa3aa (diff) |
open-behind: rewrite of internal logic
There was a critical flaw in the previous implementation of open-behind.
When an open is done in the background, it's necessary to take a
reference on the fd_t object because once we "fake" the open answer,
the fd could be destroyed. However as long as there's a reference,
the release function won't be called. So, if the application closes
the file descriptor without having actually opened it, there will
always remain at least 1 reference, causing a leak.
To avoid this problem, the previous implementation didn't take a
reference on the fd_t, so there were races where the fd could be
destroyed while it was still in use.
To fix this, I've implemented a new xlator cbk that gets called from
fuse when the application closes a file descriptor.
The whole logic of handling background opens have been simplified and
it's more efficient now. Only if the fop needs to be delayed until an
open completes, a stub is created. Otherwise no memory allocations are
needed.
Correctly handling the close request while the open is still pending
has added a bit of complexity, but overall normal operation is simpler.
Change-Id: I6376a5491368e0e1c283cc452849032636261592
Fixes: #1225
Signed-off-by: Xavi Hernandez <xhernandez@redhat.com>
Diffstat (limited to 'tests')
-rw-r--r-- | tests/basic/open-behind/open-behind.t | 183 | ||||
-rw-r--r-- | tests/basic/open-behind/tester-fd.c | 99 | ||||
-rw-r--r-- | tests/basic/open-behind/tester.c | 444 | ||||
-rw-r--r-- | tests/basic/open-behind/tester.h | 145 | ||||
-rw-r--r-- | tests/bugs/glusterfs/bug-873962-spb.t | 1 |
5 files changed, 872 insertions, 0 deletions
diff --git a/tests/basic/open-behind/open-behind.t b/tests/basic/open-behind/open-behind.t new file mode 100644 index 00000000000..5e865d602e2 --- /dev/null +++ b/tests/basic/open-behind/open-behind.t @@ -0,0 +1,183 @@ +#!/bin/bash + +WD="$(dirname "${0}")" + +. ${WD}/../../include.rc +. ${WD}/../../volume.rc + +function assign() { + local _assign_var="${1}" + local _assign_value="${2}" + + printf -v "${_assign_var}" "%s" "${_assign_value}" +} + +function pipe_create() { + local _pipe_create_var="${1}" + local _pipe_create_name + local _pipe_create_fd + + _pipe_create_name="$(mktemp -u)" + mkfifo "${_pipe_create_name}" + exec {_pipe_create_fd}<>"${_pipe_create_name}" + rm "${_pipe_create_name}" + + assign "${_pipe_create_var}" "${_pipe_create_fd}" +} + +function pipe_close() { + local _pipe_close_fd="${!1}" + + exec {_pipe_close_fd}>&- +} + +function tester_start() { + declare -ag tester + local tester_in + local tester_out + + pipe_create tester_in + pipe_create tester_out + + ${WD}/tester <&${tester_in} >&${tester_out} & + + tester=("$!" "${tester_in}" "${tester_out}") +} + +function tester_send() { + declare -ag tester + local tester_res + local tester_extra + + echo "${*}" >&${tester[1]} + + read -t 3 -u ${tester[2]} tester_res tester_extra + echo "${tester_res} ${tester_extra}" + if [[ "${tester_res}" == "OK" ]]; then + return 0 + fi + + return 1 +} + +function tester_stop() { + declare -ag tester + local tester_res + + tester_send "quit" + + tester_res=0 + if ! wait ${tester[0]}; then + tester_res=$? + fi + + unset tester + + return ${tester_res} +} + +function count_open() { + local file="$(realpath "${B0}/${V0}/${1}")" + local count="0" + local inode + local ref + + inode="$(stat -c %i "${file}")" + + for fd in /proc/${BRICK_PID}/fd/*; do + ref="$(readlink "${fd}")" + if [[ "${ref}" == "${B0}/${V0}/"* ]]; then + if [[ "$(stat -c %i "${ref}")" == "${inode}" ]]; then + count="$((${count} + 1))" + fi + fi + done + + echo "${count}" +} + +cleanup + +TEST build_tester ${WD}/tester.c ${WD}/tester-fd.c + +TEST glusterd +TEST pidof glusterd +TEST ${CLI} volume create ${V0} ${H0}:${B0}/${V0} +TEST ${CLI} volume set ${V0} flush-behind off +TEST ${CLI} volume set ${V0} write-behind off +TEST ${CLI} volume set ${V0} quick-read off +TEST ${CLI} volume set ${V0} stat-prefetch on +TEST ${CLI} volume set ${V0} io-cache off +TEST ${CLI} volume set ${V0} open-behind on +TEST ${CLI} volume set ${V0} lazy-open off +TEST ${CLI} volume set ${V0} read-after-open off +TEST ${CLI} volume start ${V0} + +TEST ${GFS} --volfile-id=/${V0} --volfile-server=${H0} ${M0}; + +BRICK_PID="$(get_brick_pid ${V0} ${H0} ${B0}/${V0})" + +TEST touch "${M0}/test" + +EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0 +TEST ${GFS} --volfile-id=/${V0} --volfile-server=${H0} ${M0}; + +TEST tester_start + +TEST tester_send fd open 0 "${M0}/test" +EXPECT_WITHIN 5 "1" count_open "/test" +TEST tester_send fd close 0 +EXPECT_WITHIN 5 "0" count_open "/test" + +EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0 +TEST ${CLI} volume set ${V0} lazy-open on +TEST ${GFS} --volfile-id=/${V0} --volfile-server=${H0} ${M0}; + +TEST tester_send fd open 0 "${M0}/test" +sleep 2 +EXPECT "0" count_open "/test" +TEST tester_send fd write 0 "test" +EXPECT "1" count_open "/test" +TEST tester_send fd close 0 +EXPECT_WITHIN 5 "0" count_open "/test" + +EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0 +TEST ${GFS} --volfile-id=/${V0} --volfile-server=${H0} ${M0}; + +TEST tester_send fd open 0 "${M0}/test" +EXPECT "0" count_open "/test" +EXPECT "test" tester_send fd read 0 64 +# Even though read-after-open is disabled, use-anonymous-fd is also disabled, +# so reads need to open the file first. +EXPECT "1" count_open "/test" +TEST tester_send fd close 0 +EXPECT "0" count_open "/test" + +EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0 +TEST ${GFS} --volfile-id=/${V0} --volfile-server=${H0} ${M0}; + +TEST tester_send fd open 0 "${M0}/test" +EXPECT "0" count_open "/test" +TEST tester_send fd open 1 "${M0}/test" +EXPECT "2" count_open "/test" +TEST tester_send fd close 0 +EXPECT_WITHIN 5 "1" count_open "/test" +TEST tester_send fd close 1 +EXPECT_WITHIN 5 "0" count_open "/test" + +EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0 +TEST ${CLI} volume set ${V0} read-after-open on +TEST ${GFS} --volfile-id=/${V0} --volfile-server=${H0} ${M0}; + +TEST tester_send fd open 0 "${M0}/test" +EXPECT "0" count_open "/test" +EXPECT "test" tester_send fd read 0 64 +EXPECT "1" count_open "/test" +TEST tester_send fd close 0 +EXPECT_WITHIN 5 "0" count_open "/test" + +EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0 + +TEST tester_stop + +cleanup diff --git a/tests/basic/open-behind/tester-fd.c b/tests/basic/open-behind/tester-fd.c new file mode 100644 index 00000000000..00f02bc5b0a --- /dev/null +++ b/tests/basic/open-behind/tester-fd.c @@ -0,0 +1,99 @@ +/* + Copyright (c) 2020 Red Hat, Inc. <http://www.redhat.com> + This file is part of GlusterFS. + + This file is licensed to you under your choice of the GNU Lesser + General Public License, version 3 or any later version (LGPLv3 or + later), or the GNU General Public License, version 2 (GPLv2), in all + cases as published by the Free Software Foundation. +*/ + +#include "tester.h" + +#include <stdlib.h> +#include <unistd.h> +#include <sys/types.h> +#include <sys/stat.h> +#include <fcntl.h> +#include <string.h> +#include <ctype.h> +#include <errno.h> + +static int32_t +fd_open(context_t *ctx, command_t *cmd) +{ + obj_t *obj; + int32_t fd; + + obj = cmd->args[0].obj.ref; + + fd = open(cmd->args[1].str.data, O_RDWR); + if (fd < 0) { + return error(errno, "open() failed"); + } + + obj->type = OBJ_TYPE_FD; + obj->fd = fd; + + out_ok("%d", fd); + + return 0; +} + +static int32_t +fd_close(context_t *ctx, command_t *cmd) +{ + obj_t *obj; + + obj = cmd->args[0].obj.ref; + obj->type = OBJ_TYPE_NONE; + + if (close(obj->fd) != 0) { + return error(errno, "close() failed"); + } + + out_ok(); + + return 0; +} + +static int32_t +fd_write(context_t *ctx, command_t *cmd) +{ + ssize_t len, ret; + + len = strlen(cmd->args[1].str.data); + ret = write(cmd->args[0].obj.ref->fd, cmd->args[1].str.data, len); + if (ret < 0) { + return error(errno, "write() failed"); + } + + out_ok("%zd", ret); + + return 0; +} + +static int32_t +fd_read(context_t *ctx, command_t *cmd) +{ + char data[cmd->args[1].num.value + 1]; + ssize_t ret; + + ret = read(cmd->args[0].obj.ref->fd, data, cmd->args[1].num.value); + if (ret < 0) { + return error(errno, "read() failed"); + } + + data[ret] = 0; + + out_ok("%zd %s", ret, data); + + return 0; +} + +command_t fd_commands[] = { + {"open", fd_open, CMD_ARGS(ARG_VAL(OBJ_TYPE_NONE), ARG_STR(1024))}, + {"close", fd_close, CMD_ARGS(ARG_VAL(OBJ_TYPE_FD))}, + {"write", fd_write, CMD_ARGS(ARG_VAL(OBJ_TYPE_FD), ARG_STR(1024))}, + {"read", fd_read, CMD_ARGS(ARG_VAL(OBJ_TYPE_FD), ARG_NUM(0, 1024))}, + CMD_END}; diff --git a/tests/basic/open-behind/tester.c b/tests/basic/open-behind/tester.c new file mode 100644 index 00000000000..b2da71c8385 --- /dev/null +++ b/tests/basic/open-behind/tester.c @@ -0,0 +1,444 @@ +/* + Copyright (c) 2020 Red Hat, Inc. <http://www.redhat.com> + This file is part of GlusterFS. + + This file is licensed to you under your choice of the GNU Lesser + General Public License, version 3 or any later version (LGPLv3 or + later), or the GNU General Public License, version 2 (GPLv2), in all + cases as published by the Free Software Foundation. +*/ + +#include "tester.h" + +#include <stdlib.h> +#include <unistd.h> +#include <string.h> +#include <ctype.h> +#include <errno.h> + +static void * +mem_alloc(size_t size) +{ + void *ptr; + + ptr = malloc(size); + if (ptr == NULL) { + error(ENOMEM, "Failed to allocate memory (%zu bytes)", size); + } + + return ptr; +} + +static void +mem_free(void *ptr) +{ + free(ptr); +} + +static bool +buffer_create(context_t *ctx, size_t size) +{ + ctx->buffer.base = mem_alloc(size); + if (ctx->buffer.base == NULL) { + return false; + } + + ctx->buffer.size = size; + ctx->buffer.len = 0; + ctx->buffer.pos = 0; + + return true; +} + +static void +buffer_destroy(context_t *ctx) +{ + mem_free(ctx->buffer.base); + ctx->buffer.size = 0; + ctx->buffer.len = 0; +} + +static int32_t +buffer_get(context_t *ctx) +{ + ssize_t len; + + if (ctx->buffer.pos >= ctx->buffer.len) { + len = read(0, ctx->buffer.base, ctx->buffer.size); + if (len < 0) { + return error(errno, "read() failed"); + } + if (len == 0) { + return 0; + } + + ctx->buffer.len = len; + ctx->buffer.pos = 0; + } + + return ctx->buffer.base[ctx->buffer.pos++]; +} + +static int32_t +str_skip_spaces(context_t *ctx, int32_t current) +{ + while ((current > 0) && (current != '\n') && isspace(current)) { + current = buffer_get(ctx); + } + + return current; +} + +static int32_t +str_token(context_t *ctx, char *buffer, uint32_t size, int32_t current) +{ + uint32_t len; + + current = str_skip_spaces(ctx, current); + + len = 0; + while ((size > 0) && (current > 0) && (current != '\n') && + !isspace(current)) { + len++; + *buffer++ = current; + size--; + current = buffer_get(ctx); + } + + if (len == 0) { + return error(ENODATA, "Expecting a token"); + } + + if (size == 0) { + return error(ENOBUFS, "Token too long"); + } + + *buffer = 0; + + return current; +} + +static int32_t +str_number(context_t *ctx, uint64_t min, uint64_t max, uint64_t *value, + int32_t current) +{ + char text[32], *ptr; + uint64_t num; + + current = str_token(ctx, text, sizeof(text), current); + if (current > 0) { + num = strtoul(text, &ptr, 0); + if ((*ptr != 0) || (num < min) || (num > max)) { + return error(ERANGE, "Invalid number"); + } + *value = num; + } + + return current; +} + +static int32_t +str_eol(context_t *ctx, int32_t current) +{ + current = str_skip_spaces(ctx, current); + if (current != '\n') { + return error(EINVAL, "Expecting end of command"); + } + + return current; +} + +static void +str_skip(context_t *ctx, int32_t current) +{ + while ((current > 0) && (current != '\n')) { + current = buffer_get(ctx); + } +} + +static int32_t +cmd_parse_obj(context_t *ctx, arg_t *arg, int32_t current) +{ + obj_t *obj; + uint64_t id; + + current = str_number(ctx, 0, ctx->obj_count, &id, current); + if (current <= 0) { + return current; + } + + obj = &ctx->objs[id]; + if (obj->type != arg->obj.type) { + if (obj->type != OBJ_TYPE_NONE) { + return error(EBUSY, "Object is in use"); + } + return error(ENOENT, "Object is not defined"); + } + + arg->obj.ref = obj; + + return current; +} + +static int32_t +cmd_parse_num(context_t *ctx, arg_t *arg, int32_t current) +{ + return str_number(ctx, arg->num.min, arg->num.max, &arg->num.value, + current); +} + +static int32_t +cmd_parse_str(context_t *ctx, arg_t *arg, int32_t current) +{ + return str_token(ctx, arg->str.data, arg->str.size, current); +} + +static int32_t +cmd_parse_args(context_t *ctx, command_t *cmd, int32_t current) +{ + arg_t *arg; + + for (arg = cmd->args; arg->type != ARG_TYPE_NONE; arg++) { + switch (arg->type) { + case ARG_TYPE_OBJ: + current = cmd_parse_obj(ctx, arg, current); + break; + case ARG_TYPE_NUM: + current = cmd_parse_num(ctx, arg, current); + break; + case ARG_TYPE_STR: + current = cmd_parse_str(ctx, arg, current); + break; + default: + return error(EINVAL, "Unknown argument type"); + } + } + + if (current < 0) { + return current; + } + + current = str_eol(ctx, current); + if (current <= 0) { + return error(EINVAL, "Syntax error"); + } + + return cmd->handler(ctx, cmd); +} + +static int32_t +cmd_parse(context_t *ctx, command_t *cmds) +{ + char text[32]; + command_t *cmd; + int32_t current; + + cmd = cmds; + do { + current = str_token(ctx, text, sizeof(text), buffer_get(ctx)); + if (current <= 0) { + return current; + } + + while (cmd->name != NULL) { + if (strcmp(cmd->name, text) == 0) { + if (cmd->handler != NULL) { + return cmd_parse_args(ctx, cmd, current); + } + cmd = cmd->cmds; + break; + } + cmd++; + } + } while (cmd->name != NULL); + + str_skip(ctx, current); + + return error(ENOTSUP, "Unknown command"); +} + +static void +cmd_fini(context_t *ctx, command_t *cmds) +{ + command_t *cmd; + arg_t *arg; + + for (cmd = cmds; cmd->name != NULL; cmd++) { + if (cmd->handler == NULL) { + cmd_fini(ctx, cmd->cmds); + } else { + for (arg = cmd->args; arg->type != ARG_TYPE_NONE; arg++) { + switch (arg->type) { + case ARG_TYPE_STR: + mem_free(arg->str.data); + arg->str.data = NULL; + break; + default: + break; + } + } + } + } +} + +static bool +cmd_init(context_t *ctx, command_t *cmds) +{ + command_t *cmd; + arg_t *arg; + + for (cmd = cmds; cmd->name != NULL; cmd++) { + if (cmd->handler == NULL) { + if (!cmd_init(ctx, cmd->cmds)) { + return false; + } + } else { + for (arg = cmd->args; arg->type != ARG_TYPE_NONE; arg++) { + switch (arg->type) { + case ARG_TYPE_STR: + arg->str.data = mem_alloc(arg->str.size); + if (arg->str.data == NULL) { + return false; + } + break; + default: + break; + } + } + } + } + + return true; +} + +static bool +objs_create(context_t *ctx, uint32_t count) +{ + uint32_t i; + + ctx->objs = mem_alloc(sizeof(obj_t) * count); + if (ctx->objs == NULL) { + return false; + } + ctx->obj_count = count; + + for (i = 0; i < count; i++) { + ctx->objs[i].type = OBJ_TYPE_NONE; + } + + return true; +} + +static int32_t +objs_destroy(context_t *ctx) +{ + uint32_t i; + int32_t err; + + err = 0; + for (i = 0; i < ctx->obj_count; i++) { + if (ctx->objs[i].type != OBJ_TYPE_NONE) { + err = error(ENOTEMPTY, "Objects not destroyed"); + break; + } + } + + mem_free(ctx->objs); + ctx->objs = NULL; + ctx->obj_count = 0; + + return err; +} + +static context_t * +init(size_t size, uint32_t objs, command_t *cmds) +{ + context_t *ctx; + + ctx = mem_alloc(sizeof(context_t)); + if (ctx == NULL) { + goto failed; + } + + if (!buffer_create(ctx, size)) { + goto failed_ctx; + } + + if (!objs_create(ctx, objs)) { + goto failed_buffer; + } + + if (!cmd_init(ctx, cmds)) { + goto failed_objs; + } + + ctx->active = true; + + return ctx; + +failed_objs: + cmd_fini(ctx, cmds); + objs_destroy(ctx); +failed_buffer: + buffer_destroy(ctx); +failed_ctx: + mem_free(ctx); +failed: + return NULL; +} + +static int32_t +fini(context_t *ctx, command_t *cmds) +{ + int32_t ret; + + cmd_fini(ctx, cmds); + buffer_destroy(ctx); + + ret = objs_destroy(ctx); + + ctx->active = false; + + return ret; +} + +static int32_t +exec_quit(context_t *ctx, command_t *cmd) +{ + ctx->active = false; + + return 0; +} + +static command_t commands[] = {{"fd", NULL, CMD_SUB(fd_commands)}, + {"quit", exec_quit, CMD_ARGS()}, + CMD_END}; + +int32_t +main(int32_t argc, char *argv[]) +{ + context_t *ctx; + int32_t res; + + ctx = init(1024, 16, commands); + if (ctx == NULL) { + return 1; + } + + do { + res = cmd_parse(ctx, commands); + if (res < 0) { + out_err(-res); + } + } while (ctx->active); + + res = fini(ctx, commands); + if (res >= 0) { + out_ok(); + return 0; + } + + out_err(-res); + + return 1; +} diff --git a/tests/basic/open-behind/tester.h b/tests/basic/open-behind/tester.h new file mode 100644 index 00000000000..64e940c78fc --- /dev/null +++ b/tests/basic/open-behind/tester.h @@ -0,0 +1,145 @@ +/* + Copyright (c) 2020 Red Hat, Inc. <http://www.redhat.com> + This file is part of GlusterFS. + + This file is licensed to you under your choice of the GNU Lesser + General Public License, version 3 or any later version (LGPLv3 or + later), or the GNU General Public License, version 2 (GPLv2), in all + cases as published by the Free Software Foundation. +*/ + +#ifndef __TESTER_H__ +#define __TESTER_H__ + +#include <stdio.h> +#include <inttypes.h> +#include <stdbool.h> + +enum _obj_type; +typedef enum _obj_type obj_type_t; + +enum _arg_type; +typedef enum _arg_type arg_type_t; + +struct _buffer; +typedef struct _buffer buffer_t; + +struct _obj; +typedef struct _obj obj_t; + +struct _context; +typedef struct _context context_t; + +struct _arg; +typedef struct _arg arg_t; + +struct _command; +typedef struct _command command_t; + +enum _obj_type { OBJ_TYPE_NONE, OBJ_TYPE_FD }; + +enum _arg_type { ARG_TYPE_NONE, ARG_TYPE_OBJ, ARG_TYPE_NUM, ARG_TYPE_STR }; + +struct _buffer { + char *base; + uint32_t size; + uint32_t len; + uint32_t pos; +}; + +struct _obj { + obj_type_t type; + union { + int32_t fd; + }; +}; + +struct _context { + obj_t *objs; + buffer_t buffer; + uint32_t obj_count; + bool active; +}; + +struct _arg { + arg_type_t type; + union { + struct { + obj_type_t type; + obj_t *ref; + } obj; + struct { + uint64_t value; + uint64_t min; + uint64_t max; + } num; + struct { + uint32_t size; + char *data; + } str; + }; +}; + +struct _command { + const char *name; + int32_t (*handler)(context_t *ctx, command_t *cmd); + union { + arg_t *args; + command_t *cmds; + }; +}; + +#define msg(_stream, _fmt, _args...) \ + do { \ + fprintf(_stream, _fmt "\n", ##_args); \ + fflush(_stream); \ + } while (0) + +#define msg_out(_fmt, _args...) msg(stdout, _fmt, ##_args) +#define msg_err(_err, _fmt, _args...) \ + ({ \ + int32_t __msg_err = (_err); \ + msg(stderr, "[%4u:%-15s] " _fmt, __LINE__, __FUNCTION__, __msg_err, \ + ##_args); \ + -__msg_err; \ + }) + +#define error(_err, _fmt, _args...) msg_err(_err, "E(%4d) " _fmt, ##_args) +#define warn(_err, _fmt, _args...) msg_err(_err, "W(%4d) " _fmt, ##_args) +#define info(_err, _fmt, _args...) msg_err(_err, "I(%4d) " _fmt, ##_args) + +#define out_ok(_args...) msg_out("OK " _args) +#define out_err(_err) msg_out("ERR %d", _err) + +#define ARG_END \ + { \ + ARG_TYPE_NONE \ + } + +#define CMD_ARGS1(_x, _args...) \ + .args = (arg_t[]) { _args } +#define CMD_ARGS(_args...) CMD_ARGS1(, ##_args, ARG_END) + +#define CMD_SUB(_cmds) .cmds = _cmds + +#define CMD_END \ + { \ + NULL, NULL, CMD_SUB(NULL) \ + } + +#define ARG_VAL(_type) \ + { \ + ARG_TYPE_OBJ, .obj = {.type = _type } \ + } +#define ARG_NUM(_min, _max) \ + { \ + ARG_TYPE_NUM, .num = {.min = _min, .max = _max } \ + } +#define ARG_STR(_size) \ + { \ + ARG_TYPE_STR, .str = {.size = _size } \ + } + +extern command_t fd_commands[]; + +#endif /* __TESTER_H__ */
\ No newline at end of file diff --git a/tests/bugs/glusterfs/bug-873962-spb.t b/tests/bugs/glusterfs/bug-873962-spb.t index db84a223089..db71cc0f6fe 100644 --- a/tests/bugs/glusterfs/bug-873962-spb.t +++ b/tests/bugs/glusterfs/bug-873962-spb.t @@ -14,6 +14,7 @@ TEST $CLI volume set $V0 performance.io-cache off TEST $CLI volume set $V0 performance.write-behind off TEST $CLI volume set $V0 performance.stat-prefetch off TEST $CLI volume set $V0 performance.read-ahead off +TEST $CLI volume set $V0 performance.open-behind off TEST $CLI volume set $V0 cluster.background-self-heal-count 0 TEST $CLI volume start $V0 TEST glusterfs --entry-timeout=0 --attribute-timeout=0 -s $H0 --volfile-id=$V0 $M0 --direct-io-mode=enable |