From c458433041aafb48ae6d6e5fcf3e1e737dc3fda3 Mon Sep 17 00:00:00 2001 From: Jeff Darcy Date: Mon, 8 Feb 2016 13:30:49 -0500 Subject: experimental: add fdl (Full Data Logging) translator NSR needs logging that is different than our existing changelog in several ways: * Full data, not just metadata * Pre-op, not post-op * High performance * Supports the concept of time-bounded "terms" Others (for example EC) might need the same thing. This patch adds such a translator. It also adds code to dump the resulting journals, and to replay them using syncops, plus (very rudimentary) tests for all of the above. Change-Id: I29680a1b4e0a9e7d5a8497fef302c46434b86636 Signed-off-by: Jeff Darcy Reviewed-on: http://review.gluster.org/12450 Smoke: Gluster Build System CentOS-regression: Gluster Build System NetBSD-regression: NetBSD Build System --- tests/features/fdl-overflow.t | 74 +++++++++++++++++++++++++++++++++++++++++++ tests/features/fdl.t | 52 ++++++++++++++++++++++++++++++ tests/features/recon.t | 62 ++++++++++++++++++++++++++++++++++++ 3 files changed, 188 insertions(+) create mode 100644 tests/features/fdl-overflow.t create mode 100644 tests/features/fdl.t create mode 100644 tests/features/recon.t (limited to 'tests/features') diff --git a/tests/features/fdl-overflow.t b/tests/features/fdl-overflow.t new file mode 100644 index 00000000000..d7633a7ca7d --- /dev/null +++ b/tests/features/fdl-overflow.t @@ -0,0 +1,74 @@ +#!/bin/bash + +. $(dirname $0)/../include.rc +. $(dirname $0)/../volume.rc + +log_base=$($CLI --print-logdir) +log_id=${B0}/${V0}-0 +log_id=${log_id:1} # Remove initial slash +log_id=${log_id//\//-} # Replace remaining slashes with dashes + +_check_sizes () { + local n=0 + local sz + local total_sz=0 + + # We don't care about the sizes of the meta files. That would be + # embedding too much of the implementation into the test. + n=$(ls ${log_base}/${log_id}-meta-*.jnl | wc -l) + [ $n = 2 ] || return 1 + + # We *do* care about the sizes of the data files, which should exactly + # reflect the amount of data written via dd. + n=0 + while read sz name; do + G_LOG "found journal ${name} size ${sz}MB" + n=$((n+1)) + total_sz=$((total_sz+sz)) + done < <(du -sm ${log_base}/${log_id}-data-*.jnl) + [ $n = 2 ] || return 1 + # On our CentOS and NetBSD regression-test systems, but not on my Fedora + # development system, each file ends up being slightly larger than its + # data size because of metadata, and 'du' rounds that up to a full extra + # megabyte. We'll allow either result, because what we're really + # looking for is a complete failure to roll over from one file to + # another at the appropriate size. + [ $total_sz = 20 -o $total_sz = $((n+20)) ] || return 1 + + return 0 +} + +check_sizes () { + set -x + _check_sizes + ret=$? + set +x + return ret +} + +if [ x"$OSTYPE" = x"NetBSD" ]; then + CREAT_OFLAG="creat," +else + CREAT_OFLAG="" +fi + +TEST rm -f ${log_base}/${log_id}-*.log +TEST glusterd +TEST pidof glusterd + +# Get a simple volume set up and mounted with FDL active. +TEST $CLI volume create $V0 ${H0}:${B0}/${V0}-0 +TEST $CLI volume set $V0 changelog.changelog off +TEST $CLI volume set $V0 features.fdl on +TEST $CLI volume start $V0 +TEST $GFS -s $H0 --volfile-id $V0 $M0 + +# Generate some I/O and unmount/stop so we can see log sizes. +TEST dd if=/dev/zero of=$M0/twentyMB bs=1048576 count=20 \ + oflag=${CREAT_OFLAG}sync +TEST umount $M0 +TEST $CLI volume stop $V0 + +TEST _check_sizes + +cleanup diff --git a/tests/features/fdl.t b/tests/features/fdl.t new file mode 100644 index 00000000000..34d6d78228a --- /dev/null +++ b/tests/features/fdl.t @@ -0,0 +1,52 @@ +#!/bin/bash + +. $(dirname $0)/../include.rc +. $(dirname $0)/../volume.rc + +log_base=$($CLI --print-logdir) +log_id=${B0}/${V0}-0 +log_id=${log_id:1} # Remove initial slash +log_id=${log_id//\//-} # Replace remaining slashes with dashes +FDL_META_FILE=${log_base}/${log_id}-meta-1.jnl +FDL_DATA_FILE=${log_base}/${log_id}-data-1.jnl + +check_logfile() { + [ $(gf_logdump $FDL_META_FILE $FDL_DATA_FILE | grep $1 | wc -l) -ge $2 ] +} + +if [ x"$OSTYPE" = x"NetBSD" ]; then + CREAT_OFLAG="creat," +else + CREAT_OFLAG="" +fi + +TEST rm -f $FDL_META_FILE $FDL_DATA_FILE +TEST glusterd +TEST pidof glusterd + +# Get a simple volume set up and mounted with FDL active. +TEST $CLI volume create $V0 ${H0}:${B0}/${V0}-0 +TEST $CLI volume set $V0 changelog.changelog off +TEST $CLI volume set $V0 features.fdl on +TEST $CLI volume start $V0 +TEST $GFS -s $H0 --volfile-id $V0 $M0 + +# Generate some I/O and unmount. +TEST mkdir -p $M0/abc/def +TEST dd if=/dev/zero of=$M0/abc/def/ghi bs=128 count=2 \ + oflag=${CREAT_OFLAG}sync +TEST chmod 314 $M0/abc/def/ghi +TEST rm -rf $M0/abc +TEST umount $M0 + +# Check that gf_logdump works, and shows the ops we just issued. There will be +# more SETATTR ops than the one corresponding to our chmod, because some are +# issued internally. We have to guess a bit about where the log will be. +TEST check_logfile GF_FOP_MKDIR 2 +TEST check_logfile GF_FOP_CREATE 1 +TEST check_logfile GF_FOP_WRITE 2 +TEST check_logfile GF_FOP_SETATTR 1 +TEST check_logfile GF_FOP_UNLINK 1 +TEST check_logfile GF_FOP_RMDIR 2 + +cleanup diff --git a/tests/features/recon.t b/tests/features/recon.t new file mode 100644 index 00000000000..7dda2a680e8 --- /dev/null +++ b/tests/features/recon.t @@ -0,0 +1,62 @@ +#!/bin/bash + +. $(dirname $0)/../include.rc +. $(dirname $0)/../volume.rc + +log_base=$($CLI --print-logdir) +log_id=${B0}/${V0}-0 +log_id=${log_id:1} # Remove initial slash +log_id=${log_id//\//-} # Replace remaining slashes with dashes +FDL_META_FILE=${log_base}/${log_id}-meta-1.jnl +FDL_DATA_FILE=${log_base}/${log_id}-data-1.jnl + +tmpdir=$(mktemp -d -t ${0##*/}.XXXXXX) +trap "rm -rf $tmpdir" EXIT + +write_file () { + echo "peekaboo" > $1 +} + +TEST rm -f $FDL_META_FILE $FDL_DATA_FILE +TEST glusterd +TEST pidof glusterd + +# Get a simple volume set up and mounted with FDL active. +TEST $CLI volume create $V0 ${H0}:${B0}/${V0}-0 +TEST $CLI volume set $V0 features.fdl on +TEST $CLI volume start $V0 +TEST $GFS -s $H0 --volfile-id $V0 $M0 + +# Generate some I/O and then copy off the journal files for later. +TEST mkdir -p $M0/abc/def +TEST write_file $M0/abc/def/ghi +#EST chmod 314 $M0/abc/def/ghi +cp ${FDL_META_FILE} ${FDL_DATA_FILE} ${tmpdir} + +# Get back to an empty state and unmount. +TEST rm -rf $M0/abc +TEST umount $M0 + +# Make sure we really are in an empty state. Otherwise the tests below could +# pass just because we never cleaned up in the first place. +TEST [ ! -d ${B0}/${V0}-0/abc ] + +# Create a stub volfile. +vol_file=${GLUSTERD_WORKDIR}/vols/${V0}/${V0}.${H0}.${log_id}.vol +vol_id_line=$(grep volume-id ${vol_file}) +cat > ${tmpdir}/recon.vol << EOF +volume recon-posix + type storage/posix + option directory ${B0}/${V0}-0 +${vol_id_line} +end-volume +EOF + +TEST gf_recon ${tmpdir}/recon.vol ${tmpdir}/$(basename ${FDL_META_FILE}) \ + ${tmpdir}/$(basename ${FDL_DATA_FILE}) + +TEST [ -d ${B0}/${V0}-0/abc/def ] +EXPECT "peekaboo" cat ${B0}/${V0}-0/abc/def/ghi +# TBD: test permissions, xattrs + +cleanup -- cgit