From e311014d94c7fcfd8c5e7abe3aca7d99f43e0d91 Mon Sep 17 00:00:00 2001 From: "Kaleb S. KEITHLEY" Date: Tue, 8 Jul 2014 08:25:36 -0400 Subject: nfs: prevent assertion error with MOUNT over UDP see http://review.gluster.org/#/c/8258/, http://review.gluster.org/8241; and BZs 1116997, 1116503, respectively. The MOUNT service over UDP runs in a separate thread. This thread does not have the correct *THIS xlator set. *THIS points to the global (base) xlator structure, but GF_CALLOC() requires it to be the NFS-xlator so that assertions can get validated correctly. This is solved by passing the NFS-xlator to the pthread function, and setting the *THIS pointer explicitly in the new thread. It seems that on occasion (needs further investigation) MOUNT over UDP does not unregister itself. There can also be issues when the kernel NLM implementation has been registered at portmap/rpcbind, so adding some unregister procedures in the cleanup of the test-cases. Change-Id: Iea3bafd65bf480f2b12da91875c73571594d52d3 BUG: 1117256 Signed-off-by: Kaleb S. KEITHLEY Reviewed-on: http://review.gluster.org/8262 Tested-by: Gluster Build System Reviewed-by: Niels de Vos Reviewed-by: Santosh Pradhan --- tests/bugs/bug-1116503.t | 24 ++++++++++++++++++++++++ tests/include.rc | 15 +++++++++++++++ tests/nfs.rc | 27 +++++++++++++++++++++++++++ tests/volume.rc | 6 ++++++ xlators/nfs/server/src/mount3.c | 2 +- xlators/nfs/server/src/mount3udp_svc.c | 9 +++++++++ 6 files changed, 82 insertions(+), 1 deletion(-) create mode 100644 tests/bugs/bug-1116503.t create mode 100644 tests/nfs.rc diff --git a/tests/bugs/bug-1116503.t b/tests/bugs/bug-1116503.t new file mode 100644 index 00000000000..0aad440560a --- /dev/null +++ b/tests/bugs/bug-1116503.t @@ -0,0 +1,24 @@ +#!/bin/bash +# +# Verify that mounting NFS over UDP (MOUNT service only) works. +# + +. $(dirname $0)/../include.rc +. $(dirname $0)/../volume.rc +. $(dirname $0)/../nfs.rc + + +cleanup; +TEST glusterd +TEST pidof glusterd + +TEST $CLI volume create $V0 $H0:$B0/$V0 +TEST $CLI volume set $V0 nfs.mount-udp on + +TEST $CLI volume start $V0 +EXPECT_WITHIN $NFS_EXPORT_TIMEOUT "1" is_nfs_export_available; + +TEST mount_nfs $H0:/$V0 $N0 nolock,mountproto=udp,proto=tcp; +EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $N0 + +cleanup; diff --git a/tests/include.rc b/tests/include.rc index bd64ed9a1a4..54427dbda50 100644 --- a/tests/include.rc +++ b/tests/include.rc @@ -6,6 +6,10 @@ V0=${V0:=patchy}; # volume name to use in tests B0=${B0:=/d/backends}; # top level of brick directories H0=${H0:=`hostname --fqdn`}; # hostname DEBUG=${DEBUG:=0} # turn on debugging? + +NFS_EXPORT_TIMEOUT=20 +UMOUNT_TIMEOUT=5 + statedumpdir=`gluster --print-statedumpdir`; # Default directory for statedump CLI="gluster --mode=script"; @@ -192,6 +196,17 @@ function cleanup() umount $m done + # unregister nfs and related services from portmapper/rpcbind + ## nfs + rpcinfo -d 100003 3 + ## mountd + rpcinfo -d 100005 1 + rpcinfo -d 100005 3 + ## nlockmgr + rpcinfo -d 100021 1 + rpcinfo -d 100021 4 + ## nfs_acl + rpcinfo -d 100027 3 LOOPDEVICES=`losetup -a | grep "$B0/" | awk '{print $1}' | tr -d :` for l in $LOOPDEVICES; diff --git a/tests/nfs.rc b/tests/nfs.rc new file mode 100644 index 00000000000..c27452395e1 --- /dev/null +++ b/tests/nfs.rc @@ -0,0 +1,27 @@ +#!/bin/bash + +# Due to portmap registration NFS takes some time to +# export all volumes. Therefore tests should start only +# after exports are visible by showmount command. This +# routine will check if showmount shows the exports or not +# +function is_nfs_export_available () +{ + local vol=$1 + + if [ "$vol" == "" ]; then + vol=$V0 + fi + + exp=$(showmount -e localhost 2> /dev/null | grep $vol | wc -l) + echo "$exp" +} + +function mount_nfs () +{ + local e=$1 + local m=$2 + local opt=$3 + if [ ! -z "$opt" ]; then opt=",$opt"; fi + mount -t nfs -o soft,intr,vers=3"$opt" $e $m +} diff --git a/tests/volume.rc b/tests/volume.rc index d942e485fa5..c3d51d9c2a6 100644 --- a/tests/volume.rc +++ b/tests/volume.rc @@ -210,3 +210,9 @@ function dht_get_layout { local my_xa=trusted.glusterfs.dht getfattr -d -e hex -n $my_xa $1 2> /dev/null | grep "$my_xa=" | cut -d= -f2 } + +function force_umount { + umount -f $1 + if [ $? -eq 0 ]; then echo "Y"; else echo "N"; fi +} + diff --git a/xlators/nfs/server/src/mount3.c b/xlators/nfs/server/src/mount3.c index cab6184f636..c4884e04fc0 100644 --- a/xlators/nfs/server/src/mount3.c +++ b/xlators/nfs/server/src/mount3.c @@ -1922,7 +1922,7 @@ mnt3svc_init (xlator_t *nfsx) } if (nfs->mount_udp) { - pthread_create (&udp_thread, NULL, mount3udp_thread, NULL); + pthread_create (&udp_thread, NULL, mount3udp_thread, nfsx); } return &mnt3prog; err: diff --git a/xlators/nfs/server/src/mount3udp_svc.c b/xlators/nfs/server/src/mount3udp_svc.c index fb59e282c8e..e49d50e9999 100644 --- a/xlators/nfs/server/src/mount3udp_svc.c +++ b/xlators/nfs/server/src/mount3udp_svc.c @@ -170,8 +170,17 @@ mountudp_program_3(struct svc_req *rqstp, register SVCXPRT *transp) void * mount3udp_thread (void *argv) { + xlator_t *nfsx = NULL; register SVCXPRT *transp = NULL; + GF_ASSERT (nfsx); + + if (glusterfs_this_set(nfsx)) { + gf_log (GF_MNT, GF_LOG_ERROR, "failed to set xlator, " + "nfs.mount-udp will not work"); + return NULL; + } + transp = svcudp_create(RPC_ANYSOCK); if (transp == NULL) { gf_log (GF_MNT, GF_LOG_ERROR, "svcudp_create error"); -- cgit