nfs: prevent assertion error with MOUNT over UDPv3.4.5beta2

see http://review.gluster.org/#/c/8258/, http://review.gluster.org/8241;
and BZs 1116997, 1116503, respectively.

The MOUNT service over UDP runs in a separate thread. This thread does
not have the correct *THIS xlator set. *THIS points to the global (base)
xlator structure, but GF_CALLOC() requires it to be the NFS-xlator so
that assertions can get validated correctly.

This is solved by passing the NFS-xlator to the pthread function, and
setting the *THIS pointer explicitly in the new thread.

It seems that on occasion (needs further investigation) MOUNT over UDP
does not unregister itself. There can also be issues when the kernel NLM
implementation has been registered at portmap/rpcbind, so adding some
unregister procedures in the cleanup of the test-cases.

Change-Id: Iea3bafd65bf480f2b12da91875c73571594d52d3
BUG: 1117256
Signed-off-by: Kaleb S. KEITHLEY <kkeithle@redhat.com>
Reviewed-on: http://review.gluster.org/8262
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Niels de Vos <ndevos@redhat.com>
Reviewed-by: Santosh Pradhan <spradhan@redhat.com>

6 files changed, 82 insertions, 1 deletions

diff --git a/tests/bugs/bug-1116503.t b/tests/bugs/bug-1116503.t
new file mode 100644
index 00000000000..0aad440560a
--- /dev/null
+++ b/tests/bugs/bug-1116503.t
@@ -0,0 +1,24 @@
+#!/bin/bash
+#
+# Verify that mounting NFS over UDP (MOUNT service only) works.
+#
+
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../volume.rc
+. $(dirname $0)/../nfs.rc
+
+
+cleanup;
+TEST glusterd
+TEST pidof glusterd
+
+TEST $CLI volume create $V0 $H0:$B0/$V0
+TEST $CLI volume set $V0 nfs.mount-udp on
+
+TEST $CLI volume start $V0
+EXPECT_WITHIN $NFS_EXPORT_TIMEOUT "1" is_nfs_export_available;
+
+TEST mount_nfs $H0:/$V0 $N0 nolock,mountproto=udp,proto=tcp;
+EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $N0
+
+cleanup;
diff --git a/tests/include.rc b/tests/include.rc
index bd64ed9a1a4..54427dbda50 100644
--- a/tests/include.rc
+++ b/tests/include.rc
@@ -6,6 +6,10 @@ V0=${V0:=patchy};             # volume name to use in tests
 B0=${B0:=/d/backends};        # top level of brick directories
 H0=${H0:=`hostname --fqdn`};  # hostname
 DEBUG=${DEBUG:=0}             # turn on debugging?
+
+NFS_EXPORT_TIMEOUT=20
+UMOUNT_TIMEOUT=5
+
 statedumpdir=`gluster --print-statedumpdir`; # Default directory for statedump
 
 CLI="gluster --mode=script";
@@ -192,6 +196,17 @@ function cleanup()
                 umount $m
         done
 
+        # unregister nfs and related services from portmapper/rpcbind
+        ## nfs
+        rpcinfo -d 100003 3
+        ## mountd
+        rpcinfo -d 100005 1
+        rpcinfo -d 100005 3
+        ## nlockmgr
+        rpcinfo -d 100021 1
+        rpcinfo -d 100021 4
+        ## nfs_acl
+        rpcinfo -d 100027 3
 
         LOOPDEVICES=`losetup -a | grep "$B0/" | awk '{print $1}' | tr -d :`
         for l in $LOOPDEVICES;
diff --git a/tests/nfs.rc b/tests/nfs.rc
new file mode 100644
index 00000000000..c27452395e1
--- /dev/null
+++ b/tests/nfs.rc
@@ -0,0 +1,27 @@
+#!/bin/bash
+
+# Due to portmap registration NFS takes some time to
+# export all volumes. Therefore tests should start only
+# after exports are visible by showmount command. This
+# routine will check if showmount shows the exports or not
+#
+function is_nfs_export_available ()
+{
+        local vol=$1
+
+        if [ "$vol" == "" ]; then
+                vol=$V0
+        fi
+
+        exp=$(showmount -e localhost 2> /dev/null | grep $vol | wc -l)
+        echo "$exp"
+}
+
+function mount_nfs ()
+{
+        local e=$1
+        local m=$2
+        local opt=$3
+        if [ ! -z "$opt" ]; then opt=",$opt"; fi
+        mount -t nfs -o soft,intr,vers=3"$opt" $e $m
+}
diff --git a/tests/volume.rc b/tests/volume.rc
index d942e485fa5..c3d51d9c2a6 100644
--- a/tests/volume.rc
+++ b/tests/volume.rc
@@ -210,3 +210,9 @@ function dht_get_layout {
 	local my_xa=trusted.glusterfs.dht
 	getfattr -d -e hex -n $my_xa $1 2> /dev/null | grep "$my_xa=" | cut -d= -f2
 }
+
+function force_umount {
+        umount -f $1
+        if [ $? -eq 0 ]; then echo "Y"; else echo "N"; fi
+}
+
diff --git a/xlators/nfs/server/src/mount3.c b/xlators/nfs/server/src/mount3.c
index cab6184f636..c4884e04fc0 100644
--- a/xlators/nfs/server/src/mount3.c
+++ b/xlators/nfs/server/src/mount3.c
@@ -1922,7 +1922,7 @@ mnt3svc_init (xlator_t *nfsx)
         }
 
         if (nfs->mount_udp) {
-                pthread_create (&udp_thread, NULL, mount3udp_thread, NULL);
+                pthread_create (&udp_thread, NULL, mount3udp_thread, nfsx);
         }
         return &mnt3prog;
 err:
diff --git a/xlators/nfs/server/src/mount3udp_svc.c b/xlators/nfs/server/src/mount3udp_svc.c
index fb59e282c8e..e49d50e9999 100644
--- a/xlators/nfs/server/src/mount3udp_svc.c
+++ b/xlators/nfs/server/src/mount3udp_svc.c
@@ -170,8 +170,17 @@ mountudp_program_3(struct svc_req *rqstp, register SVCXPRT *transp)
 void *
 mount3udp_thread (void *argv)
 {
+        xlator_t         *nfsx   = NULL;
         register SVCXPRT *transp = NULL;
 
+        GF_ASSERT (nfsx);
+
+        if (glusterfs_this_set(nfsx)) {
+                gf_log (GF_MNT, GF_LOG_ERROR, "failed to set xlator, "
+                        "nfs.mount-udp will not work");
+                return NULL;
+        }
+
         transp = svcudp_create(RPC_ANYSOCK);
         if (transp == NULL) {
                 gf_log (GF_MNT, GF_LOG_ERROR, "svcudp_create error");