fuse: add "resolve-gids" mount option to overcome 32-groups limit

Add a --resolve-gids commandline option to the glusterfs binary. This
option gets set when executing "mount -t glusterfs -o resolve-gids ...".

This option is most useful in combination with the "acl" mount option.
POSIX ACL permission checking is done on the FUSE-client side to improve
performance (in addition to the checking on the bricks).

The fuse-bridge reads /proc/$PID/status by default, and this file
contains maximum 32 groups. Any local (client-side) permission checking
that requires more than the first 32 groups will fail.

By enabling the "resolve-gids" option, the fuse-bridge will call
getgrouplist() to retrieve all the groups from the user accessing the
mountpoint. This is comparable to how "nfs.server-aux-gids" works.

Note that when a user belongs to more than ~93 groups, the volume option
server.manage-gids needs to be enabled too. Without this option, the
RPC-layer will need to reduce the number of groups to make them fit in
the RPC-header.

Change-Id: I7ede90d0e41bcf55755cced5747fa0fb1699edb2
BUG: 1246275
Signed-off-by: Niels de Vos <ndevos@redhat.com>
Reviewed-on: http://review.gluster.org/11732
Tested-by: NetBSD Build System <jenkins@build.gluster.org>
Reviewed-by: Ravishankar N <ravishankar@redhat.com>
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: jiffin tony Thottan <jthottan@redhat.com>
Reviewed-by: Kaleb KEITHLEY <kkeithle@redhat.com>

9 files changed, 228 insertions, 39 deletions

diff --git a/doc/glusterfsd.8 b/doc/glusterfsd.8
index 176d042367c..88b667cb463 100644
--- a/doc/glusterfsd.8
+++ b/doc/glusterfsd.8
@@ -101,6 +101,9 @@ Set entry timeout to SECONDS in fuse kernel module [default: 1]
 .TP
 \fB\-\-direct\-io\-mode=BOOL\fR
 Enable/Disable direct-io mode in fuse module [default: enable]
+.TP
+\fB\-\-resolve-gids\fR
+Resolve all auxilary groups in fuse translator (max 32 otherwise)
 
 .SS "Miscellaneous Options"
 .PP
diff --git a/glusterfsd/src/glusterfsd.c b/glusterfsd/src/glusterfsd.c
index fb40a1aa43e..6ebcd6ce8c4 100644
--- a/glusterfsd/src/glusterfsd.c
+++ b/glusterfsd/src/glusterfsd.c
@@ -198,6 +198,8 @@ static struct argp_option gf_options[] = {
 	{"gid-timeout", ARGP_GID_TIMEOUT_KEY, "SECONDS", 0,
 	 "Set auxilary group list timeout to SECONDS for fuse translator "
 	 "[default: 300]"},
+        {"resolve-gids", ARGP_RESOLVE_GIDS_KEY, 0, 0,
+         "Resolve all auxilary groups in fuse translator (max 32 otherwise)"},
 	{"background-qlen", ARGP_FUSE_BACKGROUND_QLEN_KEY, "N", 0,
 	 "Set fuse module's background queue length to N "
 	 "[default: 64]"},
@@ -427,6 +429,16 @@ set_fuse_mount_options (glusterfs_ctx_t *ctx, dict_t *options)
 			goto err;
 		}
 	}
+
+        if (cmd_args->resolve_gids) {
+                ret = dict_set_static_ptr (options, "resolve-gids", "on");
+                if (ret < 0) {
+                        gf_msg ("glusterfsd", GF_LOG_ERROR, 0, glusterfsd_msg_4,
+                                "resolve-gids");
+                        goto err;
+                }
+        }
+
 	if (cmd_args->background_qlen) {
 		ret = dict_set_int32 (options, "background-qlen",
                                       cmd_args->background_qlen);
@@ -1075,6 +1087,11 @@ parse_opts (int key, char *arg, struct argp_state *state)
 
 		argp_failure(state, -1, 0, "unknown group list timeout %s", arg);
 		break;
+
+        case ARGP_RESOLVE_GIDS_KEY:
+                cmd_args->resolve_gids = 1;
+                break;
+
         case ARGP_FUSE_BACKGROUND_QLEN_KEY:
                 if (!gf_string2int (arg, &cmd_args->background_qlen))
                         break;
diff --git a/glusterfsd/src/glusterfsd.h b/glusterfsd/src/glusterfsd.h
index f439cef1592..8247469e6e1 100644
--- a/glusterfsd/src/glusterfsd.h
+++ b/glusterfsd/src/glusterfsd.h
@@ -91,6 +91,7 @@ enum argp_option_keys {
         ARGP_LOG_FLUSH_TIMEOUT            = 171,
         ARGP_SECURE_MGMT_KEY              = 172,
         ARGP_GLOBAL_TIMER_WHEEL           = 173,
+        ARGP_RESOLVE_GIDS_KEY             = 174,
 };
 
 struct _gfd_vol_top_priv_t {
diff --git a/libglusterfs/src/glusterfs.h b/libglusterfs/src/glusterfs.h
index 1c1961c1880..3e3d0b02dd1 100644
--- a/libglusterfs/src/glusterfs.h
+++ b/libglusterfs/src/glusterfs.h
@@ -465,6 +465,7 @@ struct _cmd_args {
         int              congestion_threshold;
         char             *fuse_mountopts;
         int              mem_acct;
+        int              resolve_gids;
 
         /* key args */
         char            *mount_point;
diff --git a/tests/bugs/fuse/many-groups-for-acl.t b/tests/bugs/fuse/many-groups-for-acl.t
new file mode 100755
index 00000000000..d83fe12644a
--- /dev/null
+++ b/tests/bugs/fuse/many-groups-for-acl.t
@@ -0,0 +1,113 @@
+#!/bin/bash
+
+. $(dirname $0)/../../include.rc
+. $(dirname $0)/../../volume.rc
+. $(dirname $0)/../../nfs.rc
+
+cleanup
+
+# prepare the users and groups
+NEW_USER=bug1246275
+NEW_UID=1246275
+NEW_GID=1246275
+LAST_GID=1246403
+NEW_GIDS=${NEW_GID}
+
+# OS-specific overrides
+case $OSTYPE in
+NetBSD|Darwin)
+        # no ACLs, and only NGROUPS_MAX=16 secondary groups are supported
+        SKIP_TESTS
+        exit 0
+        ;;
+FreeBSD)
+        # NGROUPS_MAX=1023 (FreeBSD>=8.0), we can afford 200 groups
+        ;;
+Linux)
+        # NGROUPS_MAX=65536, we can afford 200 groups
+        ;;
+*)
+        ;;
+esac
+
+# create a user that belongs to many groups
+for GID in $(seq  -f '%6.0f' ${NEW_GID} ${LAST_GID})
+do
+        groupadd -o -g ${GID} ${NEW_USER}-${GID}
+        NEW_GIDS="${NEW_GIDS},${NEW_USER}-${GID}"
+done
+TEST useradd -o -M -u ${NEW_UID} -g ${NEW_GID} -G ${NEW_USER}-${NEW_GIDS} ${NEW_USER}
+
+# preparation done, start the tests
+
+TEST glusterd
+TEST pidof glusterd
+TEST $CLI volume create ${V0} ${H0}:${B0}/${V0}1
+# disable manage-gids on the server-side for now, gets enabled later
+TEST $CLI volume set ${V0} server.manage-gids off
+TEST $CLI volume start ${V0}
+
+EXPECT_WITHIN ${NFS_EXPORT_TIMEOUT} "1" is_nfs_export_available
+
+# mount the volume with POSIX ACL support, without --resolve-gids
+TEST glusterfs --acl --volfile-id=/${V0} --volfile-server=${H0} ${M0}
+
+# create some directories for testing
+TEST mkdir ${M0}/first-32-gids-1
+TEST setfacl -m g:${NEW_UID}:rwx ${M0}/first-32-gids-1
+TEST mkdir ${M0}/first-32-gids-2
+TEST setfacl -m g:$[NEW_UID+16]:rwx ${M0}/first-32-gids-2
+TEST mkdir ${M0}/gid-64
+TEST setfacl -m g:$[NEW_UID+64]:rwx ${M0}/gid-64
+TEST mkdir ${M0}/gid-120
+TEST setfacl -m g:$[NEW_UID+120]:rwx ${M0}/gid-120
+
+su -m ${NEW_USER} -c "touch ${M0}/first-32-gids-1/success > /dev/null"
+TEST [ $? -eq 0 ]
+
+su -m ${NEW_USER} -c "touch ${M0}/first-32-gids-2/success > /dev/null"
+TEST [ $? -eq 0 ]
+
+su -m ${NEW_USER} -c "touch ${M0}/gid-64/failure > /dev/null"
+TEST [ $? -ne 0 ]
+
+su -m ${NEW_USER} -c "touch ${M0}/gid-120/failure > /dev/null"
+TEST [ $? -ne 0 ]
+
+# unmount and remount with --resolve-gids
+EXPECT_WITHIN ${UMOUNT_TIMEOUT} "Y" force_umount ${M0}
+TEST glusterfs --acl --resolve-gids --volfile-id=/${V0} --volfile-server=${H0} ${M0}
+
+su -m ${NEW_USER} -c "touch ${M0}/gid-64/success > /dev/null"
+TEST [ $? -eq 0 ]
+
+su -m ${NEW_USER} -c "touch ${M0}/gid-120/failure > /dev/null"
+TEST [ $? -ne 0 ]
+
+# enable server-side resolving of the groups
+# stopping and starting is not really needed, but it prevents races
+TEST $CLI volume stop ${V0}
+TEST $CLI volume set ${V0} server.manage-gids on
+TEST $CLI volume start ${V0}
+EXPECT_WITHIN ${NFS_EXPORT_TIMEOUT} "1" is_nfs_export_available
+
+# unmount and remount to prevent more race conditions on test systems
+EXPECT_WITHIN ${UMOUNT_TIMEOUT} "Y" force_umount ${M0}
+TEST glusterfs --acl --resolve-gids --volfile-id=/${V0} --volfile-server=${H0} ${M0}
+
+su -m ${NEW_USER} -c "touch ${M0}/gid-120/success > /dev/null"
+TEST [ $? -eq 0 ]
+
+# cleanup
+userdel --force ${NEW_USER}
+for GID in $(seq  -f '%6.0f' ${NEW_GID} ${LAST_GID})
+do
+        groupdel ${NEW_USER}-${GID}
+done
+
+EXPECT_WITHIN ${UMOUNT_TIMEOUT} "Y" force_umount ${M0}
+
+TEST ${CLI} volume stop ${V0}
+TEST ${CLI} volume delete ${V0}
+
+cleanup
diff --git a/xlators/mount/fuse/src/fuse-bridge.c b/xlators/mount/fuse/src/fuse-bridge.c
index b67a60a76b1..d6edcb360cd 100644
--- a/xlators/mount/fuse/src/fuse-bridge.c
+++ b/xlators/mount/fuse/src/fuse-bridge.c
@@ -5427,6 +5427,8 @@ init (xlator_t *this_xl)
                 goto cleanup_exit;
         }
 
+        GF_OPTION_INIT("resolve-gids", priv->resolve_gids, bool, cleanup_exit);
+
         /* default values seemed to work fine during testing */
         GF_OPTION_INIT ("background-qlen", priv->background_qlen, int32,
                         cleanup_exit);
@@ -5665,6 +5667,10 @@ struct volume_options options[] = {
           .type = GF_OPTION_TYPE_INT,
           .default_value = "300"
         },
+        { .key = {"resolve-gids"},
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "false"
+        },
         { .key = {"acl"},
           .type = GF_OPTION_TYPE_BOOL,
           .default_value = "false"
diff --git a/xlators/mount/fuse/src/fuse-bridge.h b/xlators/mount/fuse/src/fuse-bridge.h
index a9705d6c654..850eeb60ea8 100644
--- a/xlators/mount/fuse/src/fuse-bridge.h
+++ b/xlators/mount/fuse/src/fuse-bridge.h
@@ -127,6 +127,9 @@ struct fuse_private {
         /* fini started, helps prevent multiple epoll worker threads
          * firing up the fini routine */
         gf_boolean_t fini_invoked;
+
+        /* resolve gid with getgrouplist() instead of /proc/%d/status */
+        gf_boolean_t resolve_gids;
 };
 typedef struct fuse_private fuse_private_t;
 
diff --git a/xlators/mount/fuse/src/fuse-helpers.c b/xlators/mount/fuse/src/fuse-helpers.c
index 1c888276eb5..22aa9b486d1 100644
--- a/xlators/mount/fuse/src/fuse-helpers.c
+++ b/xlators/mount/fuse/src/fuse-helpers.c
@@ -20,6 +20,8 @@
 #elif defined(CTL_KERN)
 #include <sys/sysctl.h>
 #endif
+#include <pwd.h>
+#include <grp.h>
 
 #include "fuse-bridge.h"
 
@@ -146,51 +148,87 @@ void
 frame_fill_groups (call_frame_t *frame)
 {
 #if defined(GF_LINUX_HOST_OS)
-        char         filename[32];
-        char         line[4096];
-        char        *ptr = NULL;
-        FILE        *fp = NULL;
-        int          idx = 0;
-        long int     id = 0;
-        char        *saveptr = NULL;
-        char        *endptr = NULL;
-        int          ret = 0;
-
-        ret = snprintf (filename, sizeof filename, "/proc/%d/status",
-                        frame->root->pid);
-        if (ret >= sizeof filename)
-                goto out;
+        xlator_t       *this          = frame->this;
+        fuse_private_t *priv          = this->private;
+        char            filename[32];
+        char            line[4096];
+        char           *ptr           = NULL;
+        FILE           *fp            = NULL;
+        int             idx           = 0;
+        long int        id            = 0;
+        char           *saveptr       = NULL;
+        char           *endptr        = NULL;
+        int             ret           = 0;
+        int             ngroups       = FUSE_MAX_AUX_GROUPS;
+        gid_t           mygroups[GF_MAX_AUX_GROUPS];
+
+        if (priv->resolve_gids) {
+                struct passwd    pwent;
+                char             mystrs[1024];
+                struct passwd   *result;
+
+                if (getpwuid_r (frame->root->uid, &pwent, mystrs,
+                                sizeof(mystrs), &result) != 0) {
+                        gf_log (this->name, GF_LOG_ERROR, "getpwuid_r(%u) "
+                                "failed", frame->root->uid);
+                        return;
+                }
 
-        fp = fopen (filename, "r");
-        if (!fp)
-                goto out;
+                ngroups = GF_MAX_AUX_GROUPS;
+                if (getgrouplist (result->pw_name, frame->root->gid, mygroups,
+                                  &ngroups) == -1) {
+                        gf_log (this->name, GF_LOG_ERROR, "could not map %s to "
+                                "group list (ngroups %d, max %d)",
+                                result->pw_name, ngroups, GF_MAX_AUX_GROUPS);
+                        return;
+                }
 
-        if (call_stack_alloc_groups (frame->root, FUSE_MAX_AUX_GROUPS) != 0)
-                goto out;
+                if (call_stack_alloc_groups (frame->root, ngroups) != 0)
+                        goto out;
 
-        while ((ptr = fgets (line, sizeof line, fp))) {
-                if (strncmp (ptr, "Groups:", 7) != 0)
-                        continue;
-
-                ptr = line + 8;
-
-                for (ptr = strtok_r (ptr, " \t\r\n", &saveptr);
-                     ptr;
-                     ptr = strtok_r (NULL, " \t\r\n", &saveptr)) {
-                        errno = 0;
-                        id = strtol (ptr, &endptr, 0);
-                        if (errno == ERANGE)
-                                break;
-                        if (!endptr || *endptr)
-                                break;
-                        frame->root->groups[idx++] = id;
-                        if (idx == FUSE_MAX_AUX_GROUPS)
-                                break;
+                /* Copy data to the frame. */
+                for (idx = 0; idx < ngroups; ++idx) {
+                        frame->root->groups[idx] = mygroups[idx];
                 }
+                frame->root->ngrps = ngroups;
+        } else {
+                ret = snprintf (filename, sizeof filename, "/proc/%d/status",
+                                frame->root->pid);
+                if (ret >= sizeof filename)
+                        goto out;
 
-                frame->root->ngrps = idx;
-                break;
+                fp = fopen (filename, "r");
+                if (!fp)
+                        goto out;
+
+                if (call_stack_alloc_groups (frame->root, ngroups) != 0)
+                        goto out;
+
+                while ((ptr = fgets (line, sizeof line, fp))) {
+                        if (strncmp (ptr, "Groups:", 7) != 0)
+                                continue;
+
+                        ptr = line + 8;
+
+                        for (ptr = strtok_r (ptr, " \t\r\n", &saveptr);
+                             ptr;
+                             ptr = strtok_r (NULL, " \t\r\n", &saveptr)) {
+                                errno = 0;
+                                id = strtol (ptr, &endptr, 0);
+                                if (errno == ERANGE)
+                                        break;
+                                if (!endptr || *endptr)
+                                        break;
+                                frame->root->groups[idx++] = id;
+                                if (idx == FUSE_MAX_AUX_GROUPS)
+                                        break;
+                        }
+
+                        frame->root->ngrps = idx;
+                        break;
+                }
         }
+
 out:
         if (fp)
                 fclose (fp);
diff --git a/xlators/mount/fuse/utils/mount.glusterfs.in b/xlators/mount/fuse/utils/mount.glusterfs.in
index 82660328363..112dc0a225b 100755
--- a/xlators/mount/fuse/utils/mount.glusterfs.in
+++ b/xlators/mount/fuse/utils/mount.glusterfs.in
@@ -177,6 +177,10 @@ start_glusterfs ()
         cmd_line=$(echo "$cmd_line --aux-gfid-mount");
     fi
 
+    if [ -n "$resolve_gids" ]; then
+        cmd_line=$(echo "$cmd_line --resolve-gids");
+    fi
+
     if [ -n "$no_root_squash" ]; then
         cmd_line=$(echo "$cmd_line --no-root-squash");
     fi
@@ -501,6 +505,9 @@ without_options()
                 aux_gfid_mount=1
             fi
             ;;
+        "resolve-gids")
+            resolve_gids=1
+            ;;
          # "mount -t glusterfs" sends this, but it's useless.
         "rw")
             ;;