glusterd: auth allow enhancements

* PROBLEM:

  When address-based authentication is enabled on a volume,
  the gNfs server, self-heal daemon (shd), and other operations
  such as quota, rebalance, replace-brick and geo-replication
  either stop working or the services are not started if all
  the peers' ipv{4,6} addresses or hostnames are not added in
  the "set auth.allow" operation, breaking the functionality
  of several operations.

  E.g:
    volume vol in a cluster of two peers:
    /mnt/brick1 in 192.168.1.4
    /mnt/brick2 in 192.168.1.5

    option auth.allow 192.168.1.6
    (allow connection requests only from 192.168.1.6)

    This will disrupt the nfs servers on 192.168.1.{4,5}.
    brick server processes reject connection requests from both
    nfs servers (on 4,5), because the peer addresses are not in
    the auth.allow list.

    Same holds true for local mounts (on peer machines),
    self-heal daemon, and other operations which perform
    a glusterfs mount on one of the peers.

* SOLUTION:

  Login-based authentication (username/password pairs,
  henceforth referred to as "keys") for gluster services and
  operations.

  These *per-volume* keys can be used to by-pass the addr-based
  authentication, provided none of the peers' addresses are put
  in the auth.reject list, to enable gluster services like gNfs,
  self-heal daemon and internal operations on volumes when
  auth.allow option is exercised.

* IMPLEMENTATION:

  1. Glusterd generates keys for each volume and stores it in
     memory as well as in respective volfiles.
     A new TRUSTED-FUSE volfile is generated which is
     fuse volfile + keys in protocol/client,
     and is named trusted-<volname>-fuse.vol.
     This is used by all local mounts. ANY local mount (on any peer)
     is granted the trusted-fuse volfile instead of fuse volfile
     via getspec. non-local mounts are NOT granted the trusted fuse
     volfile.

  2. The keys generated for the volume is written to each server
     volfile telling servers to allow users with these keys.

  3. NFS, self-heal daemon and replace-brick volfiles are updated
     with the volume's authentication keys.

  4. The keys are NOT written to fuse volfiles for obvious reasons.

  5. The ownership of volfiles and logfiles is restricted to root users.

  6. Merging two identical definitions of peer_info_t in auth/addr
     and rpc-lib, throwing away the one in auth/addr.

  7. Code cleanup in numerous places as appropriate.

* IMPORTANT NOTES:

  1. One SHOULD NOT put any of the peer addresses in the auth.reject
     list if one wants any of the glusterd services and features
     such as gNfs, self-heal, rebalance, geo-rep and quota.

  2. If one wants to use username/password based authentication
     to volumes, one shall append to the server, nfs and shd volfiles,
     the keys one wants to use for authentication, *while_retaining
     those_generated_by_glusterd*.
     See doc/authentication.txt file for details.

Change-Id: Ie0331d625ad000d63090e2d622fe1728fbfcc453
BUG: 789942
Signed-off-by: Rajesh Amaravathi <rajesh@redhat.com>
Reviewed-on: http://review.gluster.com/2733
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Vijay Bellur <vijay@gluster.com>

1 files changed, 121 insertions, 21 deletions

diff --git a/xlators/mgmt/glusterd/src/glusterd-volgen.c b/xlators/mgmt/glusterd/src/glusterd-volgen.c
index fe79ea48..919994b1 100644
--- a/xlators/mgmt/glusterd/src/glusterd-volgen.c
+++ b/xlators/mgmt/glusterd/src/glusterd-volgen.c
@@ -1117,8 +1117,12 @@ free_fp:
 static int
 volgen_write_volfile (volgen_graph_t *graph, char *filename)
 {
-        char *ftmp = NULL;
-        FILE *f = NULL;
+        char        *ftmp = NULL;
+        FILE        *f = NULL;
+        int          fd   = 0;
+        xlator_t    *this = NULL;
+
+        this = THIS;
 
         if (gf_asprintf (&ftmp, "%s.tmp", filename) == -1) {
                 ftmp = NULL;
@@ -1126,6 +1130,15 @@ volgen_write_volfile (volgen_graph_t *graph, char *filename)
                 goto error;
         }
 
+        fd = creat (ftmp, S_IRUSR | S_IWUSR);
+        if (fd < 0) {
+                gf_log (this->name, GF_LOG_ERROR, "%s",
+                        strerror (errno));
+                goto error;
+        }
+
+        close (fd);
+
         f = fopen (ftmp, "w");
         if (!f)
                 goto error;
@@ -1153,7 +1166,8 @@ volgen_write_volfile (volgen_graph_t *graph, char *filename)
         if (f)
                 fclose (f);
 
-        gf_log ("", GF_LOG_ERROR, "failed to create volfile %s", filename);
+        gf_log (this->name, GF_LOG_ERROR,
+                "failed to create volfile %s", filename);
 
         return -1;
 }
@@ -1497,6 +1511,7 @@ server_graph_builder (volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
         int       ret                   = 0;
         char     *xlator                = NULL;
         char     *loglevel              = NULL;
+        char      key[1024]             = {0};
 
         path = param;
         volname = volinfo->volname;
@@ -1556,6 +1571,16 @@ server_graph_builder (volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
 		if (NULL == ptranst)
 			return -1;
 
+                ret = xlator_set_option (rbxl, "username",
+                                         glusterd_auth_get_username (volinfo));
+                if (ret)
+                        return -1;
+
+                ret = xlator_set_option (rbxl, "password",
+                                         glusterd_auth_get_password (volinfo));
+                if (ret)
+                        return -1;
+
                 ret = xlator_set_option (rbxl, "transport-type", ptranst);
                 GF_FREE (ptranst);
                 if (ret)
@@ -1596,6 +1621,22 @@ server_graph_builder (volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
         if (ret)
                 return -1;
 
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "auth.login.%s.allow", path);
+        ret = xlator_set_option (xl, key,
+                                 glusterd_auth_get_username (volinfo));
+        if (ret)
+                return -1;
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "auth.login.%s.password",
+                  glusterd_auth_get_username (volinfo));
+
+        ret = xlator_set_option (xl, key,
+                                 glusterd_auth_get_password (volinfo));
+        if (ret)
+                return -1;
+
         ret = volgen_graph_set_options_generic (graph, set_dict,
                                                 (xlator && loglevel) ? (void *)set_dict : volinfo,
                                                 (xlator && loglevel) ?  &server_spec_extended_option_handler :
@@ -1922,10 +1963,12 @@ volgen_graph_build_clients (volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
 {
         int                      i                  = 0;
         int                      ret                = -1;
+        uint32_t                 client_type        = GF_CLIENT_OTHER;
         char                     transt[16]         = {0,};
         char                    *volname            = NULL;
-        glusterd_brickinfo_t    *brick = NULL;
-        xlator_t                *xl                = NULL;
+        char                    *str                = NULL;
+        glusterd_brickinfo_t    *brick              = NULL;
+        xlator_t                *xl                 = NULL;
 
         volname = volinfo->volname;
 
@@ -1968,8 +2011,26 @@ volgen_graph_build_clients (volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
                 ret = xlator_set_option (xl, "transport-type", transt);
                 if (ret)
                         goto out;
+
+                ret = dict_get_uint32 (set_dict, "trusted-client",
+                                       &client_type);
+
+                if (!ret && client_type == GF_CLIENT_TRUSTED) {
+
+                        str = glusterd_auth_get_username (volinfo);
+                        ret = xlator_set_option (xl, "username", str);
+                        if (ret)
+                                goto out;
+
+                        str = glusterd_auth_get_password (volinfo);
+                        ret = xlator_set_option (xl, "password", str);
+                        if (ret)
+                                goto out;
+                }
+
                 i++;
         }
+
         if (i != volinfo->brick_count) {
                 gf_log ("", GF_LOG_ERROR,
                         "volume inconsistency: actual number of bricks (%d) "
@@ -2501,7 +2562,7 @@ build_shd_graph (volgen_graph_t *graph, dict_t *mod_dict)
         int                ret            = 0;
         gf_boolean_t       valid_config   = _gf_false;
         xlator_t           *iostxl        = NULL;
-        int                rclusters       = 0;
+        int                rclusters      = 0;
         int                replica_count  = 0;
 
         this = THIS;
@@ -2534,6 +2595,11 @@ build_shd_graph (volgen_graph_t *graph, dict_t *mod_dict)
                 if (ret)
                         goto out;
 
+                ret = dict_set_uint32 (set_dict, "trusted-client",
+                                       GF_CLIENT_TRUSTED);
+                if (ret)
+                        goto out;
+
                 dict_copy (voliter->dict, set_dict);
                 if (mod_dict)
                         dict_copy (mod_dict, set_dict);
@@ -2650,12 +2716,21 @@ build_nfs_graph (volgen_graph_t *graph, dict_t *mod_dict)
                 if (ret)
                         goto out;
 
-                ret = dict_set_str (set_dict, "performance.client-io-threads", "off");
+                ret = dict_set_str (set_dict, "performance.client-io-threads",
+                                    "off");
                 if (ret)
                         goto out;
 
                 ret = dict_set_str (set_dict, "client-transport-type",
                                     nfs_xprt);
+                if (ret)
+                        goto out;
+
+                ret = dict_set_uint32 (set_dict, "trusted-client",
+                                       GF_CLIENT_TRUSTED);
+                if (ret)
+                        goto out;
+
                 ret = build_client_graph (&cgraph, voliter, set_dict);
                 if (ret)
                         goto out;
@@ -2814,7 +2889,7 @@ generate_brick_volfiles (glusterd_volinfo_t *volinfo)
         get_vol_tstamp_file (tstamp_file, volinfo);
 
         if (ret) {
-                ret = open (tstamp_file, O_WRONLY|O_CREAT|O_EXCL, 0644);
+                ret = open (tstamp_file, O_WRONLY|O_CREAT|O_EXCL, 0600);
                 if (ret == -1 && errno == EEXIST) {
                         gf_log ("", GF_LOG_DEBUG, "timestamp file exist");
                         ret = -2;
@@ -2889,7 +2964,8 @@ enumerate_transport_reqs (gf_transport_type type, char **types)
 }
 
 static int
-generate_client_volfiles (glusterd_volinfo_t *volinfo)
+generate_client_volfiles (glusterd_volinfo_t *volinfo,
+                          glusterd_client_type_t client_type)
 {
         char               filepath[PATH_MAX] = {0,};
         int                ret = -1;
@@ -2908,7 +2984,21 @@ generate_client_volfiles (glusterd_volinfo_t *volinfo)
                 if (ret)
                         goto out;
                 type = transport_str_to_type (types[i]);
-                glusterd_get_client_filepath (filepath, volinfo, type);
+
+                ret = dict_set_uint32 (dict, "trusted-client", client_type);
+                if (ret)
+                        goto out;
+
+                if (client_type == GF_CLIENT_TRUSTED) {
+                        glusterd_get_trusted_client_filepath (filepath,
+                                                              volinfo,
+                                                              type);
+                } else {
+                        glusterd_get_client_filepath (filepath,
+                                                      volinfo,
+                                                      type);
+                }
+
                 ret = generate_single_transport_client_volfile (volinfo,
                                                                 filepath,
                                                                 dict);
@@ -2929,7 +3019,7 @@ glusterd_create_rb_volfiles (glusterd_volinfo_t *volinfo,
 
         ret = glusterd_generate_brick_volfile (volinfo, brickinfo);
         if (!ret)
-                ret = generate_client_volfiles (volinfo);
+                ret = generate_client_volfiles (volinfo, GF_CLIENT_TRUSTED);
         if (!ret)
                 ret = glusterd_fetchspec_notify (THIS);
 
@@ -2939,23 +3029,33 @@ glusterd_create_rb_volfiles (glusterd_volinfo_t *volinfo,
 int
 glusterd_create_volfiles_and_notify_services (glusterd_volinfo_t *volinfo)
 {
-        int ret = -1;
+        int        ret  = -1;
+        xlator_t  *this = NULL;
+
+        this = THIS;
 
         ret = generate_brick_volfiles (volinfo);
         if (ret) {
-                gf_log ("", GF_LOG_ERROR,
+                gf_log (this->name, GF_LOG_ERROR,
                         "Could not generate volfiles for bricks");
                 goto out;
         }
 
-        ret = generate_client_volfiles (volinfo);
+        ret = generate_client_volfiles (volinfo, GF_CLIENT_TRUSTED);
         if (ret) {
-                gf_log ("", GF_LOG_ERROR,
-                        "Could not generate volfile for client");
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Could not generate trusted client volfiles");
                 goto out;
         }
 
-        ret = glusterd_fetchspec_notify (THIS);
+        ret = generate_client_volfiles (volinfo, GF_CLIENT_OTHER);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Could not generate client volfiles");
+                goto out;
+        }
+
+        ret = glusterd_fetchspec_notify (this);
 
 out:
         return ret;
@@ -3085,11 +3185,11 @@ out:
 
 int
 validate_clientopts (glusterd_volinfo_t *volinfo,
-                    dict_t *val_dict,
-                    char **op_errstr)
+                     dict_t *val_dict,
+                     char **op_errstr)
 {
         volgen_graph_t graph = {0,};
-        int     ret = -1;
+        int            ret   = -1;
 
         GF_ASSERT (volinfo);
 
@@ -3112,7 +3212,7 @@ validate_brickopts (glusterd_volinfo_t *volinfo,
                     char **op_errstr)
 {
         volgen_graph_t graph = {0,};
-        int     ret = -1;
+        int            ret   = -1;
 
         GF_ASSERT (volinfo);