glusterd: Introduce volume op-versions

Each volume is now associated with two op-versions,
 * op_version - the op-version of the highest op-versioned feature enabled
 * client_op_version - the op-version of the highest op-versioned feature
   enabled which affects the clients only.

These two op-versions are generated dynamically and kept updated during
runtime. Glusterd now uses the respective volumes' client-op-version during
getspec requests.

To achieve the above a new field in the vme table is introduced,
client_option, this boolean field tells if the option is a client side
option.

Change-Id: I12c83b1dd29ab506026efd50d448cebbcee53c27
BUG: 907311
Signed-off-by: Kaushal M <kaushal@redhat.com>
Reviewed-on: http://review.gluster.org/4584
Reviewed-by: Krishnan Parthasarathi <kparthas@redhat.com>
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Vijay Bellur <vbellur@redhat.com>

11 files changed, 600 insertions, 419 deletions

diff --git a/libglusterfs/src/dict.h b/libglusterfs/src/dict.h
index c7606352c..9b41b5a7d 100644
--- a/libglusterfs/src/dict.h
+++ b/libglusterfs/src/dict.h
@@ -234,5 +234,4 @@ GF_MUST_CHECK int dict_serialize_value_with_delim (dict_t *this, char *buf, int3
                                                     char delimiter);
 
 void dict_dump (dict_t *dict);
-
 #endif
diff --git a/xlators/mgmt/glusterd/src/glusterd-handshake.c b/xlators/mgmt/glusterd/src/glusterd-handshake.c
index c41a94598..9124c46ee 100644
--- a/xlators/mgmt/glusterd/src/glusterd-handshake.c
+++ b/xlators/mgmt/glusterd/src/glusterd-handshake.c
@@ -108,112 +108,140 @@ out:
         return ret;
 }
 
-int
-__server_getspec (rpcsvc_request_t *req)
+/* Get and store op-versions of the clients sending the getspec request
+ * Clients of versions <= 3.3, don't send op-versions, their op-versions are
+ * defaulted to 1
+ */
+static int
+_get_client_op_versions (gf_getspec_req *args, peer_info_t *peerinfo)
 {
-        int32_t               ret                    = -1;
-        int32_t               op_errno               = 0;
-        int32_t               spec_fd                = -1;
-        size_t                file_len               = 0;
-        char                  filename[PATH_MAX]  = {0,};
-        struct stat           stbuf                  = {0,};
-        char                 *volume                 = NULL;
-        char                 *tmp                    = NULL;
-        int                   cookie                 = 0;
-        rpc_transport_t      *trans                  = NULL;
-        gf_getspec_req        args                   = {0,};
-        gf_getspec_rsp        rsp                    = {0,};
-        char                  addrstr[RPCSVC_PEER_STRLEN] = {0};
-        dict_t               *dict                   = NULL;
-        xlator_t             *this                   = NULL;
-        glusterd_conf_t      *conf                   = NULL;
-        int                   client_min_op_version  = 1;       // OP-VERSIONs start at 1
-        int                   client_max_op_version  = 1;
-
-        this = THIS;
-        GF_ASSERT (this);
+        int    ret                   = 0;
+        int    client_max_op_version = 1;
+        int    client_min_op_version = 1;
+        dict_t *dict                 = NULL;
 
-        conf = this->private;
-        GF_ASSERT (conf);
+        GF_ASSERT (args);
+        GF_ASSERT (peerinfo);
 
-        ret = xdr_to_generic (req->msg[0], &args,
-                              (xdrproc_t)xdr_gf_getspec_req);
-        if (ret < 0) {
-                //failed to decode msg;
-                req->rpc_err = GARBAGE_ARGS;
-                goto fail;
-        }
-
-        if (!args.xdata.xdata_len) {
-                // For clients <= 3.3.0, only allow if op_version = 1
-                if (1 != conf->op_version) {
-                        ret = -1;
-                        op_errno = ENOTSUP;
-                        gf_log (this->name, GF_LOG_INFO,
-                                "Client %s doesn't support required op-version. "
-                                "Rejecting getspec request.",
-                                req->trans->peerinfo.identifier);
-                        goto fail;
-                }
-        } else {
-                // For clients > 3.3, only allow if they can support
-                // clusters' op_version
+        if (args->xdata.xdata_len) {
                 dict = dict_new ();
                 if (!dict) {
                         ret = -1;
-                        goto fail;
+                        goto out;
                 }
 
-                ret = dict_unserialize (args.xdata.xdata_val,
-                                        args.xdata.xdata_len, &dict);
+                ret = dict_unserialize (args->xdata.xdata_val,
+                                        args->xdata.xdata_len, &dict);
                 if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
+                        gf_log ("glusterd", GF_LOG_ERROR,
                                 "Failed to unserialize request dictionary");
-                        goto fail;
+                        goto out;
                 }
 
                 ret = dict_get_int32 (dict, "min-op-version",
                                       &client_min_op_version);
                 if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
+                        gf_log ("glusterd", GF_LOG_ERROR,
                                 "Failed to get client-min-op-version");
-                        goto fail;
+                        goto out;
                 }
 
                 ret = dict_get_int32 (dict, "max-op-version",
                                       &client_max_op_version);
                 if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
+                        gf_log ("glusterd", GF_LOG_ERROR,
                                 "Failed to get client-max-op-version");
-                        goto fail;
+                        goto out;
                 }
+        }
 
-                if ((client_min_op_version > conf->op_version) ||
-                    (client_max_op_version < conf->op_version)) {
-                        ret = -1;
-                        op_errno = ENOTSUP;
-                        //TODO: Add client identifier
-                        gf_log (this->name, GF_LOG_INFO,
-                                "Client %s doesn't support required op-version. "
-                                "Rejecting getspec request.",
-                                req->trans->peerinfo.identifier);
-                        goto fail;
-                }
+        peerinfo->max_op_version = client_max_op_version;
+        peerinfo->min_op_version = client_min_op_version;
 
+out:
+        return ret;
+}
+
+/* Checks if the client supports the volume, ie. client can understand all the
+ * options in the volfile
+ */
+static gf_boolean_t
+_client_supports_volume (peer_info_t *peerinfo, int32_t *op_errno)
+{
+        gf_boolean_t       ret       = _gf_true;
+        glusterd_volinfo_t *volinfo  = NULL;
+
+        GF_ASSERT (peerinfo);
+        GF_ASSERT (op_errno);
+
+
+        /* Only check when the volfile being requested is a volume. Not finding
+         * a volinfo implies that the volfile requested for is not of a gluster
+         * volume. A non volume volfile is requested by the local gluster
+         * services like shd and nfs-server. These need not be checked as they
+         * will be running at the same op-version as glusterd and will be able
+         * to support all the features
+         */
+        if ((glusterd_volinfo_find (peerinfo->volname, &volinfo) == 0) &&
+            ((peerinfo->min_op_version > volinfo->client_op_version) ||
+             (peerinfo->max_op_version < volinfo->client_op_version))) {
+                ret = _gf_false;
+                *op_errno = ENOTSUP;
+                gf_log ("glusterd", GF_LOG_INFO,
+                        "Client %s (%d -> %d) doesn't support required "
+                        "op-version (%d). Rejecting volfile request.",
+                        peerinfo->identifier, peerinfo->min_op_version,
+                        peerinfo->max_op_version, volinfo->client_op_version);
         }
 
-        // Store the op-versions supported by the client
-        req->trans->peerinfo.max_op_version = client_max_op_version;
-        req->trans->peerinfo.min_op_version = client_min_op_version;
+        return ret;
+}
 
-        volume = args.key;
+int
+__server_getspec (rpcsvc_request_t *req)
+{
+        int32_t               ret                    = -1;
+        int32_t               op_errno               = 0;
+        int32_t               spec_fd                = -1;
+        size_t                file_len               = 0;
+        char                  filename[PATH_MAX]  = {0,};
+        struct stat           stbuf                  = {0,};
+        char                 *volume                 = NULL;
+        char                 *tmp                    = NULL;
+        int                   cookie                 = 0;
+        rpc_transport_t      *trans                  = NULL;
+        gf_getspec_req        args                   = {0,};
+        gf_getspec_rsp        rsp                    = {0,};
+        char                  addrstr[RPCSVC_PEER_STRLEN] = {0};
+        peer_info_t          *peerinfo               = NULL;
 
-        // Store the name of volume  being mounted
+        ret = xdr_to_generic (req->msg[0], &args,
+                              (xdrproc_t)xdr_gf_getspec_req);
+        if (ret < 0) {
+                //failed to decode msg;
+                req->rpc_err = GARBAGE_ARGS;
+                goto fail;
+        }
+
+        peerinfo = &req->trans->peerinfo;
+
+        volume = args.key;
+        /* Need to strip leading '/' from volnames. This was introduced to
+         * support nfs style mount parameters for native gluster mount
+         */
         if (volume[0] == '/')
-                strncpy (req->trans->peerinfo.volname, &volume[1],
-                         strlen(&volume[1]));
+                strncpy (peerinfo->volname, &volume[1], strlen(&volume[1]));
         else
-                strncpy (req->trans->peerinfo.volname, volume, strlen(volume));
+                strncpy (peerinfo->volname, volume, strlen(volume));
+
+        ret = _get_client_op_versions (&args, peerinfo);
+        if (ret)
+                goto fail;
+
+        if (!_client_supports_volume (peerinfo, &op_errno)) {
+                ret = -1;
+                goto fail;
+        }
 
         trans = req->trans;
         ret = rpcsvc_transport_peername (trans, (char *)&addrstr,
diff --git a/xlators/mgmt/glusterd/src/glusterd-op-sm.c b/xlators/mgmt/glusterd/src/glusterd-op-sm.c
index 4e4f0394d..dbc23525f 100644
--- a/xlators/mgmt/glusterd/src/glusterd-op-sm.c
+++ b/xlators/mgmt/glusterd/src/glusterd-op-sm.c
@@ -1337,6 +1337,8 @@ glusterd_op_reset_volume (dict_t *dict, char **op_errstr)
                             key);
         }
 
+        gd_update_volume_op_versions (volinfo);
+
 out:
         GF_FREE (key_fixed);
         if (quorum_action)
@@ -1520,6 +1522,7 @@ glusterd_op_set_volume (dict_t *dict)
                 goto out;
         }
 
+        // TODO: Remove this once v3.3 compatability is not required
         check_op_version = dict_get_str_boolean (dict, "check-op-version",
                                                  _gf_false);
 
@@ -1648,6 +1651,7 @@ glusterd_op_set_volume (dict_t *dict)
                                 goto out;
                         }
                 }
+                gd_update_volume_op_versions (volinfo);
 
         } else {
                 list_for_each_entry (voliter, &priv->volumes, vol_list) {
@@ -1674,6 +1678,7 @@ glusterd_op_set_volume (dict_t *dict)
                                         goto out;
                                 }
                         }
+                        gd_update_volume_op_versions (volinfo);
                 }
         }
 
diff --git a/xlators/mgmt/glusterd/src/glusterd-store.c b/xlators/mgmt/glusterd/src/glusterd-store.c
index 7e26eb4a7..f89b648ed 100644
--- a/xlators/mgmt/glusterd/src/glusterd-store.c
+++ b/xlators/mgmt/glusterd/src/glusterd-store.c
@@ -2582,6 +2582,8 @@ glusterd_store_retrieve_volume (char    *volname)
         if (ret)
                 goto out;
 
+        gd_update_volume_op_versions (volinfo);
+
         list_add_tail (&volinfo->vol_list, &priv->volumes);
 
 out:
diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.c b/xlators/mgmt/glusterd/src/glusterd-utils.c
index e2d46a6a6..ced1e2367 100644
--- a/xlators/mgmt/glusterd/src/glusterd-utils.c
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.c
@@ -3031,6 +3031,8 @@ glusterd_import_friend_volume (dict_t *vols, size_t count)
         if (ret)
                 goto out;
 
+        gd_update_volume_op_versions (new_volinfo);
+
         list_add_tail (&new_volinfo->vol_list, &priv->volumes);
 out:
         gf_log ("", GF_LOG_DEBUG, "Returning with ret: %d", ret);
@@ -7488,3 +7490,43 @@ out:
         return ret;
 
 }
+
+int
+_update_volume_op_versions (dict_t *this, char *key, data_t *value, void *data)
+{
+        int                op_version = 0;
+        glusterd_volinfo_t *ctx       = NULL;
+
+        GF_ASSERT (data);
+        ctx = data;
+
+        op_version = glusterd_get_op_version_for_key (key);
+
+        if (op_version > ctx->op_version)
+                ctx->op_version = op_version;
+
+        if (gd_is_client_option (key) &&
+            (op_version > ctx->client_op_version))
+                ctx->client_op_version = op_version;
+
+        return 0;
+}
+
+void
+gd_update_volume_op_versions (glusterd_volinfo_t *volinfo)
+{
+        glusterd_conf_t    *conf = NULL;
+
+        GF_ASSERT (volinfo);
+
+        conf = THIS->private;
+        GF_ASSERT (conf);
+
+        /* Reset op-versions to minimum */
+        volinfo->op_version = 1;
+        volinfo->client_op_version = 1;
+
+        dict_foreach (volinfo->dict, _update_volume_op_versions, volinfo);
+
+        return;
+}
diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.h b/xlators/mgmt/glusterd/src/glusterd-utils.h
index 24a65aebe..047f1f510 100644
--- a/xlators/mgmt/glusterd/src/glusterd-utils.h
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.h
@@ -522,4 +522,7 @@ glusterd_copy_uuid_to_dict (uuid_t uuid, dict_t *dict, char *key);
 
 gf_boolean_t
 glusterd_is_same_address (char *name1, char *name2);
+
+void
+gd_update_volume_op_versions (glusterd_volinfo_t *volinfo);
 #endif
diff --git a/xlators/mgmt/glusterd/src/glusterd-volgen.c b/xlators/mgmt/glusterd/src/glusterd-volgen.c
index 0c8be4058..e573497bf 100644
--- a/xlators/mgmt/glusterd/src/glusterd-volgen.c
+++ b/xlators/mgmt/glusterd/src/glusterd-volgen.c
@@ -3659,3 +3659,20 @@ glusterd_get_op_version_for_key (char *key)
 
         return 0;
 }
+
+gf_boolean_t
+gd_is_client_option (char *key)
+{
+        char *completion = NULL;
+        struct volopt_map_entry *vmep = NULL;
+        int   ret = 0;
+
+        COMPLETE_OPTION(key, completion, ret);
+        for (vmep = glusterd_volopt_map; vmep->key; vmep++) {
+                if (strcmp (vmep->key, key) == 0) {
+                        return vmep->client_option;
+                }
+        }
+
+        return _gf_false;
+}
diff --git a/xlators/mgmt/glusterd/src/glusterd-volgen.h b/xlators/mgmt/glusterd/src/glusterd-volgen.h
index 0a9647bdf..746c6e92b 100644
--- a/xlators/mgmt/glusterd/src/glusterd-volgen.h
+++ b/xlators/mgmt/glusterd/src/glusterd-volgen.h
@@ -99,6 +99,10 @@ struct volopt_map_entry {
         uint32_t op_version;
         char *description;
         vme_option_validation validate_fn;
+        /* If client_option is true, the option affects clients.
+         * this is used to calculate client-op-version of volumes
+         */
+        gf_boolean_t client_option;
 };
 
 int glusterd_create_rb_volfiles (glusterd_volinfo_t *volinfo,
@@ -136,4 +140,7 @@ glusterd_check_nfs_volfile_identical (gf_boolean_t *identical);
 
 uint32_t
 glusterd_get_op_version_for_key (char *key);
+
+gf_boolean_t
+gd_is_client_option (char *key);
 #endif
diff --git a/xlators/mgmt/glusterd/src/glusterd-volume-ops.c b/xlators/mgmt/glusterd/src/glusterd-volume-ops.c
index 50b03f71e..0cae8d58b 100644
--- a/xlators/mgmt/glusterd/src/glusterd-volume-ops.c
+++ b/xlators/mgmt/glusterd/src/glusterd-volume-ops.c
@@ -1654,6 +1654,8 @@ glusterd_op_create_volume (dict_t *dict, char **op_errstr)
         volinfo->rebal.defrag_status = 0;
         list_add_tail (&volinfo->vol_list, &priv->volumes);
         vol_added = _gf_true;
+
+        gd_update_volume_op_versions (volinfo);
 out:
         GF_FREE(free_ptr);
         if (!vol_added && volinfo)
diff --git a/xlators/mgmt/glusterd/src/glusterd-volume-set.c b/xlators/mgmt/glusterd/src/glusterd-volume-set.c
index ec02680ab..b04afdf95 100644
--- a/xlators/mgmt/glusterd/src/glusterd-volume-set.c
+++ b/xlators/mgmt/glusterd/src/glusterd-volume-set.c
@@ -347,164 +347,197 @@ out:
 
 struct volopt_map_entry glusterd_volopt_map[] = {
         /* DHT xlator options */
-        { .key         = "cluster.lookup-unhashed",
-          .voltype     = "cluster/distribute",
-          .op_version  = 1
-        },
-        { .key         = "cluster.min-free-disk",
-          .voltype     = "cluster/distribute",
-          .op_version  = 1
-        },
-        { .key         = "cluster.min-free-inodes",
-          .voltype     = "cluster/distribute",
-          .op_version  = 1
-        },
-        { .key         = "cluster.rebalance-stats",
-          .voltype     = "cluster/distribute",
-          .op_version  = 2
-        },
-        { .key         = "cluster.subvols-per-directory",
-          .voltype     = "cluster/distribute",
-          .option      = "directory-layout-spread",
-          .op_version  = 2,
-          .validate_fn = validate_subvols_per_directory
-        },
-        { .key         = "cluster.readdir-optimize",
-          .voltype     = "cluster/distribute",
-          .op_version  = 2
-        },
-        { .key         = "cluster.nufa",
-          .voltype     = "cluster/distribute",
-          .option      = "!nufa",
-          .type        = NO_DOC,
-          .op_version  = 2
-        },
-        { .key         = "cluster.rsync-hash-regex",
-          .voltype     = "cluster/distribute",
-          .type        = NO_DOC,
-          .op_version  = 2
-        },
-        { .key         = "cluster.extra-hash-regex",
-          .voltype     = "cluster/distribute",
-          .type        = NO_DOC,
-          .op_version  = 2
-        },
-        { .key         = "cluster.dht-xattr-name",
-          .voltype     = "cluster/distribute",
-          .option      = "xattr-name",
-          .type        = NO_DOC,
-          .op_version  = 2
+        { .key           = "cluster.lookup-unhashed",
+          .voltype       = "cluster/distribute",
+          .op_version    = 1,
+          .client_option = _gf_true
+        },
+        { .key           = "cluster.min-free-disk",
+          .voltype       = "cluster/distribute",
+          .op_version    = 1,
+          .client_option = _gf_true
+        },
+        { .key           = "cluster.min-free-inodes",
+          .voltype       = "cluster/distribute",
+          .op_version    = 1,
+          .client_option = _gf_true
+        },
+        { .key           = "cluster.rebalance-stats",
+          .voltype       = "cluster/distribute",
+          .op_version    = 2,
+          .client_option = _gf_true
+        },
+        { .key           = "cluster.subvols-per-directory",
+          .voltype       = "cluster/distribute",
+          .option        = "directory-layout-spread",
+          .op_version    = 2,
+          .validate_fn   = validate_subvols_per_directory,
+          .client_option = _gf_true
+        },
+        { .key           = "cluster.readdir-optimize",
+          .voltype       = "cluster/distribute",
+          .op_version    = 2,
+          .client_option = _gf_true
+        },
+        { .key           = "cluster.nufa",
+          .voltype       = "cluster/distribute",
+          .option        = "!nufa",
+          .type          = NO_DOC,
+          .op_version    = 2,
+          .client_option = _gf_true
+        },
+        { .key           = "cluster.rsync-hash-regex",
+          .voltype       = "cluster/distribute",
+          .type          = NO_DOC,
+          .op_version    = 2,
+          .client_option = _gf_true
+        },
+        { .key           = "cluster.extra-hash-regex",
+          .voltype       = "cluster/distribute",
+          .type          = NO_DOC,
+          .op_version    = 2,
+          .client_option = _gf_true
+        },
+        { .key           = "cluster.dht-xattr-name",
+          .voltype       = "cluster/distribute",
+          .option        = "xattr-name",
+          .type          = NO_DOC,
+          .op_version    = 2,
+          .client_option = _gf_true
         },
 
         /* AFR xlator options */
-        { .key         = "cluster.entry-change-log",
-          .voltype     = "cluster/replicate",
-          .op_version  = 1
-        },
-        { .key         = "cluster.read-subvolume",
-          .voltype     = "cluster/replicate",
-          .op_version  = 1
-        },
-        { .key         = "cluster.read-subvolume-index",
-          .voltype     = "cluster/replicate",
-          .op_version  = 2
-        },
-        { .key         = "cluster.read-hash-mode",
-          .voltype     = "cluster/replicate",
-          .op_version  = 2
-        },
-        { .key         = "cluster.background-self-heal-count",
-          .voltype     = "cluster/replicate",
-          .op_version  = 1
-        },
-        { .key         = "cluster.metadata-self-heal",
-          .voltype     = "cluster/replicate",
-          .op_version  = 1
-        },
-        { .key         = "cluster.data-self-heal",
-          .voltype     = "cluster/replicate",
-          .op_version  = 1
-        },
-        { .key         = "cluster.entry-self-heal",
-          .voltype     = "cluster/replicate",
-          .op_version  = 1
-        },
-        { .key         = "cluster.self-heal-daemon",
-          .voltype     = "cluster/replicate",
-          .option      = "!self-heal-daemon",
-          .op_version  = 1
-        },
-        { .key         = "cluster.heal-timeout",
-          .voltype     = "cluster/replicate",
-          .option      = "!heal-timeout",
-          .op_version  = 2
-        },
-        { .key         = "cluster.strict-readdir",
-          .voltype     = "cluster/replicate",
-          .type        = NO_DOC,
-          .op_version  = 1
-        },
-        { .key         = "cluster.self-heal-window-size",
-          .voltype     = "cluster/replicate",
-          .option      = "data-self-heal-window-size",
-          .op_version  = 1
-        },
-        { .key         = "cluster.data-change-log",
-          .voltype     = "cluster/replicate",
-          .op_version  = 1
-        },
-        { .key         = "cluster.metadata-change-log",
-          .voltype     = "cluster/replicate",
-          .op_version  = 1
-        },
-        { .key         = "cluster.data-self-heal-algorithm",
-          .voltype     = "cluster/replicate",
-          .option      = "data-self-heal-algorithm",
-          .op_version  = 1
-        },
-        { .key         = "cluster.eager-lock",
-          .voltype     = "cluster/replicate",
-          .op_version  = 1
-        },
-        { .key         = "cluster.quorum-type",
-          .voltype     = "cluster/replicate",
-          .option      = "quorum-type",
-          .op_version  = 1
-        },
-        { .key         = "cluster.quorum-count",
-          .voltype     = "cluster/replicate",
-          .option      = "quorum-count",
-          .op_version  = 1
-        },
-        { .key         = "cluster.choose-local",
-          .voltype     = "cluster/replicate",
-          .op_version  = 2
-        },
-        { .key         = "cluster.self-heal-readdir-size",
-          .voltype     = "cluster/replicate",
-          .op_version  = 2
-        },
-        { .key         = "cluster.post-op-delay-secs",
-          .voltype     = "cluster/replicate",
-          .type        = NO_DOC,
-          .op_version  = 2
-        },
-        { .key         = "cluster.readdir-failover",
-          .voltype     = "cluster/replicate",
-          .op_version  = 2
+        { .key           = "cluster.entry-change-log",
+          .voltype       = "cluster/replicate",
+          .op_version    = 1,
+          .client_option = _gf_true
+        },
+        { .key           = "cluster.read-subvolume",
+          .voltype       = "cluster/replicate",
+          .op_version    = 1,
+          .client_option = _gf_true
+        },
+        { .key           = "cluster.read-subvolume-index",
+          .voltype       = "cluster/replicate",
+          .op_version    = 2,
+          .client_option = _gf_true
+        },
+        { .key           = "cluster.read-hash-mode",
+          .voltype       = "cluster/replicate",
+          .op_version    = 2,
+          .client_option = _gf_true
+        },
+        { .key           = "cluster.background-self-heal-count",
+          .voltype       = "cluster/replicate",
+          .op_version    = 1,
+          .client_option = _gf_true
+        },
+        { .key           = "cluster.metadata-self-heal",
+          .voltype       = "cluster/replicate",
+          .op_version    = 1,
+          .client_option = _gf_true
+        },
+        { .key           = "cluster.data-self-heal",
+          .voltype       = "cluster/replicate",
+          .op_version    = 1,
+          .client_option = _gf_true
+        },
+        { .key           = "cluster.entry-self-heal",
+          .voltype       = "cluster/replicate",
+          .op_version    = 1,
+          .client_option = _gf_true
+        },
+        { .key           = "cluster.self-heal-daemon",
+          .voltype       = "cluster/replicate",
+          .option        = "!self-heal-daemon",
+          .op_version    = 1
+        },
+        { .key           = "cluster.heal-timeout",
+          .voltype       = "cluster/replicate",
+          .option        = "!heal-timeout",
+          .op_version    = 2,
+          .client_option = _gf_true
+        },
+        { .key           = "cluster.strict-readdir",
+          .voltype       = "cluster/replicate",
+          .type          = NO_DOC,
+          .op_version    = 1,
+          .client_option = _gf_true
+        },
+        { .key           = "cluster.self-heal-window-size",
+          .voltype       = "cluster/replicate",
+          .option        = "data-self-heal-window-size",
+          .op_version    = 1,
+          .client_option = _gf_true
+        },
+        { .key           = "cluster.data-change-log",
+          .voltype       = "cluster/replicate",
+          .op_version    = 1,
+          .client_option = _gf_true
+        },
+        { .key           = "cluster.metadata-change-log",
+          .voltype       = "cluster/replicate",
+          .op_version    = 1,
+          .client_option = _gf_true
+        },
+        { .key           = "cluster.data-self-heal-algorithm",
+          .voltype       = "cluster/replicate",
+          .option        = "data-self-heal-algorithm",
+          .op_version    = 1,
+          .client_option = _gf_true
+        },
+        { .key           = "cluster.eager-lock",
+          .voltype       = "cluster/replicate",
+          .op_version    = 1,
+          .client_option = _gf_true
+        },
+        { .key           = "cluster.quorum-type",
+          .voltype       = "cluster/replicate",
+          .option        = "quorum-type",
+          .op_version    = 1,
+          .client_option = _gf_true
+        },
+        { .key           = "cluster.quorum-count",
+          .voltype       = "cluster/replicate",
+          .option        = "quorum-count",
+          .op_version    = 1,
+          .client_option = _gf_true
+        },
+        { .key           = "cluster.choose-local",
+          .voltype       = "cluster/replicate",
+          .op_version    = 2,
+          .client_option = _gf_true
+        },
+        { .key           = "cluster.self-heal-readdir-size",
+          .voltype       = "cluster/replicate",
+          .op_version    = 2,
+          .client_option = _gf_true
+        },
+        { .key           = "cluster.post-op-delay-secs",
+          .voltype       = "cluster/replicate",
+          .type          = NO_DOC,
+          .op_version    = 2,
+          .client_option = _gf_true
+        },
+        { .key           = "cluster.readdir-failover",
+          .voltype       = "cluster/replicate",
+          .op_version    = 2,
+          .client_option = _gf_true
         },
 
         /* Stripe xlator options */
-        { .key         = "cluster.stripe-block-size",
-          .voltype     = "cluster/stripe",
-          .option      = "block-size",
-          .op_version  = 1,
-          .validate_fn = validate_stripe
+        { .key           = "cluster.stripe-block-size",
+          .voltype       = "cluster/stripe",
+          .option        = "block-size",
+          .op_version    = 1,
+          .validate_fn = validate_stripe,
+          .client_option = _gf_true
         },
-        { .key         = "cluster.stripe-coalesce",
-          .voltype     = "cluster/stripe",
-          .option      = "coalesce",
-          .op_version  = 2
+        { .key           = "cluster.stripe-coalesce",
+          .voltype       = "cluster/stripe",
+          .option        = "coalesce",
+          .op_version    = 2,
+          .client_option = _gf_true
         },
 
         /* IO-stats xlator options */
@@ -530,48 +563,55 @@ struct volopt_map_entry glusterd_volopt_map[] = {
           .option      = "!brick-log-level",
           .op_version  = 1
         },
-        { .key         = "diagnostics.client-log-level",
-          .voltype     = "debug/io-stats",
-          .option      = "!client-log-level",
-          .op_version  = 1
+        { .key           = "diagnostics.client-log-level",
+          .voltype       = "debug/io-stats",
+          .option        = "!client-log-level",
+          .op_version    = 1,
+          .client_option = _gf_true
         },
         { .key         = "diagnostics.brick-sys-log-level",
           .voltype     = "debug/io-stats",
           .option      = "!sys-log-level",
           .op_version  = 1
         },
-        { .key         = "diagnostics.client-sys-log-level",
-          .voltype     = "debug/io-stats",
-          .option      = "!sys-log-level",
-          .op_version  = 1
+        { .key           = "diagnostics.client-sys-log-level",
+          .voltype       = "debug/io-stats",
+          .option        = "!sys-log-level",
+          .op_version    = 1,
+          .client_option = _gf_true
         },
 
         /* IO-cache xlator options */
-        { .key         = "performance.cache-max-file-size",
-          .voltype     = "performance/io-cache",
-          .option      = "max-file-size",
-          .op_version  = 1,
-          .validate_fn = validate_cache_max_min_size
-        },
-        { .key         = "performance.cache-min-file-size",
-          .voltype     = "performance/io-cache",
-          .option      = "min-file-size",
-          .op_version  = 1,
-          .validate_fn = validate_cache_max_min_size
-        },
-        { .key         = "performance.cache-refresh-timeout",
-          .voltype     = "performance/io-cache",
-          .option      = "cache-timeout",
-          .op_version  = 1
-        },
-        { .key         = "performance.cache-priority",
-          .voltype     = "performance/io-cache",
-          .option      = "priority",
-          .op_version  = 1
-        },
-        { .key         = "performance.cache-size",
-          .voltype     = "performance/io-cache",
-          .op_version  = 1
+        { .key           = "performance.cache-max-file-size",
+          .voltype       = "performance/io-cache",
+          .option        = "max-file-size",
+          .op_version    = 1,
+          .validate_fn   = validate_cache_max_min_size,
+          .client_option = _gf_true
+        },
+        { .key           = "performance.cache-min-file-size",
+          .voltype       = "performance/io-cache",
+          .option        = "min-file-size",
+          .op_version    = 1,
+          .validate_fn   = validate_cache_max_min_size,
+          .client_option = _gf_true
+        },
+        { .key           = "performance.cache-refresh-timeout",
+          .voltype       = "performance/io-cache",
+          .option        = "cache-timeout",
+          .op_version    = 1,
+          .client_option = _gf_true
+        },
+        { .key           = "performance.cache-priority",
+          .voltype       = "performance/io-cache",
+          .option        = "priority",
+          .op_version    = 1,
+          .client_option = _gf_true
+        },
+        { .key           = "performance.cache-size",
+          .voltype       = "performance/io-cache",
+          .op_version    = 1,
+          .client_option = _gf_true
         },
 
         /* IO-threads xlator options */
@@ -606,79 +646,94 @@ struct volopt_map_entry glusterd_volopt_map[] = {
         },
 
         /* Other perf xlators' options */
-        { .key         = "performance.cache-size",
-          .voltype     = "performance/quick-read",
-          .op_version  = 1
-        },
-        { .key         = "performance.flush-behind",
-          .voltype     = "performance/write-behind",
-          .option      = "flush-behind",
-          .op_version  = 1
-        },
-        { .key         = "performance.write-behind-window-size",
-          .voltype     = "performance/write-behind",
-          .option      = "cache-size",
-          .op_version  = 1
-        },
-        { .key         = "performance.strict-o-direct",
-          .voltype     = "performance/write-behind",
-          .option      = "strict-O_DIRECT",
-          .op_version  = 2
-        },
-        { .key         = "performance.strict-write-ordering",
-          .voltype     = "performance/write-behind",
-          .option      = "strict-write-ordering",
-          .op_version  = 2
-        },
-        { .key         = "performance.lazy-open",
-          .voltype     = "performance/open-behind",
-          .option      = "lazy-open",
-          .op_version =  2
-        },
-        { .key         = "performance.read-ahead-page-count",
-          .voltype     = "performance/read-ahead",
-          .option      = "page-count",
-          .op_version  = 1
-        },
-        { .key         = "performance.md-cache-timeout",
-          .voltype     = "performance/md-cache",
-          .option      = "md-cache-timeout",
-          .op_version  = 2
+        { .key           = "performance.cache-size",
+          .voltype       = "performance/quick-read",
+          .op_version    = 1,
+          .client_option = _gf_true
+        },
+        { .key           = "performance.flush-behind",
+          .voltype       = "performance/write-behind",
+          .option        = "flush-behind",
+          .op_version    = 1,
+          .client_option = _gf_true
+        },
+        { .key           = "performance.write-behind-window-size",
+          .voltype       = "performance/write-behind",
+          .option        = "cache-size",
+          .op_version    = 1,
+          .client_option = _gf_true
+        },
+        { .key           = "performance.strict-o-direct",
+          .voltype       = "performance/write-behind",
+          .option        = "strict-O_DIRECT",
+          .op_version    = 2,
+          .client_option = _gf_true
+        },
+        { .key           = "performance.strict-write-ordering",
+          .voltype       = "performance/write-behind",
+          .option        = "strict-write-ordering",
+          .op_version    = 2,
+          .client_option = _gf_true
+        },
+        { .key           = "performance.lazy-open",
+          .voltype       = "performance/open-behind",
+          .option        = "lazy-open",
+          .op_version    = 2,
+          .client_option = _gf_true
+        },
+        { .key           = "performance.read-ahead-page-count",
+          .voltype       = "performance/read-ahead",
+          .option        = "page-count",
+          .op_version    = 1,
+          .client_option = _gf_true
+        },
+        { .key           = "performance.md-cache-timeout",
+          .voltype       = "performance/md-cache",
+          .option        = "md-cache-timeout",
+          .op_version    = 2,
+          .client_option = _gf_true
         },
 
         /* Client xlator options */
-        { .key         = "network.frame-timeout",
-          .voltype     = "protocol/client",
-          .op_version  = 1
-        },
-        { .key         = "network.ping-timeout",
-          .voltype     = "protocol/client",
-          .op_version  = 1
-        },
-        { .key         = "network.tcp-window-size",
-          .voltype     = "protocol/client",
-          .op_version  = 1
-        },
-        { .key         = "features.lock-heal",
-          .voltype     = "protocol/client",
-          .option      = "lk-heal",
-          .op_version  = 1
-        },
-        { .key         = "features.grace-timeout",
-          .voltype     = "protocol/client",
-          .option      = "grace-timeout",
-          .op_version  = 1
-        },
-        { .key         = "client.ssl",
-          .voltype     = "protocol/client",
-          .option      = "transport.socket.ssl-enabled",
-          .type        = NO_DOC,
-          .op_version  = 2
-        },
-        { .key         = "network.remote-dio",
-          .voltype     = "protocol/client",
-          .option      = "filter-O_DIRECT",
-          .op_version  = 1
+        { .key           = "network.frame-timeout",
+          .voltype       = "protocol/client",
+          .op_version    = 1,
+          .client_option = _gf_true
+        },
+        { .key           = "network.ping-timeout",
+          .voltype       = "protocol/client",
+          .op_version    = 1,
+          .client_option = _gf_true
+        },
+        { .key           = "network.tcp-window-size",
+          .voltype       = "protocol/client",
+          .op_version    = 1,
+          .client_option = _gf_true
+        },
+        { .key           = "features.lock-heal",
+          .voltype       = "protocol/client",
+          .option        = "lk-heal",
+          .op_version    = 1,
+          .client_option = _gf_true
+        },
+        { .key           = "features.grace-timeout",
+          .voltype       = "protocol/client",
+          .option        = "grace-timeout",
+          .op_version    = 1,
+          .client_option = _gf_true
+        },
+        { .key           = "client.ssl",
+          .voltype       = "protocol/client",
+          .option        = "transport.socket.ssl-enabled",
+          .type          = NO_DOC,
+          .op_version    = 2,
+          .client_option = _gf_true
+        },
+        { .key           = "network.remote-dio",
+          .voltype       = "protocol/client",
+          .option        = "filter-O_DIRECT",
+          .op_version    = 1,
+          .client_option = _gf_true
         },
 
         /* Server xlator options */
@@ -743,56 +798,67 @@ struct volopt_map_entry glusterd_volopt_map[] = {
         },
 
         /* Performance xlators enable/disbable options */
-        { .key         = "performance.write-behind",
-          .voltype     = "performance/write-behind",
-          .option      = "!perf",
-          .value       = "on",
-          .op_version  = 1,
-          .description = "enable/disable write-behind translator in the volume."
-        },
-        { .key         = "performance.read-ahead",
-          .voltype     = "performance/read-ahead",
-          .option      = "!perf",
-          .value       = "on",
-          .op_version  = 1,
-          .description = "enable/disable read-ahead translator in the volume."
-        },
-        { .key         = "performance.io-cache",
-          .voltype     = "performance/io-cache",
-          .option      = "!perf",
-          .value       = "on",
-          .op_version  = 1,
-          .description = "enable/disable io-cache translator in the volume."
-        },
-        { .key         = "performance.quick-read",
-          .voltype     = "performance/quick-read",
-          .option      = "!perf",
-          .value       = "on",
-          .op_version  = 1,
-          .description = "enable/disable quick-read translator in the volume."
-        },
-        { .key         = "performance.open-behind",
-          .voltype     = "performance/open-behind",
-          .option      = "!perf",
-          .value       = "on",
-          .op_version  = 2,
-          .description = "enable/disable open-behind translator in the volume."
-        },
-        { .key         = "performance.stat-prefetch",
-          .voltype     = "performance/md-cache",
-          .option      = "!perf",
-          .value       = "on",
-          .op_version  = 1,
-          .description = "enable/disable meta-data caching translator in the "
-                         "volume."
-        },
-        { .key         = "performance.client-io-threads",
-          .voltype     = "performance/io-threads",
-          .option      = "!perf",
-          .value       = "off",
-          .op_version  = 1,
-          .description = "enable/disable io-threads translator in the client "
-                         "graph of volume."
+        { .key           = "performance.write-behind",
+          .voltype       = "performance/write-behind",
+          .option        = "!perf",
+          .value         = "on",
+          .op_version    = 1,
+          .description   = "enable/disable write-behind translator in the "
+                           "volume.",
+          .client_option = _gf_true
+        },
+        { .key           = "performance.read-ahead",
+          .voltype       = "performance/read-ahead",
+          .option        = "!perf",
+          .value         = "on",
+          .op_version    = 1,
+          .description   = "enable/disable read-ahead translator in the "
+                           "volume.",
+          .client_option = _gf_true
+        },
+        { .key           = "performance.io-cache",
+          .voltype       = "performance/io-cache",
+          .option        = "!perf",
+          .value         = "on",
+          .op_version    = 1,
+          .description   = "enable/disable io-cache translator in the volume.",
+          .client_option = _gf_true
+        },
+        { .key           = "performance.quick-read",
+          .voltype       = "performance/quick-read",
+          .option        = "!perf",
+          .value         = "on",
+          .op_version    = 1,
+          .description   = "enable/disable quick-read translator in the "
+                           "volume.",
+          .client_option = _gf_true
+        },
+        { .key           = "performance.open-behind",
+          .voltype       = "performance/open-behind",
+          .option        = "!perf",
+          .value         = "on",
+          .op_version    = 2,
+          .description   = "enable/disable open-behind translator in the "
+                           "volume.",
+          .client_option = _gf_true
+        },
+        { .key           = "performance.stat-prefetch",
+          .voltype       = "performance/md-cache",
+          .option        = "!perf",
+          .value         = "on",
+          .op_version    = 1,
+          .description   = "enable/disable meta-data caching translator in the "
+                           "volume.",
+          .client_option = _gf_true
+        },
+        { .key           = "performance.client-io-threads",
+          .voltype       = "performance/io-threads",
+          .option        = "!perf",
+          .value         = "off",
+          .op_version    = 1,
+          .description   = "enable/disable io-threads translator in the client "
+                           "graph of volume.",
+          .client_option = _gf_true
         },
         { .key         = "performance.nfs.write-behind",
           .voltype     = "performance/write-behind",
@@ -836,33 +902,37 @@ struct volopt_map_entry glusterd_volopt_map[] = {
           .type        = NO_DOC,
           .op_version  = 1
         },
-        { .key         = "performance.force-readdirp",
-          .voltype     = "performance/md-cache",
-          .option      = "force-readdirp",
-          .op_version  = 2
+        { .key           = "performance.force-readdirp",
+          .voltype       = "performance/md-cache",
+          .option        = "force-readdirp",
+          .op_version    = 2,
+          .client_option = _gf_true
         },
 
         /* Quota xlator options */
-        { .key         = VKEY_FEATURES_LIMIT_USAGE,
-          .voltype     = "features/quota",
-          .option      = "limit-set",
-          .type        = NO_DOC,
-          .op_version  = 1
-        },
-        { .key         = "features.quota-timeout",
-          .voltype     = "features/quota",
-          .option      = "timeout",
-          .value       = "0",
-          .op_version  = 1,
-          .validate_fn = validate_quota
-        },
-        { .key         = "features.quota-deem-statfs",
-          .voltype     = "features/quota",
-          .option      = "deem-statfs",
-          .value       = "off",
-          .type        = DOC,
-          .op_version  = 2,
-          .validate_fn = validate_quota
+        { .key           = VKEY_FEATURES_LIMIT_USAGE,
+          .voltype       = "features/quota",
+          .option        = "limit-set",
+          .type          = NO_DOC,
+          .op_version    = 1,
+          .client_option = _gf_true
+        },
+        { .key           = "features.quota-timeout",
+          .voltype       = "features/quota",
+          .option        = "timeout",
+          .value         = "0",
+          .op_version    = 1,
+          .validate_fn   = validate_quota,
+          .client_option = _gf_true
+        },
+        { .key           = "features.quota-deem-statfs",
+          .voltype       = "features/quota",
+          .option        = "deem-statfs",
+          .value         = "off",
+          .type          = DOC,
+          .op_version    = 2,
+          .validate_fn   = validate_quota,
+          .client_option = _gf_true
         },
 
         /* Marker xlator options */
@@ -1083,17 +1153,19 @@ struct volopt_map_entry glusterd_volopt_map[] = {
         },
 
         /* Other options which don't fit any place above */
-        { .key         = "features.read-only",
-          .voltype     = "features/read-only",
-          .option      = "!read-only",
-          .value       = "off",
-          .op_version  = 2
-        },
-        { .key         = "features.worm",
-          .voltype     = "features/worm",
-          .option      = "!worm",
-          .value       = "off",
-          .op_version  = 2
+        { .key           = "features.read-only",
+          .voltype       = "features/read-only",
+          .option        = "!read-only",
+          .value         = "off",
+          .op_version    = 2,
+          .client_option = _gf_true
+        },
+        { .key           = "features.worm",
+          .voltype       = "features/worm",
+          .option        = "!worm",
+          .value         = "off",
+          .op_version    = 2,
+          .client_option = _gf_true
         },
         { .key         = "storage.linux-aio",
           .voltype     = "storage/posix",
@@ -1113,10 +1185,11 @@ struct volopt_map_entry glusterd_volopt_map[] = {
           .voltype     = "storage/posix",
           .op_version  = 2
         },
-        { .key         = "config.memory-accounting",
-          .voltype     = "configuration",
-          .option      = "!config",
-          .op_version  = 2
+        { .key           = "config.memory-accounting",
+          .voltype       = "configuration",
+          .option        = "!config",
+          .op_version    = 2,
+          .client_option = _gf_true
         },
         { .key         = "config.transport",
           .voltype     = "configuration",
diff --git a/xlators/mgmt/glusterd/src/glusterd.h b/xlators/mgmt/glusterd/src/glusterd.h
index 4547e080d..49e7e727a 100644
--- a/xlators/mgmt/glusterd/src/glusterd.h
+++ b/xlators/mgmt/glusterd/src/glusterd.h
@@ -300,6 +300,9 @@ struct glusterd_volinfo_ {
 
         gf_boolean_t             memory_accounting;
         glusterd_vol_backend_t   backend;
+
+        int                      op_version;
+        int                      client_op_version;
 };
 
 typedef enum gd_node_type_ {