libglusterfs/dict: make 'dict_t' a opaque object

* ie, don't dereference dict_t pointer, instead use APIs everywhere
* other than dict_t only 'data_t' should be the valid export from dict.h

* added 'dict_foreach_fnmatch()' API
* changed dict_lookup() to use data_t, instead of data_pair_t

Change-Id: I400bb0dd55519a7c5d2a107e67c8e7a7207228dc
Signed-off-by: Amar Tumballi <amarts@redhat.com>
BUG: 850917
Reviewed-on: http://review.gluster.org/3829
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Anand Avati <avati@redhat.com>

37 files changed, 486 insertions, 413 deletions

diff --git a/cli/src/cli-rpc-ops.c b/cli/src/cli-rpc-ops.c
index 9d699a0f..9d8aa26e 100644
--- a/cli/src/cli-rpc-ops.c
+++ b/cli/src/cli-rpc-ops.c
@@ -478,7 +478,6 @@ gf_cli_get_volume_cbk (struct rpc_req *req, struct iovec *iov,
 {
         int                        ret                  = -1;
         int                        opt_count            = 0;
-        int                        k                    = 0;
         int32_t                    i                    = 0;
         int32_t                    j                    = 1;
         int32_t                    status               = 0;
@@ -494,7 +493,6 @@ gf_cli_get_volume_cbk (struct rpc_req *req, struct iovec *iov,
         char                      *brick                = NULL;
         char                      *volname              = NULL;
         dict_t                    *dict                 = NULL;
-        data_pair_t               *pairs                = NULL;
         data_t                    *value                = NULL;
         cli_local_t               *local                = NULL;
         char                       key[1024]            = {0};
@@ -709,12 +707,6 @@ xml_output:
                         j++;
                 }
 
-                pairs = dict->members_list;
-                if (!pairs) {
-                        ret = -1;
-                        goto out;
-                }
-
                 snprintf (key, 256, "volume%d.opt_count",i);
                 ret = dict_get_int32 (dict, key, &opt_count);
                 if (ret)
@@ -724,26 +716,26 @@ xml_output:
                         goto out;
 
                 cli_out ("Options Reconfigured:");
-                k = 0;
 
-                while (k < opt_count) {
-
-                        snprintf (key, 256, "volume%d.option.",i);
-                        while (pairs) {
-                                ptr = strstr (pairs->key, "option.");
-                                if (ptr) {
-                                        value = pairs->value;
-                                        if (!value) {
-                                                ret = -1;
-                                                goto out;
-                                        }
-                                        cli_out_options (key, pairs->key,
-                                                         value->data);
+                snprintf (key, 256, "volume%d.option.",i);
+                int _output_volinfo_opts (dict_t *d, char *k,
+                                          data_t *v, void *tmp)
+                {
+                        ptr = strstr (k, "option.");
+                        if (ptr) {
+                                value = v;
+                                if (!value) {
+                                        ret = -1;
+                                        goto internal_out;
                                 }
-                                pairs = pairs->next;
+                                cli_out_options (key, k, v->data);
                         }
-                        k++;
+                internal_out:
+                        return ret;
                 }
+                ret = dict_foreach (dict, _output_volinfo_opts, NULL);
+                if (ret)
+                        goto out;
 
                 i++;
         }
diff --git a/cli/src/cli-xml-output.c b/cli/src/cli-xml-output.c
index eebcafb9..7ca57164 100644
--- a/cli/src/cli-xml-output.c
+++ b/cli/src/cli-xml-output.c
@@ -136,7 +136,7 @@ out:
         return ret;
 }
 
-void
+int
 cli_xml_output_data_pair (dict_t *this, char *key, data_t *value,
                           void *data)
 {
@@ -148,7 +148,7 @@ cli_xml_output_data_pair (dict_t *this, char *key, data_t *value,
         ret = xmlTextWriterWriteFormatElement (*writer, (xmlChar *)key,
                                                "%s", value->data);
 
-        return;
+        return ret;
 }
 
 int
@@ -2077,17 +2077,9 @@ cli_xml_output_vol_info_options (xmlTextWriterPtr writer, dict_t *dict,
 {
         int             ret = -1;
         int             opt_count = 0;
-        data_pair_t     *pairs = 0;
         data_t          *value = 0;
         char            *ptr = NULL;
         char            key[1024] = {0,};
-        int             i = 0;
-
-        pairs = dict->members_list;
-        if (!pairs) {
-                ret = -1;
-                goto out;
-        }
 
         snprintf (key, sizeof (key), "%s.opt_count", prefix);
         ret = dict_get_int32 (dict, key, &opt_count);
@@ -2100,25 +2092,31 @@ cli_xml_output_vol_info_options (xmlTextWriterPtr writer, dict_t *dict,
         /* <options> */
         ret = xmlTextWriterStartElement (writer, (xmlChar *)"options");
         XML_RET_CHECK_AND_GOTO (ret, out);
-        while (i < opt_count) {
-                snprintf (key, sizeof (key), "%s.option.", prefix);
-                while (pairs) {
-                        ptr = strstr (pairs->key, "option.");
-                        if (ptr) {
-                                value = pairs->value;
-                                if (!value) {
-                                        ret = -1;
-                                        goto out;
-                                }
-                                ret = cli_xml_output_vol_info_option
-                                        (writer, key, pairs->key, value->data);
-                                if (ret)
-                                        goto out;
-                        }
-                        pairs = pairs->next;
+        snprintf (key, sizeof (key), "%s.option.", prefix);
+
+        int _output_vol_info_option (dict_t *d, char *k, data_t *v,
+                                     void *data)
+        {
+                int ret = 0;
+                ptr = strstr (k, "option.");
+                if (!ptr)
+                        goto internal_out;
+
+                value = v;
+                if (!value) {
+                        ret = -1;
+                        goto internal_out;
                 }
-                i++;
+                ret = cli_xml_output_vol_info_option (writer, key, k,
+                                                      v->data);
+
+        internal_out:
+                return ret;
         }
+        ret = dict_foreach (dict, _output_vol_info_option, NULL);
+        if (ret)
+                goto out;
+
         /* </options> */
         ret = xmlTextWriterEndElement (writer);
         XML_RET_CHECK_AND_GOTO (ret, out);
diff --git a/libglusterfs/src/common-utils.h b/libglusterfs/src/common-utils.h
index 3d18a29a..7e914571 100644
--- a/libglusterfs/src/common-utils.h
+++ b/libglusterfs/src/common-utils.h
@@ -171,24 +171,26 @@ void gf_print_trace (int32_t signal, glusterfs_ctx_t *ctx);
         } while (0)
 
 
-#define GF_IF_INTERNAL_XATTR_GOTO(pattern, dict, trav, op_errno, label) \
+#define GF_IF_INTERNAL_XATTR_GOTO(pattern, dict, op_errno, label)       \
         do {                                                            \
                 if (!dict) {                                            \
                         gf_log (this->name, GF_LOG_ERROR,               \
                                 "setxattr dict is null");               \
                         goto label;                                     \
                 }                                                       \
-                trav = dict->members_list;                              \
-                while (trav) {                                          \
-                        if (!fnmatch (pattern, trav->key, 0)) {         \
-                                op_errno = EPERM;                       \
-                                gf_log (this->name, GF_LOG_ERROR,       \
-                                        "attempt to set internal"       \
-                                        " xattr: %s: %s", trav->key,    \
-                                        strerror (op_errno));           \
-                                goto label;                             \
-                        }                                               \
-                        trav = trav->next;                              \
+                int _handle_keyvalue_pair (dict_t *d, char *k,          \
+                                           data_t *v, void *tmp)        \
+                {                                                       \
+                        return 0;                                       \
+                }                                                       \
+                if (dict_foreach_fnmatch (dict, pattern,                  \
+                                          _handle_keyvalue_pair, NULL) > 0) { \
+                        op_errno = EPERM;                               \
+                        gf_log (this->name, GF_LOG_ERROR,               \
+                                "attempt to set internal"               \
+                                " xattr: %s: %s", pattern,              \
+                                strerror (op_errno));                   \
+                        goto label;                                     \
                 }                                                       \
         } while (0)
 
diff --git a/libglusterfs/src/dict.c b/libglusterfs/src/dict.c
index c290e0d6..1420246e 100644
--- a/libglusterfs/src/dict.c
+++ b/libglusterfs/src/dict.c
@@ -14,6 +14,7 @@
 #include <stdio.h>
 #include <inttypes.h>
 #include <limits.h>
+#include <fnmatch.h>
 
 #ifndef _CONFIG_H
 #define _CONFIG_H
@@ -208,7 +209,7 @@ _dict_lookup (dict_t *this, char *key)
 }
 
 int32_t
-dict_lookup (dict_t *this, char *key, data_pair_t **data)
+dict_lookup (dict_t *this, char *key, data_t **data)
 {
         if (!this || !key || !data) {
                 gf_log_callingfn ("dict", GF_LOG_WARNING,
@@ -216,16 +217,18 @@ dict_lookup (dict_t *this, char *key, data_pair_t **data)
                 return -1;
         }
 
+        data_pair_t *tmp = NULL;
         LOCK (&this->lock);
         {
-                *data = _dict_lookup (this, key);
+                tmp = _dict_lookup (this, key);
         }
         UNLOCK (&this->lock);
-        if (*data)
-                return 0;
-        else
+
+        if (!tmp)
                 return -1;
 
+        *data = tmp->value;
+        return 0;
 }
 
 static int32_t
@@ -1051,47 +1054,93 @@ data_to_bin (data_t *data)
         return data->data;
 }
 
-void
+int
 dict_foreach (dict_t *dict,
-              void (*fn)(dict_t *this,
-                         char *key,
-                         data_t *value,
-                         void *data),
+              int (*fn)(dict_t *this,
+                        char *key,
+                        data_t *value,
+                        void *data),
               void *data)
 {
         if (!dict) {
                 gf_log_callingfn ("dict", GF_LOG_WARNING,
                                   "dict is NULL");
-                return;
+                return -1;
         }
 
-        data_pair_t *pairs = dict->members_list;
-        data_pair_t *next = NULL;
+        int          ret   = -1;
+        data_pair_t *pairs = NULL;
+        data_pair_t *next  = NULL;
 
+        pairs = dict->members_list;
         while (pairs) {
                 next = pairs->next;
-                fn (dict, pairs->key, pairs->value, data);
+                ret = fn (dict, pairs->key, pairs->value, data);
+                if (ret == -1)
+                        return -1;
                 pairs = next;
         }
+
+        return 0;
 }
 
+/* return values:
+   -1 = failure,
+    0 = no matches found,
+   +n = n number of matches
+*/
+int
+dict_foreach_fnmatch (dict_t *dict, char *pattern,
+                      int (*fn)(dict_t *this,
+                                char *key,
+                                data_t *value,
+                                void *data),
+                      void *data)
+{
+        if (!dict) {
+                gf_log_callingfn ("dict", GF_LOG_WARNING,
+                                  "dict is NULL");
+                return 0;
+        }
 
-static void
+        int          ret = -1;
+        int          count = 0;
+        data_pair_t *pairs = NULL;
+        data_pair_t *next  = NULL;
+
+        pairs = dict->members_list;
+        while (pairs) {
+                next = pairs->next;
+                if (!fnmatch (pattern, pairs->key, 0)) {
+                        ret = fn (dict, pairs->key, pairs->value, data);
+                        if (ret == -1)
+                                return -1;
+                        count++;
+                }
+                pairs = next;
+        }
+
+        return count;
+}
+
+
+static int
 _copy (dict_t *unused,
        char *key,
        data_t *value,
        void *newdict)
 {
-        dict_set ((dict_t *)newdict, key, (value));
+        return dict_set ((dict_t *)newdict, key, (value));
 }
 
-static void
+static int
 _remove (dict_t *dict,
          char *key,
          data_t *value,
          void *unused)
 {
         dict_del ((dict_t *)dict, key);
+        return 0;
 }
 
 
diff --git a/libglusterfs/src/dict.h b/libglusterfs/src/dict.h
index 17e9bcb4..050f7ca2 100644
--- a/libglusterfs/src/dict.h
+++ b/libglusterfs/src/dict.h
@@ -122,7 +122,7 @@ dict_t *dict_ref (dict_t *dict);
 data_t *data_ref (data_t *data);
 void data_unref (data_t *data);
 
-int32_t dict_lookup  (dict_t *this, char *key, data_pair_t **data);
+int32_t dict_lookup  (dict_t *this, char *key, data_t **data);
 /*
    TODO: provide converts for differnt byte sizes, signedness, and void *
  */
@@ -165,12 +165,19 @@ data_t * data_copy (data_t *old);
 dict_t *get_new_dict_full (int size_hint);
 dict_t *get_new_dict ();
 
-void dict_foreach (dict_t *this,
-		   void (*fn)(dict_t *this,
-			      char *key,
-			      data_t *value,
-			      void *data),
-		   void *data);
+int dict_foreach (dict_t *this,
+                  int (*fn)(dict_t *this,
+                            char *key,
+                            data_t *value,
+                            void *data),
+                  void *data);
+
+int dict_foreach_fnmatch (dict_t *dict, char *pattern,
+                          int (*fn)(dict_t *this,
+                                    char *key,
+                                    data_t *value,
+                                    void *data),
+                          void *data);
 
 dict_t *dict_copy (dict_t *this, dict_t *new);
 
diff --git a/libglusterfs/src/graph-print.c b/libglusterfs/src/graph-print.c
index 66712986..862a9300 100644
--- a/libglusterfs/src/graph-print.c
+++ b/libglusterfs/src/graph-print.c
@@ -92,7 +92,6 @@ glusterfs_graph_print (struct gf_printer *gp, glusterfs_graph_t *graph)
         } while (0)
 
         xlator_t      *trav = NULL;
-        data_pair_t   *pair = NULL;
         xlator_list_t *xch = NULL;
         int            ret = 0;
         ssize_t        len = 0;
@@ -105,11 +104,18 @@ glusterfs_graph_print (struct gf_printer *gp, glusterfs_graph_t *graph)
                 GPPRINTF (gp, "volume %s\n    type %s\n", trav->name,
                           trav->type);
 
-                for (pair =  trav->options->members_list; pair && pair->next;
-                     pair = pair->next);
-                for (; pair; pair = pair->prev)
-                        GPPRINTF (gp, "    option %s %s\n", pair->key,
-                                  pair->value->data);
+                int _print_volume_options (dict_t *d, char *k, data_t *v,
+                                           void *tmp)
+                {
+                        GPPRINTF (gp, "    option %s %s\n", k, v->data);
+                        return 0;
+                out:
+                        /* means, it is a failure */
+                        return -1;
+                }
+                ret = dict_foreach (trav->options, _print_volume_options, NULL);
+                if (ret)
+                        goto out;
 
                 if (trav->children) {
                         GPPRINTF (gp, "    subvolumes");
diff --git a/libglusterfs/src/graph.c b/libglusterfs/src/graph.c
index a42ae7cd..c18d0265 100644
--- a/libglusterfs/src/graph.c
+++ b/libglusterfs/src/graph.c
@@ -298,7 +298,7 @@ glusterfs_graph_init (glusterfs_graph_t *graph)
 }
 
 
-static void
+static int
 _log_if_unknown_option (dict_t *dict, char *key, data_t *value, void *data)
 {
         volume_option_t   *found = NULL;
@@ -313,7 +313,7 @@ _log_if_unknown_option (dict_t *dict, char *key, data_t *value, void *data)
                         "option '%s' is not recognized", key);
         }
 
-        return;
+        return 0;
 }
 
 
diff --git a/libglusterfs/src/options.c b/libglusterfs/src/options.c
index fa238528..6fa11283 100644
--- a/libglusterfs/src/options.c
+++ b/libglusterfs/src/options.c
@@ -777,7 +777,7 @@ xlator_volume_option_get (xlator_t *xl, const char *key)
 }
 
 
-static void
+static int
 xl_opt_validate (dict_t *dict, char *key, data_t *value, void *data)
 {
         xlator_t          *xl = NULL;
@@ -798,7 +798,7 @@ xl_opt_validate (dict_t *dict, char *key, data_t *value, void *data)
 
         opt = xlator_volume_option_get_list (vol_opt, key);
         if (!opt)
-                return;
+                return 0;
 
         ret = xlator_option_validate (xl, key, value->data, opt, &errstr);
         if (ret)
@@ -816,7 +816,7 @@ xl_opt_validate (dict_t *dict, char *key, data_t *value, void *data)
                 dict_set (dict, opt->key[0], value);
                 dict_del (dict, key);
         }
-        return;
+        return 0;
 }
 
 
diff --git a/xlators/cluster/afr/src/afr-inode-read.c b/xlators/cluster/afr/src/afr-inode-read.c
index 40e5abd3..4679e461 100644
--- a/xlators/cluster/afr/src/afr-inode-read.c
+++ b/xlators/cluster/afr/src/afr-inode-read.c
@@ -521,7 +521,7 @@ struct _xattr_key {
 };
 
 
-void
+int
 __gather_xattr_keys (dict_t *dict, char *key, data_t *value,
                      void *data)
 {
@@ -533,13 +533,14 @@ __gather_xattr_keys (dict_t *dict, char *key, data_t *value,
 
                 xkey = GF_CALLOC (1, sizeof (*xkey), gf_afr_mt_xattr_key);
                 if (!xkey)
-                        return;
+                        return -1;
 
                 xkey->key = key;
                 INIT_LIST_HEAD (&xkey->list);
 
                 list_add_tail (&xkey->list, list);
         }
+        return 0;
 }
 
 
diff --git a/xlators/cluster/afr/src/afr-inode-write.c b/xlators/cluster/afr/src/afr-inode-write.c
index 6ea288a5..148a939c 100644
--- a/xlators/cluster/afr/src/afr-inode-write.c
+++ b/xlators/cluster/afr/src/afr-inode-write.c
@@ -1478,17 +1478,16 @@ afr_setxattr (call_frame_t *frame, xlator_t *this,
         afr_private_t  *priv              = NULL;
         afr_local_t    *local             = NULL;
         call_frame_t   *transaction_frame = NULL;
-        data_pair_t    *trav              = NULL;
         int             ret               = -1;
         int             op_errno          = EINVAL;
 
         VALIDATE_OR_GOTO (this, out);
 
         GF_IF_INTERNAL_XATTR_GOTO ("trusted.afr.*", dict,
-                                   trav, op_errno, out);
+                                   op_errno, out);
 
         GF_IF_INTERNAL_XATTR_GOTO ("trusted.glusterfs.afr.*", dict,
-                                   trav, op_errno, out);
+                                   op_errno, out);
 
         VALIDATE_OR_GOTO (frame, out);
         VALIDATE_OR_GOTO (this->private, out);
@@ -1667,17 +1666,16 @@ afr_fsetxattr (call_frame_t *frame, xlator_t *this,
         call_frame_t     *transaction_frame = NULL;
         int               ret               = -1;
         int               op_errno          = EINVAL;
-        data_pair_t      *trav              = NULL;
 
         VALIDATE_OR_GOTO (frame, out);
         VALIDATE_OR_GOTO (this, out);
         VALIDATE_OR_GOTO (this->private, out);
 
         GF_IF_INTERNAL_XATTR_GOTO ("trusted.afr.*", dict,
-                                   trav, op_errno, out);
+                                   op_errno, out);
 
         GF_IF_INTERNAL_XATTR_GOTO ("trusted.glusterfs.afr.*", dict,
-                                   trav, op_errno, out);
+                                   op_errno, out);
 
         if (ret)
                 goto out;
diff --git a/xlators/cluster/afr/src/pump.c b/xlators/cluster/afr/src/pump.c
index ff42d8dc..e42c955d 100644
--- a/xlators/cluster/afr/src/pump.c
+++ b/xlators/cluster/afr/src/pump.c
@@ -1276,7 +1276,7 @@ struct _xattr_key {
         struct list_head list;
 };
 
-static void
+static int
 __gather_xattr_keys (dict_t *dict, char *key, data_t *value,
                      void *data)
 {
@@ -1288,13 +1288,14 @@ __gather_xattr_keys (dict_t *dict, char *key, data_t *value,
 
                 xkey = GF_CALLOC (1, sizeof (*xkey), gf_afr_mt_xattr_key);
                 if (!xkey)
-                        return;
+                        return -1;
 
                 xkey->key = key;
                 INIT_LIST_HEAD (&xkey->list);
 
                 list_add_tail (&xkey->list, list);
         }
+        return 0;
 }
 
 static void
@@ -1646,7 +1647,6 @@ pump_setxattr (call_frame_t *frame, xlator_t *this,
 	afr_private_t * priv  = NULL;
 	afr_local_t   * local = NULL;
 	call_frame_t   *transaction_frame = NULL;
-        data_pair_t   * trav  = NULL;
 	int ret = -1;
 	int op_errno = 0;
 
@@ -1655,7 +1655,7 @@ pump_setxattr (call_frame_t *frame, xlator_t *this,
 	VALIDATE_OR_GOTO (this->private, out);
 
         GF_IF_INTERNAL_XATTR_GOTO ("trusted.glusterfs.pump*", dict,
-                                   trav, op_errno, out);
+                                   op_errno, out);
 
 	priv = this->private;
         if (!priv->use_afr_in_pump) {
diff --git a/xlators/cluster/dht/src/dht-common.c b/xlators/cluster/dht/src/dht-common.c
index 4d8d326e..24f28c1f 100644
--- a/xlators/cluster/dht/src/dht-common.c
+++ b/xlators/cluster/dht/src/dht-common.c
@@ -26,13 +26,13 @@
 #include <sys/time.h>
 #include <libgen.h>
 
-void
+int
 dht_aggregate (dict_t *this, char *key, data_t *value, void *data)
 {
         dict_t  *dst  = NULL;
         int64_t *ptr  = 0, *size = NULL;
         int32_t  ret  = -1;
-        data_pair_t  *data_pair = NULL;
+        data_t  *dict_data = NULL;
 
         dst = data;
 
@@ -44,30 +44,30 @@ dht_aggregate (dict_t *this, char *key, data_t *value, void *data)
                         if (size == NULL) {
                                 gf_log ("dht", GF_LOG_WARNING,
                                         "memory allocation failed");
-                                return;
+                                return -1;
                         }
                         ret = dict_set_bin (dst, key, size, sizeof (int64_t));
                         if (ret < 0) {
                                 gf_log ("dht", GF_LOG_WARNING,
                                         "dht aggregate dict set failed");
                                 GF_FREE (size);
-                                return;
+                                return -1;
                         }
                 }
 
                 ptr = data_to_bin (value);
                 if (ptr == NULL) {
                         gf_log ("dht", GF_LOG_WARNING, "data to bin failed");
-                        return;
+                        return -1;
                 }
 
                 *size = hton64 (ntoh64 (*size) + ntoh64 (*ptr));
         } else {
                 /* compare user xattrs only */
                 if (!strncmp (key, "user.", strlen ("user."))) {
-                        ret = dict_lookup (dst, key, &data_pair); 
-                        if (!ret && data_pair && value) {
-                                ret = is_data_equal (data_pair->value, value);
+                        ret = dict_lookup (dst, key, &dict_data);
+                        if (!ret && dict_data && value) {
+                                ret = is_data_equal (dict_data, value);
                                 if (!ret)
                                         gf_log ("dht", GF_LOG_DEBUG,
                                                 "xattr mismatch for %s", key);
@@ -78,7 +78,7 @@ dht_aggregate (dict_t *this, char *key, data_t *value, void *data)
                         gf_log ("dht", GF_LOG_WARNING, "xattr dict set failed");
         }
 
-        return;
+        return 0;
 }
 
 
@@ -2100,7 +2100,6 @@ dht_fsetxattr (call_frame_t *frame, xlator_t *this,
         xlator_t     *subvol   = NULL;
         dht_local_t  *local    = NULL;
         int           op_errno = EINVAL;
-        data_pair_t  *trav     = NULL;
 
         VALIDATE_OR_GOTO (frame, err);
         VALIDATE_OR_GOTO (this, err);
@@ -2108,7 +2107,7 @@ dht_fsetxattr (call_frame_t *frame, xlator_t *this,
         VALIDATE_OR_GOTO (fd->inode, err);
 
         GF_IF_INTERNAL_XATTR_GOTO ("trusted.glusterfs.dht*", xattr,
-                                   trav, op_errno, err);
+                                   op_errno, err);
 
         local = dht_local_init (frame, NULL, fd, GF_FOP_FSETXATTR);
         if (!local) {
@@ -2206,8 +2205,6 @@ dht_setxattr (call_frame_t *frame, xlator_t *this,
         char          value[4096] = {0,};
         gf_dht_migrate_data_type_t forced_rebalance = GF_DHT_MIGRATE_DATA;
         int           call_cnt = 0;
-        data_pair_t  *trav     = NULL;
-
 
         VALIDATE_OR_GOTO (frame, err);
         VALIDATE_OR_GOTO (this, err);
@@ -2216,7 +2213,7 @@ dht_setxattr (call_frame_t *frame, xlator_t *this,
         VALIDATE_OR_GOTO (loc->path, err);
 
         GF_IF_INTERNAL_XATTR_GOTO ("trusted.glusterfs.dht*", xattr,
-                                   trav, op_errno, err);
+                                   op_errno, err);
 
         conf   = this->private;
         local = dht_local_init (frame, loc, NULL, GF_FOP_SETXATTR);
diff --git a/xlators/cluster/stripe/src/stripe-helpers.c b/xlators/cluster/stripe/src/stripe-helpers.c
index 8053f871..e85661c1 100644
--- a/xlators/cluster/stripe/src/stripe-helpers.c
+++ b/xlators/cluster/stripe/src/stripe-helpers.c
@@ -40,7 +40,7 @@ out:
 
 
 
-void
+int
 stripe_aggregate (dict_t *this, char *key, data_t *value, void *data)
 {
         dict_t  *dst  = NULL;
@@ -83,7 +83,7 @@ stripe_aggregate (dict_t *this, char *key, data_t *value, void *data)
         }
 
 out:
-        return;
+        return 0;
 }
 
 
diff --git a/xlators/cluster/stripe/src/stripe.c b/xlators/cluster/stripe/src/stripe.c
index 6588a449..76d912a0 100644
--- a/xlators/cluster/stripe/src/stripe.c
+++ b/xlators/cluster/stripe/src/stripe.c
@@ -3927,7 +3927,6 @@ int
 stripe_setxattr (call_frame_t *frame, xlator_t *this,
                  loc_t *loc, dict_t *dict, int flags, dict_t *xdata)
 {
-        data_pair_t      *pair     = NULL;
         int32_t           op_errno = EINVAL;
         xlator_list_t    *trav     = NULL;
         stripe_private_t *priv     = NULL;
@@ -3940,7 +3939,7 @@ stripe_setxattr (call_frame_t *frame, xlator_t *this,
         VALIDATE_OR_GOTO (loc->inode, err);
 
         GF_IF_INTERNAL_XATTR_GOTO ("trusted.*stripe*", dict,
-                                   pair, op_errno, err);
+                                   op_errno, err);
 
         priv = this->private;
         trav = this->children;
@@ -3992,7 +3991,6 @@ int
 stripe_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
                   dict_t *dict, int flags, dict_t *xdata)
 {
-        data_pair_t    *trav     = NULL;
         int32_t         op_ret   = -1;
         int32_t         op_errno = EINVAL;
 
@@ -4001,7 +3999,7 @@ stripe_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
         VALIDATE_OR_GOTO (fd, err);
 
         GF_IF_INTERNAL_XATTR_GOTO ("trusted.*stripe*", dict,
-                                   trav, op_errno, err);
+                                   op_errno, err);
 
         STACK_WIND (frame, stripe_fsetxattr_cbk,
                     FIRST_CHILD(this),
diff --git a/xlators/debug/io-stats/src/io-stats.c b/xlators/debug/io-stats/src/io-stats.c
index 3bb48b90..82159fca 100644
--- a/xlators/debug/io-stats/src/io-stats.c
+++ b/xlators/debug/io-stats/src/io-stats.c
@@ -2139,7 +2139,7 @@ io_stats_fsync (call_frame_t *frame, xlator_t *this,
 }
 
 
-void
+int
 conditional_dump (dict_t *dict, char *key, data_t *value, void *data)
 {
         struct {
@@ -2163,20 +2163,21 @@ conditional_dump (dict_t *dict, char *key, data_t *value, void *data)
 
                 if (!strncmp (filename, "", 1)) {
                         gf_log (this->name, GF_LOG_ERROR, "No filename given");
-                        return;
+                        return -1;
                 }
                 logfp = fopen (filename, "w+");
                 GF_ASSERT (logfp);
                 if (!logfp) {
                         gf_log (this->name, GF_LOG_ERROR, "failed to open %s "
                                 "for writing", filename);
-                        return;
+                        return -1;
                 }
                 (void) ios_dump_args_init (&args, IOS_DUMP_TYPE_FILE,
                                            logfp);
                 io_stats_dump (this, &args);
                 fclose (logfp);
         }
+        return 0;
 }
 
 
diff --git a/xlators/features/index/src/index.c b/xlators/features/index/src/index.c
index 071a25b4..a98236c1 100644
--- a/xlators/features/index/src/index.c
+++ b/xlators/features/index/src/index.c
@@ -435,20 +435,22 @@ out:
 void
 _xattrop_index_action (xlator_t *this, inode_t *inode,  dict_t *xattr)
 {
-        data_pair_t       *trav = NULL;
         gf_boolean_t      zero_xattr = _gf_true;
         index_inode_ctx_t *ctx = NULL;
         int               ret = 0;
 
-        trav = xattr->members_list;
-        while (trav && inode) {
-                if (mem_0filled ((const char*)trav->value->data,
-                                 trav->value->len)) {
+        int _check_key_is_zero_filled (dict_t *d, char *k, data_t *v,
+                                       void *tmp)
+        {
+                if (mem_0filled ((const char*)v->data, v->len)) {
                         zero_xattr = _gf_false;
-                        break;
+                        /* -1 means, no more iterations, treat as 'break' */
+                        return -1;
                 }
-                trav = trav->next;
+                return 0;
         }
+        dict_foreach (xattr, _check_key_is_zero_filled, NULL);
+
         ret = index_inode_ctx_get (inode, this, &ctx);
         if (ret) {
                 gf_log (this->name, GF_LOG_ERROR, "Not able to %s %s -> index",
diff --git a/xlators/features/marker/src/marker-common.c b/xlators/features/marker/src/marker-common.c
index f478db7a..f9e9a835 100644
--- a/xlators/features/marker/src/marker-common.c
+++ b/xlators/features/marker/src/marker-common.c
@@ -61,18 +61,10 @@ unlock: UNLOCK (&inode->lock);
         return ret;
 }
 
-void
+int
 marker_filter_quota_xattr (dict_t *dict, char *key,
                            data_t *value, void *data)
 {
-        int ret = -1;
-
-        GF_VALIDATE_OR_GOTO ("marker", dict, out);
-        GF_VALIDATE_OR_GOTO ("marker", key, out);
-
-        ret = fnmatch ("trusted.glusterfs.quota*", key, 0);
-        if (ret == 0)
-                dict_del (dict, key);
-out:
-        return;
+        dict_del (dict, key);
+        return 0;
 }
diff --git a/xlators/features/marker/src/marker-common.h b/xlators/features/marker/src/marker-common.h
index cd68ab6a..22860255 100644
--- a/xlators/features/marker/src/marker-common.h
+++ b/xlators/features/marker/src/marker-common.h
@@ -23,6 +23,6 @@
 int32_t
 marker_force_inode_ctx_get (inode_t *, xlator_t *, marker_inode_ctx_t **);
 
-void
+int
 marker_filter_quota_xattr (dict_t *, char *, data_t *, void *);
 #endif
diff --git a/xlators/features/marker/src/marker.c b/xlators/features/marker/src/marker.c
index a0c941b2..eeb208ea 100644
--- a/xlators/features/marker/src/marker.c
+++ b/xlators/features/marker/src/marker.c
@@ -275,8 +275,10 @@ marker_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 gf_log (this->name, GF_LOG_DEBUG,
                         "Filtering the quota extended attributes");
 
-                dict_foreach (dict, marker_filter_quota_xattr, NULL);
+                dict_foreach_fnmatch (dict, "trusted.glusterfs.quota*",
+                                      marker_filter_quota_xattr, NULL);
         }
+
         STACK_UNWIND_STRICT (getxattr, frame, op_ret, op_errno, dict, xdata);
         return 0;
 }
diff --git a/xlators/features/quota/src/quota.c b/xlators/features/quota/src/quota.c
index cb148325..4a2e56bb 100644
--- a/xlators/features/quota/src/quota.c
+++ b/xlators/features/quota/src/quota.c
@@ -2694,7 +2694,6 @@ int
 quota_setxattr (call_frame_t *frame, xlator_t *this,
                 loc_t *loc, dict_t *dict, int flags, dict_t *xdata)
 {
-        data_pair_t    *trav     = NULL;
         int             op_errno = EINVAL;
         int             op_ret   = -1;
 
@@ -2703,7 +2702,7 @@ quota_setxattr (call_frame_t *frame, xlator_t *this,
         VALIDATE_OR_GOTO (loc, err);
 
         GF_IF_INTERNAL_XATTR_GOTO ("trusted.glusterfs.quota*", dict,
-                                   trav, op_errno, err);
+                                   op_errno, err);
 
         STACK_WIND (frame, quota_setxattr_cbk,
                     FIRST_CHILD(this),
@@ -2727,7 +2726,6 @@ int
 quota_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
                  dict_t *dict, int flags, dict_t *xdata)
 {
-        data_pair_t    *trav     = NULL;
         int32_t         op_ret   = -1;
         int32_t         op_errno = EINVAL;
 
@@ -2736,7 +2734,7 @@ quota_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
         VALIDATE_OR_GOTO (fd, err);
 
         GF_IF_INTERNAL_XATTR_GOTO ("trusted.glusterfs.quota*", dict,
-                                   trav, op_errno, err);
+                                   op_errno, err);
 
         STACK_WIND (frame, quota_fsetxattr_cbk,
                     FIRST_CHILD(this),
diff --git a/xlators/mgmt/glusterd/src/glusterd-geo-rep.c b/xlators/mgmt/glusterd/src/glusterd-geo-rep.c
index 808b1276..2a7cfca9 100644
--- a/xlators/mgmt/glusterd/src/glusterd-geo-rep.c
+++ b/xlators/mgmt/glusterd/src/glusterd-geo-rep.c
@@ -172,7 +172,7 @@ glusterd_urltransform_add (runner_t *runner, const char *url)
         runner_add_arg (runner, url);
 }
 
-static void
+static int
 _glusterd_urltransform_add_iter (dict_t *dict, char *key, data_t *value, void *data)
 {
         runner_t *runner = (runner_t *)data;
@@ -182,6 +182,8 @@ _glusterd_urltransform_add_iter (dict_t *dict, char *key, data_t *value, void *d
         GF_ASSERT (slave);
         slave++;
         runner_add_arg (runner, slave);
+
+        return 0;
 }
 
 static void
@@ -296,7 +298,7 @@ struct dictidxmark {
         char *ikey;
 };
 
-static void
+static int
 _dict_mark_atindex (dict_t *dict, char *key, data_t *value, void *data)
 {
         struct dictidxmark *dim = data;
@@ -305,6 +307,7 @@ _dict_mark_atindex (dict_t *dict, char *key, data_t *value, void *data)
                 dim->ikey = key;
 
         dim->ithis++;
+        return 0;
 }
 
 static char *
@@ -734,7 +737,7 @@ static int
 glusterd_get_gsync_status_mst_slv (glusterd_volinfo_t *volinfo,
                                    char *slave, dict_t *rsp_dict);
 
-static void
+static int
 _get_status_mst_slv (dict_t *this, char *key, data_t *value, void *data)
 {
         glusterd_gsync_status_temp_t  *param = NULL;
@@ -750,15 +753,15 @@ _get_status_mst_slv (dict_t *this, char *key, data_t *value, void *data)
         if (slave)
                 slave ++;
         else
-                return;
+                return 0;
 
         ret = glusterd_get_gsync_status_mst_slv(param->volinfo,
                                                 slave, param->rsp_dict);
-
+        return 0;
 }
 
 
-static void
+static int
 _get_max_gsync_slave_num (dict_t *this, char *key, data_t *value, void *data)
 {
         int  tmp_slvnum = 0;
@@ -767,6 +770,8 @@ _get_max_gsync_slave_num (dict_t *this, char *key, data_t *value, void *data)
         sscanf (key, "slave%d", &tmp_slvnum);
         if (tmp_slvnum > *slvnum)
                 *slvnum = tmp_slvnum;
+
+        return 0;
 }
 
 static int
@@ -1987,7 +1992,7 @@ glusterd_do_gsync_log_rotation_mst_slv (glusterd_volinfo_t *volinfo, char *slave
         return ret;
 }
 
-static void
+static int
 _iterate_log_rotate_mst_slv (dict_t *this, char *key, data_t *value, void *data)
 {
         glusterd_gsync_status_temp_t  *param = NULL;
@@ -2004,10 +2009,11 @@ _iterate_log_rotate_mst_slv (dict_t *this, char *key, data_t *value, void *data)
         else {
                 gf_log ("", GF_LOG_ERROR, "geo-replication log-rotate: slave (%s) "
                         "not conforming to format", slave);
-                return;
+                return -1;
         }
 
         (void) glusterd_do_gsync_log_rotation_mst_slv (param->volinfo, slave, NULL);
+        return 0;
 }
 
 static int
diff --git a/xlators/mgmt/glusterd/src/glusterd-handler.c b/xlators/mgmt/glusterd/src/glusterd-handler.c
index 8804176d..180b2829 100644
--- a/xlators/mgmt/glusterd/src/glusterd-handler.c
+++ b/xlators/mgmt/glusterd/src/glusterd-handler.c
@@ -278,7 +278,6 @@ glusterd_add_volume_detail_to_dict (glusterd_volinfo_t *volinfo,
         glusterd_brickinfo_t    *brickinfo = NULL;
         char                    *buf = NULL;
         int                     i = 1;
-        data_pair_t             *pairs = NULL;
         char                    reconfig_key[256] = {0, };
         dict_t                  *dict = NULL;
         int                     opt_count = 0;
@@ -365,14 +364,15 @@ glusterd_add_volume_detail_to_dict (glusterd_volinfo_t *volinfo,
                 goto out;
         }
 
-        for (pairs = dict->members_list; pairs != NULL; pairs = pairs->next) {
-                snprintf (reconfig_key, 256, "volume%d.option.%s", count,
-                          pairs->key);
-                ret = dict_set_str  (volumes, reconfig_key, pairs->value->data);
+        int _build_option_key (dict_t *d, char *k, data_t *v, void *tmp)
+        {
+                snprintf (reconfig_key, 256, "volume%d.option.%s", count, k);
+                ret = dict_set_str  (volumes, reconfig_key, v->data);
                 if (0 == ret)
-                    opt_count++;
-
+                        opt_count++;
+                return 0;
         }
+        dict_foreach (dict, _build_option_key, NULL);
 
         snprintf (key, 256, "volume%d.opt_count", count);
         ret = dict_set_int32 (volumes, key, opt_count);
diff --git a/xlators/mgmt/glusterd/src/glusterd-op-sm.c b/xlators/mgmt/glusterd/src/glusterd-op-sm.c
index aea27500..4385f876 100644
--- a/xlators/mgmt/glusterd/src/glusterd-op-sm.c
+++ b/xlators/mgmt/glusterd/src/glusterd-op-sm.c
@@ -851,7 +851,7 @@ out:
 }
 
 
-static void
+static int
 _delete_reconfig_opt (dict_t *this, char *key, data_t *value, void *data)
 {
         int32_t        *is_force = 0;
@@ -873,7 +873,7 @@ _delete_reconfig_opt (dict_t *this, char *key, data_t *value, void *data)
                 key, value->data);
         dict_del (this, key);
 out:
-        return;
+        return 0;
 }
 
 static int
@@ -3045,7 +3045,7 @@ glusterd_op_commit_perform (glusterd_op_t op, dict_t *dict, char **op_errstr,
         return ret;
 }
 
-void
+int
 _profile_volume_add_brick_rsp (dict_t *this, char *key, data_t *value,
                              void *data)
 {
@@ -3058,6 +3058,7 @@ _profile_volume_add_brick_rsp (dict_t *this, char *key, data_t *value,
         GF_ASSERT (new_value);
         snprintf (new_key, sizeof (new_key), "%d-%s", rsp_ctx->count, key);
         dict_set (rsp_ctx->dict, new_key, new_value);
+        return 0;
 }
 
 int
@@ -3113,7 +3114,7 @@ glusterd_profile_volume_brick_rsp (void *pending_entry,
 
 //input-key: <replica-id>:<child-id>-*
 //output-key: <brick-id>-*
-void
+int
 _heal_volume_add_shd_rsp (dict_t *this, char *key, data_t *value, void *data)
 {
         char                            new_key[256] = {0,};
@@ -3170,7 +3171,7 @@ _heal_volume_add_shd_rsp (dict_t *this, char *key, data_t *value, void *data)
         dict_set (rsp_ctx->dict, new_key, new_value);
 
 out:
-        return;
+        return 0;
 }
 
 int
@@ -3206,7 +3207,7 @@ out:
         return ret;
 }
 
-void
+int
 _status_volume_add_brick_rsp (dict_t *this, char *key, data_t *value,
                               void *data)
 {
@@ -3219,7 +3220,7 @@ _status_volume_add_brick_rsp (dict_t *this, char *key, data_t *value,
         snprintf (new_key, sizeof (new_key), "brick%d.%s", rsp_ctx->count, key);
         dict_set (rsp_ctx->dict, new_key, new_value);
 
-        return;
+        return 0;
 }
 
 int
diff --git a/xlators/mgmt/glusterd/src/glusterd-rpc-ops.c b/xlators/mgmt/glusterd/src/glusterd-rpc-ops.c
index a3522a98..491a3255 100644
--- a/xlators/mgmt/glusterd/src/glusterd-rpc-ops.c
+++ b/xlators/mgmt/glusterd/src/glusterd-rpc-ops.c
@@ -942,7 +942,7 @@ out:
 
 }
 
-void
+int
 _profile_volume_add_friend_rsp (dict_t *this, char *key, data_t *value,
                                void *data)
 {
@@ -953,7 +953,7 @@ _profile_volume_add_friend_rsp (dict_t *this, char *key, data_t *value,
         char    brick_key[256];
 
         if (strcmp (key, "count") == 0)
-                return;
+                return 0;
         sscanf (key, "%d%s", &brick_count, brick_key);
         rsp_ctx = data;
         new_value = data_copy (value);
@@ -961,6 +961,7 @@ _profile_volume_add_friend_rsp (dict_t *this, char *key, data_t *value,
         snprintf (new_key, sizeof (new_key), "%d%s",
                   rsp_ctx->count + brick_count, brick_key);
         dict_set (rsp_ctx->dict, new_key, new_value);
+        return 0;
 }
 
 int
@@ -995,7 +996,7 @@ out:
         return ret;
 }
 
-void
+int
 glusterd_volume_status_add_peer_rsp (dict_t *this, char *key, data_t *value,
                                      void *data)
 {
@@ -1008,7 +1009,7 @@ glusterd_volume_status_add_peer_rsp (dict_t *this, char *key, data_t *value,
 
         if (!strcmp (key, "count") || !strcmp (key, "cmd") ||
             !strcmp (key, "brick-index-max") || !strcmp (key, "other-count"))
-                return;
+                return -1;
 
         rsp_ctx = data;
         new_value = data_copy (value);
@@ -1029,7 +1030,7 @@ glusterd_volume_status_add_peer_rsp (dict_t *this, char *key, data_t *value,
                 gf_log ("", GF_LOG_ERROR, "Unable to set key: %s in dict",
                         key);
 
-        return;
+        return 0;
 }
 
 int
diff --git a/xlators/mgmt/glusterd/src/glusterd-store.c b/xlators/mgmt/glusterd/src/glusterd-store.c
index eef42fb3..d11454bc 100644
--- a/xlators/mgmt/glusterd/src/glusterd-store.c
+++ b/xlators/mgmt/glusterd/src/glusterd-store.c
@@ -541,7 +541,7 @@ out:
         return ret;
 }
 
-static void
+static int
 _storeslaves (dict_t *this, char *key, data_t *value, void *data)
 {
         int32_t                      ret = 0;
@@ -556,12 +556,12 @@ _storeslaves (dict_t *this, char *key, data_t *value, void *data)
         GF_ASSERT (value && value->data);
 
         if ((!shandle) || (shandle->fd <= 0) || (!shandle->path))
-                return;
+                return -1;
 
         if (!key)
-                return;
+                return -1;
         if (!value || !value->data)
-                return;
+                return -1;
 
         gf_log ("", GF_LOG_DEBUG, "Storing in volinfo:key= %s, val=%s",
                 key, value->data);
@@ -570,12 +570,13 @@ _storeslaves (dict_t *this, char *key, data_t *value, void *data)
         if (ret) {
                 gf_log ("", GF_LOG_ERROR, "Unable to write into store"
                                 " handle for path: %s", shandle->path);
-                return;
+                return -1;
         }
+        return 0;
 }
 
 
-void _storeopts (dict_t *this, char *key, data_t *value, void *data)
+int _storeopts (dict_t *this, char *key, data_t *value, void *data)
 {
         int32_t                      ret = 0;
         int32_t                      exists = 0;
@@ -590,12 +591,12 @@ void _storeopts (dict_t *this, char *key, data_t *value, void *data)
         GF_ASSERT (value && value->data);
 
         if ((!shandle) || (shandle->fd <= 0) || (!shandle->path))
-                return;
+                return -1;
 
         if (!key)
-                return;
+                return -1;
         if (!value || !value->data)
-                return;
+                return -1;
 
         if (is_key_glusterd_hooks_friendly (key)) {
                 exists = 1;
@@ -611,15 +612,16 @@ void _storeopts (dict_t *this, char *key, data_t *value, void *data)
         } else {
                 gf_log ("", GF_LOG_DEBUG, "Discarding:key= %s, val=%s",
                         key, value->data);
-                return;
+                return 0;
         }
 
         ret = glusterd_store_save_value (shandle->fd, key, (char*)value->data);
         if (ret) {
                 gf_log ("", GF_LOG_ERROR, "Unable to write into store"
                                 " handle for path: %s", shandle->path);
-                return;
+                return -1;
         }
+        return 0;
 }
 
 int32_t
diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.c b/xlators/mgmt/glusterd/src/glusterd-utils.c
index 51478fb1..3a48a41d 100644
--- a/xlators/mgmt/glusterd/src/glusterd-utils.c
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.c
@@ -1615,7 +1615,7 @@ out:
         return ret;
 }
 
-void
+int
 _add_volinfo_dict_to_prdict (dict_t *this, char *key, data_t *value, void *data)
 {
         glusterd_voldict_ctx_t   *ctx = NULL;
@@ -1637,7 +1637,7 @@ _add_volinfo_dict_to_prdict (dict_t *this, char *key, data_t *value, void *data)
                         ctx->val_name, ctx->count, value->data);
         ctx->opt_count++;
 
-        return;
+        return ret;
 }
 
 int32_t
@@ -3495,7 +3495,7 @@ glusterd_restart_bricks (glusterd_conf_t *conf)
         return ret;
 }
 
-void
+int
 _local_gsyncd_start (dict_t *this, char *key, data_t *value, void *data)
 {
         char                        *slave = NULL;
@@ -3509,11 +3509,13 @@ _local_gsyncd_start (dict_t *this, char *key, data_t *value, void *data)
         if (slave)
                 slave ++;
         else
-                return;
+                return 0;
         uuid_len = (slave - value->data - 1);
 
         strncpy (uuid_str, (char*)value->data, uuid_len);
         glusterd_start_gsync (volinfo, slave, uuid_str, NULL);
+
+        return 0;
 }
 
 int
diff --git a/xlators/mgmt/glusterd/src/glusterd-volgen.c b/xlators/mgmt/glusterd/src/glusterd-volgen.c
index 28f480a3..1375bd48 100644
--- a/xlators/mgmt/glusterd/src/glusterd-volgen.c
+++ b/xlators/mgmt/glusterd/src/glusterd-volgen.c
@@ -649,14 +649,14 @@ struct opthandler_data {
         void *param;
 };
 
-static void
+static int
 process_option (dict_t *dict, char *key, data_t *value, void *param)
 {
         struct opthandler_data *odt = param;
         struct volopt_map_entry vme = {0,};
 
         if (odt->rv)
-                return;
+                return 0;
         odt->found = _gf_true;
 
         vme.key = key;
@@ -675,6 +675,7 @@ process_option (dict_t *dict, char *key, data_t *value, void *param)
                 vme.value = value->data;
 
         odt->rv = odt->handler (odt->graph, &vme, odt->param);
+        return 0;
 }
 
 static int
@@ -3637,16 +3638,18 @@ out:
         return ret;
 }
 
-static void
+static int
 _check_globalopt (dict_t *this, char *key, data_t *value, void *ret_val)
 {
         int *ret = NULL;
 
         ret = ret_val;
         if (*ret)
-                return;
+                return 0;
         if (!glusterd_check_globaloption (key))
                 *ret = 1;
+
+        return 0;
 }
 
 int
@@ -3692,16 +3695,18 @@ out:
         return ret;
 }
 
-static void
+static int
 _check_localopt (dict_t *this, char *key, data_t *value, void *ret_val)
 {
         int *ret = NULL;
 
         ret = ret_val;
         if (*ret)
-                return;
+                return 0;
         if (!glusterd_check_localoption (key))
                 *ret = 1;
+
+        return 0;
 }
 
 int
diff --git a/xlators/mgmt/glusterd/src/glusterd.c b/xlators/mgmt/glusterd/src/glusterd.c
index 17a20639..f55b3ea5 100644
--- a/xlators/mgmt/glusterd/src/glusterd.c
+++ b/xlators/mgmt/glusterd/src/glusterd.c
@@ -655,7 +655,7 @@ check_prepare_mountbroker_root (char *mountbroker_root)
         return ret;
 }
 
-static void
+static int
 _install_mount_spec (dict_t *opts, char *key, data_t *value, void *data)
 {
         glusterd_conf_t *priv           = THIS->private;
@@ -671,7 +671,7 @@ _install_mount_spec (dict_t *opts, char *key, data_t *value, void *data)
         char            *volfile_server = NULL;
 
         if (*ret == -1)
-                return;
+                return -1;
 
         label = strtail (key, "mountbroker.");
 
@@ -688,7 +688,7 @@ _install_mount_spec (dict_t *opts, char *key, data_t *value, void *data)
         }
 
         if (!label)
-                return;
+                return -1;
 
         mspec = GF_CALLOC (1, sizeof (*mspec), gf_gld_mt_mount_spec);
         if (!mspec)
@@ -727,7 +727,7 @@ _install_mount_spec (dict_t *opts, char *key, data_t *value, void *data)
 
         list_add_tail (&mspec->speclist, &priv->mount_specs);
 
-        return;
+        return 0;
  err:
 
         gf_log ("", GF_LOG_ERROR,
@@ -735,6 +735,7 @@ _install_mount_spec (dict_t *opts, char *key, data_t *value, void *data)
                 georep ? GEOREP" " : "", label, pdesc);
 
         *ret = -1;
+        return -1;
 }
 
 /*
diff --git a/xlators/mount/fuse/src/fuse-bridge.c b/xlators/mount/fuse/src/fuse-bridge.c
index 619f488d..1ad774e1 100644
--- a/xlators/mount/fuse/src/fuse-bridge.c
+++ b/xlators/mount/fuse/src/fuse-bridge.c
@@ -2871,7 +2871,6 @@ fuse_xattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         data_t         *value_data = NULL;
         int             ret = -1;
         int32_t         len = 0;
-        data_pair_t    *trav = NULL;
 
         state = frame->root->state;
         finh  = state->finh;
@@ -2900,25 +2899,33 @@ fuse_xattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         /* we need to invoke fuse_filter_xattr() twice. Once
                          * while counting size and then while filling buffer
                          */
-                        trav = dict->members_list;
-                        while (trav) {
-                                if (!fuse_filter_xattr (this, trav->key))
-                                        len += strlen (trav->key) + 1;
-                                trav = trav->next;
-                        } /* while(trav) */
+                        int _get_total_len (dict_t *d, char *k, data_t *v,
+                                            void *tmp)
+                        {
+                                if (!fuse_filter_xattr (this, k))
+                                        len += strlen (k) + 1;
+                                return 0;
+                        }
+                        dict_foreach (dict, _get_total_len, NULL);
+
                         value = alloca (len + 1);
                         if (!value)
                                 goto out;
+
                         len = 0;
-                        trav = dict->members_list;
-                        while (trav) {
-                                if (!fuse_filter_xattr (this, trav->key)) {
-                                        strcpy (value + len, trav->key);
-                                        value[len + strlen (trav->key)] = '\0';
-                                        len += strlen (trav->key) + 1;
+
+                        int _set_listxattr_keys (dict_t *d, char *k, data_t *v,
+                                                 void *tmp)
+                        {
+                                if (!fuse_filter_xattr (this, k)) {
+                                        strcpy (value + len, k);
+                                        value[len + strlen (k)] = '\0';
+                                        len += strlen (k) + 1;
                                 }
-                                trav = trav->next;
-                        } /* while(trav) */
+                                return 0;
+                        }
+                        dict_foreach (dict, _set_listxattr_keys, NULL);
+
                         send_fuse_xattr (this, finh, value, len, state->size);
                 } /* if(state->name)...else */
         } else {
diff --git a/xlators/performance/md-cache/src/md-cache.c b/xlators/performance/md-cache/src/md-cache.c
index 55923990..237acab9 100644
--- a/xlators/performance/md-cache/src/md-cache.c
+++ b/xlators/performance/md-cache/src/md-cache.c
@@ -455,7 +455,7 @@ struct updatedict {
 	int ret;
 };
 
-static void
+static int
 updatefn(dict_t *dict, char *key, data_t *value, void *data)
 {
 	struct updatedict *u = data;
@@ -472,17 +472,18 @@ updatefn(dict_t *dict, char *key, data_t *value, void *data)
 			u->dict = dict_new();
 			if (!u->dict) {
 				u->ret = -1;
-				return;
+				return -1;
 			}
 		}
 
 		if (dict_set(u->dict, key, value) < 0) {
 			u->ret = -1;
-			return;
+			return -1;
 		}
 
 		break;
 	}
+        return 0;
 }
 
 static int
@@ -651,13 +652,15 @@ is_mdc_key_satisfied (const char *key)
 }
 
 
-static void
+static int
 checkfn (dict_t *this, char *key, data_t *value, void *data)
 {
         struct checkpair *pair = data;
 
 	if (!is_mdc_key_satisfied (key))
 		pair->ret = 0;
+
+        return 0;
 }
 
 
diff --git a/xlators/protocol/server/src/authenticate.c b/xlators/protocol/server/src/authenticate.c
index d1cdebde..9c843d0b 100644
--- a/xlators/protocol/server/src/authenticate.c
+++ b/xlators/protocol/server/src/authenticate.c
@@ -31,7 +31,7 @@
 #include <errno.h>
 #include "authenticate.h"
 
-static void
+static int
 init (dict_t *this, char *key, data_t *value, void *data)
 {
         void          *handle       = NULL;
@@ -59,7 +59,7 @@ init (dict_t *this, char *key, data_t *value, void *data)
         if (-1 == ret) {
                 dict_set (this, key, data_from_dynptr (NULL, 0));
                 *error = -1;
-                return;
+                return -1;
         }
 
         handle = dlopen (auth_file, RTLD_LAZY);
@@ -69,7 +69,7 @@ init (dict_t *this, char *key, data_t *value, void *data)
                 dict_set (this, key, data_from_dynptr (NULL, 0));
                 GF_FREE (auth_file);
                 *error = -1;
-                return;
+                return -1;
         }
         GF_FREE (auth_file);
 
@@ -80,7 +80,7 @@ init (dict_t *this, char *key, data_t *value, void *data)
                 dict_set (this, key, data_from_dynptr (NULL, 0));
                 dlclose (handle);
                 *error = -1;
-                return;
+                return -1;
         }
 
         auth_handle = GF_CALLOC (1, sizeof (*auth_handle),
@@ -89,7 +89,7 @@ init (dict_t *this, char *key, data_t *value, void *data)
                 dict_set (this, key, data_from_dynptr (NULL, 0));
                 *error = -1;
                 dlclose (handle);
-                return;
+                return -1;
         }
         auth_handle->vol_opt = GF_CALLOC (1, sizeof (volume_opt_list_t),
                                           gf_common_mt_volume_opt_list_t);
@@ -98,7 +98,7 @@ init (dict_t *this, char *key, data_t *value, void *data)
                 *error = -1;
                 GF_FREE (auth_handle);
                 dlclose (handle);
-                return;
+                return -1;
         }
         auth_handle->vol_opt->given_opt = dlsym (handle, "options");
         if (auth_handle->vol_opt->given_opt == NULL) {
@@ -111,15 +111,17 @@ init (dict_t *this, char *key, data_t *value, void *data)
 
         dict_set (this, key,
                   data_from_dynptr (auth_handle, sizeof (*auth_handle)));
+        return 0;
 }
 
-static void
+static int
 fini (dict_t *this, char *key, data_t *value, void *data)
 {
         auth_handle_t *handle = data_to_ptr (value);
         if (handle) {
                 dlclose (handle->handle);
         }
+        return 0;
 }
 
 int32_t
@@ -127,31 +129,29 @@ gf_auth_init (xlator_t *xl, dict_t *auth_modules)
 {
         int ret = 0;
         auth_handle_t *handle = NULL;
-        data_pair_t *pair = NULL;
 
         dict_foreach (auth_modules, init, &ret);
         if (ret)
                 goto out;
 
-        pair = auth_modules->members_list;
-        while (pair) {
-                handle = data_to_ptr (pair->value);
-                if (!handle) {
-                        pair = pair->next;
-                        continue;
-                }
+        int _auth_option_validate (dict_t *d, char *k, data_t *v, void *tmp)
+        {
+                handle = data_to_ptr (v);
+                if (!handle)
+                        return 0;
 
                 list_add_tail (&(handle->vol_opt->list),
                                &(xl->volume_options));
                 ret = xlator_options_validate_list (xl, xl->options,
                                                     handle->vol_opt, NULL);
-
-                if (ret)
+                if (ret) {
                         gf_log ("authenticate", GF_LOG_ERROR,
                                 "volume option validation failed");
-
-                pair = pair->next;
+                        return -1;
+                }
+                return 0;
         }
+        ret = dict_foreach (auth_modules, _auth_option_validate, NULL);
 
 out:
         if (ret) {
@@ -165,7 +165,7 @@ out:
 static dict_t *__input_params;
 static dict_t *__config_params;
 
-void
+int
 map (dict_t *this, char *key, data_t *value, void *data)
 {
         dict_t *res = data;
@@ -180,15 +180,16 @@ map (dict_t *this, char *key, data_t *value, void *data)
         } else {
                 dict_set (res, key, int_to_data (AUTH_DONT_CARE));
         }
+        return 0;
 }
 
-void
+int
 reduce (dict_t *this, char *key, data_t *value, void *data)
 {
         int64_t val = 0;
         int64_t *res = data;
         if (!data)
-                return;
+                return 0;
 
         val = data_to_int64 (value);
         switch (val)
@@ -205,6 +206,7 @@ reduce (dict_t *this, char *key, data_t *value, void *data)
         case AUTH_DONT_CARE:
                 break;
         }
+        return 0;
 }
 
 
diff --git a/xlators/protocol/server/src/server-helpers.c b/xlators/protocol/server/src/server-helpers.c
index eec1684c..0adcd1cf 100644
--- a/xlators/protocol/server/src/server-helpers.c
+++ b/xlators/protocol/server/src/server-helpers.c
@@ -1388,7 +1388,6 @@ int
 gf_server_check_setxattr_cmd (call_frame_t *frame, dict_t *dict)
 {
 
-        data_pair_t      *pair = NULL;
         server_conf_t    *conf = NULL;
         rpc_transport_t  *xprt = NULL;
         uint64_t          total_read = 0;
@@ -1398,19 +1397,23 @@ gf_server_check_setxattr_cmd (call_frame_t *frame, dict_t *dict)
         if (!conf || !dict)
                 return 0;
 
-        for (pair = dict->members_list; pair; pair = pair->next) {
-                /* this exact key is used in 'io-stats' too.
-                 * But this is better place for this information dump.
-                 */
-                if (fnmatch ("*io*stat*dump", pair->key, 0) == 0) {
-                        list_for_each_entry (xprt, &conf->xprt_list, list) {
-                                total_read  += xprt->total_bytes_read;
-                                total_write += xprt->total_bytes_write;
-                        }
-                        gf_log ("stats", GF_LOG_INFO,
-                                "total-read %"PRIu64", total-write %"PRIu64,
-                                total_read, total_write);
+        /* this exact key is used in 'io-stats' too.
+         * But this is better place for this information dump.
+         */
+        int _handle_keyvalue_pair (dict_t *d, char *k,
+                                   data_t *v, void *tmp)
+        {
+                return 0;
+        }
+        if (dict_foreach_fnmatch (dict, "*io*stat*dump",
+                                  _handle_keyvalue_pair, NULL ) > 0) {
+                list_for_each_entry (xprt, &conf->xprt_list, list) {
+                        total_read  += xprt->total_bytes_read;
+                        total_write += xprt->total_bytes_write;
                 }
+                gf_log ("stats", GF_LOG_INFO,
+                        "total-read %"PRIu64", total-write %"PRIu64,
+                        total_read, total_write);
         }
 
         return 0;
diff --git a/xlators/protocol/server/src/server-rpc-fops.c b/xlators/protocol/server/src/server-rpc-fops.c
index a8f79ea7..db02aaf9 100644
--- a/xlators/protocol/server/src/server-rpc-fops.c
+++ b/xlators/protocol/server/src/server-rpc-fops.c
@@ -904,15 +904,19 @@ server_setxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                                     rsp.xdata.xdata_len, op_errno, out);
 
         if (op_ret == -1) {
-                gf_log (this->name, ((op_errno == ENOTSUP) ?
-                                     GF_LOG_DEBUG : GF_LOG_INFO),
-                        "%"PRId64": SETXATTR %s (%s) ==> %s (%s)",
-                        frame->root->unique, state->loc.path,
-                        uuid_utoa (state->resolve.gfid),
-                        ((state->dict) ? ((state->dict->members_list) ?
-                                          state->dict->members_list->key :
-                                          "(null)") : ("null")),
-                        strerror (op_errno));
+                /* print every key */
+                int _log_setxattr_failure (dict_t *d, char *k, data_t *v,
+                                           void *tmp)
+                {
+                        gf_log (this->name, ((op_errno == ENOTSUP) ?
+                                             GF_LOG_DEBUG : GF_LOG_INFO),
+                                "%"PRId64": SETXATTR %s (%s) ==> %s (%s)",
+                                frame->root->unique, state->loc.path,
+                                uuid_utoa (state->resolve.gfid), k,
+                                strerror (op_errno));
+                        return 0;
+                }
+                dict_foreach (state->dict, _log_setxattr_failure, NULL);
                 goto out;
         }
 
@@ -944,15 +948,19 @@ server_fsetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                                     rsp.xdata.xdata_len, op_errno, out);
 
         if (op_ret == -1) {
-                gf_log (this->name, ((op_errno == ENOTSUP) ?
-                                     GF_LOG_DEBUG : GF_LOG_INFO),
-                        "%"PRId64": FSETXATTR %"PRId64" (%s) ==> %s (%s)",
-                        frame->root->unique, state->resolve.fd_no,
-                        uuid_utoa (state->resolve.gfid),
-                        ((state->dict) ? ((state->dict->members_list) ?
-                                          state->dict->members_list->key :
-                                          "(null)") : "null"),
-                        strerror (op_errno));
+                /* print every key here */
+                int _log_setxattr_failure (dict_t *d, char *k, data_t *v,
+                                           void *tmp)
+                {
+                        gf_log (this->name, ((op_errno == ENOTSUP) ?
+                                             GF_LOG_DEBUG : GF_LOG_INFO),
+                                "%"PRId64": FSETXATTR %"PRId64" (%s) ==> %s (%s)",
+                                frame->root->unique, state->resolve.fd_no,
+                                uuid_utoa (state->resolve.gfid), k,
+                                strerror (op_errno));
+                        return 0;
+                }
+                dict_foreach (state->dict, _log_setxattr_failure, NULL);
                 goto out;
         }
 
diff --git a/xlators/protocol/server/src/server.c b/xlators/protocol/server/src/server.c
index b6519e59..9b1bd7cf 100644
--- a/xlators/protocol/server/src/server.c
+++ b/xlators/protocol/server/src/server.c
@@ -509,7 +509,7 @@ out:
 }
 
 
-static void
+static int
 get_auth_types (dict_t *this, char *key, data_t *value, void *data)
 {
         dict_t   *auth_dict = NULL;
@@ -546,7 +546,7 @@ get_auth_types (dict_t *this, char *key, data_t *value, void *data)
 
         GF_FREE (key_cpy);
 out:
-        return;
+        return 0;
 }
 
 
@@ -555,7 +555,6 @@ validate_auth_options (xlator_t *this, dict_t *dict)
 {
         int            error = -1;
         xlator_list_t *trav = NULL;
-        data_pair_t   *pair = NULL;
         char          *tail = NULL;
         char          *tmp_addr_list = NULL;
         char          *addr = NULL;
@@ -566,51 +565,52 @@ validate_auth_options (xlator_t *this, dict_t *dict)
 
         trav = this->children;
         while (trav) {
-                error = -1;
-                for (pair = dict->members_list; pair; pair = pair->next) {
-                        tail = strtail (pair->key, "auth.");
+                int _check_for_auth_option (dict_t *d, char *k, data_t *v,
+                                            void *tmp)
+                {
+                        int ret = 0;
+                        tail = strtail (k, "auth.");
                         if (!tail)
-                                continue;
+                                goto internal_out;
+
                         /* fast fwd thru module type */
                         tail = strchr (tail, '.');
                         if (!tail)
-                                continue;
+                                goto internal_out;
                         tail++;
 
                         tail = strtail (tail, trav->xlator->name);
                         if (!tail)
-                                continue;
+                                goto internal_out;
 
                         if (*tail == '.') {
-                                error = 0;
 
                                 /* when we are here, the key is checked for
                                  * valid auth.allow.<xlator>
                                  * Now we verify the ip address
                                  */
-                                if (!strcmp (pair->value->data, "*")) {
-                                     error = 0;
-                                     goto out;
+                                if (!strcmp (v->data, "*")) {
+                                        ret = 0;
+                                        goto internal_out;
                                 }
 
-                                tmp_addr_list = gf_strdup (pair->value->data);
-                                addr = strtok_r (tmp_addr_list, ",",
-                                                &tmp_str);
+                                tmp_addr_list = gf_strdup (v->data);
+                                addr = strtok_r (tmp_addr_list, ",", &tmp_str);
                                 if (!addr)
-                                        addr = pair->value->data;
+                                        addr = v->data;
 
                                 while (addr) {
 
-                                        if (valid_internet_address
-                                                        (addr, _gf_true)) {
-                                                error = 0;
+                                        if (valid_internet_address (addr,
+                                                                    _gf_true)) {
+                                                ret = 0;
                                         } else {
-                                                error = -1;
+                                                ret = -1;
                                                 gf_log (this->name, GF_LOG_ERROR,
                                                         "internet address '%s'"
                                                         " does not conform to"
                                                         " standards.", addr);
-                                                goto out;
+                                                goto internal_out;
 
                                         }
                                         if (tmp_str)
@@ -623,8 +623,10 @@ validate_auth_options (xlator_t *this, dict_t *dict)
                                 GF_FREE (tmp_addr_list);
                                 tmp_addr_list = NULL;
                         }
-
+                internal_out:
+                        return ret;
                 }
+                error = dict_foreach (dict, _check_for_auth_option, NULL);
 
                 if (-1 == error) {
                         gf_log (this->name, GF_LOG_ERROR,
@@ -766,31 +768,24 @@ out:
 }
 
 
-static void
-_delete_auth_opt (dict_t *this,
-                char *key,
-                data_t *value,
-                void *data)
+static int
+_delete_auth_opt (dict_t *this, char *key, data_t *value, void *data)
 {
         char *auth_option_pattern[] = { "auth.addr.*.allow",
                                         "auth.addr.*.reject"};
-        if (fnmatch ( auth_option_pattern[0], key, 0) != 0) {
+
+        if (fnmatch ( auth_option_pattern[0], key, 0) != 0)
                 dict_del (this, key);
-                return;
-        }
 
-        if (fnmatch ( auth_option_pattern[1], key, 0) != 0) {
+        if (fnmatch ( auth_option_pattern[1], key, 0) != 0)
                 dict_del (this, key);
-                return;
-        }
+
+        return 0;
 }
 
 
-static void
-_copy_auth_opt (dict_t *unused,
-                char *key,
-                data_t *value,
-                void *xl_dict)
+static int
+_copy_auth_opt (dict_t *unused, char *key, data_t *value, void *xl_dict)
 {
         char *auth_option_pattern[] = { "auth.addr.*.allow",
                                         "auth.addr.*.reject"};
@@ -799,6 +794,8 @@ _copy_auth_opt (dict_t *unused,
 
         if (fnmatch ( auth_option_pattern[1], key, 0) != 0)
                 dict_set ((dict_t *)xl_dict, key, (value));
+
+        return 0;
 }
 
 
diff --git a/xlators/storage/posix/src/posix-helpers.c b/xlators/storage/posix/src/posix-helpers.c
index ea3ad429..4169429b 100644
--- a/xlators/storage/posix/src/posix-helpers.c
+++ b/xlators/storage/posix/src/posix-helpers.c
@@ -108,7 +108,7 @@ out:
         return ignore;
 }
 
-static void
+static int
 _posix_xattr_get_set (dict_t *xattr_req,
                       char *key,
                       data_t *data,
@@ -202,7 +202,7 @@ _posix_xattr_get_set (dict_t *xattr_req,
                         value = GF_CALLOC (1, xattr_size + 1,
                                            gf_posix_mt_char);
                         if (!value)
-                                return;
+                                return -1;
 
                         xattr_size = sys_lgetxattr (filler->real_path, key, value,
                                                     xattr_size);
@@ -211,7 +211,7 @@ _posix_xattr_get_set (dict_t *xattr_req,
                                         "getxattr failed. path: %s, key: %s",
                                         filler->real_path, key);
                                 GF_FREE (value);
-                                return;
+                                return -1;
                         }
 
                         value[xattr_size] = '\0';
@@ -226,7 +226,7 @@ _posix_xattr_get_set (dict_t *xattr_req,
                 }
         }
 out:
-        return;
+        return 0;
 }
 
 
@@ -493,8 +493,8 @@ out:
 
 
 int
-posix_set_file_contents (xlator_t *this, const char *path, data_pair_t *trav,
-                         int flags)
+posix_set_file_contents (xlator_t *this, const char *path, char *keyp,
+                         data_t *value, int flags)
 {
         char *      key                        = NULL;
         char        real_path[PATH_MAX];
@@ -506,7 +506,7 @@ posix_set_file_contents (xlator_t *this, const char *path, data_pair_t *trav,
         /* XXX: does not handle assigning GFID to created files */
         return -1;
 
-        key = &(trav->key[15]);
+        key = &(keyp[15]);
         sprintf (real_path, "%s/%s", path, key);
 
         if (flags & XATTR_REPLACE) {
@@ -518,9 +518,8 @@ posix_set_file_contents (xlator_t *this, const char *path, data_pair_t *trav,
                         goto create;
                 }
 
-                if (trav->value->len) {
-                        ret = write (file_fd, trav->value->data,
-                                     trav->value->len);
+                if (value->len) {
+                        ret = write (file_fd, value->data, value->len);
                         if (ret == -1) {
                                 op_ret = -errno;
                                 gf_log (this->name, GF_LOG_ERROR,
@@ -552,7 +551,7 @@ posix_set_file_contents (xlator_t *this, const char *path, data_pair_t *trav,
                         goto out;
                 }
 
-                ret = write (file_fd, trav->value->data, trav->value->len);
+                ret = write (file_fd, value->data, value->len);
                 if (ret == -1) {
                         op_ret = -errno;
                         gf_log (this->name, GF_LOG_ERROR,
@@ -648,17 +647,17 @@ static int gf_xattr_enotsup_log;
 
 int
 posix_handle_pair (xlator_t *this, const char *real_path,
-                   data_pair_t *trav, int flags)
+                   char *key, data_t *value, int flags)
 {
         int sys_ret = -1;
         int ret     = 0;
 
-        if (ZR_FILE_CONTENT_REQUEST(trav->key)) {
-                ret = posix_set_file_contents (this, real_path, trav, flags);
+        if (ZR_FILE_CONTENT_REQUEST(key)) {
+                ret = posix_set_file_contents (this, real_path, key, value,
+                                               flags);
         } else {
-                sys_ret = sys_lsetxattr (real_path, trav->key,
-                                         trav->value->data,
-                                         trav->value->len, flags);
+                sys_ret = sys_lsetxattr (real_path, key, value->data,
+                                         value->len, flags);
 
                 if (sys_ret < 0) {
                         if (errno == ENOTSUP) {
@@ -670,7 +669,7 @@ posix_handle_pair (xlator_t *this, const char *real_path,
                                                     "flag)");
                         } else if (errno == ENOENT) {
                                 if (!posix_special_xattr (marker_xattrs,
-                                                          trav->key)) {
+                                                          key)) {
                                         gf_log (this->name, GF_LOG_ERROR,
                                                 "setxattr on %s failed: %s",
                                                 real_path, strerror (errno));
@@ -682,12 +681,12 @@ posix_handle_pair (xlator_t *this, const char *real_path,
                                         ((errno == EINVAL) ?
                                          GF_LOG_DEBUG : GF_LOG_ERROR),
                                         "%s: key:%s error:%s",
-                                        real_path, trav->key,
+                                        real_path, key,
                                         strerror (errno));
 #else /* ! DARWIN */
                                 gf_log (this->name, GF_LOG_ERROR,
                                         "%s: key:%s error:%s",
-                                        real_path, trav->key,
+                                        real_path, key,
                                         strerror (errno));
 #endif /* DARWIN */
                         }
@@ -702,13 +701,13 @@ out:
 
 int
 posix_fhandle_pair (xlator_t *this, int fd,
-                    data_pair_t *trav, int flags)
+                    char *key, data_t *value, int flags)
 {
         int sys_ret = -1;
         int ret     = 0;
 
-        sys_ret = sys_fsetxattr (fd, trav->key, trav->value->data,
-                                 trav->value->len, flags);
+        sys_ret = sys_fsetxattr (fd, key, value->data,
+                                 value->len, flags);
 
         if (sys_ret < 0) {
                 if (errno == ENOTSUP) {
@@ -729,13 +728,11 @@ posix_fhandle_pair (xlator_t *this, int fd,
                                 ((errno == EINVAL) ?
                                  GF_LOG_DEBUG : GF_LOG_ERROR),
                                 "fd=%d: key:%s error:%s",
-                                fd, trav->key,
-                                strerror (errno));
+                                fd, key, strerror (errno));
 #else /* ! DARWIN */
                         gf_log (this->name, GF_LOG_ERROR,
                                 "fd=%d: key:%s error:%s",
-                                fd, trav->key,
-                                strerror (errno));
+                                fd, key, strerror (errno));
 #endif /* DARWIN */
                 }
 
@@ -934,33 +931,31 @@ int
 posix_entry_create_xattr_set (xlator_t *this, const char *path,
                              dict_t *dict)
 {
-        data_pair_t *trav = NULL;
         int ret = -1;
 
         if (!dict)
                 goto out;
 
-        trav = dict->members_list;
-        while (trav) {
-                if (!strcmp (GFID_XATTR_KEY, trav->key) ||
-                    !strcmp ("gfid-req", trav->key) ||
-                    !strcmp ("system.posix_acl_default", trav->key) ||
-                    !strcmp ("system.posix_acl_access", trav->key) ||
-                    ZR_FILE_CONTENT_REQUEST(trav->key)) {
-                        trav = trav->next;
-                        continue;
+        int _handle_keyvalue_pair (dict_t *d, char *k, data_t *v,
+                                   void *tmp)
+        {
+                if (!strcmp (GFID_XATTR_KEY, k) ||
+                    !strcmp ("gfid-req", k) ||
+                    !strcmp ("system.posix_acl_default", k) ||
+                    !strcmp ("system.posix_acl_access", k) ||
+                    ZR_FILE_CONTENT_REQUEST(k)) {
+                        return 0;
                 }
 
-                ret = posix_handle_pair (this, path, trav, XATTR_CREATE);
+                ret = posix_handle_pair (this, path, k, v, XATTR_CREATE);
                 if (ret < 0) {
                         errno = -ret;
-                        ret = -1;
-                        goto out;
+                        return -1;
                 }
-                trav = trav->next;
+                return 0;
         }
 
-        ret = 0;
+        ret = dict_foreach (dict, _handle_keyvalue_pair, NULL);
 
 out:
         return ret;
diff --git a/xlators/storage/posix/src/posix.c b/xlators/storage/posix/src/posix.c
index df0dce72..64cfca0c 100644
--- a/xlators/storage/posix/src/posix.c
+++ b/xlators/storage/posix/src/posix.c
@@ -2375,7 +2375,6 @@ posix_setxattr (call_frame_t *frame, xlator_t *this,
         int32_t       op_ret                  = -1;
         int32_t       op_errno                = 0;
         char *        real_path               = NULL;
-        data_pair_t * trav                    = NULL;
         int           ret                     = -1;
 
         DECLARE_OLD_FS_ID_VAR;
@@ -2391,18 +2390,17 @@ posix_setxattr (call_frame_t *frame, xlator_t *this,
         op_ret = -1;
         dict_del (dict, GFID_XATTR_KEY);
 
-        trav = dict->members_list;
 
-        while (trav) {
-                ret = posix_handle_pair (this, real_path, trav, flags);
+        int _handle_every_keyvalue_pair (dict_t *d, char *k, data_t *v,
+                                         void *tmp)
+        {
+                ret = posix_handle_pair (this, real_path, k, v, flags);
                 if (ret < 0) {
                         op_errno = -ret;
-                        goto out;
                 }
-                trav = trav->next;
+                return ret;
         }
-
-        op_ret = 0;
+        op_ret = dict_foreach (dict, _handle_every_keyvalue_pair, NULL);
 
 out:
         SET_TO_OLD_FS_ID ();
@@ -2895,7 +2893,6 @@ posix_fsetxattr (call_frame_t *frame, xlator_t *this,
         int32_t            op_errno     = 0;
         struct posix_fd *  pfd          = NULL;
         int                _fd          = -1;
-        data_pair_t * trav              = NULL;
         int           ret               = -1;
 
         DECLARE_OLD_FS_ID_VAR;
@@ -2917,18 +2914,17 @@ posix_fsetxattr (call_frame_t *frame, xlator_t *this,
 
         dict_del (dict, GFID_XATTR_KEY);
 
-        trav = dict->members_list;
-
-        while (trav) {
-                ret = posix_fhandle_pair (this, _fd, trav, flags);
+        int _handle_every_keyvalue_pair (dict_t *d, char *k, data_t *v,
+                                         void *tmp)
+        {
+                ret = posix_fhandle_pair (this, _fd, k, v, flags);
                 if (ret < 0) {
                         op_errno = -ret;
-                        goto out;
                 }
-                trav = trav->next;
+                return ret;
         }
 
-        op_ret = 0;
+        op_ret = dict_foreach (dict, _handle_every_keyvalue_pair, NULL);
 
 out:
         SET_TO_OLD_FS_ID ();
@@ -3123,8 +3119,6 @@ do_xattrop (call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd,
         int              _fd = -1;
         struct posix_fd *pfd = NULL;
 
-        data_pair_t     *trav = NULL;
-
         char *    path  = NULL;
         inode_t * inode = NULL;
 
@@ -3132,8 +3126,6 @@ do_xattrop (call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd,
         VALIDATE_OR_GOTO (xattr, out);
         VALIDATE_OR_GOTO (this, out);
 
-        trav = xattr->members_list;
-
         if (fd) {
                 ret = posix_fd_ctx_get (fd, this, &pfd);
                 if (ret < 0) {
@@ -3157,19 +3149,21 @@ do_xattrop (call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd,
                 inode = fd->inode;
         }
 
-        while (trav && inode) {
-                count = trav->value->len;
-                array = GF_CALLOC (count, sizeof (char),
-                                   gf_posix_mt_char);
+        int _handle_every_keyvalue_pair (dict_t *d, char *k, data_t *v,
+                                         void *tmp)
+        {
+
+                count = v->len;
+                array = GF_CALLOC (count, sizeof (char), gf_posix_mt_char);
 
                 LOCK (&inode->lock);
                 {
                         if (loc) {
-                                size = sys_lgetxattr (real_path, trav->key, (char *)array,
-                                                      trav->value->len);
+                                size = sys_lgetxattr (real_path, k,
+                                                      (char *)array, v->len);
                         } else {
-                                size = sys_fgetxattr (_fd, trav->key, (char *)array,
-                                                      trav->value->len);
+                                size = sys_fgetxattr (_fd, k, (char *)array,
+                                                      v->len);
                         }
 
                         op_errno = errno;
@@ -3182,17 +3176,17 @@ do_xattrop (call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd,
                                                             "supported by filesystem");
                                 } else if (op_errno != ENOENT ||
                                            !posix_special_xattr (marker_xattrs,
-                                                                 trav->key)) {
+                                                                 k)) {
                                         if (loc)
                                                 gf_log (this->name, GF_LOG_ERROR,
                                                         "getxattr failed on %s while doing "
                                                         "xattrop: Key:%s (%s)", path,
-                                                        trav->key, strerror (op_errno));
+                                                        k, strerror (op_errno));
                                         else
                                                 gf_log (this->name, GF_LOG_ERROR,
                                                         "fgetxattr failed on fd=%d while doing "
                                                         "xattrop: Key:%s (%s)", _fd,
-                                                        trav->key, strerror (op_errno));
+                                                        k, strerror (op_errno));
                                 }
 
                                 op_ret = -1;
@@ -3202,13 +3196,13 @@ do_xattrop (call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd,
                         switch (optype) {
 
                         case GF_XATTROP_ADD_ARRAY:
-                                __add_array ((int32_t *) array, (int32_t *) trav->value->data,
-                                             trav->value->len / 4);
+                                __add_array ((int32_t *) array, (int32_t *) v->data,
+                                             v->len / 4);
                                 break;
 
                         case GF_XATTROP_ADD_ARRAY64:
-                                __add_long_array ((int64_t *) array, (int64_t *) trav->value->data,
-                                                  trav->value->len / 8);
+                                __add_long_array ((int64_t *) array, (int64_t *) v->data,
+                                                  v->len / 8);
                                 break;
 
                         default:
@@ -3222,11 +3216,11 @@ do_xattrop (call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd,
                         }
 
                         if (loc) {
-                                size = sys_lsetxattr (real_path, trav->key, array,
-                                                      trav->value->len, 0);
+                                size = sys_lsetxattr (real_path, k, array,
+                                                      v->len, 0);
                         } else {
-                                size = sys_fsetxattr (_fd, trav->key, (char *)array,
-                                                      trav->value->len, 0);
+                                size = sys_fsetxattr (_fd, k, (char *)array,
+                                                      v->len, 0);
                         }
                 }
         unlock:
@@ -3241,30 +3235,29 @@ do_xattrop (call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd,
                                 gf_log (this->name, GF_LOG_ERROR,
                                         "setxattr failed on %s while doing xattrop: "
                                         "key=%s (%s)", path,
-                                        trav->key, strerror (op_errno));
+                                        k, strerror (op_errno));
                         else
                                 gf_log (this->name, GF_LOG_ERROR,
                                         "fsetxattr failed on fd=%d while doing xattrop: "
                                         "key=%s (%s)", _fd,
-                                        trav->key, strerror (op_errno));
+                                        k, strerror (op_errno));
 
                         op_ret = -1;
                         goto out;
                 } else {
-                        size = dict_set_bin (xattr, trav->key, array,
-                                             trav->value->len);
+                        size = dict_set_bin (xattr, k, array, v->len);
 
                         if (size != 0) {
                                 if (loc)
                                         gf_log (this->name, GF_LOG_DEBUG,
                                                 "dict_set_bin failed (path=%s): "
                                                 "key=%s (%s)", path,
-                                                trav->key, strerror (-size));
+                                                k, strerror (-size));
                                 else
                                         gf_log (this->name, GF_LOG_DEBUG,
                                                 "dict_set_bin failed (fd=%d): "
                                                 "key=%s (%s)", _fd,
-                                                trav->key, strerror (-size));
+                                                k, strerror (-size));
 
                                 op_ret = -1;
                                 op_errno = EINVAL;
@@ -3274,8 +3267,11 @@ do_xattrop (call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd,
                 }
 
                 array = NULL;
-                trav = trav->next;
+
+        out:
+                return op_ret;
         }
+        op_ret = dict_foreach (xattr, _handle_every_keyvalue_pair, NULL);
 
 out:
         GF_FREE (array);
diff --git a/xlators/storage/posix/src/posix.h b/xlators/storage/posix/src/posix.h
index 669bd5cb..84ad85e8 100644
--- a/xlators/storage/posix/src/posix.h
+++ b/xlators/storage/posix/src/posix.h
@@ -142,14 +142,15 @@ int posix_pstat (xlator_t *this, uuid_t gfid, const char *real_path,
                  struct iatt *iatt);
 dict_t *posix_lookup_xattr_fill (xlator_t *this, const char *path,
                                  loc_t *loc, dict_t *xattr, struct iatt *buf);
-int posix_handle_pair (xlator_t *this, const char *real_path,
-                       data_pair_t *trav, int flags);
-int posix_fhandle_pair (xlator_t *this, int fd, data_pair_t *trav, int flags);
+int posix_handle_pair (xlator_t *this, const char *real_path, char *key,
+                       data_t *value, int flags);
+int posix_fhandle_pair (xlator_t *this, int fd, char *key, data_t *value,
+                        int flags);
 void posix_spawn_janitor_thread (xlator_t *this);
 int posix_get_file_contents (xlator_t *this, uuid_t pargfid,
                              const char *name, char **contents);
-int posix_set_file_contents (xlator_t *this, const char *path,
-                             data_pair_t *trav, int flags);
+int posix_set_file_contents (xlator_t *this, const char *path, char *key,
+                             data_t *value, int flags);
 int posix_acl_xattr_set (xlator_t *this, const char *path, dict_t *xattr_req);
 int posix_gfid_heal (xlator_t *this, const char *path, dict_t *xattr_req);
 int posix_entry_create_xattr_set (xlator_t *this, const char *path,