core: utilize mempool for frame->local allocations

in each translator, which uses 'frame->local', we are using
GF_CALLOC/GF_FREE, which would be costly considering the
number of allocation happening in a lifetime of 'fop'. It
would be good to utilize the mem pool framework for xlator's
local structures, so there is no allocation overhead.

Change-Id: Ida6e65039a24d9c219b380aa1c3559f36046dc94
Signed-off-by: Amar Tumballi <amar@gluster.com>
BUG: 765336
Reviewed-on: http://review.gluster.com/2772
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Anand Avati <avati@redhat.com>

13 files changed, 98 insertions, 73 deletions

diff --git a/xlators/cluster/afr/src/afr-common.c b/xlators/cluster/afr/src/afr-common.c
index 9a78f6d3d4d..d241825940f 100644
--- a/xlators/cluster/afr/src/afr-common.c
+++ b/xlators/cluster/afr/src/afr-common.c
@@ -1968,7 +1968,7 @@ afr_lookup (call_frame_t *frame, xlator_t *this,
 
         priv = this->private;
 
-        ALLOC_OR_GOTO (local, afr_local_t, out);
+        AFR_LOCAL_ALLOC_OR_GOTO (local, out);
 
         local->op_ret = -1;
 
@@ -2306,7 +2306,7 @@ afr_flush (call_frame_t *frame, xlator_t *this, fd_t *fd)
                 goto out;
         }
 
-        ALLOC_OR_GOTO (transaction_frame->local, afr_local_t, out);
+        AFR_LOCAL_ALLOC_OR_GOTO (transaction_frame->local, out);
         local = transaction_frame->local;
 
         ret = afr_local_init (local, priv, &op_errno);
@@ -2491,7 +2491,7 @@ afr_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd,
 
         priv = this->private;
 
-        ALLOC_OR_GOTO (frame->local, afr_local_t, out);
+        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
         local = frame->local;
 
         ret = afr_local_init (local, priv, &op_errno);
@@ -2570,7 +2570,7 @@ afr_fsyncdir (call_frame_t *frame, xlator_t *this, fd_t *fd,
 
         priv = this->private;
 
-        ALLOC_OR_GOTO (frame->local, afr_local_t, out);
+        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
         local = frame->local;
 
         ret = afr_local_init (local, priv, &op_errno);
@@ -2647,7 +2647,7 @@ afr_xattrop (call_frame_t *frame, xlator_t *this, loc_t *loc,
 
         priv = this->private;
 
-        ALLOC_OR_GOTO (frame->local, afr_local_t, out);
+        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
         local = frame->local;
 
         ret = afr_local_init (local, priv, &op_errno);
@@ -2725,7 +2725,7 @@ afr_fxattrop (call_frame_t *frame, xlator_t *this, fd_t *fd,
 
         priv = this->private;
 
-        ALLOC_OR_GOTO (frame->local, afr_local_t, out);
+        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
         local = frame->local;
 
         ret = afr_local_init (local, priv, &op_errno);
@@ -2801,7 +2801,7 @@ afr_inodelk (call_frame_t *frame, xlator_t *this,
 
         priv = this->private;
 
-        ALLOC_OR_GOTO (frame->local, afr_local_t, out);
+        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
         local = frame->local;
 
         ret = afr_local_init (local, priv, &op_errno);
@@ -2876,7 +2876,7 @@ afr_finodelk (call_frame_t *frame, xlator_t *this,
 
         priv = this->private;
 
-        ALLOC_OR_GOTO (frame->local, afr_local_t, out);
+        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
         local = frame->local;
 
         ret = afr_local_init (local, priv, &op_errno);
@@ -2952,7 +2952,7 @@ afr_entrylk (call_frame_t *frame, xlator_t *this,
 
         priv = this->private;
 
-        ALLOC_OR_GOTO (frame->local, afr_local_t, out);
+        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
         local = frame->local;
 
         ret = afr_local_init (local, priv, &op_errno);
@@ -3029,7 +3029,7 @@ afr_fentrylk (call_frame_t *frame, xlator_t *this,
 
         priv = this->private;
 
-        ALLOC_OR_GOTO (frame->local, afr_local_t, out);
+        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
         local = frame->local;
 
         ret = afr_local_init (local, priv, &op_errno);
@@ -3116,7 +3116,7 @@ afr_statfs (call_frame_t *frame, xlator_t *this,
         priv = this->private;
         child_count = priv->child_count;
 
-        ALLOC_OR_GOTO (frame->local, afr_local_t, out);
+        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
         local = frame->local;
 
         ret = afr_local_init (local, priv, &op_errno);
@@ -3286,7 +3286,7 @@ afr_lk (call_frame_t *frame, xlator_t *this,
 
         priv = this->private;
 
-        ALLOC_OR_GOTO (frame->local, afr_local_t, out);
+        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
         local = frame->local;
 
         ret = afr_local_init (local, priv, &op_errno);
diff --git a/xlators/cluster/afr/src/afr-dir-read.c b/xlators/cluster/afr/src/afr-dir-read.c
index 389515e3c36..ee9c5d8cc7f 100644
--- a/xlators/cluster/afr/src/afr-dir-read.c
+++ b/xlators/cluster/afr/src/afr-dir-read.c
@@ -317,7 +317,7 @@ afr_opendir (call_frame_t *frame, xlator_t *this,
 
         child_count = priv->child_count;
 
-        ALLOC_OR_GOTO (frame->local, afr_local_t, out);
+        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
         local = frame->local;
 
         ret = afr_local_init (local, priv, &op_errno);
@@ -649,7 +649,7 @@ afr_do_readdir (call_frame_t *frame, xlator_t *this,
         priv     = this->private;
         children = priv->children;
 
-        ALLOC_OR_GOTO (frame->local, afr_local_t, out);
+        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
         local = frame->local;
 
         ret = afr_local_init (local, priv, &op_errno);
diff --git a/xlators/cluster/afr/src/afr-dir-write.c b/xlators/cluster/afr/src/afr-dir-write.c
index 91aa2a9e7af..ef0025c8172 100644
--- a/xlators/cluster/afr/src/afr-dir-write.c
+++ b/xlators/cluster/afr/src/afr-dir-write.c
@@ -286,7 +286,7 @@ afr_create (call_frame_t *frame, xlator_t *this,
                 goto out;
         }
 
-        ALLOC_OR_GOTO (transaction_frame->local, afr_local_t, out);
+        AFR_LOCAL_ALLOC_OR_GOTO (transaction_frame->local, out);
         local = transaction_frame->local;
 
         ret = afr_local_init (local, priv, &op_errno);
@@ -508,7 +508,7 @@ afr_mknod (call_frame_t *frame, xlator_t *this,
                 goto out;
         }
 
-        ALLOC_OR_GOTO (transaction_frame->local, afr_local_t, out);
+        AFR_LOCAL_ALLOC_OR_GOTO (transaction_frame->local, out);
         local = transaction_frame->local;
 
         ret = afr_local_init (local, priv, &op_errno);
@@ -731,7 +731,7 @@ afr_mkdir (call_frame_t *frame, xlator_t *this,
                 goto out;
         }
 
-        ALLOC_OR_GOTO (transaction_frame->local, afr_local_t, out);
+        AFR_LOCAL_ALLOC_OR_GOTO (transaction_frame->local, out);
         local = transaction_frame->local;
 
         ret = afr_local_init (local, priv, &op_errno);
@@ -953,7 +953,7 @@ afr_link (call_frame_t *frame, xlator_t *this,
                 goto out;
         }
 
-        ALLOC_OR_GOTO (transaction_frame->local, afr_local_t, out);
+        AFR_LOCAL_ALLOC_OR_GOTO (transaction_frame->local, out);
         local = transaction_frame->local;
 
         ret = afr_local_init (local, priv, &op_errno);
@@ -1173,7 +1173,7 @@ afr_symlink (call_frame_t *frame, xlator_t *this,
                 goto out;
         }
 
-        ALLOC_OR_GOTO (transaction_frame->local, afr_local_t, out);
+        AFR_LOCAL_ALLOC_OR_GOTO (transaction_frame->local, out);
         local = transaction_frame->local;
 
         ret = afr_local_init (local, priv, &op_errno);
@@ -1388,7 +1388,7 @@ afr_rename (call_frame_t *frame, xlator_t *this,
                 goto out;
         }
 
-        ALLOC_OR_GOTO (transaction_frame->local, afr_local_t, out);
+        AFR_LOCAL_ALLOC_OR_GOTO (transaction_frame->local, out);
         local = transaction_frame->local;
 
         ret = afr_local_init (local, priv, &op_errno);
@@ -1584,7 +1584,7 @@ afr_unlink (call_frame_t *frame, xlator_t *this,
                 goto out;
         }
 
-        ALLOC_OR_GOTO (transaction_frame->local, afr_local_t, out);
+        AFR_LOCAL_ALLOC_OR_GOTO (transaction_frame->local, out);
         local = transaction_frame->local;
 
         ret = afr_local_init (local, priv, &op_errno);
@@ -1777,7 +1777,7 @@ afr_rmdir (call_frame_t *frame, xlator_t *this,
                 goto out;
         }
 
-        ALLOC_OR_GOTO (transaction_frame->local, afr_local_t, out);
+        AFR_LOCAL_ALLOC_OR_GOTO (transaction_frame->local, out);
         local = transaction_frame->local;
 
         ret = afr_local_init (local, priv, &op_errno);
diff --git a/xlators/cluster/afr/src/afr-inode-read.c b/xlators/cluster/afr/src/afr-inode-read.c
index ec0acbd3b0b..499804e6a36 100644
--- a/xlators/cluster/afr/src/afr-inode-read.c
+++ b/xlators/cluster/afr/src/afr-inode-read.c
@@ -124,7 +124,7 @@ afr_access (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t mask)
 
         children = priv->children;
 
-        ALLOC_OR_GOTO (frame->local, afr_local_t, out);
+        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
         local = frame->local;
 
         ret = afr_local_init (local, priv, &op_errno);
@@ -239,7 +239,7 @@ afr_stat (call_frame_t *frame, xlator_t *this, loc_t *loc)
 
         children = priv->children;
 
-        ALLOC_OR_GOTO (frame->local, afr_local_t, out);
+        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
         local = frame->local;
 
         ret = afr_local_init (local, priv, &op_errno);
@@ -354,7 +354,7 @@ afr_fstat (call_frame_t *frame, xlator_t *this,
 
         VALIDATE_OR_GOTO (fd->inode, out);
 
-        ALLOC_OR_GOTO (frame->local, afr_local_t, out);
+        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
         local = frame->local;
 
         ret = afr_local_init (local, priv, &op_errno);
@@ -475,7 +475,7 @@ afr_readlink (call_frame_t *frame, xlator_t *this,
 
         children = priv->children;
 
-        ALLOC_OR_GOTO (frame->local, afr_local_t, out);
+        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
         local = frame->local;
 
         ret = afr_local_init (local, priv, &op_errno);
@@ -904,7 +904,7 @@ afr_getxattr (call_frame_t *frame, xlator_t *this,
 
         children = priv->children;
 
-        ALLOC_OR_GOTO (frame->local, afr_local_t, out);
+        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
         local = frame->local;
 
         ret = afr_local_init (local, priv, &op_errno);
@@ -1111,7 +1111,7 @@ afr_fgetxattr (call_frame_t *frame, xlator_t *this,
 
         children = priv->children;
 
-        ALLOC_OR_GOTO (local, afr_local_t, out);
+        AFR_LOCAL_ALLOC_OR_GOTO (local, out);
         frame->local = local;
 
         op_ret = afr_local_init (local, priv, &op_errno);
@@ -1253,7 +1253,7 @@ afr_readv (call_frame_t *frame, xlator_t *this,
         priv     = this->private;
         children = priv->children;
 
-        ALLOC_OR_GOTO (frame->local, afr_local_t, out);
+        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
         local = frame->local;
 
         ret = afr_local_init (local, priv, &op_errno);
diff --git a/xlators/cluster/afr/src/afr-inode-write.c b/xlators/cluster/afr/src/afr-inode-write.c
index 72dcdc4785b..3dc1a418624 100644
--- a/xlators/cluster/afr/src/afr-inode-write.c
+++ b/xlators/cluster/afr/src/afr-inode-write.c
@@ -457,7 +457,7 @@ afr_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
 
         QUORUM_CHECK(writev,out);
 
-        ALLOC_OR_GOTO (frame->local, afr_local_t, out);
+        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
         local = frame->local;
 
         ret = afr_local_init (local, priv, &op_errno);
@@ -659,7 +659,7 @@ afr_truncate (call_frame_t *frame, xlator_t *this,
                 goto out;
         }
 
-        ALLOC_OR_GOTO (transaction_frame->local, afr_local_t, out);
+        AFR_LOCAL_ALLOC_OR_GOTO (transaction_frame->local, out);
         local = transaction_frame->local;
 
         ret = afr_local_init (local, priv, &op_errno);
@@ -899,7 +899,7 @@ afr_ftruncate (call_frame_t *frame, xlator_t *this,
 
         QUORUM_CHECK(ftruncate,out);
 
-        ALLOC_OR_GOTO (frame->local, afr_local_t, out);
+        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
         local = frame->local;
 
         ret = afr_local_init (local, priv, &op_errno);
@@ -1100,7 +1100,7 @@ afr_setattr (call_frame_t *frame, xlator_t *this,
                 goto out;
         }
 
-        ALLOC_OR_GOTO (transaction_frame->local, afr_local_t, out);
+        AFR_LOCAL_ALLOC_OR_GOTO (transaction_frame->local, out);
         local = transaction_frame->local;
 
         ret = afr_local_init (local, priv, &op_errno);
@@ -1302,7 +1302,7 @@ afr_fsetattr (call_frame_t *frame, xlator_t *this,
                 goto out;
         }
 
-        ALLOC_OR_GOTO (transaction_frame->local, afr_local_t, out);
+        AFR_LOCAL_ALLOC_OR_GOTO (transaction_frame->local, out);
         local = transaction_frame->local;
 
         ret = afr_local_init (local, priv, &op_errno);
@@ -1493,7 +1493,7 @@ afr_setxattr (call_frame_t *frame, xlator_t *this,
                 goto out;
         }
 
-        ALLOC_OR_GOTO (transaction_frame->local, afr_local_t, out);
+        AFR_LOCAL_ALLOC_OR_GOTO (transaction_frame->local, out);
         local = transaction_frame->local;
 
         ret = afr_local_init (local, priv, &op_errno);
@@ -1675,7 +1675,7 @@ afr_fsetxattr (call_frame_t *frame, xlator_t *this,
 
         QUORUM_CHECK(fsetxattr,out);
 
-        ALLOC_OR_GOTO (local, afr_local_t, out);
+        AFR_LOCAL_ALLOC_OR_GOTO (local, out);
 
         ret = afr_local_init (local, priv, &op_errno);
         if (ret < 0)
@@ -1865,7 +1865,7 @@ afr_removexattr (call_frame_t *frame, xlator_t *this,
                 goto out;
         }
 
-        ALLOC_OR_GOTO (transaction_frame->local, afr_local_t, out);
+        AFR_LOCAL_ALLOC_OR_GOTO (transaction_frame->local, out);
         local = transaction_frame->local;
 
         ret = afr_local_init (local, priv, &op_errno);
@@ -2040,7 +2040,7 @@ afr_fremovexattr (call_frame_t *frame, xlator_t *this,
                 goto out;
         }
 
-        ALLOC_OR_GOTO (local, afr_local_t, out);
+        AFR_LOCAL_ALLOC_OR_GOTO (local, out);
 
         ret = afr_local_init (local, priv, &op_errno);
         if (ret < 0) {
diff --git a/xlators/cluster/afr/src/afr-lk-common.c b/xlators/cluster/afr/src/afr-lk-common.c
index 2fe1349902c..e9a3a430dd0 100644
--- a/xlators/cluster/afr/src/afr-lk-common.c
+++ b/xlators/cluster/afr/src/afr-lk-common.c
@@ -2187,7 +2187,7 @@ afr_attempt_lock_recovery (xlator_t *this, int32_t child_index)
                 goto out;
         }
 
-        ALLOC_OR_GOTO (frame->local, afr_local_t, out);
+        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
         local = frame->local;
         ret = afr_local_init (local, priv, &op_errno);
         if (ret < 0) {
diff --git a/xlators/cluster/afr/src/afr-mem-types.h b/xlators/cluster/afr/src/afr-mem-types.h
index a138c967676..343260a7968 100644
--- a/xlators/cluster/afr/src/afr-mem-types.h
+++ b/xlators/cluster/afr/src/afr-mem-types.h
@@ -26,7 +26,6 @@
 enum gf_afr_mem_types_ {
         gf_afr_mt_iovec  = gf_common_mt_end + 1,
         gf_afr_mt_afr_fd_ctx_t,
-        gf_afr_mt_afr_local_t,
         gf_afr_mt_afr_private_t,
         gf_afr_mt_int32_t,
         gf_afr_mt_char,
diff --git a/xlators/cluster/afr/src/afr-open.c b/xlators/cluster/afr/src/afr-open.c
index 083e78a2a46..739def351ce 100644
--- a/xlators/cluster/afr/src/afr-open.c
+++ b/xlators/cluster/afr/src/afr-open.c
@@ -225,7 +225,7 @@ afr_open (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
                 goto out;
         }
 
-        ALLOC_OR_GOTO (frame->local, afr_local_t, out);
+        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
         local = frame->local;
 
         ret = afr_local_init (local, priv, &op_errno);
@@ -394,7 +394,7 @@ afr_fix_open (call_frame_t *frame, xlator_t *this, afr_fd_ctx_t *fd_ctx,
                         ret = -ENOMEM;
                         goto out;
                 }
-                ALLOC_OR_GOTO (open_frame->local, afr_local_t, out);
+                AFR_LOCAL_ALLOC_OR_GOTO (open_frame->local, out);
                 open_local = open_frame->local;
                 ret = afr_local_init (open_local, priv, &op_errno);
                 if (ret < 0)
diff --git a/xlators/cluster/afr/src/afr-self-heal-common.c b/xlators/cluster/afr/src/afr-self-heal-common.c
index 36a1e04c9bc..0ff959240bc 100644
--- a/xlators/cluster/afr/src/afr-self-heal-common.c
+++ b/xlators/cluster/afr/src/afr-self-heal-common.c
@@ -1040,7 +1040,7 @@ afr_impunge_frame_create (call_frame_t *frame, xlator_t *this,
                 goto out;
         }
 
-        ALLOC_OR_GOTO (impunge_local, afr_local_t, out);
+        AFR_LOCAL_ALLOC_OR_GOTO (impunge_local, out);
 
         local = frame->local;
         new_frame->local = impunge_local;
@@ -1350,7 +1350,7 @@ afr_sh_call_entry_expunge_remove (call_frame_t *frame, xlator_t *this,
                 goto out;
         }
 
-        ALLOC_OR_GOTO (expunge_local, afr_local_t, out);
+        AFR_LOCAL_ALLOC_OR_GOTO (expunge_local, out);
 
         local = frame->local;
         sh = &local->self_heal;
@@ -1926,8 +1926,7 @@ afr_local_t *afr_local_copy (afr_local_t *l, xlator_t *this)
 
         sh = &l->self_heal;
 
-        lc = GF_CALLOC (1, sizeof (afr_local_t),
-                        gf_afr_mt_afr_local_t);
+        lc = mem_get0 (this->local_pool);
         if (!lc)
                 goto out;
 
diff --git a/xlators/cluster/afr/src/afr-self-heal-entry.c b/xlators/cluster/afr/src/afr-self-heal-entry.c
index 6531615dfcf..cccce5db73a 100644
--- a/xlators/cluster/afr/src/afr-self-heal-entry.c
+++ b/xlators/cluster/afr/src/afr-self-heal-entry.c
@@ -724,7 +724,7 @@ afr_sh_entry_expunge_entry (call_frame_t *frame, xlator_t *this,
                 goto out;
         }
 
-        ALLOC_OR_GOTO (expunge_local, afr_local_t, out);
+        AFR_LOCAL_ALLOC_OR_GOTO (expunge_local, out);
 
         expunge_frame->local = expunge_local;
         expunge_sh = &expunge_local->self_heal;
@@ -1009,7 +1009,7 @@ afr_sh_entry_impunge_setattr (call_frame_t *impunge_frame, xlator_t *this)
                 op_errno = ENOMEM;
                 goto out;
         }
-        ALLOC_OR_GOTO (setattr_frame->local, afr_local_t, out);
+        AFR_LOCAL_ALLOC_OR_GOTO (setattr_frame->local, out);
         setattr_local = setattr_frame->local;
         call_count = afr_errno_count (NULL, impunge_sh->child_errno,
                                       priv->child_count, 0);
diff --git a/xlators/cluster/afr/src/afr.c b/xlators/cluster/afr/src/afr.c
index 8e2ef10080e..b73400a7d20 100644
--- a/xlators/cluster/afr/src/afr.c
+++ b/xlators/cluster/afr/src/afr.c
@@ -214,7 +214,10 @@ init (xlator_t *this)
                         "Volume is dangling.");
         }
 
-        ALLOC_OR_GOTO (this->private, afr_private_t, out);
+	this->private = GF_CALLOC (1, sizeof (afr_private_t),
+                                   gf_afr_mt_afr_private_t);
+        if (!this->private)
+                goto out;
 
         priv = this->private;
         LOCK_INIT (&priv->lock);
@@ -350,6 +353,15 @@ init (xlator_t *this)
                 goto out;
         }
 
+        /* keep more local here as we may need them for self-heal etc */
+        this->local_pool = mem_pool_new (afr_local_t, 4096);
+        if (!this->local_pool) {
+                ret = -1;
+                gf_log (this->name, GF_LOG_ERROR,
+                        "failed to create local_t's memory pool");
+                goto out;
+        }
+
         priv->first_lookup = 1;
         priv->root_inode = NULL;
 
diff --git a/xlators/cluster/afr/src/afr.h b/xlators/cluster/afr/src/afr.h
index 0f4a6d90a72..8abc4358352 100644
--- a/xlators/cluster/afr/src/afr.h
+++ b/xlators/cluster/afr/src/afr.h
@@ -728,15 +728,14 @@ typedef struct {
 
 
 /* try alloc and if it fails, goto label */
-#define ALLOC_OR_GOTO(var, type, label) do {                    \
-                var = GF_CALLOC (sizeof (type), 1,              \
-                                 gf_afr_mt_##type);             \
-                if (!var) {                                     \
-                        gf_log (this->name, GF_LOG_ERROR,       \
-                                "out of memory :(");            \
-                        op_errno = ENOMEM;                      \
-                        goto label;                             \
-                }                                               \
+#define AFR_LOCAL_ALLOC_OR_GOTO(var, label) do {                    \
+                var = mem_get0 (THIS->local_pool);                  \
+                if (!var) {                                         \
+                        gf_log (this->name, GF_LOG_ERROR,           \
+                                "out of memory :(");                \
+                        op_errno = ENOMEM;                          \
+                        goto label;                                 \
+                }                                                   \
         } while (0);
 
 
@@ -876,20 +875,24 @@ afr_launch_openfd_self_heal (call_frame_t *frame, xlator_t *this, fd_t *fd);
                         frame->local = NULL;                    \
                 }                                               \
                 STACK_UNWIND_STRICT (fop, frame, params);       \
-                afr_local_cleanup (__local, __this);            \
-                GF_FREE (__local);                              \
+                if (__local) {                                  \
+                        afr_local_cleanup (__local, __this);    \
+                        mem_put (__local);                      \
+                }                                               \
         } while (0)
 
-#define AFR_STACK_DESTROY(frame)                        \
-        do {                                            \
-                afr_local_t *__local = NULL;            \
-                xlator_t    *__this = NULL;             \
-                __local = frame->local;                 \
-                __this = frame->this;                   \
-                frame->local = NULL;                    \
-                STACK_DESTROY (frame->root);            \
-                afr_local_cleanup (__local, __this);    \
-                GF_FREE (__local);                      \
+#define AFR_STACK_DESTROY(frame)                                \
+        do {                                                    \
+                afr_local_t *__local = NULL;                    \
+                xlator_t    *__this = NULL;                     \
+                __local = frame->local;                         \
+                __this = frame->this;                           \
+                frame->local = NULL;                            \
+                STACK_DESTROY (frame->root);                    \
+                if (__local) {                                  \
+                        afr_local_cleanup (__local, __this);    \
+                        mem_put (__local);                      \
+                }                                               \
         } while (0);
 
 #define AFR_NUM_CHANGE_LOGS            3 /*data + metadata + entry*/
diff --git a/xlators/cluster/afr/src/pump.c b/xlators/cluster/afr/src/pump.c
index eae7899e9e8..18aee82ccef 100644
--- a/xlators/cluster/afr/src/pump.c
+++ b/xlators/cluster/afr/src/pump.c
@@ -1400,7 +1400,7 @@ pump_getxattr (call_frame_t *frame, xlator_t *this,
         }
 
 
-	ALLOC_OR_GOTO (frame->local, afr_local_t, out);
+	AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
 	local = frame->local;
 
         ret = afr_local_init (local, priv, &op_errno);
@@ -1670,7 +1670,7 @@ pump_setxattr (call_frame_t *frame, xlator_t *this,
         }
 
 
-	ALLOC_OR_GOTO (local, afr_local_t, out);
+	AFR_LOCAL_ALLOC_OR_GOTO (local, out);
 
 	ret = afr_local_init (local, priv, &op_errno);
 	if (ret < 0) {
@@ -2385,7 +2385,10 @@ init (xlator_t *this)
 			"Volume is dangling.");
 	}
 
-	ALLOC_OR_GOTO (this->private, afr_private_t, out);
+	this->private = GF_CALLOC (1, sizeof (afr_private_t),
+                                   gf_afr_mt_afr_private_t);
+        if (!this->private)
+                goto out;
 
 	priv = this->private;
         LOCK_INIT (&priv->lock);
@@ -2515,6 +2518,15 @@ init (xlator_t *this)
                 goto out;
         }
 
+        /* keep more local here as we may need them for self-heal etc */
+        this->local_pool = mem_pool_new (afr_local_t, 4096);
+        if (!this->local_pool) {
+                ret = -1;
+                gf_log (this->name, GF_LOG_ERROR,
+                        "failed to create local_t's memory pool");
+                goto out;
+        }
+
 	priv->pump_private = pump_priv;
 
         pump_change_state (this, PUMP_STATE_ABORT);