core: utilize mempool for frame->local allocations

in each translator, which uses 'frame->local', we are using
GF_CALLOC/GF_FREE, which would be costly considering the
number of allocation happening in a lifetime of 'fop'. It
would be good to utilize the mem pool framework for xlator's
local structures, so there is no allocation overhead.

Change-Id: Ida6e65039a24d9c219b380aa1c3559f36046dc94
Signed-off-by: Amar Tumballi <amar@gluster.com>
BUG: 765336
Reviewed-on: http://review.gluster.com/2772
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Anand Avati <avati@redhat.com>

7 files changed, 31 insertions, 30 deletions

diff --git a/xlators/protocol/client/src/client-handshake.c b/xlators/protocol/client/src/client-handshake.c
index 1896e6b6391..e1bcd4e1c55 100644
--- a/xlators/protocol/client/src/client-handshake.c
+++ b/xlators/protocol/client/src/client-handshake.c
@@ -534,7 +534,7 @@ clnt_fd_lk_local_unref (xlator_t *this, clnt_fd_lk_local_t *local)
 
         if (ref == 0) {
                 LOCK_DESTROY (&local->lock);
-                GF_FREE (local);
+                mem_put (local);
         }
         ref = 0;
 out:
@@ -1022,7 +1022,7 @@ protocol_client_reopendir (xlator_t *this, clnt_fd_ctx_t *fdctx)
                 goto out;
         }
 
-        local = GF_CALLOC (1, sizeof (*local), gf_client_mt_clnt_local_t);
+        local = mem_get0 (this->local_pool);
         if (!local) {
                 ret = -1;
                 goto out;
@@ -1108,7 +1108,7 @@ protocol_client_reopen (xlator_t *this, clnt_fd_ctx_t *fdctx)
                 goto out;
         }
 
-        local = GF_CALLOC (1, sizeof (*local), gf_client_mt_clnt_local_t);
+        local = mem_get0 (this->local_pool);
         if (!local) {
                 ret = -1;
                 goto out;
diff --git a/xlators/protocol/client/src/client-helpers.c b/xlators/protocol/client/src/client-helpers.c
index 5759f30624e..4c1ac1baa5c 100644
--- a/xlators/protocol/client/src/client-helpers.c
+++ b/xlators/protocol/client/src/client-helpers.c
@@ -115,7 +115,7 @@ client_local_wipe (clnt_local_t *local)
                         iobref_unref (local->iobref);
                 }
 
-                 GF_FREE (local);
+                 mem_put (local);
         }
 
         return 0;
diff --git a/xlators/protocol/client/src/client-lk.c b/xlators/protocol/client/src/client-lk.c
index e99fe774de6..4e87f7fcd13 100644
--- a/xlators/protocol/client/src/client-lk.c
+++ b/xlators/protocol/client/src/client-lk.c
@@ -864,7 +864,7 @@ client_attempt_lock_recovery (xlator_t *this, clnt_fd_ctx_t *fdctx)
         struct gf_flock reserve_flock;
         int ret = 0;
 
-        local = GF_CALLOC (1, sizeof (*local), gf_client_mt_clnt_local_t);
+        local = mem_get0 (this->local_pool);
         if (!local) {
                 ret = -ENOMEM;
                 goto out;
diff --git a/xlators/protocol/client/src/client-mem-types.h b/xlators/protocol/client/src/client-mem-types.h
index 6bc7daad271..e6e2c8e9fa5 100644
--- a/xlators/protocol/client/src/client-mem-types.h
+++ b/xlators/protocol/client/src/client-mem-types.h
@@ -25,7 +25,6 @@
 
 enum gf_client_mem_types_ {
         gf_client_mt_clnt_conf_t = gf_common_mt_end + 1,
-        gf_client_mt_clnt_local_t,
         gf_client_mt_clnt_req_buf_t,
         gf_client_mt_clnt_fdctx_t,
         gf_client_mt_clnt_lock_t,
diff --git a/xlators/protocol/client/src/client.c b/xlators/protocol/client/src/client.c
index acbc4829bb4..1386a2a1bd1 100644
--- a/xlators/protocol/client/src/client.c
+++ b/xlators/protocol/client/src/client.c
@@ -2371,6 +2371,14 @@ init (xlator_t *this)
                 goto out;
         }
 
+        this->local_pool = mem_pool_new (clnt_local_t, 1024);
+        if (!this->local_pool) {
+                ret = -1;
+                gf_log (this->name, GF_LOG_ERROR,
+                        "failed to create local_t's memory pool");
+                goto out;
+        }
+
         ret = client_init_rpc (this);
 out:
         if (ret)
diff --git a/xlators/protocol/client/src/client3_1-fops.c b/xlators/protocol/client/src/client3_1-fops.c
index 0bc91527aca..8007616629f 100644
--- a/xlators/protocol/client/src/client3_1-fops.c
+++ b/xlators/protocol/client/src/client3_1-fops.c
@@ -2510,7 +2510,7 @@ client3_1_lookup (call_frame_t *frame, xlator_t *this,
 
         conf = this->private;
         args = data;
-        local = GF_CALLOC (1, sizeof (*local), gf_client_mt_clnt_local_t);
+        local = mem_get0 (this->local_pool);
         if (!local) {
                 op_errno = ENOMEM;
                 goto unwind;
@@ -2960,7 +2960,7 @@ client3_1_symlink (call_frame_t *frame, xlator_t *this,
                 goto unwind;
 
         args = data;
-        local = GF_CALLOC (1, sizeof (*local), gf_client_mt_clnt_local_t);
+        local = mem_get0 (this->local_pool);
         if (!local) {
                 op_errno = ENOMEM;
                 goto unwind;
@@ -3108,7 +3108,7 @@ client3_1_link (call_frame_t *frame, xlator_t *this,
         GF_ASSERT_AND_GOTO_WITH_ERROR (this->name,
                                        !uuid_is_null (*((uuid_t*)req.newgfid)),
                                        unwind, op_errno, EINVAL);
-        local = GF_CALLOC (1, sizeof (*local), gf_client_mt_clnt_local_t);
+        local = mem_get0 (this->local_pool);
         if (!local) {
                 op_errno = ENOMEM;
                 goto unwind;
@@ -3153,7 +3153,7 @@ client3_1_mknod (call_frame_t *frame, xlator_t *this,
 
         args = data;
 
-        local = GF_CALLOC (1, sizeof (*local), gf_client_mt_clnt_local_t);
+        local = mem_get0 (this->local_pool);
         if (!local) {
                 op_errno = ENOMEM;
                 goto unwind;
@@ -3229,7 +3229,7 @@ client3_1_mkdir (call_frame_t *frame, xlator_t *this,
 
         args = data;
 
-        local = GF_CALLOC (1, sizeof (*local), gf_client_mt_clnt_local_t);
+        local = mem_get0 (this->local_pool);
         if (!local) {
                 op_errno = ENOMEM;
                 goto unwind;
@@ -3304,7 +3304,7 @@ client3_1_create (call_frame_t *frame, xlator_t *this,
 
         args = data;
 
-        local = GF_CALLOC (1, sizeof (*local), gf_client_mt_clnt_local_t);
+        local = mem_get0 (this->local_pool);
         if (!local) {
                 op_errno = ENOMEM;
                 goto unwind;
@@ -3382,7 +3382,7 @@ client3_1_open (call_frame_t *frame, xlator_t *this,
 
         args = data;
 
-        local = GF_CALLOC (1, sizeof (*local), gf_client_mt_clnt_local_t);
+        local = mem_get0 (this->local_pool);
         if (!local) {
                 op_errno = ENOMEM;
                 goto unwind;
@@ -3489,7 +3489,7 @@ client3_1_readv (call_frame_t *frame, xlator_t *this,
                 goto unwind;
         }
 
-        local = GF_CALLOC (1, sizeof (*local), gf_client_mt_clnt_local_t);
+        local = mem_get0 (this->local_pool);
         if (local == NULL) {
                 op_errno = ENOMEM;
                 goto unwind;
@@ -3594,7 +3594,7 @@ client3_1_flush (call_frame_t *frame, xlator_t *this,
 
         conf = this->private;
 
-        local = GF_CALLOC (1, sizeof (*local), gf_client_mt_clnt_local_t);
+        local = mem_get0 (this->local_pool);
         if (!local) {
                 STACK_UNWIND (frame, -1, ENOMEM);
                 return 0;
@@ -3721,7 +3721,7 @@ client3_1_opendir (call_frame_t *frame, xlator_t *this,
 
         args = data;
 
-        local = GF_CALLOC (1, sizeof (*local), gf_client_mt_clnt_local_t);
+        local = mem_get0 (this->local_pool);
         if (!local) {
                 op_errno = ENOMEM;
                 goto unwind;
@@ -3997,8 +3997,7 @@ client3_1_fgetxattr (call_frame_t *frame, xlator_t *this,
 
         CLIENT_GET_REMOTE_FD(conf, args->fd, remote_fd, op_errno, unwind);
 
-        local = GF_CALLOC (1, sizeof (*local),
-                           gf_client_mt_clnt_local_t);
+        local = mem_get0 (this->local_pool);
         if (!local) {
                 op_errno = ENOMEM;
                 goto unwind;
@@ -4100,8 +4099,7 @@ client3_1_getxattr (call_frame_t *frame, xlator_t *this,
                 goto unwind;
         }
 
-        local = GF_CALLOC (1, sizeof (*local),
-                           gf_client_mt_clnt_local_t);
+        local = mem_get0 (this->local_pool);
         if (!local) {
                 op_errno = ENOMEM;
                 goto unwind;
@@ -4226,8 +4224,7 @@ client3_1_xattrop (call_frame_t *frame, xlator_t *this,
         if (!(args->loc && args->loc->inode))
                 goto unwind;
 
-        local = GF_CALLOC (1, sizeof (*local),
-                           gf_client_mt_clnt_local_t);
+        local = mem_get0 (this->local_pool);
         if (!local) {
                 op_errno = ENOMEM;
                 goto unwind;
@@ -4346,8 +4343,7 @@ client3_1_fxattrop (call_frame_t *frame, xlator_t *this,
         req.flags  = args->flags;
         memcpy (req.gfid, args->fd->inode->gfid, 16);
 
-        local = GF_CALLOC (1, sizeof (*local),
-                           gf_client_mt_clnt_local_t);
+        local = mem_get0 (this->local_pool);
         if (!local) {
                 op_errno = ENOMEM;
                 goto unwind;
@@ -4536,7 +4532,7 @@ client3_1_lk (call_frame_t *frame, xlator_t *this,
 
         args = data;
         conf = this->private;
-        local = GF_CALLOC (1, sizeof (*local), gf_client_mt_clnt_local_t);
+        local = mem_get0 (this->local_pool);
         if (!local) {
                 op_errno = ENOMEM;
                 goto unwind;
@@ -4919,8 +4915,7 @@ client3_1_readdir (call_frame_t *frame, xlator_t *this,
 
         if ((readdir_rsp_size + GLUSTERFS_RPC_REPLY_SIZE + GLUSTERFS_RDMA_MAX_HEADER_SIZE)
             > (GLUSTERFS_RDMA_INLINE_THRESHOLD)) {
-                local = GF_CALLOC (1, sizeof (*local),
-                                   gf_client_mt_clnt_local_t);
+                local = mem_get0 (this->local_pool);
                 if (!local) {
                         op_errno = ENOMEM;
                         goto unwind;
@@ -5022,8 +5017,7 @@ client3_1_readdirp (call_frame_t *frame, xlator_t *this,
         readdirp_rsp_size = xdr_sizeof ((xdrproc_t) xdr_gfs3_readdirp_rsp, &rsp)
                 + args->size;
 
-        local = GF_CALLOC (1, sizeof (*local),
-                           gf_client_mt_clnt_local_t);
+        local = mem_get0 (this->local_pool);
         if (!local) {
                 op_errno = ENOMEM;
                 goto unwind;
diff --git a/xlators/protocol/legacy/client/src/client-protocol.c b/xlators/protocol/legacy/client/src/client-protocol.c
index 5c199e4dc22..804b93a5c14 100644
--- a/xlators/protocol/legacy/client/src/client-protocol.c
+++ b/xlators/protocol/legacy/client/src/client-protocol.c
@@ -180,7 +180,7 @@ client_local_wipe (client_local_t *local)
                 if (local->fd)
                         fd_unref (local->fd);
 
-                GF_FREE (local);
+                mem_put (local);
         }
 
         return 0;