changes to rpc

- use mem-pool for requests and saved_frames.
  - preserve the rpc_req structure till rpc invokes program's reply.
    This will enable us to store transport specific data that has to
    last till reply has come (eg., memory regions of chunk lists in
    case of rdma).
  - change signature of rpc_clnt_submit to accept rsphdr_vector and
    rsppayload_vector. The buffers pointed by these vectors will be
    from iobufs and these iobufs are added to an iobref which should
    also be passed as an arguement to rpc_clnt_submit.

Signed-off-by: Raghavendra G <raghavendra@gluster.com>
Signed-off-by: Anand V. Avati <avati@dev.gluster.com>

BUG: 875 (Implement a new protocol to provide proper backward/forward compatibility)
URL: http://bugs.gluster.com/cgi-bin/bugzilla3/show_bug.cgi?id=875

5 files changed, 16 insertions, 23 deletions

diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.c b/xlators/mgmt/glusterd/src/glusterd-utils.c
index 1d4a8d2ec69..338ebc2778a 100644
--- a/xlators/mgmt/glusterd/src/glusterd-utils.c
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.c
@@ -310,7 +310,7 @@ glusterd_submit_request (glusterd_peerinfo_t *peerinfo, void *req,
         /* Send the msg */
         ret = rpc_clnt_submit (peerinfo->rpc, prog, procnum, cbkfn,
                                &iov, count,
-                               NULL, 0, iobref, frame);
+                               NULL, 0, iobref, frame, NULL, 0, NULL, 0, NULL);
 
         if (ret == 0) {
                 pthread_mutex_lock (&peerinfo->rpc->conn.lock);
diff --git a/xlators/protocol/client/src/client.c b/xlators/protocol/client/src/client.c
index 383f52bc972..c8bbd467206 100644
--- a/xlators/protocol/client/src/client.c
+++ b/xlators/protocol/client/src/client.c
@@ -96,7 +96,7 @@ client_submit_request (xlator_t *this, void *req, call_frame_t *frame,
         }
         /* Send the msg */
         ret = rpc_clnt_submit (conf->rpc, prog, procnum, cbk, &iov, count, NULL, 0,
-                               iobref, frame);
+                               iobref, frame, NULL, 0, NULL, 0, NULL);
 
         if (ret == 0) {
                 pthread_mutex_lock (&conf->rpc->conn.lock);
diff --git a/xlators/protocol/client/src/client3_1-fops.c b/xlators/protocol/client/src/client3_1-fops.c
index 136f84bf560..52d5c093a60 100644
--- a/xlators/protocol/client/src/client3_1-fops.c
+++ b/xlators/protocol/client/src/client3_1-fops.c
@@ -80,7 +80,8 @@ client_submit_vec_request (xlator_t  *this, void *req, call_frame_t  *frame,
         }
         /* Send the msg */
         ret = rpc_clnt_submit (conf->rpc, prog, procnum, cbk, &iov, count,
-                               payload, payloadcnt, iobref, frame);
+                               payload, payloadcnt, iobref, frame, NULL, 0,
+                               NULL, 0, NULL);
 
         if (ret == 0) {
                 pthread_mutex_lock (&conf->rpc->conn.lock);
@@ -2001,13 +2002,12 @@ client3_1_readv_cbk (struct rpc_req *req, struct iovec *iov, int count,
         }
 
         if (rsp.op_ret != -1) {
-                iobref = iobref_new ();
+                iobref = req->rsp_iobref;
                 gf_stat_to_iatt (&rsp.stat, &stat);
                 vector.iov_len  = rsp.op_ret;
 
                 if (rsp.op_ret > 0) {
-                        vector.iov_base = req->rsp_procpayload->ptr;
-                        iobref_add (iobref, req->rsp_procpayload);
+                        vector.iov_base = req->rsp[1].iov_base;
                 }
         }
 out:
@@ -2015,10 +2015,6 @@ out:
                              gf_error_to_errno (rsp.op_errno), &vector, 1,
                              &stat, iobref);
 
-        if (iobref) {
-                iobref_unref (iobref);
-        }
-
         return 0;
 }
 
diff --git a/xlators/protocol/server/src/server.h b/xlators/protocol/server/src/server.h
index 81ac7edcc76..20853879f8c 100644
--- a/xlators/protocol/server/src/server.h
+++ b/xlators/protocol/server/src/server.h
@@ -162,6 +162,8 @@ struct _server_state {
 	fd_t             *fd;
 	int               flags;
         int               wbflags;
+        struct iovec      payload_vector;
+        int               payload_count;
         struct iobuf     *iobuf;
         struct iobref    *iobref;
 
diff --git a/xlators/protocol/server/src/server3_1-fops.c b/xlators/protocol/server/src/server3_1-fops.c
index 18ace876c7b..fb69cf8da28 100644
--- a/xlators/protocol/server/src/server3_1-fops.c
+++ b/xlators/protocol/server/src/server3_1-fops.c
@@ -2445,10 +2445,8 @@ server_writev_resume (call_frame_t *frame, xlator_t *bound_xl)
         if (state->resolve.op_ret != 0)
                 goto err;
 
-        iov.iov_len  = state->size;
-
-        if (state->iobuf) {
-                iov.iov_base = state->iobuf->ptr;
+        if (state->payload_count) {
+                iov = state->payload_vector;
         }
 
         STACK_WIND (frame, server_writev_cbk,
@@ -2987,10 +2985,10 @@ server_writev (rpcsvc_request_t *req)
 
 
 int
-server_writev_vec (rpcsvc_request_t *req, struct iobuf *iobuf)
+server_writev_vec (rpcsvc_request_t *req, struct iovec *payload,
+                   int payload_count, struct iobref *iobref)
 {
         server_state_t      *state  = NULL;
-        struct iobref       *iobref = NULL;
         call_frame_t        *frame  = NULL;
         gfs3_write_req       args   = {0,};
 
@@ -3022,14 +3020,11 @@ server_writev_vec (rpcsvc_request_t *req, struct iobuf *iobuf)
         state->resolve.fd_no = args.fd;
         state->offset        = args.offset;
 
-        if (iobuf) {
-                iobref = iobref_new ();
-                iobref_add (iobref, iobuf);
-
-                state->iobref = iobref;
-                state->iobuf  = iobuf_ref (iobuf);
-
+        if (payload_count != 0) {
+                state->iobref = iobref_ref (iobref);
                 state->size = req->msg[1].iov_len;
+                state->payload_count = payload_count;
+                state->payload_vector = *payload;
         }
 
         resolve_and_resume (frame, server_writev_resume);