changes to rpc

- use mem-pool for requests and saved_frames.
  - preserve the rpc_req structure till rpc invokes program's reply.
    This will enable us to store transport specific data that has to
    last till reply has come (eg., memory regions of chunk lists in
    case of rdma).
  - change signature of rpc_clnt_submit to accept rsphdr_vector and
    rsppayload_vector. The buffers pointed by these vectors will be
    from iobufs and these iobufs are added to an iobref which should
    also be passed as an arguement to rpc_clnt_submit.

Signed-off-by: Raghavendra G <raghavendra@gluster.com>
Signed-off-by: Anand V. Avati <avati@dev.gluster.com>

BUG: 875 (Implement a new protocol to provide proper backward/forward compatibility)
URL: http://bugs.gluster.com/cgi-bin/bugzilla3/show_bug.cgi?id=875

5 files changed, 16 insertions, 23 deletions

diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.c b/xlators/mgmt/glusterd/src/glusterd-utils.c
index 1d4a8d2ec..338ebc277 100644
--- a/xlators/mgmt/glusterd/src/glusterd-utils.c
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.c
@@ -310,7 +310,7 @@ glusterd_submit_request (glusterd_peerinfo_t *peerinfo, void *req,
         /* Send the msg */
         ret = rpc_clnt_submit (peerinfo->rpc, prog, procnum, cbkfn,
                                &iov, count,
-                               NULL, 0, iobref, frame);
+                               NULL, 0, iobref, frame, NULL, 0, NULL, 0, NULL);
 
         if (ret == 0) {
                 pthread_mutex_lock (&peerinfo->rpc->conn.lock);
diff --git a/xlators/protocol/client/src/client.c b/xlators/protocol/client/src/client.c
index 383f52bc9..c8bbd4672 100644
--- a/xlators/protocol/client/src/client.c
+++ b/xlators/protocol/client/src/client.c
@@ -96,7 +96,7 @@ client_submit_request (xlator_t *this, void *req, call_frame_t *frame,
         }
         /* Send the msg */
         ret = rpc_clnt_submit (conf->rpc, prog, procnum, cbk, &iov, count, NULL, 0,
-                               iobref, frame);
+                               iobref, frame, NULL, 0, NULL, 0, NULL);
 
         if (ret == 0) {
                 pthread_mutex_lock (&conf->rpc->conn.lock);
diff --git a/xlators/protocol/client/src/client3_1-fops.c b/xlators/protocol/client/src/client3_1-fops.c
index 136f84bf5..52d5c093a 100644
--- a/xlators/protocol/client/src/client3_1-fops.c
+++ b/xlators/protocol/client/src/client3_1-fops.c
@@ -80,7 +80,8 @@ client_submit_vec_request (xlator_t  *this, void *req, call_frame_t  *frame,
         }
         /* Send the msg */
         ret = rpc_clnt_submit (conf->rpc, prog, procnum, cbk, &iov, count,
-                               payload, payloadcnt, iobref, frame);
+                               payload, payloadcnt, iobref, frame, NULL, 0,
+                               NULL, 0, NULL);
 
         if (ret == 0) {
                 pthread_mutex_lock (&conf->rpc->conn.lock);
@@ -2001,13 +2002,12 @@ client3_1_readv_cbk (struct rpc_req *req, struct iovec *iov, int count,
         }
 
         if (rsp.op_ret != -1) {
-                iobref = iobref_new ();
+                iobref = req->rsp_iobref;
                 gf_stat_to_iatt (&rsp.stat, &stat);
                 vector.iov_len  = rsp.op_ret;
 
                 if (rsp.op_ret > 0) {
-                        vector.iov_base = req->rsp_procpayload->ptr;
-                        iobref_add (iobref, req->rsp_procpayload);
+                        vector.iov_base = req->rsp[1].iov_base;
                 }
         }
 out:
@@ -2015,10 +2015,6 @@ out:
                              gf_error_to_errno (rsp.op_errno), &vector, 1,
                              &stat, iobref);
 
-        if (iobref) {
-                iobref_unref (iobref);
-        }
-
         return 0;
 }
 
diff --git a/xlators/protocol/server/src/server.h b/xlators/protocol/server/src/server.h
index 81ac7edcc..20853879f 100644
--- a/xlators/protocol/server/src/server.h
+++ b/xlators/protocol/server/src/server.h
@@ -162,6 +162,8 @@ struct _server_state {
 	fd_t             *fd;
 	int               flags;
         int               wbflags;
+        struct iovec      payload_vector;
+        int               payload_count;
         struct iobuf     *iobuf;
         struct iobref    *iobref;
 
diff --git a/xlators/protocol/server/src/server3_1-fops.c b/xlators/protocol/server/src/server3_1-fops.c
index 18ace876c..fb69cf8da 100644
--- a/xlators/protocol/server/src/server3_1-fops.c
+++ b/xlators/protocol/server/src/server3_1-fops.c
@@ -2445,10 +2445,8 @@ server_writev_resume (call_frame_t *frame, xlator_t *bound_xl)
         if (state->resolve.op_ret != 0)
                 goto err;
 
-        iov.iov_len  = state->size;
-
-        if (state->iobuf) {
-                iov.iov_base = state->iobuf->ptr;
+        if (state->payload_count) {
+                iov = state->payload_vector;
         }
 
         STACK_WIND (frame, server_writev_cbk,
@@ -2987,10 +2985,10 @@ server_writev (rpcsvc_request_t *req)
 
 
 int
-server_writev_vec (rpcsvc_request_t *req, struct iobuf *iobuf)
+server_writev_vec (rpcsvc_request_t *req, struct iovec *payload,
+                   int payload_count, struct iobref *iobref)
 {
         server_state_t      *state  = NULL;
-        struct iobref       *iobref = NULL;
         call_frame_t        *frame  = NULL;
         gfs3_write_req       args   = {0,};
 
@@ -3022,14 +3020,11 @@ server_writev_vec (rpcsvc_request_t *req, struct iobuf *iobuf)
         state->resolve.fd_no = args.fd;
         state->offset        = args.offset;
 
-        if (iobuf) {
-                iobref = iobref_new ();
-                iobref_add (iobref, iobuf);
-
-                state->iobref = iobref;
-                state->iobuf  = iobuf_ref (iobuf);
-
+        if (payload_count != 0) {
+                state->iobref = iobref_ref (iobref);
                 state->size = req->msg[1].iov_len;
+                state->payload_count = payload_count;
+                state->payload_vector = *payload;
         }
 
         resolve_and_resume (frame, server_writev_resume);