nfs3: Submit multiple vectors received in read callback

There is a possibility of io-cache or read-ahead returning
a read buffer that straddles two separate pages in ioc or ra,
through two struct iovecs. Current nfs3 read reply does not
return as many vectors as received from a subvolume leading to
a short read for the NFS client.

Signed-off-by: Shehjar Tikoo <shehjart@gluster.com>
Signed-off-by: Anand V. Avati <avati@dev.gluster.com>

BUG: 902 (iozone hangs during random read throughput test)
URL: http://bugs.gluster.com/cgi-bin/bugzilla3/show_bug.cgi?id=902

3 files changed, 23 insertions, 17 deletions

diff --git a/xlators/nfs/server/src/nfs3-helpers.c b/xlators/nfs/server/src/nfs3-helpers.c
index f16bcac6f..0d0007745 100644
--- a/xlators/nfs/server/src/nfs3-helpers.c
+++ b/xlators/nfs/server/src/nfs3-helpers.c
@@ -1529,7 +1529,8 @@ nfs3_fill_read3res (read3res *res, nfsstat3 stat, count3 count,
         res->read3res_u.resok.file_attributes = poa;
         res->read3res_u.resok.count = count;
         res->read3res_u.resok.eof = is_eof;
-        res->read3res_u.resok.data.data_len = count;
+        res->read3res_u.resok.data.data_len = xdr_length_round_up (count,
+                                                                   1048576);
 
 }
 
@@ -2205,13 +2206,18 @@ nfs3_log_readlink_res (uint32_t xid, nfsstat3 stat, int pstat, char *linkpath)
 
 void
 nfs3_log_read_res (uint32_t xid, nfsstat3 stat, int pstat, count3 count,
-                   int is_eof)
+                   int is_eof, struct iovec *vec, int32_t veccount)
 {
         char    errstr[1024];
 
         nfs3_stat_to_errstr (xid, "READ", stat, pstat, errstr);
-        gf_log (GF_NFS3, GF_LOG_DEBUG, "%s, count: %"PRIu32", is_eof: %d",
-                errstr, count, is_eof);
+        if (vec)
+                gf_log (GF_NFS3, GF_LOG_DEBUG, "%s, count: %"PRIu32", is_eof:"
+                        " %d, vector: count: %d, len: %"PRIu64, errstr, count,
+                        is_eof, veccount, vec->iov_len);
+	else
+                gf_log (GF_NFS3, GF_LOG_DEBUG, "%s, count: %"PRIu32", is_eof:"
+                        " %d", errstr, count, is_eof);
 }
 
 
diff --git a/xlators/nfs/server/src/nfs3-helpers.h b/xlators/nfs/server/src/nfs3-helpers.h
index a282bce67..26bc11f6c 100644
--- a/xlators/nfs/server/src/nfs3-helpers.h
+++ b/xlators/nfs/server/src/nfs3-helpers.h
@@ -260,7 +260,7 @@ nfs3_log_readlink_res (uint32_t xid, nfsstat3 stat, int pstat, char *linkpath);
 
 extern void
 nfs3_log_read_res (uint32_t xid, nfsstat3 stat, int pstat, count3 count,
-                   int is_eof);
+                   int is_eof, struct iovec *vec, int32_t vcount);
 
 extern void
 nfs3_log_write_res (uint32_t xid, nfsstat3 stat, int pstat, count3 count,
diff --git a/xlators/nfs/server/src/nfs3.c b/xlators/nfs/server/src/nfs3.c
index a4ef1c6fc..7ecb82027 100644
--- a/xlators/nfs/server/src/nfs3.c
+++ b/xlators/nfs/server/src/nfs3.c
@@ -324,7 +324,7 @@ ret:
 int
 nfs3svc_submit_vector_reply (rpcsvc_request_t *req, void *arg,
                              nfs3_serializer sfunc, struct iovec *payload,
-                             struct iobref *piobref)
+                             int vcount, struct iobref *piobref)
 {
         struct iovec            outmsg = {0, };
         struct iobuf            *iob = NULL;
@@ -343,8 +343,8 @@ nfs3svc_submit_vector_reply (rpcsvc_request_t *req, void *arg,
         iobuf_unref (iob);
 
         if (piobref)
-                ret = rpcsvc_request_attach_vector (req, *payload, NULL, piobref
-                                                    , 1);
+                ret = rpcsvc_request_attach_vectors (req, payload, vcount,
+                                                     piobref);
 
         if (ret == -1)
                 goto err;
@@ -1362,8 +1362,8 @@ rpcerr:
 
 
 int
-nfs3_read_reply (rpcsvc_request_t *req, nfsstat3 stat,
-                 count3 count, struct iovec *vec, struct iobref *iobref,
+nfs3_read_reply (rpcsvc_request_t *req, nfsstat3 stat, count3 count,
+                 struct iovec *vec, int vcount, struct iobref *iobref,
                  struct iatt *poststat, int is_eof)
 {
         read3res                res = {0, };
@@ -1376,11 +1376,10 @@ nfs3_read_reply (rpcsvc_request_t *req, nfsstat3 stat,
                  * would be 0 and count = 0.
                  */
                 if (count != 0) {
-                        xdr_bytes_round_up (vec, 1048576);
                         nfs3svc_submit_vector_reply (req, (void *)&res,
                                                      (nfs3_serializer)
                                                   xdr_serialize_read3res_nocopy,
-                                                    vec, iobref);
+                                                    vec, vcount, iobref);
                 } else
                         nfs3svc_submit_reply (req, (void *)&res,
                                               (nfs3_serializer)
@@ -1415,8 +1414,9 @@ nfs3svc_read_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
 err:
         nfs3_log_read_res (rpcsvc_request_xid (cs->req), stat, op_errno,
-                           op_ret, is_eof);
-        nfs3_read_reply (cs->req, stat, op_ret, vector, iobref, stbuf, is_eof);
+                           op_ret, is_eof, vector, count);
+        nfs3_read_reply (cs->req, stat, op_ret, vector, count, iobref, stbuf,
+                         is_eof);
         nfs3_call_state_wipe (cs);
 
         return 0;
@@ -1445,7 +1445,7 @@ nfs3err:
         if (ret < 0) {
                 nfs3_log_common_res (rpcsvc_request_xid (cs->req), "READ", stat,
                                      -ret);
-                nfs3_read_reply (cs->req, stat, 0, NULL, NULL, NULL, 0);
+                nfs3_read_reply (cs->req, stat, 0, NULL, 0, NULL, NULL, 0);
                 nfs3_call_state_wipe (cs);
         }
 
@@ -1473,7 +1473,7 @@ nfs3err:
         if (ret < 0) {
                 nfs3_log_common_res (rpcsvc_request_xid (cs->req), "READ", stat,
                                      -ret);
-                nfs3_read_reply (cs->req, stat, 0, NULL, NULL, NULL, 0);
+                nfs3_read_reply (cs->req, stat, 0, NULL,0, NULL, NULL, 0);
                 nfs3_call_state_wipe (cs);
         }
 
@@ -1512,7 +1512,7 @@ nfs3err:
         if (ret < 0) {
                 nfs3_log_common_res (rpcsvc_request_xid (req), "READ", stat,
                                      -ret);
-                nfs3_read_reply (req, stat, 0, NULL, NULL, NULL, 0);
+                nfs3_read_reply (req, stat, 0, NULL,0, NULL, NULL, 0);
                 nfs3_call_state_wipe (cs);
                 ret = 0;
         }