Skip to contentSkip to conversation

NFSD: Add support for the v4.2 READ_PLUS operation

PATCH Superseded [email protected] 3 participants 14 messages Sep 8, 2020 – Sep 9, 2020

Subsystems (3)

v5 of 7 Superseded Latest is v7 →

lore
Share & tools
lore
#0 lore raw

v5 cover NFSD: Add support for the v4.2 READ_PLUS operation

From: Anna Schumaker <[email protected]>

These patches add server support for the READ_PLUS operation, which breaks read requests into several "data" and "hole" segments when replying to the client.

- Changes since v4:
- Fix up nfsd4_read_plus_rsize() calculation
- Return a short read if we detect the file has changed during encoding
- Rebase to v5.9-rc4

Here are the results of some performance tests I ran on some lab machines. I tested by reading various 2G files from a few different underlying filesystems and across several NFS versions. I used the vmtouch utility to make sure files were only cached when we wanted them to be. In addition to 100% data and 100% hole cases, I also tested with files that alternate between data and hole segments. These files have either 4K, 8K, 16K, or 32K segment sizes and start with either data or hole segments. So the file mixed-4d has a 4K segment size beginning with a data segment, but mixed-32h has 32K segments beginning with a hole. The units are in seconds, with the first number for each NFS version being the uncached read time and the second number is for when the file is cached on the server.

I added some extra data collection (client cpu percentage and sys time), but the extra data means I couldn't figure out a way to break this down into a concise table. I cut out v3 and v4.0 performance numbers to get the size down, but I kept v4.1 for comparison because it uses the same code that v4.2 without read plus uses.

Read Plus Results (ext4):
data
:... v4.1 ... Uncached ... 20.540 s, 105 MB/s, 0.65 s kern, 3% cpu
: :....... Cached ..... 18.252 s, 118 MB/s, 0.70 s kern, 3% cpu
:... v4.2 ... Uncached ... 20.605 s, 104 MB/s, 0.65 s kern, 3% cpu

        :....... Cached ..... 18.253 s, 118 MB/s, 0.67 s kern, 3% cpu
  hole
   :... v4.1 ... Uncached ... 18.255 s, 118 MB/s, 0.72 s kern,  3% cpu
   :    :....... Cached ..... 18.252 s, 118 MB/s, 0.72 s kern,  3% cpu
   :... v4.2 ... Uncached ...  0.847 s, 2.5 GB/s, 0.73 s kern, 86% cpu
        :....... Cached .....  0.845 s, 2.5 GB/s, 0.72 s kern, 85% cpu
  mixed-4d
   :... v4.1 ... Uncached ... 54.691 s,  39 MB/s, 0.75 s kern, 1% cpu
   :    :....... Cached ..... 18.252 s, 118 MB/s, 0.71 s kern, 3% cpu
   :... v4.2 ... Uncached ... 51.587 s,  42 MB/s, 0.75 s kern, 1% cpu
        :....... Cached .....  9.215 s, 233 MB/s, 0.67 s kern, 7% cpu
  mixed-8d
   :... v4.1 ... Uncached ... 37.072 s,  58 MB/s, 0.67 s kern, 1% cpu
   :    :....... Cached ..... 18.252 s, 118 MB/s, 0.71 s kern, 3% cpu
   :... v4.2 ... Uncached ... 33.259 s,  65 MB/s, 0.68 s kern, 2% cpu
        :....... Cached .....  9.172 s, 234 MB/s, 0.67 s kern, 7% cpu
  mixed-16d
   :... v4.1 ... Uncached ... 27.138 s,  79 MB/s, 0.73 s kern, 2% cpu
   :    :....... Cached ..... 18.252 s, 118 MB/s, 0.71 s kern, 3% cpu
   :... v4.2 ... Uncached ... 23.042 s,  93 MB/s, 0.73 s kern, 3% cpu
        :....... Cached .....  9.150 s, 235 MB/s, 0.66 s kern, 7% cpu
  mixed-32d
   :... v4.1 ... Uncached ... 25.326 s,  85 MB/s, 0.68 s kern, 2% cpu
   :    :....... Cached ..... 18.252 s, 118 MB/s, 0.70 s kern, 3% cpu
   :... v4.2 ... Uncached ... 21.125 s, 102 MB/s, 0.69 s kern, 3% cpu
        :....... Cached .....  9.140 s, 235 MB/s, 0.67 s kern, 7% cpu
  mixed-4h
   :... v4.1 ... Uncached ... 58.317 s,  37 MB/s, 0.75 s kern, 1% cpu
   :    :....... Cached ..... 18.252 s, 118 MB/s, 0.70 s kern, 3% cpu
   :... v4.2 ... Uncached ... 51.878 s,  41 MB/s, 0.74 s kern, 1% cpu
        :....... Cached .....  9.215 s, 233 MB/s, 0.68 s kern, 7% cpu
  mixed-8h
   :... v4.1 ... Uncached ... 36.855 s,  58 MB/s, 0.68 s kern, 1% cpu
   :    :....... Cached ..... 18.252 s, 118 MB/s, 0.72 s kern, 3% cpu
   :... v4.2 ... Uncached ... 29.457 s,  73 MB/s, 0.68 s kern, 2% cpu
        :....... Cached .....  9.172 s, 234 MB/s, 0.67 s kern, 7% cpu
  mixed-16h
   :... v4.1 ... Uncached ... 26.460 s,  81 MB/s, 0.74 s kern, 2% cpu
   :    :....... Cached ..... 18.252 s, 118 MB/s, 0.71 s kern, 3% cpu
   :... v4.2 ... Uncached ... 19.587 s, 110 MB/s, 0.74 s kern, 3% cpu
        :....... Cached .....  9.150 s, 235 MB/s, 0.67 s kern, 7% cpu
  mixed-32h
   :... v4.1 ... Uncached ... 25.495 s,  84 MB/s, 0.69 s kern, 2% cpu
   :    :....... Cached ..... 18.252 s, 118 MB/s, 0.65 s kern, 3% cpu
   :... v4.2 ... Uncached ... 17.634 s, 122 MB/s, 0.69 s kern, 3% cpu
        :....... Cached .....  9.140 s, 235 MB/s, 0.68 s kern, 7% cpu

Read Plus Results (xfs):
data
:... v4.1 ... Uncached ... 20.230 s, 106 MB/s, 0.65 s kern, 3% cpu
: :....... Cached ..... 18.252 s, 118 MB/s, 0.68 s kern, 3% cpu
:... v4.2 ... Uncached ... 20.724 s, 104 MB/s, 0.65 s kern, 3% cpu

        :....... Cached ..... 18.253 s, 118 MB/s, 0.67 s kern, 3% cpu
  hole
   :... v4.1 ... Uncached ... 18.255 s, 118 MB/s, 0.68 s kern,  3% cpu
   :    :....... Cached ..... 18.252 s, 118 MB/s, 0.69 s kern,  3% cpu
   :... v4.2 ... Uncached ...  0.904 s, 2.4 GB/s, 0.72 s kern, 79% cpu
        :....... Cached .....  0.908 s, 2.4 GB/s, 0.73 s kern, 80% cpu
  mixed-4d
   :... v4.1 ... Uncached ... 57.553 s,  37 MB/s, 0.77 s kern, 1% cpu
   :    :....... Cached ..... 18.252 s, 118 MB/s, 0.70 s kern, 3% cpu
   :... v4.2 ... Uncached ... 37.162 s,  58 MB/s, 0.73 s kern, 1% cpu
        :....... Cached .....  9.215 s, 233 MB/s, 0.67 s kern, 7% cpu
  mixed-8d
   :... v4.1 ... Uncached ... 36.754 s,  58 MB/s, 0.69 s kern, 1% cpu
   :    :....... Cached ..... 18.252 s, 118 MB/s, 0.68 s kern, 3% cpu
   :... v4.2 ... Uncached ... 24.454 s,  88 MB/s, 0.69 s kern, 2% cpu
        :....... Cached .....  9.172 s, 234 MB/s, 0.66 s kern, 7% cpu
  mixed-16d
   :... v4.1 ... Uncached ... 27.156 s,  79 MB/s, 0.73 s kern, 2% cpu
   :    :....... Cached ..... 18.252 s, 118 MB/s, 0.71 s kern, 3% cpu
   :... v4.2 ... Uncached ... 22.934 s,  94 MB/s, 0.72 s kern, 3% cpu
        :....... Cached .....  9.150 s, 235 MB/s, 0.68 s kern, 7% cpu
  mixed-32d
   :... v4.1 ... Uncached ... 27.849 s,  77 MB/s, 0.68 s kern, 2% cpu
   :    :....... Cached ..... 18.252 s, 118 MB/s, 0.72 s kern, 3% cpu
   :... v4.2 ... Uncached ... 23.670 s,  91 MB/s, 0.67 s kern, 2% cpu
        :....... Cached .....  9.139 s, 235 MB/s, 0.64 s kern, 7% cpu
  mixed-4h
   :... v4.1 ... Uncached ... 57.639 s,  37 MB/s, 0.72 s kern, 1% cpu
   :    :....... Cached ..... 18.252 s, 118 MB/s, 0.69 s kern, 3% cpu
   :... v4.2 ... Uncached ... 35.503 s,  61 MB/s, 0.72 s kern, 2% cpu
        :....... Cached .....  9.215 s, 233 MB/s, 0.66 s kern, 7% cpu
  mixed-8h
   :... v4.1 ... Uncached ... 37.044 s,  58 MB/s, 0.71 s kern, 1% cpu
   :    :....... Cached ..... 18.252 s, 118 MB/s, 0.68 s kern, 3% cpu
   :... v4.2 ... Uncached ... 23.779 s,  90 MB/s, 0.69 s kern, 2% cpu
        :....... Cached .....  9.172 s, 234 MB/s, 0.65 s kern, 7% cpu
  mixed-16h
   :... v4.1 ... Uncached ... 27.167 s,  79 MB/s, 0.73 s kern, 2% cpu
   :    :....... Cached ..... 18.252 s, 118 MB/s, 0.67 s kern, 3% cpu
   :... v4.2 ... Uncached ... 19.088 s, 113 MB/s, 0.75 s kern, 3% cpu
        :....... Cached .....  9.159 s, 234 MB/s, 0.66 s kern, 7% cpu
  mixed-32h
   :... v4.1 ... Uncached ... 27.592 s,  78 MB/s, 0.71 s kern, 2% cpu
   :    :....... Cached ..... 18.252 s, 118 MB/s, 0.68 s kern, 3% cpu
   :... v4.2 ... Uncached ... 19.682 s, 109 MB/s, 0.67 s kern, 3% cpu
        :....... Cached .....  9.140 s, 235 MB/s, 0.67 s kern, 7% cpu

Read Plus Results (btrfs):
data
:... v4.1 ... Uncached ... 21.317 s, 101 MB/s, 0.63 s kern, 2% cpu
: :....... Cached ..... 18.252 s, 118 MB/s, 0.67 s kern, 3% cpu
:... v4.2 ... Uncached ... 28.665 s, 75 MB/s, 0.65 s kern, 2% cpu

        :....... Cached ..... 18.253 s, 118 MB/s, 0.66 s kern, 3% cpu
  hole
   :... v4.1 ... Uncached ... 18.256 s, 118 MB/s, 0.70 s kern,  3% cpu
   :    :....... Cached ..... 18.254 s, 118 MB/s, 0.73 s kern,  4% cpu
   :... v4.2 ... Uncached ...  0.851 s, 2.5 GB/s, 0.72 s kern, 84% cpu
        :....... Cached .....  0.847 s, 2.5 GB/s, 0.73 s kern, 86% cpu
  mixed-4d
   :... v4.1 ... Uncached ... 56.857 s,  38 MB/s, 0.76 s kern, 1% cpu
   :    :....... Cached ..... 18.252 s, 118 MB/s, 0.72 s kern, 3% cpu
   :... v4.2 ... Uncached ... 54.455 s,  39 MB/s, 0.73 s kern, 1% cpu
        :....... Cached .....  9.215 s, 233 MB/s, 0.68 s kern, 7% cpu
  mixed-8d
   :... v4.1 ... Uncached ... 36.641 s,  59 MB/s, 0.68 s kern, 1% cpu
   :    :....... Cached ..... 18.252 s, 118 MB/s, 0.70 s kern, 3% cpu
   :... v4.2 ... Uncached ... 33.205 s,  65 MB/s, 0.67 s kern, 2% cpu
        :....... Cached .....  9.172 s, 234 MB/s, 0.65 s kern, 7% cpu
  mixed-16d
   :... v4.1 ... Uncached ... 28.653 s,  75 MB/s, 0.72 s kern, 2% cpu
   :    :....... Cached ..... 18.252 s, 118 MB/s, 0.70 s kern, 3% cpu
   :... v4.2 ... Uncached ... 25.748 s,  83 MB/s, 0.71 s kern, 2% cpu
        :....... Cached .....  9.150 s, 235 MB/s, 0.64 s kern, 7% cpu
  mixed-32d
   :... v4.1 ... Uncached ... 28.886 s,  74 MB/s, 0.67 s kern, 2% cpu
   :    :....... Cached ..... 18.252 s, 118 MB/s, 0.71 s kern, 3% cpu
   :... v4.2 ... Uncached ... 24.724 s,  87 MB/s, 0.74 s kern, 2% cpu
        :....... Cached .....  9.140 s, 235 MB/s, 0.63 s kern, 6% cpu
  mixed-4h
   :... v4.1 ... Uncached ...  52.181 s,  41 MB/s, 0.73 s kern, 1% cpu
   :    :....... Cached .....  18.252 s, 118 MB/s, 0.66 s kern, 3% cpu
   :... v4.2 ... Uncached ... 150.341 s,  14 MB/s, 0.72 s kern, 0% cpu
        :....... Cached .....   9.216 s, 233 MB/s, 0.63 s kern, 6% cpu
  mixed-8h
   :... v4.1 ... Uncached ... 36.945 s,  58 MB/s, 0.68 s kern, 1% cpu
   :    :....... Cached ..... 18.252 s, 118 MB/s, 0.65 s kern, 3% cpu
   :... v4.2 ... Uncached ... 79.781 s,  27 MB/s, 0.68 s kern, 0% cpu
        :....... Cached .....  9.172 s, 234 MB/s, 0.66 s kern, 7% cpu
  mixed-16h
   :... v4.1 ... Uncached ... 28.651 s,  75 MB/s, 0.73 s kern, 2% cpu
   :    :....... Cached ..... 18.252 s, 118 MB/s, 0.66 s kern, 3% cpu
   :... v4.2 ... Uncached ... 47.428 s,  45 MB/s, 0.71 s kern, 1% cpu
        :....... Cached .....  9.150 s, 235 MB/s, 0.67 s kern, 7% cpu
  mixed-32h
   :... v4.1 ... Uncached ... 28.618 s,  75 MB/s, 0.69 s kern, 2% cpu
   :    :....... Cached ..... 18.252 s, 118 MB/s, 0.70 s kern, 3% cpu
   :... v4.2 ... Uncached ... 38.813 s,  55 MB/s, 0.67 s kern, 1% cpu
        :....... Cached .....  9.140 s, 235 MB/s, 0.61 s kern, 6% cpu

Thoughts?
Anna

Anna Schumaker (5):
SUNRPC/NFSD: Implement xdr_reserve_space_vec()
NFSD: Add READ_PLUS data support
NFSD: Add READ_PLUS hole segment encoding
NFSD: Return both a hole and a data segment
NFSD: Encode a full READ_PLUS reply

 fs/nfsd/nfs4proc.c         |  20 +++++
 fs/nfsd/nfs4xdr.c          | 173 +++++++++++++++++++++++++++++++------
 include/linux/sunrpc/xdr.h |   2 +
 net/sunrpc/xdr.c           |  45 ++++++++++
 4 files changed, 213 insertions(+), 27 deletions(-)
Signature · 1 line
2.28.0
#1 lore raw

PATCH v5 1/5 SUNRPC/NFSD: Implement xdr_reserve_space_vec()

From: Anna Schumaker <[email protected]>

Reserving space for a large READ payload requires special handling when reserving space in the xdr buffer pages. One problem we can have is use of the scratch buffer, which is used to get a pointer to a contiguous region of data up to PAGE_SIZE. When using the scratch buffer, calls to xdr_commit_encode() shift the data to it's proper alignment in the xdr buffer. If we've reserved several pages in a vector, then this could potentially invalidate earlier pointers and result in incorrect READ data being sent to the client.

I get around this by looking at the amount of space left in the current page, and never reserve more than that for each entry in the read vector. This lets us place data directly where it needs to go in the buffer pages.

---
 fs/nfsd/nfs4xdr.c          | 28 +++---------------------
 include/linux/sunrpc/xdr.h |  2 ++
 net/sunrpc/xdr.c           | 45 ++++++++++++++++++++++++++++++++++++++
 3 files changed, 50 insertions(+), 25 deletions(-)
fs/nfsd/nfs4xdr.c +3 -25
index 259d5ad0e3f4..0be194de4888 100644
@@ -3814,36 +3814,14 @@ static __be32 nfsd4_encode_readv(struct nfsd4_compoundres *resp,
 {
 	struct xdr_stream *xdr = &resp->xdr;
 	u32 eof;
-	int v;
 	int starting_len = xdr->buf->len - 8;
-	long len;
-	int thislen;
 	__be32 nfserr;
 	__be32 tmp;
-	__be32 *p;
 	int pad;
 
-	/*
-	 * svcrdma requires every READ payload to start somewhere
-	 * in xdr->pages.
-	 */
-	if (xdr->iov == xdr->buf->head) {
-		xdr->iov = NULL;
-		xdr->end = xdr->p;
-	}
-
-	len = maxcount;
-	v = 0;
-	while (len) {
-		thislen = min_t(long, len, PAGE_SIZE);
-		p = xdr_reserve_space(xdr, thislen);
-		WARN_ON_ONCE(!p);
-		resp->rqstp->rq_vec[v].iov_base = p;
-		resp->rqstp->rq_vec[v].iov_len = thislen;
-		v++;
-		len -= thislen;
-	}
-	read->rd_vlen = v;
+	read->rd_vlen = xdr_reserve_space_vec(xdr, resp->rqstp->rq_vec, maxcount);
+	if (read->rd_vlen < 0)
+		return nfserr_resource;
 
 	nfserr = nfsd_readv(resp->rqstp, read->rd_fhp, file, read->rd_offset,
 			    resp->rqstp->rq_vec, read->rd_vlen, &maxcount,
include/linux/sunrpc/xdr.h +2 -0
index 5a6a81b7cd9f..6613d96a3029 100644
@@ -234,6 +234,8 @@ typedef int	(*kxdrdproc_t)(struct rpc_rqst *rqstp, struct xdr_stream *xdr,
 extern void xdr_init_encode(struct xdr_stream *xdr, struct xdr_buf *buf,
 			    __be32 *p, struct rpc_rqst *rqst);
 extern __be32 *xdr_reserve_space(struct xdr_stream *xdr, size_t nbytes);
+extern int xdr_reserve_space_vec(struct xdr_stream *xdr, struct kvec *vec,
+		size_t nbytes);
 extern void xdr_commit_encode(struct xdr_stream *xdr);
 extern void xdr_truncate_encode(struct xdr_stream *xdr, size_t len);
 extern int xdr_restrict_buflen(struct xdr_stream *xdr, int newbuflen);
net/sunrpc/xdr.c +45 -0

51 lines not on this page · raw message

Signature · 1 line
2.28.0
#2 lore raw

PATCH v5 2/5 NFSD: Add READ_PLUS data support

In reply to [email protected]

From: Anna Schumaker <[email protected]>

This patch adds READ_PLUS support for returning a single NFS4_CONTENT_DATA segment to the client. This is basically the same as the READ operation, only with the extra information about data segments.

---
v5: Fix up nfsd4_read_plus_rsize() calculation

---
 fs/nfsd/nfs4proc.c | 20 +++++++++++
 fs/nfsd/nfs4xdr.c  | 83 ++++++++++++++++++++++++++++++++++++++++++++--
 2 files changed, 101 insertions(+), 2 deletions(-)
fs/nfsd/nfs4proc.c +20 -0
index eaf50eafa935..0a3df5f10501 100644
@@ -2591,6 +2591,19 @@ static inline u32 nfsd4_read_rsize(struct svc_rqst *rqstp, struct nfsd4_op *op)
 	return (op_encode_hdr_size + 2 + XDR_QUADLEN(rlen)) * sizeof(__be32);
 }
 
+static inline u32 nfsd4_read_plus_rsize(struct svc_rqst *rqstp, struct nfsd4_op *op)
+{
+	u32 maxcount = svc_max_payload(rqstp);
+	u32 rlen = min(op->u.read.rd_length, maxcount);
+	/*
+	 * Worst case is a single large data segment, so make
+	 * sure we have enough room to encode that
+	 */
+	u32 seg_len = 1 + 2 + 1;
+
+	return (op_encode_hdr_size + 2 + seg_len + XDR_QUADLEN(rlen)) * sizeof(__be32);
+}
+
 static inline u32 nfsd4_readdir_rsize(struct svc_rqst *rqstp, struct nfsd4_op *op)
 {
 	u32 maxcount = 0, rlen = 0;
@@ -3163,6 +3176,13 @@ static const struct nfsd4_operation nfsd4_ops[] = {
 		.op_name = "OP_COPY",
 		.op_rsize_bop = nfsd4_copy_rsize,
 	},
+	[OP_READ_PLUS] = {
+		.op_func = nfsd4_read,
+		.op_release = nfsd4_read_release,
+		.op_name = "OP_READ_PLUS",
+		.op_rsize_bop = nfsd4_read_plus_rsize,
+		.op_get_currentstateid = nfsd4_get_readstateid,
+	},
 	[OP_SEEK] = {
 		.op_func = nfsd4_seek,
 		.op_name = "OP_SEEK",
fs/nfsd/nfs4xdr.c +81 -2

101 lines not on this page · raw message

Signature · 1 line
2.28.0
#3 lore raw

Re: [PATCH v5 2/5] NFSD: Add READ_PLUS data support

Quoted 21 lines · On Tue, Sep 08, 2020 at 12:25:56PM -0400, [email protected] wrote:

From: Anna Schumaker <[email protected]>

This patch adds READ_PLUS support for returning a single NFS4_CONTENT_DATA segment to the client. This is basically the same as the READ operation, only with the extra information about data segments.

---
v5: Fix up nfsd4_read_plus_rsize() calculation

---
 fs/nfsd/nfs4proc.c | 20 +++++++++++
 fs/nfsd/nfs4xdr.c  | 83 ++++++++++++++++++++++++++++++++++++++++++++--
 2 files changed, 101 insertions(+), 2 deletions(-)
fs/nfsd/nfs4proc.c +0 -0
index eaf50eafa935..0a3df5f10501 100644
@@ -2591,6 +2591,19 @@ static inline u32 nfsd4_read_rsize(struct svc_rqst *rqstp, struct nfsd4_op *op)
 	return (op_encode_hdr_size + 2 + XDR_QUADLEN(rlen)) * sizeof(__be32);

fs/nfsd/nfs4proc.c static inline u32 nfsd4_read_rsize(struct svc_rqst *rqstp, struct nfsd4_op *op) · line 2601

Quoted 10 lines
fs/nfsd/nfs4proc.c +8 -0
 }
 
+static inline u32 nfsd4_read_plus_rsize(struct svc_rqst *rqstp, struct nfsd4_op *op)
+{
+	u32 maxcount = svc_max_payload(rqstp);
+	u32 rlen = min(op->u.read.rd_length, maxcount);
+	/*
+	 * Worst case is a single large data segment, so make
+	 * sure we have enough room to encode that
+	 */

After the last patch we allow an unlimited number of segments. So a zillion 1-byte segments is also possible, and is a worse case.

Possible ways to avoid that kind of thing:

- when encoding, stop and return a short read if the xdr-encoded result would exceed the limit calculated here.
- round SEEK_HOLE results up to the nearest (say) 512 bytes, and SEEK_DATA result down to the nearest 512 bytes. May also need some logic to avoid encoding 0-length segments.
- talk to linux-fsdevel, see if we can get a minimum hole alignment guarantee from filesystems.

I'm thinking #1 is simplest.

--b.

Quoted 133 lines
fs/nfsd/nfs4proc.c +12 -0
+	u32 seg_len = 1 + 2 + 1;
+
+	return (op_encode_hdr_size + 2 + seg_len + XDR_QUADLEN(rlen)) * sizeof(__be32);
+}
+
 static inline u32 nfsd4_readdir_rsize(struct svc_rqst *rqstp, struct nfsd4_op *op)
 {
 	u32 maxcount = 0, rlen = 0;
@@ -3163,6 +3176,13 @@ static const struct nfsd4_operation nfsd4_ops[] = {
 		.op_name = "OP_COPY",
 		.op_rsize_bop = nfsd4_copy_rsize,
 	},
+	[OP_READ_PLUS] = {
+		.op_func = nfsd4_read,
+		.op_release = nfsd4_read_release,
+		.op_name = "OP_READ_PLUS",
+		.op_rsize_bop = nfsd4_read_plus_rsize,
+		.op_get_currentstateid = nfsd4_get_readstateid,
+	},
 	[OP_SEEK] = {
 		.op_func = nfsd4_seek,
 		.op_name = "OP_SEEK",
fs/nfsd/nfs4xdr.c +81 -2
index 0be194de4888..26d12e3edf33 100644
@@ -2173,7 +2173,7 @@ static const nfsd4_dec nfsd4_dec_ops[] = {
 	[OP_LAYOUTSTATS]	= (nfsd4_dec)nfsd4_decode_notsupp,
 	[OP_OFFLOAD_CANCEL]	= (nfsd4_dec)nfsd4_decode_offload_status,
 	[OP_OFFLOAD_STATUS]	= (nfsd4_dec)nfsd4_decode_offload_status,
-	[OP_READ_PLUS]		= (nfsd4_dec)nfsd4_decode_notsupp,
+	[OP_READ_PLUS]		= (nfsd4_dec)nfsd4_decode_read,
 	[OP_SEEK]		= (nfsd4_dec)nfsd4_decode_seek,
 	[OP_WRITE_SAME]		= (nfsd4_dec)nfsd4_decode_notsupp,
 	[OP_CLONE]		= (nfsd4_dec)nfsd4_decode_clone,
@@ -4597,6 +4597,85 @@ nfsd4_encode_offload_status(struct nfsd4_compoundres *resp, __be32 nfserr,
 		return nfserr_resource;
 	p = xdr_encode_hyper(p, os->count);
 	*p++ = cpu_to_be32(0);
+	return nfserr;
+}
+
+static __be32
+nfsd4_encode_read_plus_data(struct nfsd4_compoundres *resp,
+			    struct nfsd4_read *read,
+			    unsigned long maxcount,  u32 *eof)
+{
+	struct xdr_stream *xdr = &resp->xdr;
+	struct file *file = read->rd_nf->nf_file;
+	int starting_len = xdr->buf->len;
+	__be32 nfserr;
+	__be32 *p, tmp;
+	__be64 tmp64;
+
+	/* Content type, offset, byte count */
+	p = xdr_reserve_space(xdr, 4 + 8 + 4);
+	if (!p)
+		return nfserr_resource;
+
+	read->rd_vlen = xdr_reserve_space_vec(xdr, resp->rqstp->rq_vec, maxcount);
+	if (read->rd_vlen < 0)
+		return nfserr_resource;
+
+	nfserr = nfsd_readv(resp->rqstp, read->rd_fhp, file, read->rd_offset,
+			    resp->rqstp->rq_vec, read->rd_vlen, &maxcount, eof);
+	if (nfserr)
+		return nfserr;
+
+	tmp = htonl(NFS4_CONTENT_DATA);
+	write_bytes_to_xdr_buf(xdr->buf, starting_len,      &tmp,   4);
+	tmp64 = cpu_to_be64(read->rd_offset);
+	write_bytes_to_xdr_buf(xdr->buf, starting_len + 4,  &tmp64, 8);
+	tmp = htonl(maxcount);
+	write_bytes_to_xdr_buf(xdr->buf, starting_len + 12, &tmp,   4);
+	return nfs_ok;
+}
+
+static __be32
+nfsd4_encode_read_plus(struct nfsd4_compoundres *resp, __be32 nfserr,
+		       struct nfsd4_read *read)
+{
+	unsigned long maxcount;
+	struct xdr_stream *xdr = &resp->xdr;
+	struct file *file;
+	int starting_len = xdr->buf->len;
+	int segments = 0;
+	__be32 *p, tmp;
+	u32 eof;
+
+	if (nfserr)
+		return nfserr;
+	file = read->rd_nf->nf_file;
+
+	/* eof flag, segment count */
+	p = xdr_reserve_space(xdr, 4 + 4);
+	if (!p)
+		return nfserr_resource;
+	xdr_commit_encode(xdr);
+
+	maxcount = svc_max_payload(resp->rqstp);
+	maxcount = min_t(unsigned long, maxcount,
+			 (xdr->buf->buflen - xdr->buf->len));
+	maxcount = min_t(unsigned long, maxcount, read->rd_length);
+
+	eof = read->rd_offset >= i_size_read(file_inode(file));
+	if (!eof) {
+		nfserr = nfsd4_encode_read_plus_data(resp, read, maxcount, &eof);
+		segments++;
+	}
+
+	if (nfserr)
+		xdr_truncate_encode(xdr, starting_len);
+	else {
+		tmp = htonl(eof);
+		write_bytes_to_xdr_buf(xdr->buf, starting_len,     &tmp, 4);
+		tmp = htonl(segments);
+		write_bytes_to_xdr_buf(xdr->buf, starting_len + 4, &tmp, 4);
+	}
 
 	return nfserr;
 }
@@ -4974,7 +5053,7 @@ static const nfsd4_enc nfsd4_enc_ops[] = {
 	[OP_LAYOUTSTATS]	= (nfsd4_enc)nfsd4_encode_noop,
 	[OP_OFFLOAD_CANCEL]	= (nfsd4_enc)nfsd4_encode_noop,
 	[OP_OFFLOAD_STATUS]	= (nfsd4_enc)nfsd4_encode_offload_status,
-	[OP_READ_PLUS]		= (nfsd4_enc)nfsd4_encode_noop,
+	[OP_READ_PLUS]		= (nfsd4_enc)nfsd4_encode_read_plus,
 	[OP_SEEK]		= (nfsd4_enc)nfsd4_encode_seek,
 	[OP_WRITE_SAME]		= (nfsd4_enc)nfsd4_encode_noop,
 	[OP_CLONE]		= (nfsd4_enc)nfsd4_encode_noop,
Signature · 1 line
2.28.0
#4 lore raw

Re: [PATCH v5 2/5] NFSD: Add READ_PLUS data support

Quoted 37 lines · J. Bruce Fields
Quoted 21 lines · On Tue, Sep 08, 2020 at 12:25:56PM -0400, [email protected] wrote:

From: Anna Schumaker <[email protected]>

This patch adds READ_PLUS support for returning a single NFS4_CONTENT_DATA segment to the client. This is basically the same as the READ operation, only with the extra information about data segments.

---
v5: Fix up nfsd4_read_plus_rsize() calculation

---
 fs/nfsd/nfs4proc.c | 20 +++++++++++
 fs/nfsd/nfs4xdr.c  | 83 ++++++++++++++++++++++++++++++++++++++++++++--
 2 files changed, 101 insertions(+), 2 deletions(-)
fs/nfsd/nfs4proc.c +0 -0
index eaf50eafa935..0a3df5f10501 100644
@@ -2591,6 +2591,19 @@ static inline u32 nfsd4_read_rsize(struct svc_rqst *rqstp, struct nfsd4_op *op)
      return (op_encode_hdr_size + 2 + XDR_QUADLEN(rlen)) * sizeof(__be32);

fs/nfsd/nfs4proc.c static inline u32 nfsd4_read_rsize(struct svc_rqst *rqstp, struct nfsd4_op *op) · line 2601

Quoted 10 lines
fs/nfsd/nfs4proc.c +8 -0
 }
 
+static inline u32 nfsd4_read_plus_rsize(struct svc_rqst *rqstp, struct nfsd4_op *op)
+{
+     u32 maxcount = svc_max_payload(rqstp);
+     u32 rlen = min(op->u.read.rd_length, maxcount);
+     /*
+      * Worst case is a single large data segment, so make
+      * sure we have enough room to encode that
+      */

After the last patch we allow an unlimited number of segments. So a zillion 1-byte segments is also possible, and is a worse case.

Quoted 4 lines

Possible ways to avoid that kind of thing:

- when encoding, stop and return a short read if the xdr-encoded result would exceed the limit calculated here.

Doesn't this happen automatically through calls to xdr_reserve_space()?

Quoted 144 lines

- round SEEK_HOLE results up to the nearest (say) 512 bytes, and SEEK_DATA result down to the nearest 512 bytes. May also need some logic to avoid encoding 0-length segments.
- talk to linux-fsdevel, see if we can get a minimum hole alignment guarantee from filesystems.

I'm thinking #1 is simplest.

--b.

Quoted 133 lines
fs/nfsd/nfs4proc.c +12 -0
+     u32 seg_len = 1 + 2 + 1;
+
+     return (op_encode_hdr_size + 2 + seg_len + XDR_QUADLEN(rlen)) * sizeof(__be32);
+}
+
 static inline u32 nfsd4_readdir_rsize(struct svc_rqst *rqstp, struct nfsd4_op *op)
 {
      u32 maxcount = 0, rlen = 0;
@@ -3163,6 +3176,13 @@ static const struct nfsd4_operation nfsd4_ops[] = {
              .op_name = "OP_COPY",
              .op_rsize_bop = nfsd4_copy_rsize,
      },
+     [OP_READ_PLUS] = {
+             .op_func = nfsd4_read,
+             .op_release = nfsd4_read_release,
+             .op_name = "OP_READ_PLUS",
+             .op_rsize_bop = nfsd4_read_plus_rsize,
+             .op_get_currentstateid = nfsd4_get_readstateid,
+     },
      [OP_SEEK] = {
              .op_func = nfsd4_seek,
              .op_name = "OP_SEEK",
fs/nfsd/nfs4xdr.c +81 -3
index 0be194de4888..26d12e3edf33 100644
@@ -2173,7 +2173,7 @@ static const nfsd4_dec nfsd4_dec_ops[] = {
      [OP_LAYOUTSTATS]        = (nfsd4_dec)nfsd4_decode_notsupp,
      [OP_OFFLOAD_CANCEL]     = (nfsd4_dec)nfsd4_decode_offload_status,
      [OP_OFFLOAD_STATUS]     = (nfsd4_dec)nfsd4_decode_offload_status,
-     [OP_READ_PLUS]          = (nfsd4_dec)nfsd4_decode_notsupp,
+     [OP_READ_PLUS]          = (nfsd4_dec)nfsd4_decode_read,
      [OP_SEEK]               = (nfsd4_dec)nfsd4_decode_seek,
      [OP_WRITE_SAME]         = (nfsd4_dec)nfsd4_decode_notsupp,
      [OP_CLONE]              = (nfsd4_dec)nfsd4_decode_clone,
@@ -4597,6 +4597,85 @@ nfsd4_encode_offload_status(struct nfsd4_compoundres *resp, __be32 nfserr,
              return nfserr_resource;
      p = xdr_encode_hyper(p, os->count);
      *p++ = cpu_to_be32(0);
+     return nfserr;
+}
+
+static __be32
+nfsd4_encode_read_plus_data(struct nfsd4_compoundres *resp,
+                         struct nfsd4_read *read,
+                         unsigned long maxcount,  u32 *eof)
+{
+     struct xdr_stream *xdr = &resp->xdr;
+     struct file *file = read->rd_nf->nf_file;
+     int starting_len = xdr->buf->len;
+     __be32 nfserr;
+     __be32 *p, tmp;
+     __be64 tmp64;
+
+     /* Content type, offset, byte count */
+     p = xdr_reserve_space(xdr, 4 + 8 + 4);
+     if (!p)
+             return nfserr_resource;
+
+     read->rd_vlen = xdr_reserve_space_vec(xdr, resp->rqstp->rq_vec, maxcount);
+     if (read->rd_vlen < 0)
+             return nfserr_resource;
+
+     nfserr = nfsd_readv(resp->rqstp, read->rd_fhp, file, read->rd_offset,
+                         resp->rqstp->rq_vec, read->rd_vlen, &maxcount, eof);
+     if (nfserr)
+             return nfserr;
+
+     tmp = htonl(NFS4_CONTENT_DATA);
+     write_bytes_to_xdr_buf(xdr->buf, starting_len,      &tmp,   4);
+     tmp64 = cpu_to_be64(read->rd_offset);
+     write_bytes_to_xdr_buf(xdr->buf, starting_len + 4,  &tmp64, 8);
+     tmp = htonl(maxcount);
+     write_bytes_to_xdr_buf(xdr->buf, starting_len + 12, &tmp,   4);
+     return nfs_ok;
+}
+
+static __be32
+nfsd4_encode_read_plus(struct nfsd4_compoundres *resp, __be32 nfserr,
+                    struct nfsd4_read *read)
+{
+     unsigned long maxcount;
+     struct xdr_stream *xdr = &resp->xdr;
+     struct file *file;
+     int starting_len = xdr->buf->len;
+     int segments = 0;
+     __be32 *p, tmp;
+     u32 eof;
+
+     if (nfserr)
+             return nfserr;
+     file = read->rd_nf->nf_file;
+
+     /* eof flag, segment count */
+     p = xdr_reserve_space(xdr, 4 + 4);
+     if (!p)
+             return nfserr_resource;
+     xdr_commit_encode(xdr);
+
+     maxcount = svc_max_payload(resp->rqstp);
+     maxcount = min_t(unsigned long, maxcount,
+                      (xdr->buf->buflen - xdr->buf->len));
+     maxcount = min_t(unsigned long, maxcount, read->rd_length);
+
+     eof = read->rd_offset >= i_size_read(file_inode(file));
+     if (!eof) {
+             nfserr = nfsd4_encode_read_plus_data(resp, read, maxcount, &eof);
+             segments++;
+     }
+
+     if (nfserr)
+             xdr_truncate_encode(xdr, starting_len);
+     else {
+             tmp = htonl(eof);
+             write_bytes_to_xdr_buf(xdr->buf, starting_len,     &tmp, 4);
+             tmp = htonl(segments);
+             write_bytes_to_xdr_buf(xdr->buf, starting_len + 4, &tmp, 4);
+     }
 
      return nfserr;
 }
@@ -4974,7 +5053,7 @@ static const nfsd4_enc nfsd4_enc_ops[] = {
      [OP_LAYOUTSTATS]        = (nfsd4_enc)nfsd4_encode_noop,
      [OP_OFFLOAD_CANCEL]     = (nfsd4_enc)nfsd4_encode_noop,
      [OP_OFFLOAD_STATUS]     = (nfsd4_enc)nfsd4_encode_offload_status,
-     [OP_READ_PLUS]          = (nfsd4_enc)nfsd4_encode_noop,
+     [OP_READ_PLUS]          = (nfsd4_enc)nfsd4_encode_read_plus,
      [OP_SEEK]               = (nfsd4_enc)nfsd4_encode_seek,
      [OP_WRITE_SAME]         = (nfsd4_enc)nfsd4_encode_noop,
      [OP_CLONE]              = (nfsd4_enc)nfsd4_encode_noop,
--

2.28.0

#5 lore raw

Re: [PATCH v5 2/5] NFSD: Add READ_PLUS data support

Quoted 38 lines · Anna Schumaker
Quoted 37 lines · J. Bruce Fields
Quoted 21 lines · On Tue, Sep 08, 2020 at 12:25:56PM -0400, [email protected] wrote:

From: Anna Schumaker <[email protected]>

This patch adds READ_PLUS support for returning a single NFS4_CONTENT_DATA segment to the client. This is basically the same as the READ operation, only with the extra information about data segments.

---
v5: Fix up nfsd4_read_plus_rsize() calculation

---
 fs/nfsd/nfs4proc.c | 20 +++++++++++
 fs/nfsd/nfs4xdr.c  | 83 ++++++++++++++++++++++++++++++++++++++++++++--
 2 files changed, 101 insertions(+), 2 deletions(-)
fs/nfsd/nfs4proc.c +0 -0
index eaf50eafa935..0a3df5f10501 100644
@@ -2591,6 +2591,19 @@ static inline u32 nfsd4_read_rsize(struct svc_rqst *rqstp, struct nfsd4_op *op)
      return (op_encode_hdr_size + 2 + XDR_QUADLEN(rlen)) * sizeof(__be32);

fs/nfsd/nfs4proc.c static inline u32 nfsd4_read_rsize(struct svc_rqst *rqstp, struct nfsd4_op *op) · line 2601

Quoted 10 lines
fs/nfsd/nfs4proc.c +8 -0
 }
 
+static inline u32 nfsd4_read_plus_rsize(struct svc_rqst *rqstp, struct nfsd4_op *op)
+{
+     u32 maxcount = svc_max_payload(rqstp);
+     u32 rlen = min(op->u.read.rd_length, maxcount);
+     /*
+      * Worst case is a single large data segment, so make
+      * sure we have enough room to encode that
+      */

After the last patch we allow an unlimited number of segments. So a zillion 1-byte segments is also possible, and is a worse case.

Quoted 6 lines
Quoted 4 lines

Possible ways to avoid that kind of thing:

- when encoding, stop and return a short read if the xdr-encoded result would exceed the limit calculated here.

Doesn't this happen automatically through calls to xdr_reserve_space()?

No, xdr_reserve_space() will keep us from running out of buffer completely, but it won't check that ops come in under the estimates made in read_plus_rsize().

--b.

Quoted 145 lines
Quoted 145 lines

- round SEEK_HOLE results up to the nearest (say) 512 bytes, and SEEK_DATA result down to the nearest 512 bytes. May also need some logic to avoid encoding 0-length segments.
- talk to linux-fsdevel, see if we can get a minimum hole alignment guarantee from filesystems.

I'm thinking #1 is simplest.

--b.

Quoted 133 lines
fs/nfsd/nfs4proc.c +12 -0
+     u32 seg_len = 1 + 2 + 1;
+
+     return (op_encode_hdr_size + 2 + seg_len + XDR_QUADLEN(rlen)) * sizeof(__be32);
+}
+
 static inline u32 nfsd4_readdir_rsize(struct svc_rqst *rqstp, struct nfsd4_op *op)
 {
      u32 maxcount = 0, rlen = 0;
@@ -3163,6 +3176,13 @@ static const struct nfsd4_operation nfsd4_ops[] = {
              .op_name = "OP_COPY",
              .op_rsize_bop = nfsd4_copy_rsize,
      },
+     [OP_READ_PLUS] = {
+             .op_func = nfsd4_read,
+             .op_release = nfsd4_read_release,
+             .op_name = "OP_READ_PLUS",
+             .op_rsize_bop = nfsd4_read_plus_rsize,
+             .op_get_currentstateid = nfsd4_get_readstateid,
+     },
      [OP_SEEK] = {
              .op_func = nfsd4_seek,
              .op_name = "OP_SEEK",
fs/nfsd/nfs4xdr.c +81 -3
index 0be194de4888..26d12e3edf33 100644
@@ -2173,7 +2173,7 @@ static const nfsd4_dec nfsd4_dec_ops[] = {
      [OP_LAYOUTSTATS]        = (nfsd4_dec)nfsd4_decode_notsupp,
      [OP_OFFLOAD_CANCEL]     = (nfsd4_dec)nfsd4_decode_offload_status,
      [OP_OFFLOAD_STATUS]     = (nfsd4_dec)nfsd4_decode_offload_status,
-     [OP_READ_PLUS]          = (nfsd4_dec)nfsd4_decode_notsupp,
+     [OP_READ_PLUS]          = (nfsd4_dec)nfsd4_decode_read,
      [OP_SEEK]               = (nfsd4_dec)nfsd4_decode_seek,
      [OP_WRITE_SAME]         = (nfsd4_dec)nfsd4_decode_notsupp,
      [OP_CLONE]              = (nfsd4_dec)nfsd4_decode_clone,
@@ -4597,6 +4597,85 @@ nfsd4_encode_offload_status(struct nfsd4_compoundres *resp, __be32 nfserr,
              return nfserr_resource;
      p = xdr_encode_hyper(p, os->count);
      *p++ = cpu_to_be32(0);
+     return nfserr;
+}
+
+static __be32
+nfsd4_encode_read_plus_data(struct nfsd4_compoundres *resp,
+                         struct nfsd4_read *read,
+                         unsigned long maxcount,  u32 *eof)
+{
+     struct xdr_stream *xdr = &resp->xdr;
+     struct file *file = read->rd_nf->nf_file;
+     int starting_len = xdr->buf->len;
+     __be32 nfserr;
+     __be32 *p, tmp;
+     __be64 tmp64;
+
+     /* Content type, offset, byte count */
+     p = xdr_reserve_space(xdr, 4 + 8 + 4);
+     if (!p)
+             return nfserr_resource;
+
+     read->rd_vlen = xdr_reserve_space_vec(xdr, resp->rqstp->rq_vec, maxcount);
+     if (read->rd_vlen < 0)
+             return nfserr_resource;
+
+     nfserr = nfsd_readv(resp->rqstp, read->rd_fhp, file, read->rd_offset,
+                         resp->rqstp->rq_vec, read->rd_vlen, &maxcount, eof);
+     if (nfserr)
+             return nfserr;
+
+     tmp = htonl(NFS4_CONTENT_DATA);
+     write_bytes_to_xdr_buf(xdr->buf, starting_len,      &tmp,   4);
+     tmp64 = cpu_to_be64(read->rd_offset);
+     write_bytes_to_xdr_buf(xdr->buf, starting_len + 4,  &tmp64, 8);
+     tmp = htonl(maxcount);
+     write_bytes_to_xdr_buf(xdr->buf, starting_len + 12, &tmp,   4);
+     return nfs_ok;
+}
+
+static __be32
+nfsd4_encode_read_plus(struct nfsd4_compoundres *resp, __be32 nfserr,
+                    struct nfsd4_read *read)
+{
+     unsigned long maxcount;
+     struct xdr_stream *xdr = &resp->xdr;
+     struct file *file;
+     int starting_len = xdr->buf->len;
+     int segments = 0;
+     __be32 *p, tmp;
+     u32 eof;
+
+     if (nfserr)
+             return nfserr;
+     file = read->rd_nf->nf_file;
+
+     /* eof flag, segment count */
+     p = xdr_reserve_space(xdr, 4 + 4);
+     if (!p)
+             return nfserr_resource;
+     xdr_commit_encode(xdr);
+
+     maxcount = svc_max_payload(resp->rqstp);
+     maxcount = min_t(unsigned long, maxcount,
+                      (xdr->buf->buflen - xdr->buf->len));
+     maxcount = min_t(unsigned long, maxcount, read->rd_length);
+
+     eof = read->rd_offset >= i_size_read(file_inode(file));
+     if (!eof) {
+             nfserr = nfsd4_encode_read_plus_data(resp, read, maxcount, &eof);
+             segments++;
+     }
+
+     if (nfserr)
+             xdr_truncate_encode(xdr, starting_len);
+     else {
+             tmp = htonl(eof);
+             write_bytes_to_xdr_buf(xdr->buf, starting_len,     &tmp, 4);
+             tmp = htonl(segments);
+             write_bytes_to_xdr_buf(xdr->buf, starting_len + 4, &tmp, 4);
+     }
 
      return nfserr;
 }
@@ -4974,7 +5053,7 @@ static const nfsd4_enc nfsd4_enc_ops[] = {
      [OP_LAYOUTSTATS]        = (nfsd4_enc)nfsd4_encode_noop,
      [OP_OFFLOAD_CANCEL]     = (nfsd4_enc)nfsd4_encode_noop,
      [OP_OFFLOAD_STATUS]     = (nfsd4_enc)nfsd4_encode_offload_status,
-     [OP_READ_PLUS]          = (nfsd4_enc)nfsd4_encode_noop,
+     [OP_READ_PLUS]          = (nfsd4_enc)nfsd4_encode_read_plus,
      [OP_SEEK]               = (nfsd4_enc)nfsd4_encode_seek,
      [OP_WRITE_SAME]         = (nfsd4_enc)nfsd4_encode_noop,
      [OP_CLONE]              = (nfsd4_enc)nfsd4_encode_noop,
--

2.28.0

#6 lore raw

Re: [PATCH v5 2/5] NFSD: Add READ_PLUS data support

Quoted 41 lines · bfields
Quoted 40 lines · Anna Schumaker
Quoted 39 lines · J. Bruce Fields
Quoted 21 lines · On Tue, Sep 08, 2020 at 12:25:56PM -0400, [email protected] wrote:

From: Anna Schumaker <[email protected]>

This patch adds READ_PLUS support for returning a single NFS4_CONTENT_DATA segment to the client. This is basically the same as the READ operation, only with the extra information about data segments.

---
v5: Fix up nfsd4_read_plus_rsize() calculation

---
 fs/nfsd/nfs4proc.c | 20 +++++++++++
 fs/nfsd/nfs4xdr.c  | 83 ++++++++++++++++++++++++++++++++++++++++++++--
 2 files changed, 101 insertions(+), 2 deletions(-)
fs/nfsd/nfs4proc.c +0 -0
index eaf50eafa935..0a3df5f10501 100644
@@ -2591,6 +2591,19 @@ static inline u32 nfsd4_read_rsize(struct svc_rqst *rqstp, struct nfsd4_op *op)
      return (op_encode_hdr_size + 2 + XDR_QUADLEN(rlen)) * sizeof(__be32);

fs/nfsd/nfs4proc.c static inline u32 nfsd4_read_rsize(struct svc_rqst *rqstp, struct nfsd4_op *op) · line 2601

Quoted 10 lines
fs/nfsd/nfs4proc.c +8 -0
 }
 
+static inline u32 nfsd4_read_plus_rsize(struct svc_rqst *rqstp, struct nfsd4_op *op)
+{
+     u32 maxcount = svc_max_payload(rqstp);
+     u32 rlen = min(op->u.read.rd_length, maxcount);
+     /*
+      * Worst case is a single large data segment, so make
+      * sure we have enough room to encode that
+      */

After the last patch we allow an unlimited number of segments. So a zillion 1-byte segments is also possible, and is a worse case.

Possible ways to avoid that kind of thing:

Quoted 8 lines
Quoted 4 lines
Quoted 2 lines

- when encoding, stop and return a short read if the xdr-encoded result would exceed the limit calculated here.

Doesn't this happen automatically through calls to xdr_reserve_space()?

No, xdr_reserve_space() will keep us from running out of buffer completely, but it won't check that ops come in under the estimates made in read_plus_rsize().

If it's easier, another option might be just: if we ever get a "small" hole (say, less than 512 bytes), just give up and encode the rest of the result as a single big data segment.

--b.

#7 lore raw

PATCH v5 3/5 NFSD: Add READ_PLUS hole segment encoding

In reply to [email protected]

From: Anna Schumaker <[email protected]>

However, we still only reply to the READ_PLUS call with a single segment at this time.

---
 fs/nfsd/nfs4xdr.c | 40 +++++++++++++++++++++++++++++++++++++++-
 1 file changed, 39 insertions(+), 1 deletion(-)
fs/nfsd/nfs4xdr.c +39 -1
index 26d12e3edf33..45159bd9e9a4 100644
@@ -4608,10 +4608,14 @@ nfsd4_encode_read_plus_data(struct nfsd4_compoundres *resp,
 	struct xdr_stream *xdr = &resp->xdr;
 	struct file *file = read->rd_nf->nf_file;
 	int starting_len = xdr->buf->len;
+	loff_t hole_pos = vfs_llseek(file, read->rd_offset, SEEK_HOLE);
 	__be32 nfserr;
 	__be32 *p, tmp;
 	__be64 tmp64;
 
+	if (hole_pos > read->rd_offset)
+		maxcount = min_t(unsigned long, maxcount, hole_pos - read->rd_offset);
+
 	/* Content type, offset, byte count */
 	p = xdr_reserve_space(xdr, 4 + 8 + 4);
 	if (!p)
@@ -4635,6 +4639,27 @@ nfsd4_encode_read_plus_data(struct nfsd4_compoundres *resp,
 	return nfs_ok;
 }
 
+static __be32
+nfsd4_encode_read_plus_hole(struct nfsd4_compoundres *resp,
+			    struct nfsd4_read *read,
+			    unsigned long maxcount, u32 *eof)
+{
+	struct file *file = read->rd_nf->nf_file;
+	__be32 *p;
+
+	/* Content type, offset, byte count */
+	p = xdr_reserve_space(&resp->xdr, 4 + 8 + 8);
+	if (!p)
+		return nfserr_resource;
+
+	*p++ = htonl(NFS4_CONTENT_HOLE);
+	 p   = xdr_encode_hyper(p, read->rd_offset);
+	 p   = xdr_encode_hyper(p, maxcount);
+
+	*eof = (read->rd_offset + maxcount) >= i_size_read(file_inode(file));
+	return nfs_ok;
+}
+
 static __be32
 nfsd4_encode_read_plus(struct nfsd4_compoundres *resp, __be32 nfserr,
 		       struct nfsd4_read *read)
@@ -4645,6 +4670,7 @@ nfsd4_encode_read_plus(struct nfsd4_compoundres *resp, __be32 nfserr,
 	int starting_len = xdr->buf->len;
 	int segments = 0;
 	__be32 *p, tmp;
+	loff_t pos;
 	u32 eof;
 
 	if (nfserr)
@@ -4663,11 +4689,23 @@ nfsd4_encode_read_plus(struct nfsd4_compoundres *resp, __be32 nfserr,
 	maxcount = min_t(unsigned long, maxcount, read->rd_length);
 
 	eof = read->rd_offset >= i_size_read(file_inode(file));
-	if (!eof) {
+	if (eof)
+		goto out;
+
+	pos = vfs_llseek(file, read->rd_offset, SEEK_DATA);
+	if (pos == -ENXIO)
+		pos = i_size_read(file_inode(file));
+
+	if (pos > read->rd_offset) {
+		maxcount = pos - read->rd_offset;
+		nfserr = nfsd4_encode_read_plus_hole(resp, read, maxcount, &eof);
+		segments++;
+	} else {
 		nfserr = nfsd4_encode_read_plus_data(resp, read, maxcount, &eof);
 		segments++;
 	}
 
+out:
 	if (nfserr)
 		xdr_truncate_encode(xdr, starting_len);
 	else {
Signature · 1 line
2.28.0
#8 lore raw

PATCH v5 4/5 NFSD: Return both a hole and a data segment

In reply to [email protected]

From: Anna Schumaker <[email protected]>

But only one of each right now. We'll expand on this in the next patch.

---
v5: If we've already encoded a segment, then return a short read if
later segments return an error for some reason.

---
 fs/nfsd/nfs4xdr.c | 54 ++++++++++++++++++++++++++++++++++-------------
 1 file changed, 39 insertions(+), 15 deletions(-)
fs/nfsd/nfs4xdr.c +39 -15
index 45159bd9e9a4..856606263c1d 100644
@@ -4603,7 +4603,7 @@ nfsd4_encode_offload_status(struct nfsd4_compoundres *resp, __be32 nfserr,
 static __be32
 nfsd4_encode_read_plus_data(struct nfsd4_compoundres *resp,
 			    struct nfsd4_read *read,
-			    unsigned long maxcount,  u32 *eof)
+			    unsigned long *maxcount, u32 *eof)
 {
 	struct xdr_stream *xdr = &resp->xdr;
 	struct file *file = read->rd_nf->nf_file;
@@ -4614,19 +4614,19 @@ nfsd4_encode_read_plus_data(struct nfsd4_compoundres *resp,
 	__be64 tmp64;
 
 	if (hole_pos > read->rd_offset)
-		maxcount = min_t(unsigned long, maxcount, hole_pos - read->rd_offset);
+		*maxcount = min_t(unsigned long, *maxcount, hole_pos - read->rd_offset);
 
 	/* Content type, offset, byte count */
 	p = xdr_reserve_space(xdr, 4 + 8 + 4);
 	if (!p)
 		return nfserr_resource;
 
-	read->rd_vlen = xdr_reserve_space_vec(xdr, resp->rqstp->rq_vec, maxcount);
+	read->rd_vlen = xdr_reserve_space_vec(xdr, resp->rqstp->rq_vec, *maxcount);
 	if (read->rd_vlen < 0)
 		return nfserr_resource;
 
 	nfserr = nfsd_readv(resp->rqstp, read->rd_fhp, file, read->rd_offset,
-			    resp->rqstp->rq_vec, read->rd_vlen, &maxcount, eof);
+			    resp->rqstp->rq_vec, read->rd_vlen, maxcount, eof);
 	if (nfserr)
 		return nfserr;
 
@@ -4634,7 +4634,7 @@ nfsd4_encode_read_plus_data(struct nfsd4_compoundres *resp,
 	write_bytes_to_xdr_buf(xdr->buf, starting_len,      &tmp,   4);
 	tmp64 = cpu_to_be64(read->rd_offset);
 	write_bytes_to_xdr_buf(xdr->buf, starting_len + 4,  &tmp64, 8);
-	tmp = htonl(maxcount);
+	tmp = htonl(*maxcount);
 	write_bytes_to_xdr_buf(xdr->buf, starting_len + 12, &tmp,   4);
 	return nfs_ok;
 }
@@ -4642,11 +4642,19 @@ nfsd4_encode_read_plus_data(struct nfsd4_compoundres *resp,
 static __be32
 nfsd4_encode_read_plus_hole(struct nfsd4_compoundres *resp,
 			    struct nfsd4_read *read,
-			    unsigned long maxcount, u32 *eof)
+			    unsigned long *maxcount, u32 *eof)
 {
 	struct file *file = read->rd_nf->nf_file;
+	loff_t data_pos = vfs_llseek(file, read->rd_offset, SEEK_DATA);
+	unsigned long count;
 	__be32 *p;
 
+	if (data_pos == -ENXIO)
+		data_pos = i_size_read(file_inode(file));
+	else if (data_pos <= read->rd_offset)
+		return nfserr_resource;
+	count = data_pos - read->rd_offset;
+
 	/* Content type, offset, byte count */
 	p = xdr_reserve_space(&resp->xdr, 4 + 8 + 8);
 	if (!p)
@@ -4654,9 +4662,10 @@ nfsd4_encode_read_plus_hole(struct nfsd4_compoundres *resp,
 
 	*p++ = htonl(NFS4_CONTENT_HOLE);
 	 p   = xdr_encode_hyper(p, read->rd_offset);
-	 p   = xdr_encode_hyper(p, maxcount);
+	 p   = xdr_encode_hyper(p, count);
 
-	*eof = (read->rd_offset + maxcount) >= i_size_read(file_inode(file));
+	*eof = (read->rd_offset + count) >= i_size_read(file_inode(file));
+	*maxcount = min_t(unsigned long, count, *maxcount);
 	return nfs_ok;
 }
 
@@ -4664,7 +4673,7 @@ static __be32
 nfsd4_encode_read_plus(struct nfsd4_compoundres *resp, __be32 nfserr,
 		       struct nfsd4_read *read)
 {
-	unsigned long maxcount;
+	unsigned long maxcount, count;
 	struct xdr_stream *xdr = &resp->xdr;
 	struct file *file;
 	int starting_len = xdr->buf->len;
@@ -4687,6 +4696,7 @@ nfsd4_encode_read_plus(struct nfsd4_compoundres *resp, __be32 nfserr,
 	maxcount = min_t(unsigned long, maxcount,
 			 (xdr->buf->buflen - xdr->buf->len));
 	maxcount = min_t(unsigned long, maxcount, read->rd_length);
+	count    = maxcount;
 
 	eof = read->rd_offset >= i_size_read(file_inode(file));
 	if (eof)
@@ -4695,24 +4705,38 @@ nfsd4_encode_read_plus(struct nfsd4_compoundres *resp, __be32 nfserr,
 	pos = vfs_llseek(file, read->rd_offset, SEEK_DATA);
 	if (pos == -ENXIO)
 		pos = i_size_read(file_inode(file));
+	else if (pos < 0)
+		pos = read->rd_offset;
 
-	if (pos > read->rd_offset) {
-		maxcount = pos - read->rd_offset;
-		nfserr = nfsd4_encode_read_plus_hole(resp, read, maxcount, &eof);
+	if (pos == read->rd_offset) {
+		maxcount = count;
+		nfserr = nfsd4_encode_read_plus_data(resp, read, &maxcount, &eof);
+		if (nfserr)
+			goto out;
+		count -= maxcount;
+		read->rd_offset += maxcount;
 		segments++;
-	} else {
-		nfserr = nfsd4_encode_read_plus_data(resp, read, maxcount, &eof);
+	}
+
+	if (count > 0 && !eof) {
+		maxcount = count;
+		nfserr = nfsd4_encode_read_plus_hole(resp, read, &maxcount, &eof);
+		if (nfserr)
+			goto out;
+		count -= maxcount;
+		read->rd_offset += maxcount;
 		segments++;
 	}
 
 out:
-	if (nfserr)
+	if (nfserr && segments == 0)
 		xdr_truncate_encode(xdr, starting_len);
 	else {
 		tmp = htonl(eof);
 		write_bytes_to_xdr_buf(xdr->buf, starting_len,     &tmp, 4);
 		tmp = htonl(segments);
 		write_bytes_to_xdr_buf(xdr->buf, starting_len + 4, &tmp, 4);
+		nfserr = nfs_ok;
 	}
 
 	return nfserr;
Signature · 1 line
2.28.0
#9 lore raw

Re: [PATCH v5 4/5] NFSD: Return both a hole and a data segment

Quoted 63 lines · On Tue, Sep 08, 2020 at 12:25:58PM -0400, [email protected] wrote:

From: Anna Schumaker <[email protected]>

But only one of each right now. We'll expand on this in the next patch.

---
v5: If we've already encoded a segment, then return a short read if
later segments return an error for some reason.

---
 fs/nfsd/nfs4xdr.c | 54 ++++++++++++++++++++++++++++++++++-------------
 1 file changed, 39 insertions(+), 15 deletions(-)
fs/nfsd/nfs4xdr.c +6 -6
index 45159bd9e9a4..856606263c1d 100644
@@ -4603,7 +4603,7 @@ nfsd4_encode_offload_status(struct nfsd4_compoundres *resp, __be32 nfserr,
 static __be32
 nfsd4_encode_read_plus_data(struct nfsd4_compoundres *resp,
 			    struct nfsd4_read *read,
-			    unsigned long maxcount,  u32 *eof)
+			    unsigned long *maxcount, u32 *eof)
 {
 	struct xdr_stream *xdr = &resp->xdr;
 	struct file *file = read->rd_nf->nf_file;
@@ -4614,19 +4614,19 @@ nfsd4_encode_read_plus_data(struct nfsd4_compoundres *resp,
 	__be64 tmp64;
 
 	if (hole_pos > read->rd_offset)
-		maxcount = min_t(unsigned long, maxcount, hole_pos - read->rd_offset);
+		*maxcount = min_t(unsigned long, *maxcount, hole_pos - read->rd_offset);
 
 	/* Content type, offset, byte count */
 	p = xdr_reserve_space(xdr, 4 + 8 + 4);
 	if (!p)
 		return nfserr_resource;
 
-	read->rd_vlen = xdr_reserve_space_vec(xdr, resp->rqstp->rq_vec, maxcount);
+	read->rd_vlen = xdr_reserve_space_vec(xdr, resp->rqstp->rq_vec, *maxcount);
 	if (read->rd_vlen < 0)
 		return nfserr_resource;
 
 	nfserr = nfsd_readv(resp->rqstp, read->rd_fhp, file, read->rd_offset,
-			    resp->rqstp->rq_vec, read->rd_vlen, &maxcount, eof);
+			    resp->rqstp->rq_vec, read->rd_vlen, maxcount, eof);
 	if (nfserr)
 		return nfserr;
 
@@ -4634,7 +4634,7 @@ nfsd4_encode_read_plus_data(struct nfsd4_compoundres *resp,
 	write_bytes_to_xdr_buf(xdr->buf, starting_len,      &tmp,   4);
 	tmp64 = cpu_to_be64(read->rd_offset);
 	write_bytes_to_xdr_buf(xdr->buf, starting_len + 4,  &tmp64, 8);
-	tmp = htonl(maxcount);
+	tmp = htonl(*maxcount);
 	write_bytes_to_xdr_buf(xdr->buf, starting_len + 12, &tmp,   4);
 	return nfs_ok;
 }
@@ -4642,11 +4642,19 @@ nfsd4_encode_read_plus_data(struct nfsd4_compoundres *resp,
 static __be32
 nfsd4_encode_read_plus_hole(struct nfsd4_compoundres *resp,
 			    struct nfsd4_read *read,
-			    unsigned long maxcount, u32 *eof)
+			    unsigned long *maxcount, u32 *eof)

fs/nfsd/nfs4xdr.c nfsd4_encode_read_plus_data(struct nfsd4_compoundres *resp, · line 4655

Quoted 10 lines
fs/nfsd/nfs4xdr.c +6 -0
 {
 	struct file *file = read->rd_nf->nf_file;
+	loff_t data_pos = vfs_llseek(file, read->rd_offset, SEEK_DATA);
+	unsigned long count;
 	__be32 *p;
 
+	if (data_pos == -ENXIO)
+		data_pos = i_size_read(file_inode(file));
+	else if (data_pos <= read->rd_offset)
+		return nfserr_resource;

I think there's still a race here:

vfs_llseek(.,0,SEEK_HOLE) returns 1024
read 1024 bytes of data
another process fills the hole
vfs_llseek(.,1024,SEEK_DATA) returns 1024
code above returns nfserr_resource

We end up returning an error to the client when we should have just returned more data.

--b.

Quoted 83 lines
fs/nfsd/nfs4xdr.c +27 -9
+	count = data_pos - read->rd_offset;
+
 	/* Content type, offset, byte count */
 	p = xdr_reserve_space(&resp->xdr, 4 + 8 + 8);
 	if (!p)
@@ -4654,9 +4662,10 @@ nfsd4_encode_read_plus_hole(struct nfsd4_compoundres *resp,
 
 	*p++ = htonl(NFS4_CONTENT_HOLE);
 	 p   = xdr_encode_hyper(p, read->rd_offset);
-	 p   = xdr_encode_hyper(p, maxcount);
+	 p   = xdr_encode_hyper(p, count);
 
-	*eof = (read->rd_offset + maxcount) >= i_size_read(file_inode(file));
+	*eof = (read->rd_offset + count) >= i_size_read(file_inode(file));
+	*maxcount = min_t(unsigned long, count, *maxcount);
 	return nfs_ok;
 }
 
@@ -4664,7 +4673,7 @@ static __be32
 nfsd4_encode_read_plus(struct nfsd4_compoundres *resp, __be32 nfserr,
 		       struct nfsd4_read *read)
 {
-	unsigned long maxcount;
+	unsigned long maxcount, count;
 	struct xdr_stream *xdr = &resp->xdr;
 	struct file *file;
 	int starting_len = xdr->buf->len;
@@ -4687,6 +4696,7 @@ nfsd4_encode_read_plus(struct nfsd4_compoundres *resp, __be32 nfserr,
 	maxcount = min_t(unsigned long, maxcount,
 			 (xdr->buf->buflen - xdr->buf->len));
 	maxcount = min_t(unsigned long, maxcount, read->rd_length);
+	count    = maxcount;
 
 	eof = read->rd_offset >= i_size_read(file_inode(file));
 	if (eof)
@@ -4695,24 +4705,38 @@ nfsd4_encode_read_plus(struct nfsd4_compoundres *resp, __be32 nfserr,
 	pos = vfs_llseek(file, read->rd_offset, SEEK_DATA);
 	if (pos == -ENXIO)
 		pos = i_size_read(file_inode(file));
+	else if (pos < 0)
+		pos = read->rd_offset;
 
-	if (pos > read->rd_offset) {
-		maxcount = pos - read->rd_offset;
-		nfserr = nfsd4_encode_read_plus_hole(resp, read, maxcount, &eof);
+	if (pos == read->rd_offset) {
+		maxcount = count;
+		nfserr = nfsd4_encode_read_plus_data(resp, read, &maxcount, &eof);
+		if (nfserr)
+			goto out;
+		count -= maxcount;
+		read->rd_offset += maxcount;
 		segments++;
-	} else {
-		nfserr = nfsd4_encode_read_plus_data(resp, read, maxcount, &eof);
+	}
+
+	if (count > 0 && !eof) {
+		maxcount = count;
+		nfserr = nfsd4_encode_read_plus_hole(resp, read, &maxcount, &eof);
+		if (nfserr)
+			goto out;
+		count -= maxcount;
+		read->rd_offset += maxcount;
 		segments++;
 	}
 
 out:
-	if (nfserr)
+	if (nfserr && segments == 0)
 		xdr_truncate_encode(xdr, starting_len);
 	else {
 		tmp = htonl(eof);
 		write_bytes_to_xdr_buf(xdr->buf, starting_len,     &tmp, 4);
 		tmp = htonl(segments);
 		write_bytes_to_xdr_buf(xdr->buf, starting_len + 4, &tmp, 4);
+		nfserr = nfs_ok;
 	}
 
 	return nfserr;
Signature · 1 line
2.28.0
#10 lore raw

Re: [PATCH v5 4/5] NFSD: Return both a hole and a data segment

Quoted 78 lines · J. Bruce Fields
Quoted 63 lines · On Tue, Sep 08, 2020 at 12:25:58PM -0400, [email protected] wrote:

From: Anna Schumaker <[email protected]>

But only one of each right now. We'll expand on this in the next patch.

---
v5: If we've already encoded a segment, then return a short read if
later segments return an error for some reason.

---
 fs/nfsd/nfs4xdr.c | 54 ++++++++++++++++++++++++++++++++++-------------
 1 file changed, 39 insertions(+), 15 deletions(-)
fs/nfsd/nfs4xdr.c +6 -6
index 45159bd9e9a4..856606263c1d 100644
@@ -4603,7 +4603,7 @@ nfsd4_encode_offload_status(struct nfsd4_compoundres *resp, __be32 nfserr,
 static __be32
 nfsd4_encode_read_plus_data(struct nfsd4_compoundres *resp,
                          struct nfsd4_read *read,
-                         unsigned long maxcount,  u32 *eof)
+                         unsigned long *maxcount, u32 *eof)
 {
      struct xdr_stream *xdr = &resp->xdr;
      struct file *file = read->rd_nf->nf_file;
@@ -4614,19 +4614,19 @@ nfsd4_encode_read_plus_data(struct nfsd4_compoundres *resp,
      __be64 tmp64;
 
      if (hole_pos > read->rd_offset)
-             maxcount = min_t(unsigned long, maxcount, hole_pos - read->rd_offset);
+             *maxcount = min_t(unsigned long, *maxcount, hole_pos - read->rd_offset);
 
      /* Content type, offset, byte count */
      p = xdr_reserve_space(xdr, 4 + 8 + 4);
      if (!p)
              return nfserr_resource;
 
-     read->rd_vlen = xdr_reserve_space_vec(xdr, resp->rqstp->rq_vec, maxcount);
+     read->rd_vlen = xdr_reserve_space_vec(xdr, resp->rqstp->rq_vec, *maxcount);
      if (read->rd_vlen < 0)
              return nfserr_resource;
 
      nfserr = nfsd_readv(resp->rqstp, read->rd_fhp, file, read->rd_offset,
-                         resp->rqstp->rq_vec, read->rd_vlen, &maxcount, eof);
+                         resp->rqstp->rq_vec, read->rd_vlen, maxcount, eof);
      if (nfserr)
              return nfserr;
 
@@ -4634,7 +4634,7 @@ nfsd4_encode_read_plus_data(struct nfsd4_compoundres *resp,
      write_bytes_to_xdr_buf(xdr->buf, starting_len,      &tmp,   4);
      tmp64 = cpu_to_be64(read->rd_offset);
      write_bytes_to_xdr_buf(xdr->buf, starting_len + 4,  &tmp64, 8);
-     tmp = htonl(maxcount);
+     tmp = htonl(*maxcount);
      write_bytes_to_xdr_buf(xdr->buf, starting_len + 12, &tmp,   4);
      return nfs_ok;
 }
@@ -4642,11 +4642,19 @@ nfsd4_encode_read_plus_data(struct nfsd4_compoundres *resp,
 static __be32
 nfsd4_encode_read_plus_hole(struct nfsd4_compoundres *resp,
                          struct nfsd4_read *read,
-                         unsigned long maxcount, u32 *eof)
+                         unsigned long *maxcount, u32 *eof)

fs/nfsd/nfs4xdr.c nfsd4_encode_read_plus_data(struct nfsd4_compoundres *resp, · line 4655

Quoted 10 lines
fs/nfsd/nfs4xdr.c +6 -0
 {
      struct file *file = read->rd_nf->nf_file;
+     loff_t data_pos = vfs_llseek(file, read->rd_offset, SEEK_DATA);
+     unsigned long count;
      __be32 *p;
 
+     if (data_pos == -ENXIO)
+             data_pos = i_size_read(file_inode(file));
+     else if (data_pos <= read->rd_offset)
+             return nfserr_resource;

I think there's still a race here:

Quoted 8 lines

vfs_llseek(.,0,SEEK_HOLE) returns 1024
read 1024 bytes of data
another process fills the hole
vfs_llseek(.,1024,SEEK_DATA) returns 1024
code above returns nfserr_resource

We end up returning an error to the client when we should have just returned more data.

As long as we've encoded at least one segment successfully, we'll actually return a short read in this case (as of the most recent patches). I tried implementing a check for what each segment actually was before encoding, but it lead to a lot of extra lseeks (so potential for races / performance problems on btrfs). Returning a short read seemed like a better approach to me.

Anna

Quoted 87 lines

--b.

Quoted 83 lines
fs/nfsd/nfs4xdr.c +27 -10
+     count = data_pos - read->rd_offset;
+
      /* Content type, offset, byte count */
      p = xdr_reserve_space(&resp->xdr, 4 + 8 + 8);
      if (!p)
@@ -4654,9 +4662,10 @@ nfsd4_encode_read_plus_hole(struct nfsd4_compoundres *resp,
 
      *p++ = htonl(NFS4_CONTENT_HOLE);
       p   = xdr_encode_hyper(p, read->rd_offset);
-      p   = xdr_encode_hyper(p, maxcount);
+      p   = xdr_encode_hyper(p, count);
 
-     *eof = (read->rd_offset + maxcount) >= i_size_read(file_inode(file));
+     *eof = (read->rd_offset + count) >= i_size_read(file_inode(file));
+     *maxcount = min_t(unsigned long, count, *maxcount);
      return nfs_ok;
 }
 
@@ -4664,7 +4673,7 @@ static __be32
 nfsd4_encode_read_plus(struct nfsd4_compoundres *resp, __be32 nfserr,
                     struct nfsd4_read *read)
 {
-     unsigned long maxcount;
+     unsigned long maxcount, count;
      struct xdr_stream *xdr = &resp->xdr;
      struct file *file;
      int starting_len = xdr->buf->len;
@@ -4687,6 +4696,7 @@ nfsd4_encode_read_plus(struct nfsd4_compoundres *resp, __be32 nfserr,
      maxcount = min_t(unsigned long, maxcount,
                       (xdr->buf->buflen - xdr->buf->len));
      maxcount = min_t(unsigned long, maxcount, read->rd_length);
+     count    = maxcount;
 
      eof = read->rd_offset >= i_size_read(file_inode(file));
      if (eof)
@@ -4695,24 +4705,38 @@ nfsd4_encode_read_plus(struct nfsd4_compoundres *resp, __be32 nfserr,
      pos = vfs_llseek(file, read->rd_offset, SEEK_DATA);
      if (pos == -ENXIO)
              pos = i_size_read(file_inode(file));
+     else if (pos < 0)
+             pos = read->rd_offset;
 
-     if (pos > read->rd_offset) {
-             maxcount = pos - read->rd_offset;
-             nfserr = nfsd4_encode_read_plus_hole(resp, read, maxcount, &eof);
+     if (pos == read->rd_offset) {
+             maxcount = count;
+             nfserr = nfsd4_encode_read_plus_data(resp, read, &maxcount, &eof);
+             if (nfserr)
+                     goto out;
+             count -= maxcount;
+             read->rd_offset += maxcount;
              segments++;
-     } else {
-             nfserr = nfsd4_encode_read_plus_data(resp, read, maxcount, &eof);
+     }
+
+     if (count > 0 && !eof) {
+             maxcount = count;
+             nfserr = nfsd4_encode_read_plus_hole(resp, read, &maxcount, &eof);
+             if (nfserr)
+                     goto out;
+             count -= maxcount;
+             read->rd_offset += maxcount;
              segments++;
      }
 
 out:
-     if (nfserr)
+     if (nfserr && segments == 0)
              xdr_truncate_encode(xdr, starting_len);
      else {
              tmp = htonl(eof);
              write_bytes_to_xdr_buf(xdr->buf, starting_len,     &tmp, 4);
              tmp = htonl(segments);
              write_bytes_to_xdr_buf(xdr->buf, starting_len + 4, &tmp, 4);
+             nfserr = nfs_ok;
      }
 
      return nfserr;
--

2.28.0

#11 lore raw

Re: [PATCH v5 4/5] NFSD: Return both a hole and a data segment

Quoted 88 lines · Anna Schumaker
Quoted 78 lines · J. Bruce Fields
Quoted 63 lines · On Tue, Sep 08, 2020 at 12:25:58PM -0400, [email protected] wrote:

From: Anna Schumaker <[email protected]>

But only one of each right now. We'll expand on this in the next patch.

---
v5: If we've already encoded a segment, then return a short read if
later segments return an error for some reason.

---
 fs/nfsd/nfs4xdr.c | 54 ++++++++++++++++++++++++++++++++++-------------
 1 file changed, 39 insertions(+), 15 deletions(-)
fs/nfsd/nfs4xdr.c +6 -6
index 45159bd9e9a4..856606263c1d 100644
@@ -4603,7 +4603,7 @@ nfsd4_encode_offload_status(struct nfsd4_compoundres *resp, __be32 nfserr,
 static __be32
 nfsd4_encode_read_plus_data(struct nfsd4_compoundres *resp,
                          struct nfsd4_read *read,
-                         unsigned long maxcount,  u32 *eof)
+                         unsigned long *maxcount, u32 *eof)
 {
      struct xdr_stream *xdr = &resp->xdr;
      struct file *file = read->rd_nf->nf_file;
@@ -4614,19 +4614,19 @@ nfsd4_encode_read_plus_data(struct nfsd4_compoundres *resp,
      __be64 tmp64;
 
      if (hole_pos > read->rd_offset)
-             maxcount = min_t(unsigned long, maxcount, hole_pos - read->rd_offset);
+             *maxcount = min_t(unsigned long, *maxcount, hole_pos - read->rd_offset);
 
      /* Content type, offset, byte count */
      p = xdr_reserve_space(xdr, 4 + 8 + 4);
      if (!p)
              return nfserr_resource;
 
-     read->rd_vlen = xdr_reserve_space_vec(xdr, resp->rqstp->rq_vec, maxcount);
+     read->rd_vlen = xdr_reserve_space_vec(xdr, resp->rqstp->rq_vec, *maxcount);
      if (read->rd_vlen < 0)
              return nfserr_resource;
 
      nfserr = nfsd_readv(resp->rqstp, read->rd_fhp, file, read->rd_offset,
-                         resp->rqstp->rq_vec, read->rd_vlen, &maxcount, eof);
+                         resp->rqstp->rq_vec, read->rd_vlen, maxcount, eof);
      if (nfserr)
              return nfserr;
 
@@ -4634,7 +4634,7 @@ nfsd4_encode_read_plus_data(struct nfsd4_compoundres *resp,
      write_bytes_to_xdr_buf(xdr->buf, starting_len,      &tmp,   4);
      tmp64 = cpu_to_be64(read->rd_offset);
      write_bytes_to_xdr_buf(xdr->buf, starting_len + 4,  &tmp64, 8);
-     tmp = htonl(maxcount);
+     tmp = htonl(*maxcount);
      write_bytes_to_xdr_buf(xdr->buf, starting_len + 12, &tmp,   4);
      return nfs_ok;
 }
@@ -4642,11 +4642,19 @@ nfsd4_encode_read_plus_data(struct nfsd4_compoundres *resp,
 static __be32
 nfsd4_encode_read_plus_hole(struct nfsd4_compoundres *resp,
                          struct nfsd4_read *read,
-                         unsigned long maxcount, u32 *eof)
+                         unsigned long *maxcount, u32 *eof)

fs/nfsd/nfs4xdr.c nfsd4_encode_read_plus_data(struct nfsd4_compoundres *resp, · line 4655

Quoted 10 lines
fs/nfsd/nfs4xdr.c +6 -0
 {
      struct file *file = read->rd_nf->nf_file;
+     loff_t data_pos = vfs_llseek(file, read->rd_offset, SEEK_DATA);
+     unsigned long count;
      __be32 *p;
 
+     if (data_pos == -ENXIO)
+             data_pos = i_size_read(file_inode(file));
+     else if (data_pos <= read->rd_offset)
+             return nfserr_resource;

I think there's still a race here:

Quoted 8 lines

vfs_llseek(.,0,SEEK_HOLE) returns 1024
read 1024 bytes of data
another process fills the hole
vfs_llseek(.,1024,SEEK_DATA) returns 1024
code above returns nfserr_resource

We end up returning an error to the client when we should have just returned more data.

Quoted 6 lines

As long as we've encoded at least one segment successfully, we'll actually return a short read in this case (as of the most recent patches). I tried implementing a check for what each segment actually was before encoding, but it lead to a lot of extra lseeks (so potential for races / performance problems on btrfs). Returning a short read seemed like a better approach to me.

Argh, right, I forgot the "if (nfserr && segments == 0)" at the end of nfsd4_encode_read_plus().

It's still possible to get a spurious error return if this happens at the very start of the READ_PLUS, though. Hm. Might be better to just encode another data segment?

--b.

Quoted 90 lines

Anna

Quoted 87 lines

--b.

Quoted 83 lines
fs/nfsd/nfs4xdr.c +27 -10
+     count = data_pos - read->rd_offset;
+
      /* Content type, offset, byte count */
      p = xdr_reserve_space(&resp->xdr, 4 + 8 + 8);
      if (!p)
@@ -4654,9 +4662,10 @@ nfsd4_encode_read_plus_hole(struct nfsd4_compoundres *resp,
 
      *p++ = htonl(NFS4_CONTENT_HOLE);
       p   = xdr_encode_hyper(p, read->rd_offset);
-      p   = xdr_encode_hyper(p, maxcount);
+      p   = xdr_encode_hyper(p, count);
 
-     *eof = (read->rd_offset + maxcount) >= i_size_read(file_inode(file));
+     *eof = (read->rd_offset + count) >= i_size_read(file_inode(file));
+     *maxcount = min_t(unsigned long, count, *maxcount);
      return nfs_ok;
 }
 
@@ -4664,7 +4673,7 @@ static __be32
 nfsd4_encode_read_plus(struct nfsd4_compoundres *resp, __be32 nfserr,
                     struct nfsd4_read *read)
 {
-     unsigned long maxcount;
+     unsigned long maxcount, count;
      struct xdr_stream *xdr = &resp->xdr;
      struct file *file;
      int starting_len = xdr->buf->len;
@@ -4687,6 +4696,7 @@ nfsd4_encode_read_plus(struct nfsd4_compoundres *resp, __be32 nfserr,
      maxcount = min_t(unsigned long, maxcount,
                       (xdr->buf->buflen - xdr->buf->len));
      maxcount = min_t(unsigned long, maxcount, read->rd_length);
+     count    = maxcount;
 
      eof = read->rd_offset >= i_size_read(file_inode(file));
      if (eof)
@@ -4695,24 +4705,38 @@ nfsd4_encode_read_plus(struct nfsd4_compoundres *resp, __be32 nfserr,
      pos = vfs_llseek(file, read->rd_offset, SEEK_DATA);
      if (pos == -ENXIO)
              pos = i_size_read(file_inode(file));
+     else if (pos < 0)
+             pos = read->rd_offset;
 
-     if (pos > read->rd_offset) {
-             maxcount = pos - read->rd_offset;
-             nfserr = nfsd4_encode_read_plus_hole(resp, read, maxcount, &eof);
+     if (pos == read->rd_offset) {
+             maxcount = count;
+             nfserr = nfsd4_encode_read_plus_data(resp, read, &maxcount, &eof);
+             if (nfserr)
+                     goto out;
+             count -= maxcount;
+             read->rd_offset += maxcount;
              segments++;
-     } else {
-             nfserr = nfsd4_encode_read_plus_data(resp, read, maxcount, &eof);
+     }
+
+     if (count > 0 && !eof) {
+             maxcount = count;
+             nfserr = nfsd4_encode_read_plus_hole(resp, read, &maxcount, &eof);
+             if (nfserr)
+                     goto out;
+             count -= maxcount;
+             read->rd_offset += maxcount;
              segments++;
      }
 
 out:
-     if (nfserr)
+     if (nfserr && segments == 0)
              xdr_truncate_encode(xdr, starting_len);
      else {
              tmp = htonl(eof);
              write_bytes_to_xdr_buf(xdr->buf, starting_len,     &tmp, 4);
              tmp = htonl(segments);
              write_bytes_to_xdr_buf(xdr->buf, starting_len + 4, &tmp, 4);
+             nfserr = nfs_ok;
      }
 
      return nfserr;
--

2.28.0

#12 lore raw

Re: [PATCH v5 4/5] NFSD: Return both a hole and a data segment

Quoted 20 lines · bfields
Quoted 12 lines · Anna Schumaker
Quoted 10 lines · J. Bruce Fields

I think there's still a race here:

vfs_llseek(.,0,SEEK_HOLE) returns 1024
read 1024 bytes of data
another process fills the hole
vfs_llseek(.,1024,SEEK_DATA) returns 1024
code above returns nfserr_resource

We end up returning an error to the client when we should have just returned more data.

Quoted 6 lines

As long as we've encoded at least one segment successfully, we'll actually return a short read in this case (as of the most recent patches). I tried implementing a check for what each segment actually was before encoding, but it lead to a lot of extra lseeks (so potential for races / performance problems on btrfs). Returning a short read seemed like a better approach to me.

Quoted 2 lines

Argh, right, I forgot the "if (nfserr && segments == 0)" at the end of nfsd4_encode_read_plus().

(Oops, it's actually the "if (nfserr)" below that handles that case.)

Quoted 3 lines

It's still possible to get a spurious error return if this happens at the very start of the READ_PLUS, though. Hm. Might be better to just encode another data segment?

So, I mean, if nfsd4_encode_read_plus_hole() doesn't find a hole after all, just keep going, and create a data segment.

--b.

Quoted 93 lines

--b.

Quoted 90 lines

Anna

Quoted 87 lines

--b.

Quoted 83 lines
fs/nfsd/nfs4xdr.c +27 -10
+     count = data_pos - read->rd_offset;
+
      /* Content type, offset, byte count */
      p = xdr_reserve_space(&resp->xdr, 4 + 8 + 8);
      if (!p)
@@ -4654,9 +4662,10 @@ nfsd4_encode_read_plus_hole(struct nfsd4_compoundres *resp,
 
      *p++ = htonl(NFS4_CONTENT_HOLE);
       p   = xdr_encode_hyper(p, read->rd_offset);
-      p   = xdr_encode_hyper(p, maxcount);
+      p   = xdr_encode_hyper(p, count);
 
-     *eof = (read->rd_offset + maxcount) >= i_size_read(file_inode(file));
+     *eof = (read->rd_offset + count) >= i_size_read(file_inode(file));
+     *maxcount = min_t(unsigned long, count, *maxcount);
      return nfs_ok;
 }
 
@@ -4664,7 +4673,7 @@ static __be32
 nfsd4_encode_read_plus(struct nfsd4_compoundres *resp, __be32 nfserr,
                     struct nfsd4_read *read)
 {
-     unsigned long maxcount;
+     unsigned long maxcount, count;
      struct xdr_stream *xdr = &resp->xdr;
      struct file *file;
      int starting_len = xdr->buf->len;
@@ -4687,6 +4696,7 @@ nfsd4_encode_read_plus(struct nfsd4_compoundres *resp, __be32 nfserr,
      maxcount = min_t(unsigned long, maxcount,
                       (xdr->buf->buflen - xdr->buf->len));
      maxcount = min_t(unsigned long, maxcount, read->rd_length);
+     count    = maxcount;
 
      eof = read->rd_offset >= i_size_read(file_inode(file));
      if (eof)
@@ -4695,24 +4705,38 @@ nfsd4_encode_read_plus(struct nfsd4_compoundres *resp, __be32 nfserr,
      pos = vfs_llseek(file, read->rd_offset, SEEK_DATA);
      if (pos == -ENXIO)
              pos = i_size_read(file_inode(file));
+     else if (pos < 0)
+             pos = read->rd_offset;
 
-     if (pos > read->rd_offset) {
-             maxcount = pos - read->rd_offset;
-             nfserr = nfsd4_encode_read_plus_hole(resp, read, maxcount, &eof);
+     if (pos == read->rd_offset) {
+             maxcount = count;
+             nfserr = nfsd4_encode_read_plus_data(resp, read, &maxcount, &eof);
+             if (nfserr)
+                     goto out;
+             count -= maxcount;
+             read->rd_offset += maxcount;
              segments++;
-     } else {
-             nfserr = nfsd4_encode_read_plus_data(resp, read, maxcount, &eof);
+     }
+
+     if (count > 0 && !eof) {
+             maxcount = count;
+             nfserr = nfsd4_encode_read_plus_hole(resp, read, &maxcount, &eof);
+             if (nfserr)
+                     goto out;
+             count -= maxcount;
+             read->rd_offset += maxcount;
              segments++;
      }
 
 out:
-     if (nfserr)
+     if (nfserr && segments == 0)
              xdr_truncate_encode(xdr, starting_len);
      else {
              tmp = htonl(eof);
              write_bytes_to_xdr_buf(xdr->buf, starting_len,     &tmp, 4);
              tmp = htonl(segments);
              write_bytes_to_xdr_buf(xdr->buf, starting_len + 4, &tmp, 4);
+             nfserr = nfs_ok;
      }
 
      return nfserr;
--

2.28.0

#13 lore raw

PATCH v5 5/5 NFSD: Encode a full READ_PLUS reply

In reply to [email protected]

From: Anna Schumaker <[email protected]>

Reply to the client with multiple hole and data segments. I use the result of the first vfs_llseek() call for encoding as an optimization so we don't have to immediately repeat the call.

---
v5: Truncate the encode to the last segment length if we're returning a
short read

---
 fs/nfsd/nfs4xdr.c | 40 ++++++++++++++++++++--------------------
 1 file changed, 20 insertions(+), 20 deletions(-)
fs/nfsd/nfs4xdr.c +20 -20
index 856606263c1d..eec23a7d5ca0 100644
@@ -4603,16 +4603,18 @@ nfsd4_encode_offload_status(struct nfsd4_compoundres *resp, __be32 nfserr,
 static __be32
 nfsd4_encode_read_plus_data(struct nfsd4_compoundres *resp,
 			    struct nfsd4_read *read,
-			    unsigned long *maxcount, u32 *eof)
+			    unsigned long *maxcount, u32 *eof,
+			    loff_t *pos)
 {
 	struct xdr_stream *xdr = &resp->xdr;
 	struct file *file = read->rd_nf->nf_file;
 	int starting_len = xdr->buf->len;
-	loff_t hole_pos = vfs_llseek(file, read->rd_offset, SEEK_HOLE);
+	loff_t hole_pos;
 	__be32 nfserr;
 	__be32 *p, tmp;
 	__be64 tmp64;
 
+	hole_pos = pos ? *pos : vfs_llseek(file, read->rd_offset, SEEK_HOLE);
 	if (hole_pos > read->rd_offset)
 		*maxcount = min_t(unsigned long, *maxcount, hole_pos - read->rd_offset);
 
@@ -4677,8 +4679,10 @@ nfsd4_encode_read_plus(struct nfsd4_compoundres *resp, __be32 nfserr,
 	struct xdr_stream *xdr = &resp->xdr;
 	struct file *file;
 	int starting_len = xdr->buf->len;
+	int last_segment = xdr->buf->len;
 	int segments = 0;
 	__be32 *p, tmp;
+	bool is_data;
 	loff_t pos;
 	u32 eof;
 
@@ -4702,29 +4706,22 @@ nfsd4_encode_read_plus(struct nfsd4_compoundres *resp, __be32 nfserr,
 	if (eof)
 		goto out;
 
-	pos = vfs_llseek(file, read->rd_offset, SEEK_DATA);
-	if (pos == -ENXIO)
-		pos = i_size_read(file_inode(file));
-	else if (pos < 0)
-		pos = read->rd_offset;
+	pos = vfs_llseek(file, read->rd_offset, SEEK_HOLE);
+	is_data = pos > read->rd_offset;
 
-	if (pos == read->rd_offset) {
+	while (count > 0 && !eof) {
 		maxcount = count;
-		nfserr = nfsd4_encode_read_plus_data(resp, read, &maxcount, &eof);
-		if (nfserr)
-			goto out;
-		count -= maxcount;
-		read->rd_offset += maxcount;
-		segments++;
-	}
-
-	if (count > 0 && !eof) {
-		maxcount = count;
-		nfserr = nfsd4_encode_read_plus_hole(resp, read, &maxcount, &eof);
+		if (is_data)
+			nfserr = nfsd4_encode_read_plus_data(resp, read, &maxcount, &eof,
+						segments == 0 ? &pos : NULL);
+		else
+			nfserr = nfsd4_encode_read_plus_hole(resp, read, &maxcount, &eof);
 		if (nfserr)
 			goto out;
 		count -= maxcount;
 		read->rd_offset += maxcount;
+		is_data = !is_data;
+		last_segment = xdr->buf->len;
 		segments++;
 	}
 
@@ -4736,7 +4733,10 @@ nfsd4_encode_read_plus(struct nfsd4_compoundres *resp, __be32 nfserr,
 		write_bytes_to_xdr_buf(xdr->buf, starting_len,     &tmp, 4);
 		tmp = htonl(segments);
 		write_bytes_to_xdr_buf(xdr->buf, starting_len + 4, &tmp, 4);
-		nfserr = nfs_ok;
+		if (nfserr) {
+			xdr_truncate_encode(xdr, last_segment);
+			nfserr = nfs_ok;
+		}
 	}
 
 	return nfserr;
Signature · 1 line
2.28.0