Move rq_vec[] and rq_bvec[] out of svc_rqst
Subsystems (3)
[email protected]
maintainercover Move rq_vec[] and rq_bvec[] out of svc_rqst
From: Chuck Lever <[email protected]>
In order to make RPCSVC_MAXPAYLOAD larger (or variable in size), we need to do something clever with the payload arrays embedded in struct svc_rqst. Here's one way of dealing with two of them.
My preference is to keep these arrays allocated all the time because allocating them on demand increases the risk of a memory allocation failure during a large I/O. This is a quick-and-dirty approach that might be replaced once NFSD is converted to use large folios.
The downside of this design choice is that it pins a few pages per NFSD thread (and that's the current situation already). But note that because RPCSVC_MAXPAGES is 259, each array is just over a page in size, making the allocation waste quite a bit of memory beyond the end of the array due to power-of-2 allocator round up. This gets worse as the MAXPAGES value is doubled or quadrupled.
I plan to look at rq_pages[] next.
Chuck Lever (2):
sunrpc: Replace the rq_bvec array with dynamically-allocated memory
sunrpc: Replace the rq_vec array with dynamically-allocated memory
fs/nfsd/nfs4proc.c | 2 +- fs/nfsd/vfs.c | 2 +- include/linux/sunrpc/svc.h | 4 ++-- net/sunrpc/svc.c | 14 +++++++++++++- net/sunrpc/svcsock.c | 7 +++---- 5 files changed, 20 insertions(+), 9 deletions(-)
Signature · 1 line
2.49.0
[email protected]
maintainerRFC PATCH 1/2 sunrpc: Replace the rq_bvec array with dynamically-allocated memory
From: Chuck Lever <[email protected]>
As a step towards making NFSD's maximum rsize and wsize variable, replace the fixed-size rq_bvec[] array in struct svc_rqst with a chunk of dynamically-allocated memory.
On a system with 8-byte pointers and 4KB pages, pahole reports that the rq_bvec[] array is 4144 bytes. Replacing it with a single pointer reduces the size of struct svc_rqst to about 7500 bytes.
--- include/linux/sunrpc/svc.h | 2 +- net/sunrpc/svc.c | 6 ++++++ net/sunrpc/svcsock.c | 7 +++---- 3 files changed, 10 insertions(+), 5 deletions(-)
include/linux/sunrpc/svc.h +1 -1
@@ -195,7 +195,7 @@ struct svc_rqst {struct folio_batch rq_fbatch;struct kvec rq_vec[RPCSVC_MAXPAGES ];/* generally useful.. */ -struct bio_vec rq_bvec[RPCSVC_MAXPAGES ]; +struct bio_vec *rq_bvec;__be32 rq_xid;/* transmission id */ u32 rq_prog;/* program number */
net/sunrpc/svc.c +6 -0
@@ -673,6 +673,7 @@ static void svc_rqst_free(struct svc_rqst *rqstp) {folio_batch_release (&rqstp->rq_fbatch); +kfree (rqstp->rq_bvec);svc_release_buffer (rqstp);if (rqstp->rq_scratch_page)put_page (rqstp->rq_scratch_page);
@@ -711,6 +712,11 @@ svc_prepare_thread(struct svc_serv *serv, struct svc_pool *pool, int node)if (!svc_init_buffer (rqstp, serv->sv_max_mesg, node)) goto out_enomem; + rqstp->rq_bvec =kcalloc_node (RPCSVC_MAXPAGES ,sizeof (struct bio_vec ), +GFP_KERNEL , node); +if (!rqstp->rq_bvec) + goto out_enomem; + rqstp->rq_err = -EAGAIN ;/* No error yet */ serv->sv_nrthreads +=1 ;
net/sunrpc/svcsock.c +3 -4
@@ -713,8 +713,7 @@ static int svc_udp_sendto(struct svc_rqst *rqstp)if (svc_xprt_is_dead (xprt)) goto out_notconn; - count =xdr_buf_to_bvec (rqstp->rq_bvec, -ARRAY_SIZE (rqstp->rq_bvec), xdr); + count =xdr_buf_to_bvec (rqstp->rq_bvec,RPCSVC_MAXPAGES , xdr);iov_iter_bvec (&msg.msg_iter,ITER_SOURCE , rqstp->rq_bvec, count, rqstp->rq_res.len);
@@ -1219,8 +1218,8 @@ static int svc_tcp_sendmsg(struct svc_sock *svsk, struct svc_rqst *rqstp,memcpy (buf, &marker,sizeof (marker));bvec_set_virt (rqstp->rq_bvec, buf,sizeof (marker)); - count =xdr_buf_to_bvec (rqstp->rq_bvec +1 , -ARRAY_SIZE (rqstp->rq_bvec) -1 , &rqstp->rq_res); + count =xdr_buf_to_bvec (rqstp->rq_bvec +1 ,RPCSVC_MAXPAGES , + &rqstp->rq_res);iov_iter_bvec (&msg.msg_iter,ITER_SOURCE , rqstp->rq_bvec,1 + count,sizeof (marker) + rqstp->rq_res.len);
Signature · 1 line
2.49.0
[email protected]
maintainerRFC PATCH 2/2 sunrpc: Replace the rq_vec array with dynamically-allocated memory
In reply to [email protected]From: Chuck Lever <[email protected]>
As a step towards making NFSD's maximum rsize and wsize variable, replace the fixed-size rq_vec[] array in struct svc_rqst with a chunk of dynamically-allocated memory.
On a system with 8-byte pointers and 4KB pages, pahole reports that the rq_vec[] array is 4144 bytes. Replacing it with a single pointer reduces the size of struct svc_rqst to about 3300 bytes.
--- fs/nfsd/nfs4proc.c | 2 +- fs/nfsd/vfs.c | 2 +- include/linux/sunrpc/svc.h | 2 +- net/sunrpc/svc.c | 8 +++++++- 4 files changed, 10 insertions(+), 4 deletions(-)
fs/nfsd/nfs4proc.c +1 -1
@@ -1228,7 +1228,7 @@ nfsd4_write(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, write->wr_how_written = write->wr_stable_how; nvecs =svc_fill_write_vector (rqstp, &write->wr_payload); -WARN_ON_ONCE (nvecs >ARRAY_SIZE (rqstp->rq_vec)); +/* WARN_ON_ONCE(nvecs > ARRAY_SIZE(rqstp->rq_vec)); */ status = nfsd_vfs_write(rqstp, &cstate->current_fh, nf, write->wr_offset, rqstp->rq_vec, nvecs, &cnt,
fs/nfsd/vfs.c +1 -1
@@ -1094,7 +1094,7 @@ __be32 nfsd_iter_read(struct svc_rqst *rqstp, struct svc_fh *fhp, ++v; base =0 ; } -WARN_ON_ONCE (v >ARRAY_SIZE (rqstp->rq_vec)); +WARN_ON_ONCE (v >RPCSVC_MAXPAGES );trace_nfsd_read_vector (rqstp, fhp, offset, *count);iov_iter_kvec (&iter,ITER_DEST , rqstp->rq_vec, v, *count);
include/linux/sunrpc/svc.h +1 -1
@@ -194,7 +194,7 @@ struct svc_rqst {struct page * *rq_page_end;/* one past the last page */ struct folio_batch rq_fbatch; -struct kvec rq_vec[RPCSVC_MAXPAGES ];/* generally useful.. */ +struct kvec *rq_vec;struct bio_vec *rq_bvec;__be32 rq_xid;/* transmission id */
net/sunrpc/svc.c +7 -1
@@ -674,6 +674,7 @@ svc_rqst_free(struct svc_rqst *rqstp) {folio_batch_release (&rqstp->rq_fbatch);kfree (rqstp->rq_bvec); +kfree (rqstp->rq_vec);svc_release_buffer (rqstp);if (rqstp->rq_scratch_page)put_page (rqstp->rq_scratch_page);
@@ -712,6 +713,11 @@ svc_prepare_thread(struct svc_serv *serv, struct svc_pool *pool, int node)if (!svc_init_buffer (rqstp, serv->sv_max_mesg, node)) goto out_enomem; + rqstp->rq_vec =kcalloc_node (RPCSVC_MAXPAGES ,sizeof (struct kvec ), +GFP_KERNEL , node); +if (!rqstp->rq_vec) + goto out_enomem; + rqstp->rq_bvec =kcalloc_node (RPCSVC_MAXPAGES ,sizeof (struct bio_vec ),GFP_KERNEL , node);if (!rqstp->rq_bvec)
@@ -1754,7 +1760,7 @@ unsigned int svc_fill_write_vector(struct svc_rqst *rqstp, ++pages; } -WARN_ON_ONCE (i >ARRAY_SIZE (rqstp->rq_vec)); +WARN_ON_ONCE (i >RPCSVC_MAXPAGES );return i; }EXPORT_SYMBOL_GPL (svc_fill_write_vector);
Signature · 1 line
2.49.0
Jeff Layton
maintainerRe: [RFC PATCH 1/2] sunrpc: Replace the rq_bvec array with dynamically-allocated memory
In reply to [email protected]Quoted 22 lines · On Wed, 2025-04-16 at 11:28 -0400, [email protected] wrote:
From: Chuck Lever <[email protected]>
As a step towards making NFSD's maximum rsize and wsize variable, replace the fixed-size rq_bvec[] array in struct
svc_rqstwith a chunk of dynamically-allocated memory.On a system with 8-byte pointers and 4KB pages, pahole reports that the rq_bvec[] array is 4144 bytes. Replacing it with a single pointer reduces the size of struct
svc_rqstto about 7500 bytes.Signed-off-by Chuck Lever <[email protected]>--- include/linux/sunrpc/svc.h | 2 +- net/sunrpc/svc.c | 6 ++++++ net/sunrpc/svcsock.c | 7 +++---- 3 files changed, 10 insertions(+), 5 deletions(-)include/linux/sunrpc/svc.h +0
-0
include/linux/sunrpc/svc.h struct svc_rqst { · line 198
Quoted 5 lines
include/linux/sunrpc/svc.h +1
-1@@ -195,7 +195,7 @@ struct svc_rqst {struct folio_batch rq_fbatch;struct kvec rq_vec[RPCSVC_MAXPAGES ];/* generally useful.. */ -struct bio_vec rq_bvec[RPCSVC_MAXPAGES ]; +struct bio_vec *rq_bvec;
It's a reasonable start.
What would also be good to do here is to replace the invocations of RPCSVC_MAXPAGES that involve this array with a helper function that returns the length of it.
For now it could just return RPCSVC_MAXPAGES, but eventually you could add (e.g.) a rqstp->rq_bvec_len field and use that to indicate how many entries there are in rq_bvec.
Quoted 52 lines
include/linux/sunrpc/svc.h +0
-0__be32 rq_xid;/* transmission id */ u32 rq_prog;/* program number */ net/sunrpc/svc.c +6
-0@@ -673,6 +673,7 @@ static void svc_rqst_free(struct svc_rqst *rqstp) {folio_batch_release (&rqstp->rq_fbatch); +kfree (rqstp->rq_bvec);svc_release_buffer (rqstp);if (rqstp->rq_scratch_page)put_page (rqstp->rq_scratch_page);@@ -711,6 +712,11 @@ svc_prepare_thread(struct svc_serv *serv, struct svc_pool *pool, int node)if (!svc_init_buffer (rqstp, serv->sv_max_mesg, node)) goto out_enomem; + rqstp->rq_bvec =kcalloc_node (RPCSVC_MAXPAGES ,sizeof (struct bio_vec ), +GFP_KERNEL , node); +if (!rqstp->rq_bvec) + goto out_enomem; + rqstp->rq_err = -EAGAIN ;/* No error yet */ serv->sv_nrthreads +=1 ;net/sunrpc/svcsock.c +3
-4@@ -713,8 +713,7 @@ static int svc_udp_sendto(struct svc_rqst *rqstp)if (svc_xprt_is_dead (xprt)) goto out_notconn; - count =xdr_buf_to_bvec (rqstp->rq_bvec, -ARRAY_SIZE (rqstp->rq_bvec), xdr); + count =xdr_buf_to_bvec (rqstp->rq_bvec,RPCSVC_MAXPAGES , xdr);iov_iter_bvec (&msg.msg_iter,ITER_SOURCE , rqstp->rq_bvec, count, rqstp->rq_res.len);@@ -1219,8 +1218,8 @@ static int svc_tcp_sendmsg(struct svc_sock *svsk, struct svc_rqst *rqstp,memcpy (buf, &marker,sizeof (marker));bvec_set_virt (rqstp->rq_bvec, buf,sizeof (marker)); - count =xdr_buf_to_bvec (rqstp->rq_bvec +1 , -ARRAY_SIZE (rqstp->rq_bvec) -1 , &rqstp->rq_res); + count =xdr_buf_to_bvec (rqstp->rq_bvec +1 ,RPCSVC_MAXPAGES , + &rqstp->rq_res);iov_iter_bvec (&msg.msg_iter,ITER_SOURCE , rqstp->rq_bvec,1 + count,sizeof (marker) + rqstp->rq_res.len);
Signature · 1 line
Jeff Layton <[email protected]>
Re: [RFC PATCH 1/2] sunrpc: Replace the rq_bvec array with dynamically-allocated memory
Quoted 31 lines · Jeff Layton
Quoted 22 lines · On Wed, 2025-04-16 at 11:28 -0400, [email protected] wrote:
From: Chuck Lever <[email protected]>
As a step towards making NFSD's maximum rsize and wsize variable, replace the fixed-size rq_bvec[] array in struct
svc_rqstwith a chunk of dynamically-allocated memory.On a system with 8-byte pointers and 4KB pages, pahole reports that the rq_bvec[] array is 4144 bytes. Replacing it with a single pointer reduces the size of struct
svc_rqstto about 7500 bytes.Signed-off-by Chuck Lever <[email protected]>--- include/linux/sunrpc/svc.h | 2 +- net/sunrpc/svc.c | 6 ++++++ net/sunrpc/svcsock.c | 7 +++---- 3 files changed, 10 insertions(+), 5 deletions(-)include/linux/sunrpc/svc.h +0
-0include/linux/sunrpc/svc.h struct svc_rqst { · line 198
Quoted 5 lines
include/linux/sunrpc/svc.h +1
-1@@ -195,7 +195,7 @@ struct svc_rqst {struct folio_batch rq_fbatch;struct kvec rq_vec[RPCSVC_MAXPAGES ];/* generally useful.. */ -struct bio_vec rq_bvec[RPCSVC_MAXPAGES ]; +struct bio_vec *rq_bvec;It's a reasonable start.
Quoted 7 lines
What would also be good to do here is to replace the invocations of
RPCSVC_MAXPAGESthat involve this array with a helper function that returns the length of it.For now it could just return
RPCSVC_MAXPAGES, but eventually you could add (e.g.) arqstp->rq_bvec_lenfield and use that to indicate how many entries there are inrq_bvec.
rq_vec, rq_pages, and rq_bvec all have the same entry count (plus or minus one) so only one new field is necessary. There are a few other places that allocate arrays of size RPCSVC_MAXPAGES that will need similar treatment.
Stay tuned for v2.
Quoted 52 lines
Quoted 51 lines
include/linux/sunrpc/svc.h +0
-0__be32 rq_xid;/* transmission id */ u32 rq_prog;/* program number */ net/sunrpc/svc.c +6
-0@@ -673,6 +673,7 @@ static void svc_rqst_free(struct svc_rqst *rqstp) {folio_batch_release (&rqstp->rq_fbatch); +kfree (rqstp->rq_bvec);svc_release_buffer (rqstp);if (rqstp->rq_scratch_page)put_page (rqstp->rq_scratch_page);@@ -711,6 +712,11 @@ svc_prepare_thread(struct svc_serv *serv, struct svc_pool *pool, int node)if (!svc_init_buffer (rqstp, serv->sv_max_mesg, node)) goto out_enomem; + rqstp->rq_bvec =kcalloc_node (RPCSVC_MAXPAGES ,sizeof (struct bio_vec ), +GFP_KERNEL , node); +if (!rqstp->rq_bvec) + goto out_enomem; + rqstp->rq_err = -EAGAIN ;/* No error yet */ serv->sv_nrthreads +=1 ;net/sunrpc/svcsock.c +3
-4@@ -713,8 +713,7 @@ static int svc_udp_sendto(struct svc_rqst *rqstp)if (svc_xprt_is_dead (xprt)) goto out_notconn; - count =xdr_buf_to_bvec (rqstp->rq_bvec, -ARRAY_SIZE (rqstp->rq_bvec), xdr); + count =xdr_buf_to_bvec (rqstp->rq_bvec,RPCSVC_MAXPAGES , xdr);iov_iter_bvec (&msg.msg_iter,ITER_SOURCE , rqstp->rq_bvec, count, rqstp->rq_res.len);@@ -1219,8 +1218,8 @@ static int svc_tcp_sendmsg(struct svc_sock *svsk, struct svc_rqst *rqstp,memcpy (buf, &marker,sizeof (marker));bvec_set_virt (rqstp->rq_bvec, buf,sizeof (marker)); - count =xdr_buf_to_bvec (rqstp->rq_bvec +1 , -ARRAY_SIZE (rqstp->rq_bvec) -1 , &rqstp->rq_res); + count =xdr_buf_to_bvec (rqstp->rq_bvec +1 ,RPCSVC_MAXPAGES , + &rqstp->rq_res);iov_iter_bvec (&msg.msg_iter,ITER_SOURCE , rqstp->rq_bvec,1 + count,sizeof (marker) + rqstp->rq_res.len);
Signature · 1 line
Chuck Lever
Jeff Layton
maintainerRe: [RFC PATCH 1/2] sunrpc: Replace the rq_bvec array with dynamically-allocated memory
Quoted 40 lines · Chuck Lever
Quoted 31 lines · Jeff Layton
Quoted 22 lines · On Wed, 2025-04-16 at 11:28 -0400, [email protected] wrote:
From: Chuck Lever <[email protected]>
As a step towards making NFSD's maximum rsize and wsize variable, replace the fixed-size rq_bvec[] array in struct
svc_rqstwith a chunk of dynamically-allocated memory.On a system with 8-byte pointers and 4KB pages, pahole reports that the rq_bvec[] array is 4144 bytes. Replacing it with a single pointer reduces the size of struct
svc_rqstto about 7500 bytes.Signed-off-by Chuck Lever <[email protected]>--- include/linux/sunrpc/svc.h | 2 +- net/sunrpc/svc.c | 6 ++++++ net/sunrpc/svcsock.c | 7 +++---- 3 files changed, 10 insertions(+), 5 deletions(-)include/linux/sunrpc/svc.h +0
-0include/linux/sunrpc/svc.h struct svc_rqst { · line 198
Quoted 5 lines
include/linux/sunrpc/svc.h +1
-1@@ -195,7 +195,7 @@ struct svc_rqst {struct folio_batch rq_fbatch;struct kvec rq_vec[RPCSVC_MAXPAGES ];/* generally useful.. */ -struct bio_vec rq_bvec[RPCSVC_MAXPAGES ]; +struct bio_vec *rq_bvec;It's a reasonable start.
Quoted 7 lines
What would also be good to do here is to replace the invocations of
RPCSVC_MAXPAGESthat involve this array with a helper function that returns the length of it.For now it could just return
RPCSVC_MAXPAGES, but eventually you could add (e.g.) arqstp->rq_bvec_lenfield and use that to indicate how many entries there are inrq_bvec.
Quoted 7 lines
rq_vec,rq_pages, andrq_bvecall have the same entry count (plus or minus one) so only one new field is necessary. There are a few other places that allocate arrays of sizeRPCSVC_MAXPAGESthat will need similar treatment.Stay tuned for v2.
Ok. I think I didn't articulate this well. Let me try again:
If you're looking to break the assumption that the length of these arrays is RPCSVC_MAXPAGES, then the thing to do is to eliminate the places where we make that assumption.
In particular, the two places where you're adding new RPCSVC_MAXPAGES invocations would be better replaced with a helper function that we can change the return value of later.
Quoted 56 lines
Quoted 55 lines
Quoted 54 lines
include/linux/sunrpc/svc.h +0
-0__be32 rq_xid;/* transmission id */ u32 rq_prog;/* program number */ net/sunrpc/svc.c +6
-0@@ -673,6 +673,7 @@ static void svc_rqst_free(struct svc_rqst *rqstp) {folio_batch_release (&rqstp->rq_fbatch); +kfree (rqstp->rq_bvec);svc_release_buffer (rqstp);if (rqstp->rq_scratch_page)put_page (rqstp->rq_scratch_page);@@ -711,6 +712,11 @@ svc_prepare_thread(struct svc_serv *serv, struct svc_pool *pool, int node)if (!svc_init_buffer (rqstp, serv->sv_max_mesg, node)) goto out_enomem; + rqstp->rq_bvec =kcalloc_node (RPCSVC_MAXPAGES ,sizeof (struct bio_vec ), +GFP_KERNEL , node); +if (!rqstp->rq_bvec) + goto out_enomem; + rqstp->rq_err = -EAGAIN ;/* No error yet */ serv->sv_nrthreads +=1 ;net/sunrpc/svcsock.c +3
-4@@ -713,8 +713,7 @@ static int svc_udp_sendto(struct svc_rqst *rqstp)if (svc_xprt_is_dead (xprt)) goto out_notconn; - count =xdr_buf_to_bvec (rqstp->rq_bvec, -ARRAY_SIZE (rqstp->rq_bvec), xdr); + count =xdr_buf_to_bvec (rqstp->rq_bvec,RPCSVC_MAXPAGES , xdr);iov_iter_bvec (&msg.msg_iter,ITER_SOURCE , rqstp->rq_bvec, count, rqstp->rq_res.len);@@ -1219,8 +1218,8 @@ static int svc_tcp_sendmsg(struct svc_sock *svsk, struct svc_rqst *rqstp,memcpy (buf, &marker,sizeof (marker));bvec_set_virt (rqstp->rq_bvec, buf,sizeof (marker)); - count =xdr_buf_to_bvec (rqstp->rq_bvec +1 , -ARRAY_SIZE (rqstp->rq_bvec) -1 , &rqstp->rq_res); + count =xdr_buf_to_bvec (rqstp->rq_bvec +1 ,RPCSVC_MAXPAGES , + &rqstp->rq_res);net/sunrpc/svcsock.c +0
-0iov_iter_bvec (&msg.msg_iter,ITER_SOURCE , rqstp->rq_bvec,1 + count,sizeof (marker) + rqstp->rq_res.len);
Signature · 1 line
Jeff Layton <[email protected]>