Skip to contentSkip to conversation

svcrdma: Introduce Receive buffer arenas

RFC PATCH Superseded Chuck Lever 2 participants 2 messages Aug 8, 2025 – Aug 9, 2025

Subsystems (4)

v1 of 3 Superseded Latest is v3 →

lore
Share & tools
lore

Chuck Lever

author maintainer
#0 lore raw

RFC PATCH 1/1 svcrdma: Introduce Receive buffer arenas

From: Chuck Lever <[email protected]>

Reduce the per-connection footprint in the host's and RNIC's memory management TLBs by combining each connection's Receive buffers into a single IOVA.

I don't have a good way to measure whether this approach is effective.

---
 include/linux/sunrpc/svc_rdma.h         |   3 +
 net/sunrpc/xprtrdma/Makefile            |   2 +-
 net/sunrpc/xprtrdma/pool.c              | 162 ++++++++++++++++++++++++
 net/sunrpc/xprtrdma/pool.h              |  25 ++++
 net/sunrpc/xprtrdma/svc_rdma_recvfrom.c |  43 +++----
 5 files changed, 210 insertions(+), 25 deletions(-)
 create mode 100644 net/sunrpc/xprtrdma/pool.c
 create mode 100644 net/sunrpc/xprtrdma/pool.h
include/linux/sunrpc/svc_rdma.h +3 -0
index 22704c2e5b9b..b4f3c01f1b94 100644
@@ -73,6 +73,8 @@ extern struct percpu_counter svcrdma_stat_recv;
 extern struct percpu_counter svcrdma_stat_sq_starve;
 extern struct percpu_counter svcrdma_stat_write;
 
+struct rpcrdma_pool;
+
 struct svcxprt_rdma {
 	struct svc_xprt      sc_xprt;		/* SVC transport structure */
 	struct rdma_cm_id    *sc_cm_id;		/* RDMA connection id */
@@ -112,6 +114,7 @@ struct svcxprt_rdma {
 	unsigned long	     sc_flags;
 	struct work_struct   sc_work;
 
+	struct rpcrdma_pool  *sc_recv_pool;
 	struct llist_head    sc_recv_ctxts;
 
 	atomic_t	     sc_completion_ids;
net/sunrpc/xprtrdma/Makefile +1 -1
index 3232aa23cdb4..f69456dffe87 100644
@@ -1,7 +1,7 @@
 # SPDX-License-Identifier: GPL-2.0
 obj-$(CONFIG_SUNRPC_XPRT_RDMA) += rpcrdma.o
 
-rpcrdma-y := transport.o rpc_rdma.o verbs.o frwr_ops.o ib_client.o \
+rpcrdma-y := transport.o rpc_rdma.o verbs.o frwr_ops.o ib_client.o pool.o \
 	svc_rdma.o svc_rdma_backchannel.o svc_rdma_transport.o \
 	svc_rdma_sendto.o svc_rdma_recvfrom.o svc_rdma_rw.o \
 	svc_rdma_pcl.o module.o
net/sunrpc/xprtrdma/pool.c new file +162 -0

162 lines not on this page · raw message

net/sunrpc/xprtrdma/pool.h new file +25 -0
new file mode 100644
index 000000000000..666543e22b5b
@@ -0,0 +1,25 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (c) 2025, Oracle and/or its affiliates.
+ *
+ * Pools for Send and Receive buffers.
+ */
+
+#ifndef RPCRDMA_POOL_H
+#define RPCRDMA_POOL_H
+
+struct rpcrdma_pool_args {
+	struct ib_device	*pa_device;
+	size_t			pa_bufsize;
+	enum dma_data_direction	pa_direction;
+};
+
+struct rpcrdma_pool;
+
+struct rpcrdma_pool *
+rpcrdma_pool_create(struct rpcrdma_pool_args *args, gfp_t flags);
+void rpcrdma_pool_destroy(struct rpcrdma_pool *pool);
+bool rpcrdma_pool_alloc_buffer(struct rpcrdma_pool *pool, gfp_t flags,
+			       void **cpu_addr, dma_addr_t *dma_addr);
+
+#endif /* RPCRDMA_POOL_H */
net/sunrpc/xprtrdma/svc_rdma_recvfrom.c +19 -24

101 lines not on this page · raw message

Signature · 1 line
2.50.0
#1 lore raw
kernel test robot noticed the following build errors:

Hi Chuck,

[This is a private test report for your RFC patch.] kernel test robot noticed the following build errors:

[auto build test ERROR on trondmy-nfs/linux-next]
[also build test ERROR on linus/master v6.16 next-20250808] [If your patch is applied to the wrong git tree, kindly drop us a note. And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch#_base_tree_information]

url:    https://github.com/intel-lab-lkp/linux/commits/Chuck-Lever/svcrdma-Introduce-Receive-buffer-arenas/20250809-025410
base:   git://git.linux-nfs.org/projects/trondmy/linux-nfs.git linux-next
patch link:    https://lore.kernel.org/r/20250808184648.120866-1-cel%40kernel.org
patch subject: [RFC PATCH] svcrdma: Introduce Receive buffer arenas
config: m68k-allmodconfig (https://download.01.org/0day-ci/archive/20250810/[email protected]/config)
compiler: m68k-linux-gcc (GCC) 15.1.0
reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20250810/[email protected]/reproduce)

If you fix the issue in a separate patch/commit (i.e. not just a new version of the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <[email protected]>
| Closes: https://lore.kernel.org/oe-kbuild-all/[email protected]/

All errors (new ones prefixed by >>):

net/sunrpc/xprtrdma/svc_rdma_recvfrom.c: In function 'svc_rdma_recv_ctxt_alloc':

>> net/sunrpc/xprtrdma/svc_rdma_recvfrom.c:136:40: error: passing argument 4 of 'rpcrdma_pool_alloc_buffer' from incompatible pointer type [-Wincompatible-pointer-types]
     136 |                                        &ctxt->rc_recv_sge.addr))
         |                                        ^~~~~~~~~~~~~~~~~~~~~~~
         |                                        |
         |                                        u64 * {aka long long unsigned int *}
   In file included from net/sunrpc/xprtrdma/svc_rdma_recvfrom.c:107:
   net/sunrpc/xprtrdma/pool.h:23:61: note: expected 'dma_addr_t *' {aka 'unsigned int *'} but argument is of type 'u64 *' {aka 'long long unsigned int *'}
      23 |                                void **cpu_addr, dma_addr_t *dma_addr);
         |                                                 ~~~~~~~~~~~~^~~~~~~~

vim +/rpcrdma_pool_alloc_buffer +136 net/sunrpc/xprtrdma/svc_rdma_recvfrom.c

119
120 static struct svc_rdma_recv_ctxt *
121 svc_rdma_recv_ctxt_alloc(struct svcxprt_rdma *rdma)
122 {
123 int node = ibdev_to_node(rdma->sc_cm_id->device);
124 struct svc_rdma_recv_ctxt *ctxt;
125 unsigned long pages;
126
127 pages = svc_serv_maxpages(rdma->sc_xprt.xpt_server);
128 ctxt = kzalloc_node(struct_size(ctxt, rc_pages, pages),
129 GFP_KERNEL, node);
130 if (!ctxt)
131 goto fail0;
132 ctxt->rc_maxpages = pages;
133
134 if (!rpcrdma_pool_alloc_buffer(rdma->sc_recv_pool, GFP_KERNEL,
135 &ctxt->rc_recv_buf,

 > 136					       &ctxt->rc_recv_sge.addr))

137 goto fail1;
138
139 svc_rdma_recv_cid_init(rdma, &ctxt->rc_cid);
140 pcl_init(&ctxt->rc_call_pcl);
141 pcl_init(&ctxt->rc_read_pcl);
142 pcl_init(&ctxt->rc_write_pcl);
143 pcl_init(&ctxt->rc_reply_pcl);
144
145 ctxt->rc_recv_wr.next = NULL;
146 ctxt->rc_recv_wr.wr_cqe = &ctxt->rc_cqe;
147 ctxt->rc_recv_wr.sg_list = &ctxt->rc_recv_sge;
148 ctxt->rc_recv_wr.num_sge = 1;
149 ctxt->rc_cqe.done = svc_rdma_wc_receive;
150 ctxt->rc_recv_sge.length = rdma->sc_max_req_size;
151 ctxt->rc_recv_sge.lkey = rdma->sc_pd->local_dma_lkey;
152 svc_rdma_cc_init(rdma, &ctxt->rc_cc);
153 return ctxt;
154
155 fail1:
156 kfree(ctxt);
157 fail0:
158 return NULL;
159 }
160

Signature · 2 lines
0-DAY CI Kernel Test Service
https://github.com/intel/lkp-tests/wiki