Make RPCRDMA_MAX_RECV_BATCH configurable.
Subsystems (4)
PATCH 1/1 Make RPCRDMA_MAX_RECV_BATCH configurable.
Bumped up rpcrdma_max_recv_batch to 64.
Added param to change to it, it becomes handy to use higher value to avoid hung.
Signed-off-by Gaurav Gangalwar <[email protected]>
--- net/sunrpc/xprtrdma/frwr_ops.c | 2 +- net/sunrpc/xprtrdma/module.c | 6 ++++++ net/sunrpc/xprtrdma/svc_rdma_transport.c | 2 +- net/sunrpc/xprtrdma/verbs.c | 2 +- net/sunrpc/xprtrdma/xprt_rdma.h | 4 +--- 5 files changed, 10 insertions(+), 6 deletions(-)
net/sunrpc/xprtrdma/frwr_ops.c +1 -1
@@ -246,7 +246,7 @@ int frwr_query_device(struct rpcrdma_ep *ep, const struct ib_device *device) ep->re_attr.cap.max_send_wr +=1 ;/* for ib_drain_sq */ ep->re_attr.cap.max_recv_wr = ep->re_max_requests; ep->re_attr.cap.max_recv_wr +=RPCRDMA_BACKWARD_WRS ; - ep->re_attr.cap.max_recv_wr +=RPCRDMA_MAX_RECV_BATCH ; + ep->re_attr.cap.max_recv_wr += rpcrdma_max_recv_batch; ep->re_attr.cap.max_recv_wr +=1 ;/* for ib_drain_rq */ ep->re_max_rdma_segs =
net/sunrpc/xprtrdma/module.c +6 -0
@@ -27,6 +27,12 @@ MODULE_ALIAS("svcrdma");MODULE_ALIAS ("xprtrdma" );MODULE_ALIAS ("rpcrdma6" ); +unsigned int rpcrdma_max_recv_batch =64 ; +module_param_named (max_recv_batch, rpcrdma_max_recv_batch, uint,0644 ); +MODULE_PARM_DESC (max_recv_batch, +"Maximum number of Receive WRs to post in a batch " +"(default: 64, set to 0 to disable batching)" ); + static void __exit rpc_rdma_cleanup(void) {xprt_rdma_cleanup ();
net/sunrpc/xprtrdma/svc_rdma_transport.c +1 -1
@@ -440,7 +440,7 @@ static struct svc_xprt *svc_rdma_accept(struct svc_xprt *xprt) newxprt->sc_max_req_size = svcrdma_max_req_size; newxprt->sc_max_requests = svcrdma_max_requests; newxprt->sc_max_bc_requests = svcrdma_max_bc_requests; - newxprt->sc_recv_batch =RPCRDMA_MAX_RECV_BATCH ; + newxprt->sc_recv_batch = rpcrdma_max_recv_batch; newxprt->sc_fc_credits =cpu_to_be32 (newxprt->sc_max_requests);/* Qualify the transport's resource defaults with the
net/sunrpc/xprtrdma/verbs.c +1 -1
@@ -1359,7 +1359,7 @@ void rpcrdma_post_recvs(struct rpcrdma_xprt *r_xprt, int needed)if (likely (ep->re_receive_count > needed)) goto out; needed -= ep->re_receive_count; - needed +=RPCRDMA_MAX_RECV_BATCH ; + needed += rpcrdma_max_recv_batch;if (atomic_inc_return (&ep->re_receiving) >1 ) goto out;
net/sunrpc/xprtrdma/xprt_rdma.h +1 -3
@@ -216,9 +216,7 @@ struct rpcrdma_rep {* * Setting this to zero disables Receive post batching. */ -enum { -RPCRDMA_MAX_RECV_BATCH =7 , -}; +extern unsigned int rpcrdma_max_recv_batch;/* struct rpcrdma_sendctx - DMA mapped SGEs to unmap after Send completes */
Signature · 1 line
2.43.7
Quoted 3 lines · Gaurav Gangalwar
Bumped up
rpcrdma_max_recv_batchto 64.
Added param to change to it, it becomes handy to use higher value to avoid hung.
NAK until we have a full root cause analysis. Please explain why this change helps.
Quoted 80 lines
Signed-off-by Gaurav Gangalwar <[email protected]>--- net/sunrpc/xprtrdma/frwr_ops.c | 2 +- net/sunrpc/xprtrdma/module.c | 6 ++++++ net/sunrpc/xprtrdma/svc_rdma_transport.c | 2 +- net/sunrpc/xprtrdma/verbs.c | 2 +- net/sunrpc/xprtrdma/xprt_rdma.h | 4 +--- 5 files changed, 10 insertions(+), 6 deletions(-)net/sunrpc/xprtrdma/frwr_ops.c +1
-1@@ -246,7 +246,7 @@ int frwr_query_device(struct rpcrdma_ep *ep, const struct ib_device *device) ep->re_attr.cap.max_send_wr +=1 ;/* for ib_drain_sq */ ep->re_attr.cap.max_recv_wr = ep->re_max_requests; ep->re_attr.cap.max_recv_wr +=RPCRDMA_BACKWARD_WRS ; - ep->re_attr.cap.max_recv_wr +=RPCRDMA_MAX_RECV_BATCH ; + ep->re_attr.cap.max_recv_wr += rpcrdma_max_recv_batch; ep->re_attr.cap.max_recv_wr +=1 ;/* for ib_drain_rq */ ep->re_max_rdma_segs =net/sunrpc/xprtrdma/module.c +6
-0@@ -27,6 +27,12 @@ MODULE_ALIAS("svcrdma");MODULE_ALIAS ("xprtrdma" );MODULE_ALIAS ("rpcrdma6" ); +unsigned int rpcrdma_max_recv_batch =64 ; +module_param_named (max_recv_batch, rpcrdma_max_recv_batch, uint,0644 ); +MODULE_PARM_DESC (max_recv_batch, +"Maximum number of Receive WRs to post in a batch " +"(default: 64, set to 0 to disable batching)" ); + static void __exit rpc_rdma_cleanup(void) {xprt_rdma_cleanup ();net/sunrpc/xprtrdma/svc_rdma_transport.c +1
-1@@ -440,7 +440,7 @@ static struct svc_xprt *svc_rdma_accept(struct svc_xprt *xprt) newxprt->sc_max_req_size = svcrdma_max_req_size; newxprt->sc_max_requests = svcrdma_max_requests; newxprt->sc_max_bc_requests = svcrdma_max_bc_requests; - newxprt->sc_recv_batch =RPCRDMA_MAX_RECV_BATCH ; + newxprt->sc_recv_batch = rpcrdma_max_recv_batch; newxprt->sc_fc_credits =cpu_to_be32 (newxprt->sc_max_requests);/* Qualify the transport's resource defaults with the net/sunrpc/xprtrdma/verbs.c +1
-1@@ -1359,7 +1359,7 @@ void rpcrdma_post_recvs(struct rpcrdma_xprt *r_xprt, int needed)if (likely (ep->re_receive_count > needed)) goto out; needed -= ep->re_receive_count; - needed +=RPCRDMA_MAX_RECV_BATCH ; + needed += rpcrdma_max_recv_batch;if (atomic_inc_return (&ep->re_receiving) >1 ) goto out;net/sunrpc/xprtrdma/xprt_rdma.h +1
-3@@ -216,9 +216,7 @@ struct rpcrdma_rep {* * Setting this to zero disables Receive post batching. */ -enum { -RPCRDMA_MAX_RECV_BATCH =7 , -}; +extern unsigned int rpcrdma_max_recv_batch;/* struct rpcrdma_sendctx - DMA mapped SGEs to unmap after Send completes */
Signature · 1 line
Chuck Lever