NFS/RDMA Connectathon fixes
Subsystems (3)
cover NFS/RDMA Connectathon fixes
The following patch series are the fixes from Connectathon. With the first two, we achieved full interop between Linux and OpenSolaris, over Infiniband, in both directions. (Linux-to-Linux was also tested successfully with both Infiniband and iWARP.)
The first patch merely removes a bogus assertion in the server rdma chunk manipulation. This was triggered by the slightly different way that the OpenSolaris client marshals certain requests.
The second patch fixes a defect in the client chunk marshaling, which caused a bad XDR encode in (only) NFSv4 symlink requests.
The third and fourth patches are a usability improvement, to dynamically load the xprtrdma client module on-demand when mounting nfs over rdma.
I'm hoping they'll all be acceptable for the 2.6.30 merge window.
Comments are welcome, etc.
---
Tom Talpey (4):
NFS: load the rpc/rdma transport module automatically
SUNRPC: dynamically load RPC transport modules on-demand
XPRTRDMA: correct an rpc/rdma inline send marshaling error
SVCRDMA: remove faulty assertions in rpc/rdma chunk validation.
fs/nfs/super.c | 4 +++- include/linux/sunrpc/xprt.h | 1 + net/sunrpc/xprt.c | 31 +++++++++++++++++++++++++++++++ net/sunrpc/xprtrdma/rpc_rdma.c | 26 ++++++++++++++------------ net/sunrpc/xprtrdma/svc_rdma_sendto.c | 8 +++++--- 5 files changed, 54 insertions(+), 16 deletions(-)
Signature · 1 line
Tom.
A message that is not in the archive (sent off-list or not yet received): 20090305210738.6517.67862.stgit-bi+AKbBUZKY6gyzm1THtWbp2dZbC/[email protected]
PATCH 1/4 SVCRDMA: remove faulty assertions in rpc/rdma chunk validation.
Certain client-provided RPCRDMA chunk alignments result in an additional scatter/gather entry, which triggered nfs/rdma server assertions incorrectly. OpenSolaris nfs/rdma client connectathon testing was blocked by these in the special/locking section.
---
net/sunrpc/xprtrdma/svc_rdma_sendto.c | 8 +++++--- 1 files changed, 5 insertions(+), 3 deletions(-)
net/sunrpc/xprtrdma/svc_rdma_sendto.c +5 -3
@@ -191,7 +191,6 @@ static int map_xdr(struct svcxprt_rdma *xprt,struct xdr_buf *xdr, struct svc_rdma_req_map *vec) { - int sge_max = (xdr->len+PAGE_SIZE-1) / PAGE_SIZE + 3; int sge_no;u32 sge_bytes;u32 page_bytes;
@@ -235,7 +234,11 @@ static int map_xdr(struct svcxprt_rdma *xprt, sge_no++; } -BUG_ON (sge_no > sge_max); +dprintk ("svcrdma: map_xdr: sge_no %d page_no %d " +"page_base %zd page_len %zd head_len %d tail_len %d\n" , + sge_no, page_no, xdr->page_base, xdr->page_len, + xdr->head[0 ].iov_len, xdr->tail[0 ].iov_len); + vec->count = sge_no;return 0 ; }
@@ -579,7 +582,6 @@ static int send_reply(struct svcxprt_rdma *rdma, ctxt->sge[page_no+1 ].length =0 ; }BUG_ON (sge_no > rdma->sc_max_sge); -BUG_ON (sge_no > ctxt->count);memset (&send_wr,0 ,sizeof send_wr); ctxt->wr_op =IB_WR_SEND ; send_wr.wr_id = (unsigned long )ctxt;
PATCH 2/4 XPRTRDMA: correct an rpc/rdma inline send marshaling error
In reply to a message not in the archiveCertain client rpc's which contain both lengthy page-contained metadata and a non-empty xdr_tail buffer require careful handling to avoid overlapped memory copying. Rearranging of existing rpcrdma marshaling code avoids it; this fixes an NFSv4 symlink creation error detected with connectathon basic/test8 to multiple servers.
---
net/sunrpc/xprtrdma/rpc_rdma.c | 26 ++++++++++++++------------ 1 files changed, 14 insertions(+), 12 deletions(-)
net/sunrpc/xprtrdma/rpc_rdma.c +14 -12
@@ -310,6 +310,19 @@ rpcrdma_inline_pullup(struct rpc_rqst *rqst, int pad) __func__, pad, destp, rqst->rq_slen, curlen); copy_len = rqst->rq_snd_buf.page_len; + +if (rqst->rq_snd_buf.tail[0 ].iov_len) { + curlen = rqst->rq_snd_buf.tail[0 ].iov_len; +if (destp + copy_len != rqst->rq_snd_buf.tail[0 ].iov_base) { +memmove (destp + copy_len, + rqst->rq_snd_buf.tail[0 ].iov_base, curlen); + r_xprt->rx_stats.pullup_copy_count += curlen; + } +dprintk ("RPC: %s: tail destp 0x%p len %d\n" , + __func__, destp + copy_len, curlen); + rqst->rq_svec[0 ].iov_len += curlen; + } + r_xprt->rx_stats.pullup_copy_count += copy_len; npages =PAGE_ALIGN (rqst->rq_snd_buf.page_base+copy_len) >>PAGE_SHIFT ;for (i =0 ; copy_len && i < npages; i++) {
@@ -332,17 +345,6 @@ rpcrdma_inline_pullup(struct rpc_rqst *rqst, int pad) destp += curlen; copy_len -= curlen; } -if (rqst->rq_snd_buf.tail[0 ].iov_len) { - curlen = rqst->rq_snd_buf.tail[0 ].iov_len; -if (destp != rqst->rq_snd_buf.tail[0 ].iov_base) { -memcpy (destp, - rqst->rq_snd_buf.tail[0 ].iov_base, curlen); - r_xprt->rx_stats.pullup_copy_count += curlen; - } -dprintk ("RPC: %s: tail destp 0x%p len %d curlen %d\n" , - __func__, destp, copy_len, curlen); - rqst->rq_svec[0 ].iov_len += curlen; - }/* header now contains entire send message */ return pad; }
@@ -656,7 +658,7 @@ rpcrdma_inline_fixup(struct rpc_rqst *rqst, char *srcp, int copy_len, int pad)if (curlen > rqst->rq_rcv_buf.tail[0 ].iov_len) curlen = rqst->rq_rcv_buf.tail[0 ].iov_len;if (rqst->rq_rcv_buf.tail[0 ].iov_base != srcp) -memcpy (rqst->rq_rcv_buf.tail[0 ].iov_base, srcp, curlen); +memmove (rqst->rq_rcv_buf.tail[0 ].iov_base, srcp, curlen);dprintk ("RPC: %s: tail srcp 0x%p len %d curlen %d\n" , __func__, srcp, copy_len, curlen); rqst->rq_rcv_buf.tail[0 ].iov_len = curlen;
PATCH 3/4 SUNRPC: dynamically load RPC transport modules on-demand
In reply to a message not in the archiveProvide an api to attempt to load any necessary kernel RPC client transport module automatically. By convention, the desired module name is "xprt"+"transport name". For example, when NFS mounting with "-o proto=rdma", attempt to load the "xprtrdma" module.
---
include/linux/sunrpc/xprt.h | 1 + net/sunrpc/xprt.c | 31 +++++++++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 0 deletions(-)
include/linux/sunrpc/xprt.h +1 -0
@@ -235,6 +235,7 @@ static inline __be32 *xprt_skip_transport_header(struct rpc_xprt *xprt, __be32 **/ int xprt_register_transport (struct xprt_class *type);int xprt_unregister_transport (struct xprt_class *type); +int xprt_load_transport (const char *);void xprt_set_retrans_timeout_def (struct rpc_task *task);void xprt_set_retrans_timeout_rtt (struct rpc_task *task);void xprt_wake_pending_tasks (struct rpc_xprt *xprt,int status);
net/sunrpc/xprt.c +31 -0
@@ -152,6 +152,37 @@ out:EXPORT_SYMBOL_GPL (xprt_unregister_transport);/** +* xprt_load_transport - load a transport implementation +* @transport_name: transport to load +* +* Returns: +* 0: transport successfully loaded +* -ENOENT: transport module not available +*/ +int xprt_load_transport (const char *transport_name) +{ +struct xprt_class *t; +char module_name[sizeof t->name +5 ]; +int result; + + result =0 ; +spin_lock (&xprt_list_lock); +list_for_each_entry (t, &xprt_list, list) { +if (strcmp (t->name, transport_name) ==0 ) { +spin_unlock (&xprt_list_lock); + goto out; + } + } +spin_unlock (&xprt_list_lock); +strcpy (module_name,"xprt" ); +strncat (module_name, transport_name,sizeof t->name); + result =request_module (module_name); +out: +return result; +} +EXPORT_SYMBOL_GPL (xprt_load_transport); + +/** * xprt_reserve_xprt - serialize write access to transports * @task: task that is requesting access to the transport *
PATCH 4/4 NFS: load the rpc/rdma transport module automatically
In reply to a message not in the archiveWhen mounting an NFS/RDMA server with the "-o proto=rdma" or "-o rdma" options, attempt to dynamically load the necessary "xprtrdma" client transport module. Doing so improves usability, while avoiding a static module dependency and any unnecesary resources.
---
fs/nfs/super.c | 4 +++- 1 files changed, 3 insertions(+), 1 deletions(-)
fs/nfs/super.c +3 -1
@@ -1018,6 +1018,7 @@ static int nfs_parse_mount_options(char *raw,case Opt_rdma: mnt->flags |=NFS_MOUNT_TCP ;/* for side protocols */ mnt->nfs_server.protocol =XPRT_TRANSPORT_RDMA ; +xprt_load_transport (p);break ;case Opt_acl: mnt->flags &= ~NFS_MOUNT_NOACL ;
@@ -1190,7 +1191,6 @@ static int nfs_parse_mount_options(char *raw, goto out_nomem; token =match_token (string, nfs_xprt_protocol_tokens, args); -kfree (string);switch (token) {case Opt_xprt_udp:
@@ -1205,12 +1205,14 @@ static int nfs_parse_mount_options(char *raw,/* vector side protocols to TCP */ mnt->flags |=NFS_MOUNT_TCP ; mnt->nfs_server.protocol =XPRT_TRANSPORT_RDMA ; +xprt_load_transport (string);break ;default : errors++;dfprintk (MOUNT ,"NFS: unrecognized " "transport protocol\n" ); } +kfree (string);break ;case Opt_mountproto: string =match_strdup (args);