Fix ADDR_CHANGE event handling for NFSD
Subsystems (4)
[email protected]
maintainercover Fix ADDR_CHANGE event handling for NFSD
From: Chuck Lever <[email protected]>
Sagi recently pointed out that the CM ADDR_CHANGE event handler in svcrdma has a bug similar to one he fixed in the NVMe target. This series attempts to address that issue.
Chuck Lever (2):
svcrdma: Refactor the creation of listener CMA ID
svcrdma: Handle ADDR_CHANGE CM event properly
net/sunrpc/xprtrdma/svc_rdma_transport.c | 83 ++++++++++++++++-------- 1 file changed, 55 insertions(+), 28 deletions(-)
Signature · 1 line
2.45.1
[email protected]
maintainerPATCH 1/2 svcrdma: Refactor the creation of listener CMA ID
From: Chuck Lever <[email protected]>
In a moment, I will add a second consumer of CMA ID creation in svcrdma. Refactor so this code can be reused.
--- net/sunrpc/xprtrdma/svc_rdma_transport.c | 67 ++++++++++++++---------- 1 file changed, 40 insertions(+), 27 deletions(-)
net/sunrpc/xprtrdma/svc_rdma_transport.c +40 -27
@@ -65,6 +65,8 @@static struct svcxprt_rdma *svc_rdma_create_xprt (struct svc_serv *serv,struct net *net,int node); +static int svc_rdma_listen_handler (struct rdma_cm_id *cma_id, +struct rdma_cm_event *event); static struct svc_xprt *svc_rdma_create(struct svc_serv *serv, struct net *net, struct sockaddr *sa, int salen,
@@ -122,6 +124,41 @@ static void qp_event_handler(struct ib_event *event, void *context) } } +static struct rdma_cm_id * +svc_rdma_create_listen_id (struct net *net,struct sockaddr *sap, +void *context) +{ +struct rdma_cm_id *listen_id; +int ret; + + listen_id =rdma_create_id (net, svc_rdma_listen_handler, context, +RDMA_PS_TCP ,IB_QPT_RC ); +if (IS_ERR (listen_id)) +return listen_id; + +/* Allow both IPv4 and IPv6 sockets to bind a single port +* at the same time. +*/ +#if IS_ENABLED (CONFIG_IPV6 ) + ret =rdma_set_afonly (listen_id,1 ); +if (ret) + goto out_destroy; +#endif + ret =rdma_bind_addr (listen_id, sap); +if (ret) + goto out_destroy; + + ret =rdma_listen (listen_id,RPCRDMA_LISTEN_BACKLOG ); +if (ret) + goto out_destroy; + +return listen_id; + +out_destroy: +rdma_destroy_id (listen_id); +return ERR_PTR (ret); +} +static struct svcxprt_rdma *svc_rdma_create_xprt (struct svc_serv *serv,struct net *net,int node) {
@@ -307,7 +344,6 @@ static struct svc_xprt *svc_rdma_create(struct svc_serv *serv, {struct rdma_cm_id *listen_id;struct svcxprt_rdma *cma_xprt; -int ret;if (sa->sa_family !=AF_INET && sa->sa_family !=AF_INET6 )return ERR_PTR (-EAFNOSUPPORT );
@@ -317,30 +353,13 @@ static struct svc_xprt *svc_rdma_create(struct svc_serv *serv,set_bit (XPT_LISTENER , &cma_xprt->sc_xprt.xpt_flags);strcpy (cma_xprt->sc_xprt.xpt_remotebuf,"listener" ); - listen_id =rdma_create_id (net, svc_rdma_listen_handler, cma_xprt, -RDMA_PS_TCP ,IB_QPT_RC ); + listen_id =svc_rdma_create_listen_id (net, sa, cma_xprt);if (IS_ERR (listen_id)) { - ret =PTR_ERR (listen_id); - goto err0; +kfree (cma_xprt); +return (struct svc_xprt *)listen_id; } - -/* Allow both IPv4 and IPv6 sockets to bind a single port -* at the same time. -*/ -#if IS_ENABLED (CONFIG_IPV6 ) - ret =rdma_set_afonly (listen_id,1 ); -if (ret) - goto err1; -#endif - ret =rdma_bind_addr (listen_id, sa); -if (ret) - goto err1; cma_xprt->sc_cm_id = listen_id; - ret =rdma_listen (listen_id,RPCRDMA_LISTEN_BACKLOG ); -if (ret) - goto err1; -/* * We need to use the address from the cm_id in case the * caller specified 0 for the port number.
@@ -349,12 +368,6 @@ static struct svc_xprt *svc_rdma_create(struct svc_serv *serv,svc_xprt_set_local (&cma_xprt->sc_xprt, sa, salen);return &cma_xprt->sc_xprt; - - err1: -rdma_destroy_id (listen_id); - err0: -kfree (cma_xprt); -return ERR_PTR (ret); }/*
Signature · 1 line
2.45.1
Re: [PATCH 1/2] svcrdma: Refactor the creation of listener CMA ID
Quoted 28 lines · On 5/31/24 21:15, [email protected] wrote:
From: Chuck Lever <[email protected]>
In a moment, I will add a second consumer of CMA ID creation in svcrdma. Refactor so this code can be reused.
Signed-off-by Chuck Lever <[email protected]>--- net/sunrpc/xprtrdma/svc_rdma_transport.c | 67 ++++++++++++++---------- 1 file changed, 40 insertions(+), 27 deletions(-)net/sunrpc/xprtrdma/svc_rdma_transport.c +3
-0@@ -65,6 +65,8 @@static struct svcxprt_rdma *svc_rdma_create_xprt (struct svc_serv *serv,struct net *net,int node); +static int svc_rdma_listen_handler (struct rdma_cm_id *cma_id, +struct rdma_cm_event *event); static struct svc_xprt *svc_rdma_create(struct svc_serv *serv, struct net *net, struct sockaddr *sa, int salen,@@ -122,6 +124,41 @@ static void qp_event_handler(struct ib_event *event, void *context) } } +static struct rdma_cm_id *
net/sunrpc/xprtrdma/svc_rdma_transport.c static void qp_event_handler(struct ib_event *event, void *context) · line 137
Quoted 10 lines
net/sunrpc/xprtrdma/svc_rdma_transport.c +10
-0+svc_rdma_create_listen_id(struct net *net, struct sockaddr *sap, + void *context) +{ + struct rdma_cm_id *listen_id; +int ret; + + listen_id =rdma_create_id (net, svc_rdma_listen_handler, context, +RDMA_PS_TCP ,IB_QPT_RC ); +if (IS_ERR (listen_id)) +return listen_id;
I am wondering if above need to return PTR_ERR(listen_id), and I find some callers (in net/rds/, nvme etc) return PTR_ERR(id) while others (rtrs-srv, ib_isert.c) return ERR_PTR(ret) with ret is set to PTR_ERR(id).
Thanks,
Guoqing
Re: [PATCH 1/2] svcrdma: Refactor the creation of listener CMA ID
Quoted 33 lines · Guoqing Jiang
Quoted 30 lines · On 5/31/24 21:15, [email protected] wrote:
From: Chuck Lever <[email protected]>
In a moment, I will add a second consumer of CMA ID creation in svcrdma. Refactor so this code can be reused.
Signed-off-by Chuck Lever <[email protected]>--- net/sunrpc/xprtrdma/svc_rdma_transport.c | 67 ++++++++++++++---------- 1 file changed, 40 insertions(+), 27 deletions(-)net/sunrpc/xprtrdma/svc_rdma_transport.c +7
-0@@ -65,6 +65,8 @@static struct svcxprt_rdma *svc_rdma_create_xprt (struct svc_serv *serv,struct net *net,int node); +static int svc_rdma_listen_handler (struct rdma_cm_id *cma_id, +struct rdma_cm_event *event); static struct svc_xprt *svc_rdma_create(struct svc_serv *serv, struct net *net, struct sockaddr *sa, int salen,@@ -122,6 +124,41 @@ static void qp_event_handler(struct ib_event *event, void *context) } } +static struct rdma_cm_id * +svc_rdma_create_listen_id (struct net *net,struct sockaddr *sap, +void *context) +{ +struct rdma_cm_id *listen_id;
Quoted 8 lines
net/sunrpc/xprtrdma/svc_rdma_transport.c static void qp_event_handler(struct ib_event *event, void *context) · line 137
Quoted 6 lines
net/sunrpc/xprtrdma/svc_rdma_transport.c +6
-0+int ret; + + listen_id =rdma_create_id (net, svc_rdma_listen_handler, context, +RDMA_PS_TCP ,IB_QPT_RC ); +if (IS_ERR (listen_id)) +return listen_id;I am wondering if above need to return
PTR_ERR(listen_id),
PTR_ERR would convert the listen_id error to an integer, but svc_rdma_create_listen_id() returns a pointer or an ERR_PTR. Thus using PTR_ERR() would be wrong in this case.
Quoted 4 lines
and I find some
callers (innet/rds/, nvme etc)
returnPTR_ERR(id)while others (rtrs-srv,ib_isert.c) returnERR_PTR(ret)with ret is set toPTR_ERR(id).
These functions use PTR_ERR only when the calling function returns an int.
Signature · 1 line
Chuck Lever
Re: [PATCH 1/2] svcrdma: Refactor the creation of listener CMA ID
Quoted 45 lines · Chuck Lever
Quoted 31 lines · Guoqing Jiang
Quoted 29 lines · On 5/31/24 21:15, [email protected] wrote:
From: Chuck Lever <[email protected]>
In a moment, I will add a second consumer of CMA ID creation in svcrdma. Refactor so this code can be reused.
Signed-off-by Chuck Lever <[email protected]>--- net/sunrpc/xprtrdma/svc_rdma_transport.c | 67 ++++++++++++++---------- 1 file changed, 40 insertions(+), 27 deletions(-)net/sunrpc/xprtrdma/svc_rdma_transport.c +6
-0@@ -65,6 +65,8 @@static struct svcxprt_rdma *svc_rdma_create_xprt (struct svc_serv *serv,struct net *net,int node); +static int svc_rdma_listen_handler (struct rdma_cm_id *cma_id, +struct rdma_cm_event *event); static struct svc_xprt *svc_rdma_create(struct svc_serv *serv, struct net *net, struct sockaddr *sa, int salen,@@ -122,6 +124,41 @@ static void qp_event_handler(struct ib_event *event, void *context) } } +static struct rdma_cm_id * +svc_rdma_create_listen_id (struct net *net,struct sockaddr *sap, +void *context) +{Quoted 8 lines
net/sunrpc/xprtrdma/svc_rdma_transport.c static void qp_event_handler(struct ib_event *event, void *context) · line 137
Quoted 7 lines
net/sunrpc/xprtrdma/svc_rdma_transport.c +7
-0+struct rdma_cm_id *listen_id; +int ret; + + listen_id =rdma_create_id (net, svc_rdma_listen_handler, context, +RDMA_PS_TCP ,IB_QPT_RC ); +if (IS_ERR (listen_id)) +return listen_id;I am wondering if above need to return
PTR_ERR(listen_id),
PTR_ERRwould convert thelisten_iderror to an integer, butsvc_rdma_create_listen_id()returns a pointer or anERR_PTR. Thus usingPTR_ERR()would be wrong in this case.
Quoted 6 lines
Quoted 4 lines
and I find some
callers (innet/rds/, nvme etc)
returnPTR_ERR(id)while others (rtrs-srv,ib_isert.c) returnERR_PTR(ret)with ret is set toPTR_ERR(id).These functions use
PTR_ERRonly when the calling function returns an int.
Thanks for the explanation!
Guoqing
[email protected]
maintainerPATCH 2/2 svcrdma: Handle ADDR_CHANGE CM event properly
In reply to [email protected]From: Chuck Lever <[email protected]>
Sagi tells me that when a bonded device reports an address change, the consumer must destroy its listener IDs and create new ones.
See commit a032e4f6d60d ("nvmet-rdma: fix bonding failover possible NULL deref").
--- net/sunrpc/xprtrdma/svc_rdma_transport.c | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-)
net/sunrpc/xprtrdma/svc_rdma_transport.c +15 -1
@@ -284,17 +284,31 @@ static void handle_connect_req(struct rdma_cm_id *new_cma_id,* * Return values: * %0: Do not destroy @cma_id -* %1: Destroy @cma_id (never returned here) +* %1: Destroy @cma_id * * NB: There is never a DEVICE_REMOVAL event for INADDR_ANY listeners. */ static int svc_rdma_listen_handler (struct rdma_cm_id *cma_id,struct rdma_cm_event *event) { +struct svcxprt_rdma *cma_xprt = cma_id->context; +struct svc_xprt *cma_rdma = &cma_xprt->sc_xprt; +struct sockaddr *sap = (struct sockaddr *)&cma_id->route.addr.src_addr; +struct rdma_cm_id *listen_id; +switch (event->event) {case RDMA_CM_EVENT_CONNECT_REQUEST :handle_connect_req (cma_id, &event->param.conn);break ; +case RDMA_CM_EVENT_ADDR_CHANGE : + listen_id =svc_rdma_create_listen_id (cma_rdma->xpt_net, + sap, cma_xprt); +if (IS_ERR (listen_id)) { +pr_err ("Listener dead, address change failed for device %s\n" , + cma_id->device->name); + }else + cma_xprt->sc_cm_id = listen_id; +return 1 ;default :break ; }
Signature · 1 line
2.45.1
Re: [PATCH 2/2] svcrdma: Handle ADDR_CHANGE CM event properly
Quoted 24 lines · On 31.05.24 15:15, [email protected] wrote:
From: Chuck Lever <[email protected]>
Sagi tells me that when a bonded device reports an address change, the consumer must destroy its listener IDs and create new ones.
See commit
a032e4f6d60d("nvmet-rdma: fix bonding failover possible NULL deref").Suggested-by Sagi Grimberg <[email protected]>Signed-off-by Chuck Lever <[email protected]>--- net/sunrpc/xprtrdma/svc_rdma_transport.c | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-)net/sunrpc/xprtrdma/svc_rdma_transport.c +1
-1@@ -284,17 +284,31 @@ static void handle_connect_req(struct rdma_cm_id *new_cma_id,* * Return values: * %0: Do not destroy @cma_id -* %1: Destroy @cma_id (never returned here) +* %1: Destroy @cma_id
net/sunrpc/xprtrdma/svc_rdma_transport.c static void handle_connect_req(struct rdma_cm_id *new_cma_id, · line 297
Quoted 10 lines
net/sunrpc/xprtrdma/svc_rdma_transport.c +4
-0* * NB: There is never a DEVICE_REMOVAL event for INADDR_ANY listeners. */ static int svc_rdma_listen_handler (struct rdma_cm_id *cma_id,struct rdma_cm_event *event) { +struct svcxprt_rdma *cma_xprt = cma_id->context; +struct svc_xprt *cma_rdma = &cma_xprt->sc_xprt; +struct sockaddr *sap = (struct sockaddr *)&cma_id->route.addr.src_addr; +struct rdma_cm_id *listen_id;
I am not sure whether I need to suggest "Reverse Christmas Tree" or not. This is a trivial problem. ^_^
You can ignore it. And it is not mandatory.
But if we follow this rule, the source code will become more readable.
Zhu Yanjun
Quoted 17 lines
net/sunrpc/xprtrdma/svc_rdma_transport.c +10
-0+switch (event->event) {case RDMA_CM_EVENT_CONNECT_REQUEST :handle_connect_req (cma_id, &event->param.conn);break ; +case RDMA_CM_EVENT_ADDR_CHANGE : + listen_id =svc_rdma_create_listen_id (cma_rdma->xpt_net, + sap, cma_xprt); +if (IS_ERR (listen_id)) { +pr_err ("Listener dead, address change failed for device %s\n" , + cma_id->device->name); + }else + cma_xprt->sc_cm_id = listen_id; +return 1 ;default :break ; }
Re: [PATCH 2/2] svcrdma: Handle ADDR_CHANGE CM event properly
Quoted 31 lines · Zhu Yanjun
Quoted 30 lines · On 31.05.24 15:15, [email protected] wrote:
From: Chuck Lever <[email protected]>
Sagi tells me that when a bonded device reports an address change, the consumer must destroy its listener IDs and create new ones.
See commit
a032e4f6d60d("nvmet-rdma: fix bonding failover possible NULL deref").Suggested-by Sagi Grimberg <[email protected]>Signed-off-by Chuck Lever <[email protected]>--- net/sunrpc/xprtrdma/svc_rdma_transport.c | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-)net/sunrpc/xprtrdma/svc_rdma_transport.c +1
-1@@ -284,17 +284,31 @@ static void handle_connect_req(struct rdma_cm_id *new_cma_id,* * Return values: * %0: Do not destroy @cma_id -* %1: Destroy @cma_id (never returned here) +* %1: Destroy @cma_id * * NB: There is never a DEVICE_REMOVAL event for INADDR_ANY listeners. */ static int svc_rdma_listen_handler (struct rdma_cm_id *cma_id,struct rdma_cm_event *event) {
Quoted 8 lines
net/sunrpc/xprtrdma/svc_rdma_transport.c static void handle_connect_req(struct rdma_cm_id *new_cma_id, · line 297
Quoted 4 lines
net/sunrpc/xprtrdma/svc_rdma_transport.c +4
-0+struct svcxprt_rdma *cma_xprt = cma_id->context; +struct svc_xprt *cma_rdma = &cma_xprt->sc_xprt; +struct sockaddr *sap = (struct sockaddr *)&cma_id->route.addr.src_addr; +struct rdma_cm_id *listen_id;I am not sure whether I need to suggest "Reverse Christmas Tree" or not. This is a trivial problem. ^_^
You can ignore it. And it is not mandatory.
Oops, I missed that @sap can be declared first. It must have gotten lost while I was trying out different solutions for this issue.
Fixed.
Quoted 22 lines
But if we follow this rule, the source code will become more readable.
Zhu Yanjun
Quoted 17 lines
net/sunrpc/xprtrdma/svc_rdma_transport.c +10
-0+switch (event->event) {case RDMA_CM_EVENT_CONNECT_REQUEST :handle_connect_req (cma_id, &event->param.conn);break ; +case RDMA_CM_EVENT_ADDR_CHANGE : + listen_id =svc_rdma_create_listen_id (cma_rdma->xpt_net, + sap, cma_xprt); +if (IS_ERR (listen_id)) { +pr_err ("Listener dead, address change failed for device %s\n" , + cma_id->device->name); + }else + cma_xprt->sc_cm_id = listen_id; +return 1 ;default :break ; }
Signature · 1 line
Chuck Lever
Quoted 6 lines · On 31/05/2024 16:15, [email protected] wrote:
From: Chuck Lever <[email protected]>
Sagi recently pointed out that the CM
ADDR_CHANGEevent handler in svcrdma has a bug similar to one he fixed in the NVMe target. This series attempts to address that issue.
Quoted 7 lines
Chuck Lever (2):
svcrdma: Refactor the creation of listener CMA ID
svcrdma: HandleADDR_CHANGECM event properlynet/sunrpc/xprtrdma/svc_rdma_transport.c | 83 ++++++++++++++++-------- 1 file changed, 55 insertions(+), 28 deletions(-)
Looks good, For the series:
Chuck Lever III
AppliedQuoted 17 lines · Sagi Grimberg
Quoted 10 lines · On 31/05/2024 16:15, [email protected] wrote:
From: Chuck Lever <[email protected]>
Sagi recently pointed out that the CM
ADDR_CHANGEevent handler in svcrdma has a bug similar to one he fixed in the NVMe target. This series attempts to address that issue.Chuck Lever (2):
svcrdma: Refactor the creation of listener CMA ID
svcrdma: HandleADDR_CHANGECM event properly
Quoted 7 lines
Quoted 3 lines
net/sunrpc/xprtrdma/svc_rdma_transport.c | 83 ++++++++++++++++-------- 1 file changed, 55 insertions(+), 28 deletions(-)Looks good, For the series:
Reviewed-by Sagi Grimberg <[email protected]>
Thank you! Applied to nfsd-next (for 6.11).
--
Chuck Lever