Skip to contentSkip to conversation

Fix ADDR_CHANGE event handling for NFSD

PATCH Applied [email protected] 5 participants 10 messages May 31, 2024 – Jun 3, 2024

Subsystems (4)
lore
Share & tools
lore

[email protected]

author maintainer
#0 lore raw

cover Fix ADDR_CHANGE event handling for NFSD

From: Chuck Lever <[email protected]>

Sagi recently pointed out that the CM ADDR_CHANGE event handler in svcrdma has a bug similar to one he fixed in the NVMe target. This series attempts to address that issue.

Chuck Lever (2):
svcrdma: Refactor the creation of listener CMA ID
svcrdma: Handle ADDR_CHANGE CM event properly

 net/sunrpc/xprtrdma/svc_rdma_transport.c | 83 ++++++++++++++++--------
 1 file changed, 55 insertions(+), 28 deletions(-)
Signature · 1 line
2.45.1
author maintainer
#1 lore raw

PATCH 1/2 svcrdma: Refactor the creation of listener CMA ID

From: Chuck Lever <[email protected]>

In a moment, I will add a second consumer of CMA ID creation in svcrdma. Refactor so this code can be reused.

---
 net/sunrpc/xprtrdma/svc_rdma_transport.c | 67 ++++++++++++++----------
 1 file changed, 40 insertions(+), 27 deletions(-)
net/sunrpc/xprtrdma/svc_rdma_transport.c +40 -27
index 2b1c16b9547d..fa50b7494a0a 100644
@@ -65,6 +65,8 @@
 
 static struct svcxprt_rdma *svc_rdma_create_xprt(struct svc_serv *serv,
 						 struct net *net, int node);
+static int svc_rdma_listen_handler(struct rdma_cm_id *cma_id,
+				   struct rdma_cm_event *event);
 static struct svc_xprt *svc_rdma_create(struct svc_serv *serv,
 					struct net *net,
 					struct sockaddr *sa, int salen,
@@ -122,6 +124,41 @@ static void qp_event_handler(struct ib_event *event, void *context)
 	}
 }
 
+static struct rdma_cm_id *
+svc_rdma_create_listen_id(struct net *net, struct sockaddr *sap,
+			  void *context)
+{
+	struct rdma_cm_id *listen_id;
+	int ret;
+
+	listen_id = rdma_create_id(net, svc_rdma_listen_handler, context,
+				   RDMA_PS_TCP, IB_QPT_RC);
+	if (IS_ERR(listen_id))
+		return listen_id;
+
+	/* Allow both IPv4 and IPv6 sockets to bind a single port
+	 * at the same time.
+	 */
+#if IS_ENABLED(CONFIG_IPV6)
+	ret = rdma_set_afonly(listen_id, 1);
+	if (ret)
+		goto out_destroy;
+#endif
+	ret = rdma_bind_addr(listen_id, sap);
+	if (ret)
+		goto out_destroy;
+
+	ret = rdma_listen(listen_id, RPCRDMA_LISTEN_BACKLOG);
+	if (ret)
+		goto out_destroy;
+
+	return listen_id;
+
+out_destroy:
+	rdma_destroy_id(listen_id);
+	return ERR_PTR(ret);
+}
+
 static struct svcxprt_rdma *svc_rdma_create_xprt(struct svc_serv *serv,
 						 struct net *net, int node)
 {
@@ -307,7 +344,6 @@ static struct svc_xprt *svc_rdma_create(struct svc_serv *serv,
 {
 	struct rdma_cm_id *listen_id;
 	struct svcxprt_rdma *cma_xprt;
-	int ret;
 
 	if (sa->sa_family != AF_INET && sa->sa_family != AF_INET6)
 		return ERR_PTR(-EAFNOSUPPORT);
@@ -317,30 +353,13 @@ static struct svc_xprt *svc_rdma_create(struct svc_serv *serv,
 	set_bit(XPT_LISTENER, &cma_xprt->sc_xprt.xpt_flags);
 	strcpy(cma_xprt->sc_xprt.xpt_remotebuf, "listener");
 
-	listen_id = rdma_create_id(net, svc_rdma_listen_handler, cma_xprt,
-				   RDMA_PS_TCP, IB_QPT_RC);
+	listen_id = svc_rdma_create_listen_id(net, sa, cma_xprt);
 	if (IS_ERR(listen_id)) {
-		ret = PTR_ERR(listen_id);
-		goto err0;
+		kfree(cma_xprt);
+		return (struct svc_xprt *)listen_id;
 	}
-
-	/* Allow both IPv4 and IPv6 sockets to bind a single port
-	 * at the same time.
-	 */
-#if IS_ENABLED(CONFIG_IPV6)
-	ret = rdma_set_afonly(listen_id, 1);
-	if (ret)
-		goto err1;
-#endif
-	ret = rdma_bind_addr(listen_id, sa);
-	if (ret)
-		goto err1;
 	cma_xprt->sc_cm_id = listen_id;
 
-	ret = rdma_listen(listen_id, RPCRDMA_LISTEN_BACKLOG);
-	if (ret)
-		goto err1;
-
 	/*
 	 * We need to use the address from the cm_id in case the
 	 * caller specified 0 for the port number.
@@ -349,12 +368,6 @@ static struct svc_xprt *svc_rdma_create(struct svc_serv *serv,
 	svc_xprt_set_local(&cma_xprt->sc_xprt, sa, salen);
 
 	return &cma_xprt->sc_xprt;
-
- err1:
-	rdma_destroy_id(listen_id);
- err0:
-	kfree(cma_xprt);
-	return ERR_PTR(ret);
 }
 
 /*
Signature · 1 line
2.45.1
#2 lore raw

Re: [PATCH 1/2] svcrdma: Refactor the creation of listener CMA ID

Quoted 28 lines · On 5/31/24 21:15, [email protected] wrote:

From: Chuck Lever <[email protected]>

In a moment, I will add a second consumer of CMA ID creation in svcrdma. Refactor so this code can be reused.

---
  net/sunrpc/xprtrdma/svc_rdma_transport.c | 67 ++++++++++++++----------
  1 file changed, 40 insertions(+), 27 deletions(-)
net/sunrpc/xprtrdma/svc_rdma_transport.c +3 -0
index 2b1c16b9547d..fa50b7494a0a 100644
@@ -65,6 +65,8 @@
 
  static struct svcxprt_rdma *svc_rdma_create_xprt(struct svc_serv *serv,
  						 struct net *net, int node);
+static int svc_rdma_listen_handler(struct rdma_cm_id *cma_id,
+				   struct rdma_cm_event *event);
  static struct svc_xprt *svc_rdma_create(struct svc_serv *serv,
  					struct net *net,
  					struct sockaddr *sa, int salen,
@@ -122,6 +124,41 @@ static void qp_event_handler(struct ib_event *event, void *context)
  	}
  }
 
+static struct rdma_cm_id *

net/sunrpc/xprtrdma/svc_rdma_transport.c static void qp_event_handler(struct ib_event *event, void *context) · line 137

Quoted 10 lines
net/sunrpc/xprtrdma/svc_rdma_transport.c +10 -0
+svc_rdma_create_listen_id(struct net *net, struct sockaddr *sap,
+			  void *context)
+{
+	struct rdma_cm_id *listen_id;
+	int ret;
+
+	listen_id = rdma_create_id(net, svc_rdma_listen_handler, context,
+				   RDMA_PS_TCP, IB_QPT_RC);
+	if (IS_ERR(listen_id))
+		return listen_id;

I am wondering if above need to return PTR_ERR(listen_id), and I find some callers (in net/rds/, nvme etc) return PTR_ERR(id) while others (rtrs-srv, ib_isert.c) return ERR_PTR(ret) with ret is set to PTR_ERR(id).

Thanks,
Guoqing

#3 lore raw

Re: [PATCH 1/2] svcrdma: Refactor the creation of listener CMA ID

Quoted 33 lines · Guoqing Jiang
Quoted 30 lines · On 5/31/24 21:15, [email protected] wrote:

From: Chuck Lever <[email protected]>

In a moment, I will add a second consumer of CMA ID creation in svcrdma. Refactor so this code can be reused.

---
  net/sunrpc/xprtrdma/svc_rdma_transport.c | 67 ++++++++++++++----------
  1 file changed, 40 insertions(+), 27 deletions(-)
net/sunrpc/xprtrdma/svc_rdma_transport.c +7 -0
index 2b1c16b9547d..fa50b7494a0a 100644
@@ -65,6 +65,8 @@
  static struct svcxprt_rdma *svc_rdma_create_xprt(struct svc_serv *serv,
  						 struct net *net, int node);
+static int svc_rdma_listen_handler(struct rdma_cm_id *cma_id,
+				   struct rdma_cm_event *event);
  static struct svc_xprt *svc_rdma_create(struct svc_serv *serv,
  					struct net *net,
  					struct sockaddr *sa, int salen,
@@ -122,6 +124,41 @@ static void qp_event_handler(struct ib_event *event, void *context)
  	}
  }
+static struct rdma_cm_id *
+svc_rdma_create_listen_id(struct net *net, struct sockaddr *sap,
+			  void *context)
+{
+	struct rdma_cm_id *listen_id;
Quoted 8 lines

net/sunrpc/xprtrdma/svc_rdma_transport.c static void qp_event_handler(struct ib_event *event, void *context) · line 137

Quoted 6 lines
net/sunrpc/xprtrdma/svc_rdma_transport.c +6 -0
+	int ret;
+
+	listen_id = rdma_create_id(net, svc_rdma_listen_handler, context,
+				   RDMA_PS_TCP, IB_QPT_RC);
+	if (IS_ERR(listen_id))
+		return listen_id;

I am wondering if above need to return PTR_ERR(listen_id),

PTR_ERR would convert the listen_id error to an integer, but svc_rdma_create_listen_id() returns a pointer or an ERR_PTR. Thus using PTR_ERR() would be wrong in this case.

Quoted 4 lines

and I find some
callers (in net/rds/, nvme etc)
return PTR_ERR(id) while others (rtrs-srv, ib_isert.c) return ERR_PTR(ret) with ret is set to PTR_ERR(id).

These functions use PTR_ERR only when the calling function returns an int.

Signature · 1 line
Chuck Lever
#4 lore raw

Re: [PATCH 1/2] svcrdma: Refactor the creation of listener CMA ID

Quoted 45 lines · Chuck Lever
Quoted 31 lines · Guoqing Jiang
Quoted 29 lines · On 5/31/24 21:15, [email protected] wrote:

From: Chuck Lever <[email protected]>

In a moment, I will add a second consumer of CMA ID creation in svcrdma. Refactor so this code can be reused.

---
   net/sunrpc/xprtrdma/svc_rdma_transport.c | 67 ++++++++++++++----------
   1 file changed, 40 insertions(+), 27 deletions(-)
net/sunrpc/xprtrdma/svc_rdma_transport.c +6 -0
index 2b1c16b9547d..fa50b7494a0a 100644
@@ -65,6 +65,8 @@
   static struct svcxprt_rdma *svc_rdma_create_xprt(struct svc_serv *serv,
   						 struct net *net, int node);
+static int svc_rdma_listen_handler(struct rdma_cm_id *cma_id,
+				   struct rdma_cm_event *event);
   static struct svc_xprt *svc_rdma_create(struct svc_serv *serv,
   					struct net *net,
   					struct sockaddr *sa, int salen,
@@ -122,6 +124,41 @@ static void qp_event_handler(struct ib_event *event, void *context)
   	}
   }
+static struct rdma_cm_id *
+svc_rdma_create_listen_id(struct net *net, struct sockaddr *sap,
+			  void *context)
+{
Quoted 8 lines

net/sunrpc/xprtrdma/svc_rdma_transport.c static void qp_event_handler(struct ib_event *event, void *context) · line 137

Quoted 7 lines
net/sunrpc/xprtrdma/svc_rdma_transport.c +7 -0
+	struct rdma_cm_id *listen_id;
+	int ret;
+
+	listen_id = rdma_create_id(net, svc_rdma_listen_handler, context,
+				   RDMA_PS_TCP, IB_QPT_RC);
+	if (IS_ERR(listen_id))
+		return listen_id;

I am wondering if above need to return PTR_ERR(listen_id),

PTR_ERR would convert the listen_id error to an integer, but svc_rdma_create_listen_id() returns a pointer or an ERR_PTR. Thus using PTR_ERR() would be wrong in this case.

Quoted 6 lines
Quoted 4 lines

and I find some
callers (in net/rds/, nvme etc)
return PTR_ERR(id) while others (rtrs-srv, ib_isert.c) return ERR_PTR(ret) with ret is set to PTR_ERR(id).

These functions use PTR_ERR only when the calling function returns an int.

Thanks for the explanation!

Guoqing

author maintainer
#5 lore raw

PATCH 2/2 svcrdma: Handle ADDR_CHANGE CM event properly

In reply to [email protected]

From: Chuck Lever <[email protected]>

Sagi tells me that when a bonded device reports an address change, the consumer must destroy its listener IDs and create new ones.

See commit a032e4f6d60d ("nvmet-rdma: fix bonding failover possible NULL deref").

---
 net/sunrpc/xprtrdma/svc_rdma_transport.c | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)
net/sunrpc/xprtrdma/svc_rdma_transport.c +15 -1
index fa50b7494a0a..a003b62fb7d5 100644
@@ -284,17 +284,31 @@ static void handle_connect_req(struct rdma_cm_id *new_cma_id,
  *
  * Return values:
  *     %0: Do not destroy @cma_id
- *     %1: Destroy @cma_id (never returned here)
+ *     %1: Destroy @cma_id
  *
  * NB: There is never a DEVICE_REMOVAL event for INADDR_ANY listeners.
  */
 static int svc_rdma_listen_handler(struct rdma_cm_id *cma_id,
 				   struct rdma_cm_event *event)
 {
+	struct svcxprt_rdma *cma_xprt = cma_id->context;
+	struct svc_xprt *cma_rdma = &cma_xprt->sc_xprt;
+	struct sockaddr *sap = (struct sockaddr *)&cma_id->route.addr.src_addr;
+	struct rdma_cm_id *listen_id;
+
 	switch (event->event) {
 	case RDMA_CM_EVENT_CONNECT_REQUEST:
 		handle_connect_req(cma_id, &event->param.conn);
 		break;
+	case RDMA_CM_EVENT_ADDR_CHANGE:
+		listen_id = svc_rdma_create_listen_id(cma_rdma->xpt_net,
+						      sap, cma_xprt);
+		if (IS_ERR(listen_id)) {
+			pr_err("Listener dead, address change failed for device %s\n",
+				cma_id->device->name);
+		} else
+			cma_xprt->sc_cm_id = listen_id;
+		return 1;
 	default:
 		break;
 	}
Signature · 1 line
2.45.1
#6 lore raw

Re: [PATCH 2/2] svcrdma: Handle ADDR_CHANGE CM event properly

Quoted 24 lines · On 31.05.24 15:15, [email protected] wrote:

From: Chuck Lever <[email protected]>

Sagi tells me that when a bonded device reports an address change, the consumer must destroy its listener IDs and create new ones.

See commit a032e4f6d60d ("nvmet-rdma: fix bonding failover possible NULL deref").

---
  net/sunrpc/xprtrdma/svc_rdma_transport.c | 16 +++++++++++++++-
  1 file changed, 15 insertions(+), 1 deletion(-)
net/sunrpc/xprtrdma/svc_rdma_transport.c +1 -1
index fa50b7494a0a..a003b62fb7d5 100644
@@ -284,17 +284,31 @@ static void handle_connect_req(struct rdma_cm_id *new_cma_id,
   *
   * Return values:
   *     %0: Do not destroy @cma_id
- *     %1: Destroy @cma_id (never returned here)
+ *     %1: Destroy @cma_id

net/sunrpc/xprtrdma/svc_rdma_transport.c static void handle_connect_req(struct rdma_cm_id *new_cma_id, · line 297

Quoted 10 lines
net/sunrpc/xprtrdma/svc_rdma_transport.c +4 -0
   *
   * NB: There is never a DEVICE_REMOVAL event for INADDR_ANY listeners.
   */
  static int svc_rdma_listen_handler(struct rdma_cm_id *cma_id,
  				   struct rdma_cm_event *event)
  {
+	struct svcxprt_rdma *cma_xprt = cma_id->context;
+	struct svc_xprt *cma_rdma = &cma_xprt->sc_xprt;
+	struct sockaddr *sap = (struct sockaddr *)&cma_id->route.addr.src_addr;
+	struct rdma_cm_id *listen_id;

I am not sure whether I need to suggest "Reverse Christmas Tree" or not. This is a trivial problem. ^_^
You can ignore it. And it is not mandatory.

But if we follow this rule, the source code will become more readable.

Zhu Yanjun

Quoted 17 lines
net/sunrpc/xprtrdma/svc_rdma_transport.c +10 -0
+
  	switch (event->event) {
  	case RDMA_CM_EVENT_CONNECT_REQUEST:
  		handle_connect_req(cma_id, &event->param.conn);
  		break;
+	case RDMA_CM_EVENT_ADDR_CHANGE:
+		listen_id = svc_rdma_create_listen_id(cma_rdma->xpt_net,
+						      sap, cma_xprt);
+		if (IS_ERR(listen_id)) {
+			pr_err("Listener dead, address change failed for device %s\n",
+				cma_id->device->name);
+		} else
+			cma_xprt->sc_cm_id = listen_id;
+		return 1;
  	default:
  		break;
  	}
#7 lore raw

Re: [PATCH 2/2] svcrdma: Handle ADDR_CHANGE CM event properly

Quoted 31 lines · Zhu Yanjun
Quoted 30 lines · On 31.05.24 15:15, [email protected] wrote:

From: Chuck Lever <[email protected]>

Sagi tells me that when a bonded device reports an address change, the consumer must destroy its listener IDs and create new ones.

See commit a032e4f6d60d ("nvmet-rdma: fix bonding failover possible NULL deref").

---
  net/sunrpc/xprtrdma/svc_rdma_transport.c | 16 +++++++++++++++-
  1 file changed, 15 insertions(+), 1 deletion(-)
net/sunrpc/xprtrdma/svc_rdma_transport.c +1 -1
index fa50b7494a0a..a003b62fb7d5 100644
@@ -284,17 +284,31 @@ static void handle_connect_req(struct rdma_cm_id *new_cma_id,
   *
   * Return values:
   *     %0: Do not destroy @cma_id
- *     %1: Destroy @cma_id (never returned here)
+ *     %1: Destroy @cma_id
   *
   * NB: There is never a DEVICE_REMOVAL event for INADDR_ANY listeners.
   */
  static int svc_rdma_listen_handler(struct rdma_cm_id *cma_id,
  				   struct rdma_cm_event *event)
  {
Quoted 8 lines

net/sunrpc/xprtrdma/svc_rdma_transport.c static void handle_connect_req(struct rdma_cm_id *new_cma_id, · line 297

Quoted 4 lines
net/sunrpc/xprtrdma/svc_rdma_transport.c +4 -0
+	struct svcxprt_rdma *cma_xprt = cma_id->context;
+	struct svc_xprt *cma_rdma = &cma_xprt->sc_xprt;
+	struct sockaddr *sap = (struct sockaddr *)&cma_id->route.addr.src_addr;
+	struct rdma_cm_id *listen_id;

I am not sure whether I need to suggest "Reverse Christmas Tree" or not. This is a trivial problem. ^_^
You can ignore it. And it is not mandatory.

Oops, I missed that @sap can be declared first. It must have gotten lost while I was trying out different solutions for this issue.

Fixed.

Quoted 22 lines

But if we follow this rule, the source code will become more readable.

Zhu Yanjun

Quoted 17 lines
net/sunrpc/xprtrdma/svc_rdma_transport.c +10 -0
+
  	switch (event->event) {
  	case RDMA_CM_EVENT_CONNECT_REQUEST:
  		handle_connect_req(cma_id, &event->param.conn);
  		break;
+	case RDMA_CM_EVENT_ADDR_CHANGE:
+		listen_id = svc_rdma_create_listen_id(cma_rdma->xpt_net,
+						      sap, cma_xprt);
+		if (IS_ERR(listen_id)) {
+			pr_err("Listener dead, address change failed for device %s\n",
+				cma_id->device->name);
+		} else
+			cma_xprt->sc_cm_id = listen_id;
+		return 1;
  	default:
  		break;
  	}
Signature · 1 line
Chuck Lever
Reviewed-by on the v1 series
#8 lore raw
In reply to [email protected]
Quoted 6 lines · On 31/05/2024 16:15, [email protected] wrote:

From: Chuck Lever <[email protected]>

Sagi recently pointed out that the CM ADDR_CHANGE event handler in svcrdma has a bug similar to one he fixed in the NVMe target. This series attempts to address that issue.

Quoted 7 lines

Chuck Lever (2):
svcrdma: Refactor the creation of listener CMA ID
svcrdma: Handle ADDR_CHANGE CM event properly

  net/sunrpc/xprtrdma/svc_rdma_transport.c | 83 ++++++++++++++++--------
  1 file changed, 55 insertions(+), 28 deletions(-)

Looks good, For the series:

#9 lore raw
Quoted 17 lines · Sagi Grimberg
Quoted 10 lines · On 31/05/2024 16:15, [email protected] wrote:

From: Chuck Lever <[email protected]>

Sagi recently pointed out that the CM ADDR_CHANGE event handler in svcrdma has a bug similar to one he fixed in the NVMe target. This series attempts to address that issue.

Chuck Lever (2):
svcrdma: Refactor the creation of listener CMA ID
svcrdma: Handle ADDR_CHANGE CM event properly

Quoted 7 lines
Quoted 3 lines
 net/sunrpc/xprtrdma/svc_rdma_transport.c | 83 ++++++++++++++++--------
 1 file changed, 55 insertions(+), 28 deletions(-)

Looks good, For the series:

Thank you! Applied to nfsd-next (for 6.11).

--
Chuck Lever