Skip to contentSkip to conversation

RDMA/rxe: drive UDP tunnel socket lifetime from the GID table

PATCH New Serhat Kumral 2 participants 4 messages Sep 10

Subsystems (2)
lore
Share & tools
lore
#0 lore raw

PATCH v2 1/2 RDMA/rxe: drive UDP tunnel socket lifetime from the GID table

rxe creates the shared per-netns UDP tunnel sockets in rxe_newlink() and releases them from dellink, the NETDEV_UNREGISTER notifier and a pernet exit hook, with driver side bookkeeping deciding when the last user is gone. Keeping that in sync with the core GID table has been a recurring source of lifetime bugs, most recently a refcount underflow / use-after-free reported by syzkaller when those paths raced.

Implement add_gid/del_gid and key the sockets off the GID entries
instead: the first RoCEv2 entry of an address family in a netns creates the wildcard socket, the last one releases it. The core pairs every installed GID entry with one del_gid, so the counts have a single owner and cannot underflow.

The per-netns state is a small global list keyed by netns; no pernet storage and no pernet hook are needed. rxe_ns.[ch], the sk_refcnt magic in rxe_net_del() and the .dellink hook are gone.

One behavior change: the core does not propagate add_gid errors, so a busy port 4791 or a failed allocation no longer fails "rdma link add", it leaves the GID entry missing.

Fixes f1327abd6abe ("RDMA/rxe: Support RDMA link creation and destruction per net namespace")
Reported-by [email protected]
Assisted-by Claude:claude-fable-5

---
v2:
- Dropped backstop.

v1:https://lore.kernel.org/linux-rdma/[email protected]/

 drivers/infiniband/sw/rxe/Makefile    |   3 +-
 drivers/infiniband/sw/rxe/rxe.c       |  26 +--
 drivers/infiniband/sw/rxe/rxe_net.c   | 289 +++++++++++++++-----------
 drivers/infiniband/sw/rxe/rxe_net.h   |   5 +-
 drivers/infiniband/sw/rxe/rxe_ns.c    | 124 -----------
 drivers/infiniband/sw/rxe/rxe_ns.h    |  26 ---
 drivers/infiniband/sw/rxe/rxe_verbs.c |   2 +
 7 files changed, 171 insertions(+), 304 deletions(-)
 delete mode 100644 drivers/infiniband/sw/rxe/rxe_ns.c
 delete mode 100644 drivers/infiniband/sw/rxe/rxe_ns.h
drivers/infiniband/sw/rxe/Makefile +1 -2
index e097c1ca1874..18ff60542744 100644
@@ -23,7 +23,6 @@ rdma_rxe-y := \
 	rxe_task.o \
 	rxe_net.o \
 	rxe_hw_counters.o \
-	rxe_mad.o \
-	rxe_ns.o
+	rxe_mad.o
 
 rdma_rxe-$(CONFIG_INFINIBAND_ON_DEMAND_PAGING) += rxe_odp.o
drivers/infiniband/sw/rxe/rxe.c +1 -25
index af39209d0fcf..66d5986dac99 100644
@@ -9,7 +9,6 @@
 #include "rxe.h"
 #include "rxe_loc.h"
 #include "rxe_net.h"
-#include "rxe_ns.h"
 
 MODULE_AUTHOR("Bob Pearson, Frank Zago, John Groves, Kamal Heib");
 MODULE_DESCRIPTION("Soft RDMA transport");
@@ -202,8 +201,6 @@ void rxe_set_mtu(struct rxe_dev *rxe, unsigned int ndev_mtu)
 	port->mtu_cap = ib_mtu_enum_to_int(mtu);
 }
 
-static struct rdma_link_ops rxe_link_ops;
-
 /* called by ifc layer to create new rxe device.
  * The caller should allocate memory for rxe by calling ib_alloc_device.
  */
@@ -212,7 +209,6 @@ int rxe_add(struct rxe_dev *rxe, unsigned int mtu, const char *ibdev_name,
 {
 	rxe_init(rxe, ndev);
 	rxe_set_mtu(rxe, mtu);
-	rxe->ib_dev.link_ops = &rxe_link_ops;
 
 	return rxe_register_device(rxe, ibdev_name, ndev);
 }
@@ -236,10 +232,6 @@ static int rxe_newlink(const char *ibdev_name, struct net_device *ndev)
 		goto err;
 	}
 
-	err = rxe_net_init(ndev);
-	if (err)
-		return err;
-
 	err = rxe_net_add(ibdev_name, ndev);
 	if (err) {
 		rxe_err("failed to add %s\n", ndev->name);
@@ -249,17 +241,9 @@ static int rxe_newlink(const char *ibdev_name, struct net_device *ndev)
 	return err;
 }
 
-static int rxe_dellink(struct ib_device *dev)
-{
-	rxe_net_del(dev);
-
-	return 0;
-}
-
 static struct rdma_link_ops rxe_link_ops = {
 	.type = "rxe",
 	.newlink = rxe_newlink,
-	.dellink = rxe_dellink,
 };
 
 static int __init rxe_module_init(void)
@@ -270,21 +254,15 @@ static int __init rxe_module_init(void)
 	if (err)
 		return err;
 
-	err = rxe_namespace_init();
-	if (err)
-		goto err_destroy_wq;
-
 	err = rxe_register_notifier();
 	if (err)
-		goto err_namespace_exit;
+		goto err_destroy_wq;
 
 	rdma_link_register(&rxe_link_ops);
 
 	pr_info("loaded\n");
 	return 0;
 
-err_namespace_exit:
-	rxe_namespace_exit();
 err_destroy_wq:
 	rxe_destroy_wq();
 	return err;
@@ -297,8 +275,6 @@ static void __exit rxe_module_exit(void)
 	rxe_net_exit();
 	rxe_destroy_wq();
 
-	rxe_namespace_exit();
-
 	pr_info("unloaded\n");
 }
 
drivers/infiniband/sw/rxe/rxe_net.c +164 -125

337 lines not on this page · raw message

drivers/infiniband/sw/rxe/rxe_net.h +3 -2
index 56249677d692..b564ff5c83f3 100644
@@ -12,10 +12,11 @@
 #include <linux/module.h>
 
 int rxe_net_add(const char *ibdev_name, struct net_device *ndev);
-void rxe_net_del(struct ib_device *dev);
+
+int rxe_add_gid(const struct ib_gid_attr *attr, void **context);
+int rxe_del_gid(const struct ib_gid_attr *attr, void **context);
 
 int rxe_register_notifier(void);
-int rxe_net_init(struct net_device *ndev);
 void rxe_net_exit(void);
 
 #endif /* RXE_NET_H */
drivers/infiniband/sw/rxe/rxe_ns.c deleted +0 -124

124 lines not on this page · raw message

drivers/infiniband/sw/rxe/rxe_ns.h deleted +0 -26
deleted file mode 100644
index 4da2709e6b71..000000000000
@@ -1,26 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
-
-#ifndef RXE_NS_H
-#define RXE_NS_H
-
-struct sock *rxe_ns_pernet_sk4(struct net *net);
-void rxe_ns_pernet_set_sk4(struct net *net, struct sock *sk);
-
-#if IS_ENABLED(CONFIG_IPV6)
-void rxe_ns_pernet_set_sk6(struct net *net, struct sock *sk);
-struct sock *rxe_ns_pernet_sk6(struct net *net);
-#else /* IPv6 */
-static inline struct sock *rxe_ns_pernet_sk6(struct net *net)
-{
-	return NULL;
-}
-
-static inline void rxe_ns_pernet_set_sk6(struct net *net, struct sock *sk)
-{
-}
-#endif /* IPv6 */
-
-int rxe_namespace_init(void);
-void rxe_namespace_exit(void);
-
-#endif /* RXE_NS_H */
drivers/infiniband/sw/rxe/rxe_verbs.c +2 -0
index 96c7716057fe..278da342ca1e 100644
@@ -1460,6 +1460,7 @@ static const struct ib_device_ops rxe_dev_ops = {
 	.driver_id = RDMA_DRIVER_RXE,
 	.uverbs_abi_ver = RXE_UVERBS_ABI_VERSION,
 
+	.add_gid = rxe_add_gid,
 	.alloc_hw_port_stats = rxe_ib_alloc_hw_port_stats,
 	.alloc_mr = rxe_alloc_mr,
 	.alloc_mw = rxe_alloc_mw,
@@ -1475,6 +1476,7 @@ static const struct ib_device_ops rxe_dev_ops = {
 	.dealloc_mw = rxe_dealloc_mw,
 	.dealloc_pd = rxe_dealloc_pd,
 	.dealloc_ucontext = rxe_dealloc_ucontext,
+	.del_gid = rxe_del_gid,
 	.dereg_mr = rxe_dereg_mr,
 	.destroy_ah = rxe_destroy_ah,
 	.destroy_cq = rxe_destroy_cq,

base-commit: cee9395acd8043be0644b25c34bfa86623f2b935

Signature · 1 line
2.53.0
#1 lore raw

PATCH v2 2/2 RDMA/nldev: remove the unused dellink link op

rxe was the only driver implementing rdma_link_ops.dellink and does not need it anymore now that its UDP tunnel sockets follow the GID table. Remove the callback, the nldev code invoking it, the mutex that serialized it and the now unused ib_device::link_ops field the invocation looked it up through.

Assisted-by Claude:claude-fable-5
---
 drivers/infiniband/core/nldev.c | 15 ---------------
 include/rdma/ib_verbs.h         |  2 --
 include/rdma/rdma_netlink.h     |  1 -
 3 files changed, 18 deletions(-)
drivers/infiniband/core/nldev.c +0 -15
index a4014a230639..8b5b9bfb1345 100644
@@ -51,7 +51,6 @@
  * a controlled QKEY.
  */
 static bool privileged_qkey;
-static DEFINE_MUTEX(nldev_dellink_mutex);
 
 typedef int (*res_fill_func_t)(struct sk_buff*, bool,
 			       struct rdma_restrack_entry*, uint32_t);
@@ -1885,20 +1884,6 @@ static int nldev_dellink(struct sk_buff *skb, struct nlmsghdr *nlh,
 		return -EINVAL;
 	}
 
-	/*
-	 * This path is triggered by the 'rdma link delete' administrative command.
-	 * For Soft-RoCE (RXE), we ensure that transport sockets are closed here.
-	 * Note: iWARP driver does not implement .dellink, so this logic is
-	 * implicitly scoped to the driver supporting dynamic link deletion like RXE.
-	 */
-	if (device->link_ops && device->link_ops->dellink) {
-		mutex_lock(&nldev_dellink_mutex);
-		err = device->link_ops->dellink(device);
-		mutex_unlock(&nldev_dellink_mutex);
-		if (err)
-			return err;
-	}
-
 	ib_unregister_device_and_put(device);
 	return 0;
 }
include/rdma/ib_verbs.h +0 -2
index cb3b6163961b..4930def66c70 100644
@@ -3034,8 +3034,6 @@ struct ib_device {
 	struct completion unreg_completion;
 	struct work_struct unregistration_work;
 
-	const struct rdma_link_ops *link_ops;
-
 	/* Protects compat_devs xarray modifications */
 	struct mutex compat_devs_mutex;
 	/* Maintains compat devices for each net namespace */
include/rdma/rdma_netlink.h +0 -1
index 2fd1358ea57d..d1a7186f7436 100644
@@ -127,7 +127,6 @@ struct rdma_link_ops {
 	struct list_head list;
 	const char *type;
 	int (*newlink)(const char *ibdev_name, struct net_device *ndev);
-	int (*dellink)(struct ib_device *dev);
 };
 
 void rdma_link_register(struct rdma_link_ops *ops);
Signature · 1 line
2.53.0
#2 lore raw

Re: [PATCH v2 2/2] RDMA/nldev: remove the unused dellink link op

Sashiko has reviewed this patch and found no issues. It looks great!
Quoted 8 lines

rxe was the only driver implementing rdma_link_ops.dellink and does not need it anymore now that its UDP tunnel sockets follow the GID table. Remove the callback, the nldev code invoking it, the mutex that serialized it and the now unused ib_device::link_ops field the invocation looked it up through.

Assisted-by Claude:claude-fable-5

Sashiko has reviewed this patch and found no issues. It looks great!

Signature · 1 line
Sashiko AI review · https://sashiko.dev/#/patchset/[email protected]?part=2
#3 lore raw
In reply to Serhat Kumral
Thank you for your contribution! Sashiko AI review found 1 potential issue(s) to consider: - [Critical] Asynchronous GID deletion outlives netns teardown, causing a Use-After-Free panic and state…

Thank you for your contribution! Sashiko AI review found 1 potential issue(s) to consider:
- [Critical] Asynchronous GID deletion outlives netns teardown, causing a Use-After-Free panic and state corruption. --

commit 2c5402d7d2ec6f9905d17f75a69f7841c9e31efb
Author: Serhat Kumral <[email protected]>

RDMA/rxe: drive UDP tunnel socket lifetime from the GID table

This commit keys the lifetime of shared per-netns UDP tunnel sockets off the GID table instead of driver-side bookkeeping. It implements add_gid/del_gid to manage the wildcard sockets, keeping the per-netns state in a global list and removing the pernet storage and hooks.

Quoted 4 lines
drivers/infiniband/sw/rxe/rxe_net.c +0 -0
index 53daaf4c1eb2e..fa1cf49c13e33 100644

[ ... ]

Quoted 1 line
drivers/infiniband/sw/rxe/rxe_net.c +0 -0
@@ -288,10 +283,165 @@ static struct socket *rxe_setup_udp_tunnel(struct net *net, __be16 port,

[ ... ]

Quoted 2 lines
drivers/infiniband/sw/rxe/rxe_net.c +2 -0
+int rxe_add_gid(const struct ib_gid_attr *attr, void **context)
+{

[ ... ]

drivers/infiniband/sw/rxe/rxe_net.c static struct socket *rxe_setup_udp_tunnel(struct net *net, __be16 port, · line 394

Quoted 5 lines
drivers/infiniband/sw/rxe/rxe_net.c +5 -0
+	mutex_unlock(&rxe_ns_lock);
+
+	put_net(net);
+	*context = ns;
+	return 0;

[Severity: Critical]
Is it safe to drop the active netns reference here?

A code comment mentions that passive net references keep the net pointer valid as a list key until the sockets are released. However, kernel sockets use passive references precisely so they do not keep the netns alive. If a user deletes the namespace, the net structure can be freed while its pointer remains in ns->net.

If a new netns is allocated at the same memory address before the old GIDs are completely removed, could a concurrent rxe_add_gid() find this stale rxe_ns_sock entry, leading to state corruption and cross-namespace socket sharing?

Quoted 2 lines
drivers/infiniband/sw/rxe/rxe_net.c +2 -0
+
+out_free:

[ ... ]

Quoted 2 lines
drivers/infiniband/sw/rxe/rxe_net.c +2 -0
+int rxe_del_gid(const struct ib_gid_attr *attr, void **context)
+{

[ ... ]

drivers/infiniband/sw/rxe/rxe_net.c static struct socket *rxe_setup_udp_tunnel(struct net *net, __be16 port, · line 436

Quoted 7 lines
drivers/infiniband/sw/rxe/rxe_net.c +7 -0
+	/*
+	 * Release under the lock: a concurrent rxe_add_gid() must not
+	 * see a zero count while the old socket still holds the port,
+	 * or its bind() fails with -EADDRINUSE.
+	 */
+	if (sock)
+		udp_tunnel_sock_release(sock->sk);

[Severity: Critical]
Can this lead to a use-after-free panic when the namespace is torn down?

When a net namespace is deleted, the NETDEV_UNREGISTER event triggers an asynchronous unregistration of the IB device via ib_unreg_wq. Because this is asynchronous, the netns teardown can complete and free the net structure memory before rxe_del_gid() is executed.

When udp_tunnel_sock_release() is eventually called here, the socket release path (such as udp_lib_unhash) will dereference sock_net(sk). Since that points to the freed netns memory, won't this cause a fatal use-after-free panic?

Quoted 9 lines
drivers/infiniband/sw/rxe/rxe_net.c +9 -0
+
+	if (!ns->nr4 && !ns->nr6) {
+		list_del(&ns->node);
+		kfree(ns);
+	}
+	mutex_unlock(&rxe_ns_lock);
+
+	return 0;
+}
Signature · 1 line
Sashiko AI review · https://sashiko.dev/#/patchset/[email protected]?part=1