RDMA/rxe: drive UDP tunnel socket lifetime from the GID table
Subsystems (2)
PATCH v2 1/2 RDMA/rxe: drive UDP tunnel socket lifetime from the GID table
rxe creates the shared per-netns UDP tunnel sockets in rxe_newlink() and releases them from dellink, the NETDEV_UNREGISTER notifier and a pernet exit hook, with driver side bookkeeping deciding when the last user is gone. Keeping that in sync with the core GID table has been a recurring source of lifetime bugs, most recently a refcount underflow / use-after-free reported by syzkaller when those paths raced.
Implement add_gid/del_gid and key the sockets off the GID entries
instead: the first RoCEv2 entry of an address family in a netns creates the wildcard socket, the last one releases it. The core pairs every installed GID entry with one del_gid, so the counts have a single owner and cannot underflow.
The per-netns state is a small global list keyed by netns; no pernet storage and no pernet hook are needed. rxe_ns.[ch], the sk_refcnt magic in rxe_net_del() and the .dellink hook are gone.
One behavior change: the core does not propagate add_gid errors, so a busy port 4791 or a failed allocation no longer fails "rdma link add", it leaves the GID entry missing.
---
v2:
- Dropped backstop.
v1:https://lore.kernel.org/linux-rdma/[email protected]/
drivers/infiniband/sw/rxe/Makefile | 3 +- drivers/infiniband/sw/rxe/rxe.c | 26 +-- drivers/infiniband/sw/rxe/rxe_net.c | 289 +++++++++++++++----------- drivers/infiniband/sw/rxe/rxe_net.h | 5 +- drivers/infiniband/sw/rxe/rxe_ns.c | 124 ----------- drivers/infiniband/sw/rxe/rxe_ns.h | 26 --- drivers/infiniband/sw/rxe/rxe_verbs.c | 2 + 7 files changed, 171 insertions(+), 304 deletions(-) delete mode 100644 drivers/infiniband/sw/rxe/rxe_ns.c delete mode 100644 drivers/infiniband/sw/rxe/rxe_ns.h
drivers/infiniband/sw/rxe/Makefile +1 -2
@@ -23,7 +23,6 @@ rdma_rxe-y := \ rxe_task.o \ rxe_net.o \ rxe_hw_counters.o \ - rxe_mad.o \ - rxe_ns.o + rxe_mad.o rdma_rxe-$(CONFIG_INFINIBAND_ON_DEMAND_PAGING) += rxe_odp.o
drivers/infiniband/sw/rxe/rxe.c +1 -25
@@ -9,7 +9,6 @@#include "rxe.h" #include "rxe_loc.h" #include "rxe_net.h" -#include "rxe_ns.h" MODULE_AUTHOR ("Bob Pearson, Frank Zago, John Groves, Kamal Heib" );MODULE_DESCRIPTION ("Soft RDMA transport" );
@@ -202,8 +201,6 @@ void rxe_set_mtu(struct rxe_dev *rxe, unsigned int ndev_mtu) port->mtu_cap =ib_mtu_enum_to_int (mtu); } -static struct rdma_link_ops rxe_link_ops; -/* called by ifc layer to create new rxe device. * The caller should allocate memory for rxe by calling ib_alloc_device. */
@@ -212,7 +209,6 @@ int rxe_add(struct rxe_dev *rxe, unsigned int mtu, const char *ibdev_name, {rxe_init (rxe, ndev);rxe_set_mtu (rxe, mtu); - rxe->ib_dev.link_ops = &rxe_link_ops;return rxe_register_device (rxe, ibdev_name, ndev); }
@@ -236,10 +232,6 @@ static int rxe_newlink(const char *ibdev_name, struct net_device *ndev) goto err; } - err =rxe_net_init (ndev); -if (err) -return err; - err =rxe_net_add (ibdev_name, ndev);if (err) {rxe_err ("failed to add %s\n" , ndev->name);
@@ -249,17 +241,9 @@ static int rxe_newlink(const char *ibdev_name, struct net_device *ndev)return err; } -static int rxe_dellink (struct ib_device *dev) -{ -rxe_net_del (dev); - -return 0 ; -} -static struct rdma_link_ops rxe_link_ops = { .type ="rxe" , .newlink = rxe_newlink, - .dellink = rxe_dellink, };static int __init rxe_module_init (void )
@@ -270,21 +254,15 @@ static int __init rxe_module_init(void)if (err)return err; - err =rxe_namespace_init (); -if (err) - goto err_destroy_wq; - err =rxe_register_notifier ();if (err) - goto err_namespace_exit; + goto err_destroy_wq;rdma_link_register (&rxe_link_ops);pr_info ("loaded\n" );return 0 ; -err_namespace_exit: -rxe_namespace_exit (); err_destroy_wq:rxe_destroy_wq ();return err;
@@ -297,8 +275,6 @@ static void __exit rxe_module_exit(void)rxe_net_exit ();rxe_destroy_wq (); -rxe_namespace_exit (); -pr_info ("unloaded\n" ); }
drivers/infiniband/sw/rxe/rxe_net.c +164 -125
337 lines not on this page · raw message
drivers/infiniband/sw/rxe/rxe_net.h +3 -2
@@ -12,10 +12,11 @@#include <linux/module.h> int rxe_net_add (const char *ibdev_name,struct net_device *ndev); -void rxe_net_del (struct ib_device *dev); + +int rxe_add_gid (const struct ib_gid_attr *attr,void **context); +int rxe_del_gid (const struct ib_gid_attr *attr,void **context);int rxe_register_notifier (void ); -int rxe_net_init (struct net_device *ndev);void rxe_net_exit (void );#endif /* RXE_NET_H */
drivers/infiniband/sw/rxe/rxe_ns.c deleted +0 -124
124 lines not on this page · raw message
drivers/infiniband/sw/rxe/rxe_ns.h deleted +0 -26
@@ -1,26 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ - -#ifndef RXE_NS_H -#define RXE_NS_H - -struct sock *rxe_ns_pernet_sk4 (struct net *net); -void rxe_ns_pernet_set_sk4 (struct net *net,struct sock *sk); - -#if IS_ENABLED (CONFIG_IPV6 ) -void rxe_ns_pernet_set_sk6 (struct net *net,struct sock *sk); -struct sock *rxe_ns_pernet_sk6 (struct net *net); -#else /* IPv6 */ -static inline struct sock *rxe_ns_pernet_sk6 (struct net *net) -{ -return NULL ; -} - -static inline void rxe_ns_pernet_set_sk6 (struct net *net,struct sock *sk) -{ -} -#endif /* IPv6 */ - -int rxe_namespace_init (void ); -void rxe_namespace_exit (void ); - -#endif /* RXE_NS_H */
drivers/infiniband/sw/rxe/rxe_verbs.c +2 -0
@@ -1460,6 +1460,7 @@ static const struct ib_device_ops rxe_dev_ops = { .driver_id = RDMA_DRIVER_RXE, .uverbs_abi_ver = RXE_UVERBS_ABI_VERSION, + .add_gid = rxe_add_gid, .alloc_hw_port_stats = rxe_ib_alloc_hw_port_stats, .alloc_mr = rxe_alloc_mr, .alloc_mw = rxe_alloc_mw,
@@ -1475,6 +1476,7 @@ static const struct ib_device_ops rxe_dev_ops = { .dealloc_mw = rxe_dealloc_mw, .dealloc_pd = rxe_dealloc_pd, .dealloc_ucontext = rxe_dealloc_ucontext, + .del_gid = rxe_del_gid, .dereg_mr = rxe_dereg_mr, .destroy_ah = rxe_destroy_ah, .destroy_cq = rxe_destroy_cq,
base-commit: cee9395acd8043be0644b25c34bfa86623f2b935
Signature · 1 line
2.53.0
PATCH v2 2/2 RDMA/nldev: remove the unused dellink link op
rxe was the only driver implementing rdma_link_ops.dellink and does not need it anymore now that its UDP tunnel sockets follow the GID table. Remove the callback, the nldev code invoking it, the mutex that serialized it and the now unused ib_device::link_ops field the invocation looked it up through.
--- drivers/infiniband/core/nldev.c | 15 --------------- include/rdma/ib_verbs.h | 2 -- include/rdma/rdma_netlink.h | 1 - 3 files changed, 18 deletions(-)
drivers/infiniband/core/nldev.c +0 -15
@@ -51,7 +51,6 @@* a controlled QKEY. */ static bool privileged_qkey; -static DEFINE_MUTEX (nldev_dellink_mutex);typedef int (*res_fill_func_t )(struct sk_buff *,bool ,struct rdma_restrack_entry *,uint32_t );
@@ -1885,20 +1884,6 @@ static int nldev_dellink(struct sk_buff *skb, struct nlmsghdr *nlh,return -EINVAL ; } -/* -* This path is triggered by the 'rdma link delete' administrative command. -* For Soft-RoCE (RXE), we ensure that transport sockets are closed here. -* Note: iWARP driver does not implement .dellink, so this logic is -* implicitly scoped to the driver supporting dynamic link deletion like RXE. -*/ -if (device->link_ops && device->link_ops->dellink) { -mutex_lock (&nldev_dellink_mutex); - err = device->link_ops->dellink (device); -mutex_unlock (&nldev_dellink_mutex); -if (err) -return err; - } -ib_unregister_device_and_put (device);return 0 ; }
include/rdma/ib_verbs.h +0 -2
@@ -3034,8 +3034,6 @@ struct ib_device {struct completion unreg_completion;struct work_struct unregistration_work; -const struct rdma_link_ops *link_ops; -/* Protects compat_devs xarray modifications */ struct mutex compat_devs_mutex;/* Maintains compat devices for each net namespace */
include/rdma/rdma_netlink.h +0 -1
@@ -127,7 +127,6 @@ struct rdma_link_ops {struct list_head list;const char *type;int (*newlink)(const char *ibdev_name,struct net_device *ndev); -int (*dellink)(struct ib_device *dev); };void rdma_link_register (struct rdma_link_ops *ops);
Signature · 1 line
2.53.0
[email protected]
AI reviewRe: [PATCH v2 2/2] RDMA/nldev: remove the unused dellink link op
Sashiko has reviewed this patch and found no issues. It looks great!
Quoted 8 lines
rxe was the only driver implementing rdma_link_ops.dellink and does not need it anymore now that its UDP tunnel sockets follow the GID table. Remove the callback, the nldev code invoking it, the mutex that serialized it and the now unused
ib_device::link_opsfield the invocation looked it up through.Assisted-by Claude:claude-fable-5Signed-off-by Serhat Kumral <[email protected]>
Sashiko has reviewed this patch and found no issues. It looks great!
Signature · 1 line
Sashiko AI review · https://sashiko.dev/#/patchset/[email protected]?part=2
[email protected]
AI reviewThank you for your contribution! Sashiko AI review found 1 potential issue(s) to consider: - [Critical] Asynchronous GID deletion outlives netns teardown, causing a Use-After-Free panic and state…
Thank you for your contribution! Sashiko AI review found 1 potential issue(s) to consider:
- [Critical] Asynchronous GID deletion outlives netns teardown, causing a Use-After-Free panic and state corruption. --
commit 2c5402d7d2ec6f9905d17f75a69f7841c9e31efb
Author: Serhat Kumral <[email protected]>
RDMA/rxe: drive UDP tunnel socket lifetime from the GID table
This commit keys the lifetime of shared per-netns UDP tunnel sockets off the GID table instead of driver-side bookkeeping. It implements add_gid/del_gid to manage the wildcard sockets, keeping the per-netns state in a global list and removing the pernet storage and hooks.
Quoted 4 lines
drivers/infiniband/sw/rxe/rxe_net.c +0
-0
[ ... ]
Quoted 1 line
drivers/infiniband/sw/rxe/rxe_net.c +0
-0@@ -288,10 +283,165 @@ static struct socket *rxe_setup_udp_tunnel(struct net *net, __be16 port,
[ ... ]
Quoted 2 lines
drivers/infiniband/sw/rxe/rxe_net.c +2
-0+int rxe_add_gid (const struct ib_gid_attr *attr,void **context) +{
[ ... ]
drivers/infiniband/sw/rxe/rxe_net.c static struct socket *rxe_setup_udp_tunnel(struct net *net, __be16 port, · line 394
Quoted 5 lines
drivers/infiniband/sw/rxe/rxe_net.c +5
-0+mutex_unlock (&rxe_ns_lock); + +put_net (net); + *context = ns; +return 0 ;
[Severity: Critical]
Is it safe to drop the active netns reference here?
A code comment mentions that passive net references keep the net pointer valid as a list key until the sockets are released. However, kernel sockets use passive references precisely so they do not keep the netns alive. If a user deletes the namespace, the net structure can be freed while its pointer remains in ns->net.
If a new netns is allocated at the same memory address before the old GIDs are completely removed, could a concurrent rxe_add_gid() find this stale rxe_ns_sock entry, leading to state corruption and cross-namespace socket sharing?
Quoted 2 lines
drivers/infiniband/sw/rxe/rxe_net.c +2
-0+ +out_free:
[ ... ]
Quoted 2 lines
drivers/infiniband/sw/rxe/rxe_net.c +2
-0+int rxe_del_gid (const struct ib_gid_attr *attr,void **context) +{
[ ... ]
drivers/infiniband/sw/rxe/rxe_net.c static struct socket *rxe_setup_udp_tunnel(struct net *net, __be16 port, · line 436
Quoted 7 lines
drivers/infiniband/sw/rxe/rxe_net.c +7
-0+/* +* Release under the lock: a concurrent rxe_add_gid() must not +* see a zero count while the old socket still holds the port, +* or its bind() fails with -EADDRINUSE. +*/ +if (sock) +udp_tunnel_sock_release (sock->sk);
[Severity: Critical]
Can this lead to a use-after-free panic when the namespace is torn down?
When a net namespace is deleted, the NETDEV_UNREGISTER event triggers an asynchronous unregistration of the IB device via ib_unreg_wq. Because this is asynchronous, the netns teardown can complete and free the net structure memory before rxe_del_gid() is executed.
When udp_tunnel_sock_release() is eventually called here, the socket release path (such as udp_lib_unhash) will dereference sock_net(sk). Since that points to the freed netns memory, won't this cause a fatal use-after-free panic?
Quoted 9 lines
drivers/infiniband/sw/rxe/rxe_net.c +9
-0+ +if (!ns->nr4 && !ns->nr6) { +list_del (&ns->node); +kfree (ns); + } +mutex_unlock (&rxe_ns_lock); + +return 0 ; +}
Signature · 1 line
Sashiko AI review · https://sashiko.dev/#/patchset/[email protected]?part=1