rds: synchronize info callbacks with connection teardown
Subsystems (5)
PATCH 1/1 rds: synchronize info callbacks with connection teardown
rds_info_getsockopt() reads an info callback without holding rds_info_lock across its invocation. At the same time, rds_exit() destroys all connections before unregistering the socket info callbacks.
This permits the following interleaving:
CPU0 CPU1
rds_info_getsockopt()
func = rds6_sock_inc_info
rds_exit()
rds_conn_exit()
rds_conn_destroy()
kmem_cache_free(conn)
func()
rds6_inc_info_copy()
inc->i_conn->c_tosThe receive queue lock keeps the incoming message on its socket queue, but it does not keep inc->i_conn alive. KASAN reports:
BUG: KASAN: slab-use-after-free in rds6_inc_info_copy+0x459/0x530 [rds] Read of size 1 at addr ffff888106031c50 by task poc/101 Call Trace: rds6_inc_info_copy+0x459/0x530 [rds] rds6_sock_inc_info+0x2b9/0x3c0 [rds] rds_info_getsockopt+0x19d/0x380 [rds] do_sock_getsockopt+0x2ac/0x480 __sys_getsockopt+0x128/0x210 Allocated by task 99: kmem_cache_alloc_noprof+0x11a/0x360 __rds_conn_create+0x7e/0xd60 [rds] rds_conn_create_outgoing+0x61/0x80 [rds] rds_sendmsg+0xb83/0x1d00 [rds] Freed by task 102: kmem_cache_free+0x1b5/0x3d0 rds_conn_destroy+0x484/0x600 [rds] rds_loop_exit_net+0x32/0x50 [rds] unregister_pernet_device+0x2c/0x50 rds_conn_exit+0x13/0xa0 [rds] rds_exit+0x1a/0xc40 [rds] __do_sys_delete_module+0x30a/0x4d0
Publish and clear callback pointers with SRCU, and keep the SRCU read-side critical section across callback execution. SRCU permits callbacks such as RDS_INFO_COUNTERS to sleep, unlike classic RCU. After clearing a slot, synchronize_srcu() waits for every in-flight callback.
Move socket callback deregistration ahead of protocol and connection teardown. This prevents callbacks from reaching a connection after it is freed.
--- net/rds/af_rds.c | 12 ++++++------ net/rds/info.c | 20 ++++++++++++++------ 2 files changed, 20 insertions(+), 12 deletions(-)
net/rds/af_rds.c +6 -6
@@ -925,6 +925,12 @@ static void rds6_sock_info(struct socket *sock, unsigned int len,static void rds_exit (void ) {sock_unregister (rds_family_ops.family); +rds_info_deregister_func (RDS_INFO_SOCKETS , rds_sock_info); +rds_info_deregister_func (RDS_INFO_RECV_MESSAGES , rds_sock_inc_info); +#if IS_ENABLED (CONFIG_IPV6 ) +rds_info_deregister_func (RDS6_INFO_SOCKETS , rds6_sock_info); +rds_info_deregister_func (RDS6_INFO_RECV_MESSAGES , rds6_sock_inc_info); +#endif proto_unregister (&rds_proto);rds_conn_exit ();rds_cong_exit ();
@@ -933,12 +939,6 @@ static void rds_exit(void)rds_stats_exit ();rds_page_exit ();rds_bind_lock_destroy (); -rds_info_deregister_func (RDS_INFO_SOCKETS , rds_sock_info); -rds_info_deregister_func (RDS_INFO_RECV_MESSAGES , rds_sock_inc_info); -#if IS_ENABLED (CONFIG_IPV6 ) -rds_info_deregister_func (RDS6_INFO_SOCKETS , rds6_sock_info); -rds_info_deregister_func (RDS6_INFO_RECV_MESSAGES , rds6_sock_inc_info); -#endif }module_exit (rds_exit);
net/rds/info.c +14 -6
@@ -32,6 +32,7 @@*/ #include <linux/percpu.h> #include <linux/seq_file.h> +#include <linux/srcu.h> #include <linux/slab.h> #include <linux/proc_fs.h> #include <linux/export.h>
@@ -68,8 +69,9 @@ struct rds_info_iterator {unsigned long offset; }; +DEFINE_STATIC_SRCU (rds_info_srcu);static DEFINE_SPINLOCK (rds_info_lock); -static rds_info_func rds_info_funcs[RDS_INFO_LAST -RDS_INFO_FIRST +1 ]; +static rds_info_func __rcu rds_info_funcs[RDS_INFO_LAST -RDS_INFO_FIRST +1 ];void rds_info_register_func (int optname,rds_info_func func) {
@@ -78,8 +80,8 @@ void rds_info_register_func(int optname, rds_info_func func)BUG_ON (optname <RDS_INFO_FIRST || optname >RDS_INFO_LAST );spin_lock (&rds_info_lock); -BUG_ON (rds_info_funcs[offset]); - rds_info_funcs[offset] = func; +BUG_ON (rcu_access_pointer (rds_info_funcs[offset])); +rcu_assign_pointer (rds_info_funcs[offset], func);spin_unlock (&rds_info_lock); }EXPORT_SYMBOL_GPL (rds_info_register_func);
@@ -91,9 +93,10 @@ void rds_info_deregister_func(int optname, rds_info_func func)BUG_ON (optname <RDS_INFO_FIRST || optname >RDS_INFO_LAST );spin_lock (&rds_info_lock); -BUG_ON (rds_info_funcs[offset] != func); - rds_info_funcs[offset] =NULL ; +BUG_ON (rcu_access_pointer (rds_info_funcs[offset]) != func); +RCU_INIT_POINTER (rds_info_funcs[offset],NULL );spin_unlock (&rds_info_lock); +synchronize_srcu (&rds_info_srcu); }EXPORT_SYMBOL_GPL (rds_info_deregister_func);
@@ -165,6 +168,7 @@ int rds_info_getsockopt(struct socket *sock, int optname, sockopt_t *opt)int npages =0 ;int ret;int len; +int srcu_idx;int total; len = opt->optlen;
@@ -214,8 +218,11 @@ int rds_info_getsockopt(struct socket *sock, int optname, sockopt_t *opt)rdsdebug ("len %d nr_pages %lu\n" , len, nr_pages); call_func: - func = rds_info_funcs[optname -RDS_INFO_FIRST ]; + srcu_idx =srcu_read_lock (&rds_info_srcu); + func =srcu_dereference (rds_info_funcs[optname -RDS_INFO_FIRST ], + &rds_info_srcu);if (!func) { +srcu_read_unlock (&rds_info_srcu, srcu_idx); ret = -ENOPROTOOPT ; goto out; }
@@ -225,6 +232,7 @@ int rds_info_getsockopt(struct socket *sock, int optname, sockopt_t *opt) iter.offset = offset0;func (sock, len, &iter, &lens); +srcu_read_unlock (&rds_info_srcu, srcu_idx);BUG_ON (lens.each ==0 ); total = lens.nr * lens.each;
Signature · 1 line
2.43.0
Allison Henderson
maintainerQuoted 42 lines · Chengfeng Ye
rds_info_getsockopt()reads an info callback without holdingrds_info_lockacross its invocation. At the same time,rds_exit()destroys all connections before unregistering the socket info callbacks.This permits the following interleaving:
CPU0 CPU1
rds_info_getsockopt()func = rds6_sock_inc_info rds_exit() rds_conn_exit() rds_conn_destroy() kmem_cache_free(conn) func() rds6_inc_info_copy() inc->i_conn->c_tosThe receive queue lock keeps the incoming message on its socket queue, but it does not keep
inc->i_connalive. KASAN reports:BUG: KASAN: slab-use-after-free in rds6_inc_info_copy+0x459/0x530 [rds] Read of size 1 at addr ffff888106031c50 by task poc/101 Call Trace: rds6_inc_info_copy+0x459/0x530 [rds] rds6_sock_inc_info+0x2b9/0x3c0 [rds] rds_info_getsockopt+0x19d/0x380 [rds] do_sock_getsockopt+0x2ac/0x480 __sys_getsockopt+0x128/0x210 Allocated by task 99: kmem_cache_alloc_noprof+0x11a/0x360 __rds_conn_create+0x7e/0xd60 [rds] rds_conn_create_outgoing+0x61/0x80 [rds] rds_sendmsg+0xb83/0x1d00 [rds] Freed by task 102: kmem_cache_free+0x1b5/0x3d0 rds_conn_destroy+0x484/0x600 [rds] rds_loop_exit_net+0x32/0x50 [rds] unregister_pernet_device+0x2c/0x50 rds_conn_exit+0x13/0xa0 [rds] rds_exit+0x1a/0xc40 [rds] __do_sys_delete_module+0x30a/0x4d0
Quoted 7 lines
Publish and clear callback pointers with SRCU, and keep the SRCU read-side critical section across callback execution. SRCU permits callbacks such as
RDS_INFO_COUNTERSto sleep, unlike classic RCU. After clearing a slot,synchronize_srcu()waits for every in-flight callback.
Move socket callback deregistration ahead of protocol and connection teardown. This prevents callbacks from reaching a connection after it is freed.
Hi Chengfeng,
Thanks for the report and the patch, this is a great start. The UAF analysis is correct, but there are a few bumps to smooth out before this can go in. First, the build bots caught some sparse errors that need to be addressed:
https://netdev-ctrl.bots.linux.dev/logs/build/1131097/14698942/build_allmodconfig_warn/stderr
Sparse can't apply the __rcu notation through a function-pointer typedef, so all of the rcu_*() macros trip over it. The simplest fix is to drop the __rcu annotation and just use smp_load_acquire/smp_store_release on a plain array in register/deregister. Also, checkpatch warns about the two BUG_ON()s you touched. We can clean those up by converting them to a WARN_ON_ONCE().
On to the fix itself: this patch has two separate changes going on that I think would be better done as two separate patches. Comments below:
net/rds/af_rds.c
Quoted 13 lines
Fixes 7d0a06586b26 ("net/rds: Fix info leak in rds6_inc_info_copy()")Signed-off-by Chengfeng Ye <[email protected]>--- net/rds/af_rds.c | 12 ++++++------ net/rds/info.c | 20 ++++++++++++++------ 2 files changed, 20 insertions(+), 12 deletions(-)net/rds/af_rds.c +0
-0
First, the changes in af_rds.c changes don't quite work the way you might think. net/rds builds as three separate modules: rds.ko, rds_tcp.ko and rds_rdma.ko. rds_exit() only runs for "rmmod rds". rds_tcp_exit() only runs for "rmmod rds_tcp".
"rmmod rds_tcp" deregisters RDS_INFO_TCP_SOCKETS, and then destroys and frees all the TCP connections. The rds module stays loaded with RDS_INFO_RECV_MESSAGES still registered, so "rmmod rds_tcp" still reproduces the splat since rds_exit() never runs in that case.
Ideally what we want here is a ref counting infrastructure that isn't upstream yet. The good news is, I've been working on one, so I can push a tag for you to base a revised fix on.
https://github.com/allisonhenderson/rds_work/releases/tag/rds-conn-kref-base-v1
The branch may still see some review churn, but I'll leave this tag here for you to base your fix on. On top of it, the fix is just to make every holder of inc->i_conn own a reference. Really what we're doing here is porting a small UEK patch:
https://github.com/oracle/linux-uek/commit/99b9a3715419
Upstream's assignment sites differ a little from UEK's, but basically the idea is to use rds_conn_get() everywhere i_conn is assigned.
This can target net-next since every remaining trigger needs CAP_SYS_MODULE or physical device removal. So it isn't reachable by an unprivileged attacker.
Once you've worked out the patch, I'll carry it with yourself as the author as part of that series so it lands in the right order. Alternately if you'd rather not take on a port, that's fine too, just let me know and I would tag you for the Reported-by.
net/rds/info.c
Quoted 30 lines
net/rds/af_rds.c +6
-6@@ -925,6 +925,12 @@ static void rds6_sock_info(struct socket *sock, unsigned int len,static void rds_exit (void ) {sock_unregister (rds_family_ops.family); +rds_info_deregister_func (RDS_INFO_SOCKETS , rds_sock_info); +rds_info_deregister_func (RDS_INFO_RECV_MESSAGES , rds_sock_inc_info); +#if IS_ENABLED (CONFIG_IPV6 ) +rds_info_deregister_func (RDS6_INFO_SOCKETS , rds6_sock_info); +rds_info_deregister_func (RDS6_INFO_RECV_MESSAGES , rds6_sock_inc_info); +#endif proto_unregister (&rds_proto);rds_conn_exit ();rds_cong_exit ();@@ -933,12 +939,6 @@ static void rds_exit(void)rds_stats_exit ();rds_page_exit ();rds_bind_lock_destroy (); -rds_info_deregister_func (RDS_INFO_SOCKETS , rds_sock_info); -rds_info_deregister_func (RDS_INFO_RECV_MESSAGES , rds_sock_inc_info); -#if IS_ENABLED (CONFIG_IPV6 ) -rds_info_deregister_func (RDS6_INFO_SOCKETS , rds6_sock_info); -rds_info_deregister_func (RDS6_INFO_RECV_MESSAGES , rds6_sock_inc_info); -#endif }module_exit (rds_exit);net/rds/info.c +0
-0
These SRCU conversions in info.c are actually fixing a completely separate bug
independent of the connections. A getsockopt(RDS_INFO_TCP_SOCKETS)
racing against a "rmmod rds_tcp" could call the callback pointer of a freed module. Which this patch fixes. So please resend the info.c changes against net-next, with the build fixes, and a commit message describing that race.
I think that about covers it. Let me know if you get stuck or have questions
Thanks again, this was a good find.
Allison
Quoted 72 lines
net/rds/info.c +14
-6@@ -32,6 +32,7 @@*/ #include <linux/percpu.h> #include <linux/seq_file.h> +#include <linux/srcu.h> #include <linux/slab.h> #include <linux/proc_fs.h> #include <linux/export.h> @@ -68,8 +69,9 @@ struct rds_info_iterator {unsigned long offset; }; +DEFINE_STATIC_SRCU (rds_info_srcu);static DEFINE_SPINLOCK (rds_info_lock); -static rds_info_func rds_info_funcs[RDS_INFO_LAST -RDS_INFO_FIRST +1 ]; +static rds_info_func __rcu rds_info_funcs[RDS_INFO_LAST -RDS_INFO_FIRST +1 ];void rds_info_register_func (int optname,rds_info_func func) {@@ -78,8 +80,8 @@ void rds_info_register_func(int optname, rds_info_func func)BUG_ON (optname <RDS_INFO_FIRST || optname >RDS_INFO_LAST );spin_lock (&rds_info_lock); -BUG_ON (rds_info_funcs[offset]); - rds_info_funcs[offset] = func; +BUG_ON (rcu_access_pointer (rds_info_funcs[offset])); +rcu_assign_pointer (rds_info_funcs[offset], func);spin_unlock (&rds_info_lock); }EXPORT_SYMBOL_GPL (rds_info_register_func);@@ -91,9 +93,10 @@ void rds_info_deregister_func(int optname, rds_info_func func)BUG_ON (optname <RDS_INFO_FIRST || optname >RDS_INFO_LAST );spin_lock (&rds_info_lock); -BUG_ON (rds_info_funcs[offset] != func); - rds_info_funcs[offset] =NULL ; +BUG_ON (rcu_access_pointer (rds_info_funcs[offset]) != func); +RCU_INIT_POINTER (rds_info_funcs[offset],NULL );spin_unlock (&rds_info_lock); +synchronize_srcu (&rds_info_srcu); }EXPORT_SYMBOL_GPL (rds_info_deregister_func);@@ -165,6 +168,7 @@ int rds_info_getsockopt(struct socket *sock, int optname, sockopt_t *opt)int npages =0 ;int ret;int len; +int srcu_idx;int total; len = opt->optlen;@@ -214,8 +218,11 @@ int rds_info_getsockopt(struct socket *sock, int optname, sockopt_t *opt)rdsdebug ("len %d nr_pages %lu\n" , len, nr_pages); call_func: - func = rds_info_funcs[optname -RDS_INFO_FIRST ]; + srcu_idx =srcu_read_lock (&rds_info_srcu); + func =srcu_dereference (rds_info_funcs[optname -RDS_INFO_FIRST ], + &rds_info_srcu);if (!func) { +srcu_read_unlock (&rds_info_srcu, srcu_idx); ret = -ENOPROTOOPT ; goto out; }@@ -225,6 +232,7 @@ int rds_info_getsockopt(struct socket *sock, int optname, sockopt_t *opt) iter.offset = offset0;func (sock, len, &iter, &lens); +srcu_read_unlock (&rds_info_srcu, srcu_idx);BUG_ON (lens.each ==0 ); total = lens.nr * lens.each;
Quoted 232 lines · Allison Henderson
Quoted 42 lines · Chengfeng Ye
rds_info_getsockopt()reads an info callback without holdingrds_info_lockacross its invocation. At the same time,rds_exit()destroys all connections before unregistering the socket info callbacks.This permits the following interleaving:
CPU0 CPU1
rds_info_getsockopt()func = rds6_sock_inc_info rds_exit() rds_conn_exit() rds_conn_destroy() kmem_cache_free(conn) func() rds6_inc_info_copy() inc->i_conn->c_tosThe receive queue lock keeps the incoming message on its socket queue, but it does not keep
inc->i_connalive. KASAN reports:BUG: KASAN: slab-use-after-free in rds6_inc_info_copy+0x459/0x530 [rds] Read of size 1 at addr ffff888106031c50 by task poc/101 Call Trace: rds6_inc_info_copy+0x459/0x530 [rds] rds6_sock_inc_info+0x2b9/0x3c0 [rds] rds_info_getsockopt+0x19d/0x380 [rds] do_sock_getsockopt+0x2ac/0x480 __sys_getsockopt+0x128/0x210 Allocated by task 99: kmem_cache_alloc_noprof+0x11a/0x360 __rds_conn_create+0x7e/0xd60 [rds] rds_conn_create_outgoing+0x61/0x80 [rds] rds_sendmsg+0xb83/0x1d00 [rds] Freed by task 102: kmem_cache_free+0x1b5/0x3d0 rds_conn_destroy+0x484/0x600 [rds] rds_loop_exit_net+0x32/0x50 [rds] unregister_pernet_device+0x2c/0x50 rds_conn_exit+0x13/0xa0 [rds] rds_exit+0x1a/0xc40 [rds] __do_sys_delete_module+0x30a/0x4d0Quoted 7 lines
Publish and clear callback pointers with SRCU, and keep the SRCU read-side critical section across callback execution. SRCU permits callbacks such as
RDS_INFO_COUNTERSto sleep, unlike classic RCU. After clearing a slot,synchronize_srcu()waits for every in-flight callback.
Move socket callback deregistration ahead of protocol and connection teardown. This prevents callbacks from reaching a connection after it is freed.Hi Chengfeng,
Thanks for the report and the patch, this is a great start. The UAF analysis is correct, but there are a few bumps to smooth out before this can go in. First, the build bots caught some sparse errors that need to be addressed:
https://netdev-ctrl.bots.linux.dev/logs/build/1131097/14698942/build_allmodconfig_warn/stderr
Sparse can't apply the
__rcunotation through a function-pointer typedef, so all of the rcu_*() macros trip over it. The simplest fix is to drop the__rcuannotation and just use smp_load_acquire/smp_store_release on a plain array in register/deregister. Also, checkpatch warns about the two BUG_ON()s you touched. We can clean those up by converting them to aWARN_ON_ONCE().On to the fix itself: this patch has two separate changes going on that I think would be better done as two separate patches. Comments below:
net/rds/af_rds.c
Quoted 13 lines
Fixes 7d0a06586b26 ("net/rds: Fix info leak in rds6_inc_info_copy()")Signed-off-by Chengfeng Ye <[email protected]>--- net/rds/af_rds.c | 12 ++++++------ net/rds/info.c | 20 ++++++++++++++------ 2 files changed, 20 insertions(+), 12 deletions(-)net/rds/af_rds.c +0
-0First, the changes in
af_rds.cchanges don't quite work the way you might think. net/rds builds as three separate modules: rds.ko, rds_tcp.ko and rds_rdma.ko.rds_exit()only runs for "rmmod rds".rds_tcp_exit()only runs for "rmmodrds_tcp"."rmmod
rds_tcp" deregistersRDS_INFO_TCP_SOCKETS, and then destroys and frees all the TCP connections. The rds module stays loaded withRDS_INFO_RECV_MESSAGESstill registered, so "rmmodrds_tcp" still reproduces the splat sincerds_exit()never runs in that case.Ideally what we want here is a ref counting infrastructure that isn't upstream yet. The good news is, I've been working on one, so I can push a tag for you to base a revised fix on.
https://github.com/allisonhenderson/rds_work/releases/tag/rds-conn-kref-base-v1
The branch may still see some review churn, but I'll leave this tag here for you to base your fix on. On top of it, the fix is just to make every holder of
inc->i_connown a reference. Really what we're doing here is porting a small UEK patch:https://github.com/oracle/linux-uek/commit/99b9a3715419
Upstream's assignment sites differ a little from UEK's, but basically the idea is to use
rds_conn_get()everywherei_connis assigned.This can target net-next since every remaining trigger needs
CAP_SYS_MODULEor physical device removal. So it isn't reachable by an unprivileged attacker.Once you've worked out the patch, I'll carry it with yourself as the author as part of that series so it lands in the right order. Alternately if you'd rather not take on a port, that's fine too, just let me know and I would tag you for the Reported-by.
net/rds/info.c
Quoted 30 lines
net/rds/af_rds.c +6
-6@@ -925,6 +925,12 @@ static void rds6_sock_info(struct socket *sock, unsigned int len,static void rds_exit (void ) {sock_unregister (rds_family_ops.family); +rds_info_deregister_func (RDS_INFO_SOCKETS , rds_sock_info); +rds_info_deregister_func (RDS_INFO_RECV_MESSAGES , rds_sock_inc_info); +#if IS_ENABLED (CONFIG_IPV6 ) +rds_info_deregister_func (RDS6_INFO_SOCKETS , rds6_sock_info); +rds_info_deregister_func (RDS6_INFO_RECV_MESSAGES , rds6_sock_inc_info); +#endif proto_unregister (&rds_proto);rds_conn_exit ();rds_cong_exit ();@@ -933,12 +939,6 @@ static void rds_exit(void)rds_stats_exit ();rds_page_exit ();rds_bind_lock_destroy (); -rds_info_deregister_func (RDS_INFO_SOCKETS , rds_sock_info); -rds_info_deregister_func (RDS_INFO_RECV_MESSAGES , rds_sock_inc_info); -#if IS_ENABLED (CONFIG_IPV6 ) -rds_info_deregister_func (RDS6_INFO_SOCKETS , rds6_sock_info); -rds_info_deregister_func (RDS6_INFO_RECV_MESSAGES , rds6_sock_inc_info); -#endif }module_exit (rds_exit);net/rds/info.c +0
-0These SRCU conversions in
info.care actually fixing a completely separate bug
independent of the connections. Agetsockopt(RDS_INFO_TCP_SOCKETS)
racing against a "rmmodrds_tcp" could call the callback pointer of a freed module. Which this patch fixes. So please resend theinfo.cchanges against net-next, with the build fixes, and a commit message describing that race.I think that about covers it. Let me know if you get stuck or have questions
Thanks again, this was a good find.
Allison
Quoted 67 lines
net/rds/info.c +13
-6@@ -32,6 +32,7 @@*/ #include <linux/percpu.h> #include <linux/seq_file.h> +#include <linux/srcu.h> #include <linux/slab.h> #include <linux/proc_fs.h> #include <linux/export.h> @@ -68,8 +69,9 @@ struct rds_info_iterator {unsigned long offset; }; +DEFINE_STATIC_SRCU (rds_info_srcu);static DEFINE_SPINLOCK (rds_info_lock); -static rds_info_func rds_info_funcs[RDS_INFO_LAST -RDS_INFO_FIRST +1 ]; +static rds_info_func __rcu rds_info_funcs[RDS_INFO_LAST -RDS_INFO_FIRST +1 ];void rds_info_register_func (int optname,rds_info_func func) {@@ -78,8 +80,8 @@ void rds_info_register_func(int optname, rds_info_func func)BUG_ON (optname <RDS_INFO_FIRST || optname >RDS_INFO_LAST );spin_lock (&rds_info_lock); -BUG_ON (rds_info_funcs[offset]); - rds_info_funcs[offset] = func; +BUG_ON (rcu_access_pointer (rds_info_funcs[offset])); +rcu_assign_pointer (rds_info_funcs[offset], func);spin_unlock (&rds_info_lock); }EXPORT_SYMBOL_GPL (rds_info_register_func);@@ -91,9 +93,10 @@ void rds_info_deregister_func(int optname, rds_info_func func)BUG_ON (optname <RDS_INFO_FIRST || optname >RDS_INFO_LAST );spin_lock (&rds_info_lock); -BUG_ON (rds_info_funcs[offset] != func); - rds_info_funcs[offset] =NULL ; +BUG_ON (rcu_access_pointer (rds_info_funcs[offset]) != func); +RCU_INIT_POINTER (rds_info_funcs[offset],NULL );spin_unlock (&rds_info_lock); +synchronize_srcu (&rds_info_srcu); }EXPORT_SYMBOL_GPL (rds_info_deregister_func);@@ -165,6 +168,7 @@ int rds_info_getsockopt(struct socket *sock, int optname, sockopt_t *opt)int npages =0 ;int ret;int len; +int srcu_idx;int total; len = opt->optlen;@@ -214,8 +218,11 @@ int rds_info_getsockopt(struct socket *sock, int optname, sockopt_t *opt)rdsdebug ("len %d nr_pages %lu\n" , len, nr_pages); call_func: - func = rds_info_funcs[optname -RDS_INFO_FIRST ]; + srcu_idx =srcu_read_lock (&rds_info_srcu); + func =srcu_dereference (rds_info_funcs[optname -RDS_INFO_FIRST ], + &rds_info_srcu);if (!func) { +srcu_read_unlock (&rds_info_srcu, srcu_idx); ret = -ENOPROTOOPT ; goto out; }@@ -225,6 +232,7 @@ int rds_info_getsockopt(struct socket *sock, int optname, sockopt_t *opt) iter.offset = offset0;
Quoted 6 lines
net/rds/info.c int rds_info_getsockopt(struct socket *sock, int optname, sockopt_t *opt) · line 238
Quoted 5 lines
net/rds/info.c +1
-0func (sock, len, &iter, &lens); +srcu_read_unlock (&rds_info_srcu, srcu_idx);BUG_ON (lens.each ==0 ); total = lens.nr * lens.each;
Hi Allison,
Thank you for the detailed review and for confirming the bug.
I would prefer to leave the connection-reference port to you, since it depends on the new refcounting infrastructure. Please feel free to carry the fix in your series with:
I will resend the info.c changes against net-next.
Best regards,
Chengfeng
PATCH v2 1/1 rds: synchronize info callbacks with module unload
In reply to Chengfeng Yerds_info_getsockopt() reads a callback from rds_info_funcs and invokes it without protecting the callback's lifetime. Transport modules register functions stored in this array. For example, rds_tcp.ko registers rds_tcp_tc_info() for RDS_INFO_TCP_SOCKETS.
This permits the following interleaving:
CPU0 CPU1
rds_info_getsockopt()
func = rds_tcp_tc_info
rmmod rds_tcp
rds_tcp_exit()
rds_info_deregister_func()
rds_info_funcs[offset] = NULL
free rds_tcp module text
func()
The reader can therefore branch to an address in unloaded module text.
Protect callback invocation with SRCU. Enter the SRCU read-side critical section before loading the callback and leave it only after the callback returns. Clear the callback with release semantics and call synchronize_srcu() before deregistration returns, preventing module unload from freeing its text while an old reader is still executing it. SRCU is required because callbacks such as RDS_INFO_COUNTERS can sleep.
Keep the callback array unannotated and use acquire and release operations for publication so sparse does not have to apply __rcu through the function-pointer typedef. Replace the two callback-slot BUG_ON() checks with WARN_ON_ONCE() and return without changing the slot on mismatch.
---
Changes in v2:
- Split the info callback/module-unload race from the independent inc->i_conn lifetime bug.
- Keep the callback array unannotated and use smp_load_acquire() and smp_store_release() to resolve the sparse errors.
- Replace the two touched callback-slot BUG_ON() checks with WARN_ON_ONCE() error paths.
- Rewrite the commit message for the callback race and omit the unrelated inc->i_conn KASAN report.
net/rds/info.c | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-)
net/rds/info.c +23 -5
@@ -32,6 +32,7 @@*/ #include <linux/percpu.h> #include <linux/seq_file.h> +#include <linux/srcu.h> #include <linux/slab.h> #include <linux/proc_fs.h> #include <linux/export.h>
@@ -68,6 +69,7 @@ struct rds_info_iterator {unsigned long offset; }; +DEFINE_STATIC_SRCU (rds_info_srcu);static DEFINE_SPINLOCK (rds_info_lock);static rds_info_func rds_info_funcs[RDS_INFO_LAST -RDS_INFO_FIRST +1 ];
@@ -78,8 +80,13 @@ void rds_info_register_func(int optname, rds_info_func func)BUG_ON (optname <RDS_INFO_FIRST || optname >RDS_INFO_LAST );spin_lock (&rds_info_lock); -BUG_ON (rds_info_funcs[offset]); - rds_info_funcs[offset] = func; +/* Validate the latest published slot value. */ +if (WARN_ON_ONCE (smp_load_acquire (&rds_info_funcs[offset]))) { +spin_unlock (&rds_info_lock); +return ; + } +/* Pair with lockless callback lookup. */ +smp_store_release (&rds_info_funcs[offset], func);spin_unlock (&rds_info_lock); }EXPORT_SYMBOL_GPL (rds_info_register_func);
@@ -91,9 +98,15 @@ void rds_info_deregister_func(int optname, rds_info_func func)BUG_ON (optname <RDS_INFO_FIRST || optname >RDS_INFO_LAST );spin_lock (&rds_info_lock); -BUG_ON (rds_info_funcs[offset] != func); - rds_info_funcs[offset] =NULL ; +/* Validate the latest published slot value. */ +if (WARN_ON_ONCE (smp_load_acquire (&rds_info_funcs[offset]) != func)) { +spin_unlock (&rds_info_lock); +return ; + } +/* Hide the callback before waiting for old readers. */ +smp_store_release (&rds_info_funcs[offset],NULL );spin_unlock (&rds_info_lock); +synchronize_srcu (&rds_info_srcu); }EXPORT_SYMBOL_GPL (rds_info_deregister_func);
@@ -165,6 +178,7 @@ int rds_info_getsockopt(struct socket *sock, int optname, sockopt_t *opt)int npages =0 ;int ret;int len; +int srcu_idx;int total; len = opt->optlen;
@@ -214,8 +228,11 @@ int rds_info_getsockopt(struct socket *sock, int optname, sockopt_t *opt)rdsdebug ("len %d nr_pages %lu\n" , len, nr_pages); call_func: - func = rds_info_funcs[optname -RDS_INFO_FIRST ]; + srcu_idx =srcu_read_lock (&rds_info_srcu); +/* Pair with callback slot publication and removal. */ + func =smp_load_acquire (&rds_info_funcs[optname -RDS_INFO_FIRST ]);if (!func) { +srcu_read_unlock (&rds_info_srcu, srcu_idx); ret = -ENOPROTOOPT ; goto out; }
@@ -225,6 +242,7 @@ int rds_info_getsockopt(struct socket *sock, int optname, sockopt_t *opt) iter.offset = offset0;func (sock, len, &iter, &lens); +srcu_read_unlock (&rds_info_srcu, srcu_idx);BUG_ON (lens.each ==0 ); total = lens.nr * lens.each;
Signature · 1 line
2.43.0
Re: [PATCH net-next v2] rds: synchronize info callbacks with module unload
Quoted 31 lines · Chengfeng Ye
rds_info_getsockopt()reads a callback fromrds_info_funcsand invokes it without protecting the callback's lifetime. Transport modules register functions stored in this array. For example, rds_tcp.ko registersrds_tcp_tc_info()forRDS_INFO_TCP_SOCKETS.This permits the following interleaving:
CPU0 CPU1
rds_info_getsockopt()
func =rds_tcp_tc_info
rmmodrds_tcp
rds_tcp_exit()
rds_info_deregister_func()
rds_info_funcs[offset] = NULL
freerds_tcpmodule text
func()The reader can therefore branch to an address in unloaded module text.
Protect callback invocation with SRCU. Enter the SRCU read-side critical section before loading the callback and leave it only after the callback returns. Clear the callback with release semantics and call
synchronize_srcu()before deregistration returns, preventing module unload from freeing its text while an old reader is still executing it. SRCU is required because callbacks such asRDS_INFO_COUNTERScan sleep.Keep the callback array unannotated and use acquire and release operations for publication so sparse does not have to apply
__rcuthrough the function-pointer typedef. Replace the two callback-slotBUG_ON()checks withWARN_ON_ONCE()and return without changing the slot on mismatch.
Quoted 4 lines
Fixes 70041088e3b9 ("RDS: Add TCP transport to RDS")Suggested-by Allison Henderson <[email protected]>Signed-off-by Chengfeng Ye <[email protected]>
Hi Chengfeng,
Thanks for the updates, this looks good to me.
Allison
Quoted 100 lines
---
Changes in v2:
- Split the info callback/module-unload race from the independentinc->i_connlifetime bug.
- Keep the callback array unannotated and usesmp_load_acquire()andsmp_store_release()to resolve the sparse errors.
- Replace the two touched callback-slotBUG_ON()checks withWARN_ON_ONCE()error paths.
- Rewrite the commit message for the callback race and omit the unrelatedinc->i_connKASAN report.net/rds/info.c | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-)net/rds/info.c +23
-5@@ -32,6 +32,7 @@*/ #include <linux/percpu.h> #include <linux/seq_file.h> +#include <linux/srcu.h> #include <linux/slab.h> #include <linux/proc_fs.h> #include <linux/export.h> @@ -68,6 +69,7 @@ struct rds_info_iterator {unsigned long offset; }; +DEFINE_STATIC_SRCU (rds_info_srcu);static DEFINE_SPINLOCK (rds_info_lock);static rds_info_func rds_info_funcs[RDS_INFO_LAST -RDS_INFO_FIRST +1 ];@@ -78,8 +80,13 @@ void rds_info_register_func(int optname, rds_info_func func)BUG_ON (optname <RDS_INFO_FIRST || optname >RDS_INFO_LAST );spin_lock (&rds_info_lock); -BUG_ON (rds_info_funcs[offset]); - rds_info_funcs[offset] = func; +/* Validate the latest published slot value. */ +if (WARN_ON_ONCE (smp_load_acquire (&rds_info_funcs[offset]))) { +spin_unlock (&rds_info_lock); +return ; + } +/* Pair with lockless callback lookup. */ +smp_store_release (&rds_info_funcs[offset], func);spin_unlock (&rds_info_lock); }EXPORT_SYMBOL_GPL (rds_info_register_func);@@ -91,9 +98,15 @@ void rds_info_deregister_func(int optname, rds_info_func func)BUG_ON (optname <RDS_INFO_FIRST || optname >RDS_INFO_LAST );spin_lock (&rds_info_lock); -BUG_ON (rds_info_funcs[offset] != func); - rds_info_funcs[offset] =NULL ; +/* Validate the latest published slot value. */ +if (WARN_ON_ONCE (smp_load_acquire (&rds_info_funcs[offset]) != func)) { +spin_unlock (&rds_info_lock); +return ; + } +/* Hide the callback before waiting for old readers. */ +smp_store_release (&rds_info_funcs[offset],NULL );spin_unlock (&rds_info_lock); +synchronize_srcu (&rds_info_srcu); }EXPORT_SYMBOL_GPL (rds_info_deregister_func);@@ -165,6 +178,7 @@ int rds_info_getsockopt(struct socket *sock, int optname, sockopt_t *opt)int npages =0 ;int ret;int len; +int srcu_idx;int total; len = opt->optlen;@@ -214,8 +228,11 @@ int rds_info_getsockopt(struct socket *sock, int optname, sockopt_t *opt)rdsdebug ("len %d nr_pages %lu\n" , len, nr_pages); call_func: - func = rds_info_funcs[optname -RDS_INFO_FIRST ]; + srcu_idx =srcu_read_lock (&rds_info_srcu); +/* Pair with callback slot publication and removal. */ + func =smp_load_acquire (&rds_info_funcs[optname -RDS_INFO_FIRST ]);if (!func) { +srcu_read_unlock (&rds_info_srcu, srcu_idx); ret = -ENOPROTOOPT ; goto out; }@@ -225,6 +242,7 @@ int rds_info_getsockopt(struct socket *sock, int optname, sockopt_t *opt) iter.offset = offset0;func (sock, len, &iter, &lens); +srcu_read_unlock (&rds_info_srcu, srcu_idx);BUG_ON (lens.each ==0 ); total = lens.nr * lens.each;
Paolo Abeni
maintainerRe: [PATCH net-next v2] rds: synchronize info callbacks with module unload
In reply to Chengfeng YeQuoted 31 lines · Chengfeng Ye
rds_info_getsockopt()reads a callback fromrds_info_funcsand invokes it without protecting the callback's lifetime. Transport modules register functions stored in this array. For example, rds_tcp.ko registersrds_tcp_tc_info()forRDS_INFO_TCP_SOCKETS.This permits the following interleaving:
CPU0 CPU1
rds_info_getsockopt()
func =rds_tcp_tc_info
rmmodrds_tcp
rds_tcp_exit()
rds_info_deregister_func()
rds_info_funcs[offset] = NULL
freerds_tcpmodule text
func()The reader can therefore branch to an address in unloaded module text.
Protect callback invocation with SRCU. Enter the SRCU read-side critical section before loading the callback and leave it only after the callback returns. Clear the callback with release semantics and call
synchronize_srcu()before deregistration returns, preventing module unload from freeing its text while an old reader is still executing it. SRCU is required because callbacks such asRDS_INFO_COUNTERScan sleep.Keep the callback array unannotated and use acquire and release operations for publication so sparse does not have to apply
__rcuthrough the function-pointer typedef. Replace the two callback-slotBUG_ON()checks withWARN_ON_ONCE()and return without changing the slot on mismatch.
Quoted 4 lines
Fixes 70041088e3b9 ("RDS: Add TCP transport to RDS")Suggested-by Allison Henderson <[email protected]>Signed-off-by Chengfeng Ye <[email protected]>
@Alison: I understand you prefer this on net-next, and I'm fine with such target, but the current guidance for net-next patches is no fixes tag unless the buggy code is on net-next only.
I'm going to strip the fixes tag while applying this one.
Thanks,
Paolo
Paolo Abeni
maintainerRe: [PATCH net-next v2] rds: synchronize info callbacks with module unload
In reply to Chengfeng YeHi,
It looks like I was too optimistic in my previous reply.
net/rds/info.c void rds_info_register_func(int optname, rds_info_func func) · line 84
Quoted 8 lines · Chengfeng Ye
net/rds/info.c +2
-2@@ -78,8 +80,13 @@ void rds_info_register_func(int optname, rds_info_func func)BUG_ON (optname <RDS_INFO_FIRST || optname >RDS_INFO_LAST );spin_lock (&rds_info_lock); -BUG_ON (rds_info_funcs[offset]); - rds_info_funcs[offset] = func; +/* Validate the latest published slot value. */ +if (WARN_ON_ONCE (smp_load_acquire (&rds_info_funcs[offset]))) {
This read is under the same lock that perform the store. It's totally unclear why smp_load_acquire() would be needed here. Very reasonably it's not.
Quoted 9 lines
net/rds/info.c +5
-0+spin_unlock (&rds_info_lock); +return ; + } +/* Pair with lockless callback lookup. */ +smp_store_release (&rds_info_funcs[offset], func);spin_unlock (&rds_info_lock); }EXPORT_SYMBOL_GPL (rds_info_register_func);
net/rds/info.c void rds_info_deregister_func(int optname, rds_info_func func) · line 102
Quoted 7 lines
net/rds/info.c +2
-2@@ -91,9 +98,15 @@ void rds_info_deregister_func(int optname, rds_info_func func)BUG_ON (optname <RDS_INFO_FIRST || optname >RDS_INFO_LAST );spin_lock (&rds_info_lock); -BUG_ON (rds_info_funcs[offset] != func); - rds_info_funcs[offset] =NULL ; +/* Validate the latest published slot value. */ +if (WARN_ON_ONCE (smp_load_acquire (&rds_info_funcs[offset]) != func)) {
Same here.
/P
Re: [PATCH net-next v2] rds: synchronize info callbacks with module unload
Quoted 38 lines · Paolo Abeni
Hi,
It looks like I was too optimistic in my previous reply.
net/rds/info.c void rds_info_register_func(int optname, rds_info_func func) · line 84
Quoted 8 lines · Chengfeng Ye
net/rds/info.c +2
-2@@ -78,8 +80,13 @@ void rds_info_register_func(int optname, rds_info_func func)BUG_ON (optname <RDS_INFO_FIRST || optname >RDS_INFO_LAST );spin_lock (&rds_info_lock); -BUG_ON (rds_info_funcs[offset]); - rds_info_funcs[offset] = func; +/* Validate the latest published slot value. */ +if (WARN_ON_ONCE (smp_load_acquire (&rds_info_funcs[offset]))) {This read is under the same lock that perform the store. It's totally unclear why
smp_load_acquire()would be needed here. Very reasonably it's not.Quoted 9 lines
net/rds/info.c +5
-0+spin_unlock (&rds_info_lock); +return ; + } +/* Pair with lockless callback lookup. */ +smp_store_release (&rds_info_funcs[offset], func);spin_unlock (&rds_info_lock); }EXPORT_SYMBOL_GPL (rds_info_register_func);net/rds/info.c void rds_info_deregister_func(int optname, rds_info_func func) · line 102
Quoted 7 lines
net/rds/info.c +2
-2@@ -91,9 +98,15 @@ void rds_info_deregister_func(int optname, rds_info_func func)BUG_ON (optname <RDS_INFO_FIRST || optname >RDS_INFO_LAST );spin_lock (&rds_info_lock); -BUG_ON (rds_info_funcs[offset] != func); - rds_info_funcs[offset] =NULL ; +/* Validate the latest published slot value. */ +if (WARN_ON_ONCE (smp_load_acquire (&rds_info_funcs[offset]) != func)) {Same here.
Quoted 2 lines
/P
Hi Paolo,
You are right, that two smp_load_acquire() are unnecessary. Should I send a v3 patch to change that two smp_load_acquire() under locked to use back plain read?
Thanks, Chengfeng
Allison Henderson
maintainerRe: [PATCH net-next v2] rds: synchronize info callbacks with module unload
In reply to Paolo AbeniQuoted 40 lines · Paolo Abeni
Quoted 31 lines · Chengfeng Ye
rds_info_getsockopt()reads a callback fromrds_info_funcsand invokes it without protecting the callback's lifetime. Transport modules register functions stored in this array. For example, rds_tcp.ko registersrds_tcp_tc_info()forRDS_INFO_TCP_SOCKETS.This permits the following interleaving:
CPU0 CPU1
rds_info_getsockopt()
func =rds_tcp_tc_info
rmmodrds_tcp
rds_tcp_exit()
rds_info_deregister_func()
rds_info_funcs[offset] = NULL
freerds_tcpmodule text
func()The reader can therefore branch to an address in unloaded module text.
Protect callback invocation with SRCU. Enter the SRCU read-side critical section before loading the callback and leave it only after the callback returns. Clear the callback with release semantics and call
synchronize_srcu()before deregistration returns, preventing module unload from freeing its text while an old reader is still executing it. SRCU is required because callbacks such asRDS_INFO_COUNTERScan sleep.Keep the callback array unannotated and use acquire and release operations for publication so sparse does not have to apply
__rcuthrough the function-pointer typedef. Replace the two callback-slotBUG_ON()checks withWARN_ON_ONCE()and return without changing the slot on mismatch.Quoted 4 lines
Fixes 70041088e3b9 ("RDS: Add TCP transport to RDS")Suggested-by Allison Henderson <[email protected]>Signed-off-by Chengfeng Ye <[email protected]>@Alison: I understand you prefer this on net-next, and I'm fine with such target, but the current guidance for net-next patches is no fixes tag unless the buggy code is on net-next only.
Quoted 6 lines
I'm going to strip the fixes tag while applying this one.
Thanks,
Paolo
I see, sure that would be fine. Thank you!
Allison
kernel test robot
build botkernel test robot noticed the following build warnings:
Hi Chengfeng,
kernel test robot noticed the following build warnings:
[auto build test WARNING on net/main]
[also build test WARNING on linus/master horms-ipvs/master v7.2-rc7] [cannot apply to net-next/main next-20260812]
[If your patch is applied to the wrong git tree, kindly drop us a note. And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch#_base_tree_information]
url: https://github.com/intel-lab-lkp/linux/commits/Chengfeng-Ye/rds-synchronize-info-callbacks-with-connection-teardown/20260812-225931 base: net/main patch link: https://lore.kernel.org/r/20260720184955.3008978-1-nicoyip.dev%40gmail.com patch subject: [PATCH] rds: synchronize info callbacks with connection teardown config: mips-randconfig-r134-20260813 (https://download.01.org/0day-ci/archive/20260814/[email protected]/config) compiler: mips-linux-gcc (GCC) 11.5.0 sparse: v0.6.5-rc1 reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20260814/[email protected]/reproduce)
If you fix the issue in a separate patch/commit (i.e. not just a new version of the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <[email protected]>
| Closes: https://lore.kernel.org/oe-kbuild-all/[email protected]/
sparse warnings: (new ones prefixed by >>)
>> net/rds/info.c:83:9: sparse: sparse: cast removes address space '__rcu' of expression >> net/rds/info.c:83:9: sparse: sparse: incompatible types in comparison expression (different address spaces):
net/rds/info.c:83:9: sparse: void ( [noderef] __rcu * )( ... )
net/rds/info.c:83:9: sparse: void ( * )( ... )
net/rds/info.c:84:9: sparse: sparse: incompatible types in comparison expression (different address spaces):
net/rds/info.c:84:9: sparse: void ( [noderef] __rcu * )( ... )
net/rds/info.c:84:9: sparse: void ( * )( ... )
net/rds/info.c:96:9: sparse: sparse: cast removes address space '__rcu' of expression net/rds/info.c:96:9: sparse: sparse: incompatible types in comparison expression (different address spaces):
net/rds/info.c:96:9: sparse: void ( [noderef] __rcu * )( ... )
net/rds/info.c:96:9: sparse: void ( * )( ... )
net/rds/info.c:97:9: sparse: sparse: incompatible types in comparison expression (different address spaces):
net/rds/info.c:97:9: sparse: void ( [noderef] __rcu * )( ... )
net/rds/info.c:97:9: sparse: void ( * )( ... )
net/rds/info.c:222:16: sparse: sparse: cast removes address space '__rcu' of expression net/rds/info.c:222:16: sparse: sparse: incompatible types in comparison expression (different address spaces):
net/rds/info.c:222:16: sparse: void ( [noderef] __rcu * )( ... )
net/rds/info.c:222:16: sparse: void ( * )( ... )
vim +/__rcu +83 net/rds/info.c
75
76 void rds_info_register_func(int optname, rds_info_func func)
77 {
78 int offset = optname - RDS_INFO_FIRST;
79
80 BUG_ON(optname < RDS_INFO_FIRST || optname > RDS_INFO_LAST);
81
82 spin_lock(&rds_info_lock);
> 83 BUG_ON(rcu_access_pointer(rds_info_funcs[offset]));
84 rcu_assign_pointer(rds_info_funcs[offset], func);
85 spin_unlock(&rds_info_lock);
86 }
87 EXPORT_SYMBOL_GPL(rds_info_register_func);
88 --
0-DAY CI Kernel Test Service
https://github.com/intel/lkp-tests/wiki