net: protect egress device access in the output path with rcu_read_lock
Subsystems (5)
PATCH v2 1/1 net: protect egress device access in the output path with rcu_read_lock
ip_mc_output(), __ip_local_out(), xfrm4_output(), raw_send_hdrinc(), xfrm_output_resume() and vrf_output() read the egress network device from rt->dst.dev / skb_dst(skb)->dev / skb_dst_dev(skb) and then pass that pointer as the 'out' device argument of NF_HOOK without holding rcu_read_lock.
dst->dev is a field protected by RCU. When a device is unregistered its value is replaced with blackhole_netdev, and the previous device is freed after an RCU grace period. A section that reads dst->dev and uses that pointer must therefore hold rcu_read_lock. The functions above are missing this protection, so when the egress device is unregistered concurrently with transmission, a LOCAL_OUT / POST_ROUTING hook (nft meta oif, selinux_ip_postroute_compat, etc.) can reference a device pointer that is no longer valid.
In the leaves above, wrap the section that reads the device and runs NF_HOOK in rcu_read_lock(), and read the device through the RCU accessors (skb_dst_dev_rcu() / dst_dev_rcu()). As a result, the RCU grace period cannot complete while a transmission is in flight, so the egress device is not freed and the hook always references a valid device.
---
Changes in v2:
- Changed to a net-wide patch that also fixes the issue in raw_send_hdrinc(), xfrm_output_resume() and vrf_output().
- v1: https://lore.kernel.org/all/ahqIg6vURwYI0LJ5@v4bel/
--- drivers/net/vrf.c | 16 +++++++++++----- net/ipv4/ip_output.c | 27 +++++++++++++++++++-------- net/ipv4/raw.c | 4 +++- net/ipv4/xfrm4_output.c | 13 +++++++++---- net/xfrm/xfrm_output.c | 4 +++- 5 files changed, 45 insertions(+), 19 deletions(-)
drivers/net/vrf.c +11 -5
@@ -833,17 +833,23 @@ static int vrf_finish_output(struct net *net, struct sock *sk, struct sk_buff *sstatic int vrf_output (struct net *net,struct sock *sk,struct sk_buff *skb) { -struct net_device *dev =skb_dst (skb)->dev; +struct net_device *dev; +int ret; + +rcu_read_lock (); + dev =skb_dst_dev_rcu (skb);IP_UPD_PO_STATS (net,IPSTATS_MIB_OUT , skb->len); skb->dev = dev; skb->protocol =htons (ETH_P_IP ); -return NF_HOOK_COND (NFPROTO_IPV4 ,NF_INET_POST_ROUTING , - net, sk, skb,NULL , dev, - vrf_finish_output, - !(IPCB (skb)->flags &IPSKB_REROUTED )); + ret =NF_HOOK_COND (NFPROTO_IPV4 ,NF_INET_POST_ROUTING , + net, sk, skb,NULL , dev, + vrf_finish_output, + !(IPCB (skb)->flags &IPSKB_REROUTED )); +rcu_read_unlock (); +return ret; }/* set dst on skb to send packet to us via dev_xmit path. Allows
net/ipv4/ip_output.c +19 -8
@@ -102,6 +102,7 @@ EXPORT_SYMBOL(ip_send_check);int __ip_local_out (struct net *net,struct sock *sk,struct sk_buff *skb) {struct iphdr *iph =ip_hdr (skb); +int err;IP_INC_STATS (net,IPSTATS_MIB_OUTREQUESTS );
@@ -117,9 +118,12 @@ int __ip_local_out(struct net *net, struct sock *sk, struct sk_buff *skb) skb->protocol =htons (ETH_P_IP ); -return nf_hook (NFPROTO_IPV4 ,NF_INET_LOCAL_OUT , - net, sk, skb,NULL ,skb_dst_dev (skb), - dst_output); +rcu_read_lock (); + err =nf_hook (NFPROTO_IPV4 ,NF_INET_LOCAL_OUT , + net, sk, skb,NULL ,skb_dst_dev_rcu (skb), + dst_output); +rcu_read_unlock (); +return err; }int ip_local_out (struct net *net,struct sock *sk,struct sk_buff *skb)
@@ -368,7 +372,11 @@ static int ip_mc_finish_output(struct net *net, struct sock *sk,int ip_mc_output (struct net *net,struct sock *sk,struct sk_buff *skb) {struct rtable *rt =skb_rtable (skb); -struct net_device *dev = rt->dst.dev; +struct net_device *dev; +int ret; + +rcu_read_lock (); + dev =dst_dev_rcu (&rt->dst);/* * If the indicated interface is up and running, send the packet.
@@ -407,6 +415,7 @@ int ip_mc_output(struct net *net, struct sock *sk, struct sk_buff *skb)if (ip_hdr (skb)->ttl ==0 ) {kfree_skb (skb); +rcu_read_unlock ();return 0 ; } }
@@ -419,10 +428,12 @@ int ip_mc_output(struct net *net, struct sock *sk, struct sk_buff *skb) ip_mc_finish_output); } -return NF_HOOK_COND (NFPROTO_IPV4 ,NF_INET_POST_ROUTING , - net, sk, skb,NULL , skb->dev, - ip_finish_output, - !(IPCB (skb)->flags &IPSKB_REROUTED )); + ret =NF_HOOK_COND (NFPROTO_IPV4 ,NF_INET_POST_ROUTING , + net, sk, skb,NULL , skb->dev, + ip_finish_output, + !(IPCB (skb)->flags &IPSKB_REROUTED )); +rcu_read_unlock (); +return ret; }int ip_output (struct net *net,struct sock *sk,struct sk_buff *skb)
net/ipv4/raw.c +3 -1
@@ -410,9 +410,11 @@ static int raw_send_hdrinc(struct sock *sk, struct flowi4 *fl4, skb_transport_header(skb))->type); } +rcu_read_lock (); err =NF_HOOK (NFPROTO_IPV4 ,NF_INET_LOCAL_OUT , - net, sk, skb,NULL , rt->dst.dev, + net, sk, skb,NULL ,skb_dst_dev_rcu (skb), dst_output); +rcu_read_unlock ();if (err >0 ) err =net_xmit_errno (err);if (err)
net/ipv4/xfrm4_output.c +9 -4
@@ -30,10 +30,15 @@ static int __xfrm4_output(struct net *net, struct sock *sk, struct sk_buff *skb)int xfrm4_output (struct net *net,struct sock *sk,struct sk_buff *skb) { -return NF_HOOK_COND (NFPROTO_IPV4 ,NF_INET_POST_ROUTING , - net, sk, skb, skb->dev,skb_dst_dev (skb), - __xfrm4_output, - !(IPCB (skb)->flags &IPSKB_REROUTED )); +int ret; + +rcu_read_lock (); + ret =NF_HOOK_COND (NFPROTO_IPV4 ,NF_INET_POST_ROUTING , + net, sk, skb, skb->dev,skb_dst_dev_rcu (skb), + __xfrm4_output, + !(IPCB (skb)->flags &IPSKB_REROUTED )); +rcu_read_unlock (); +return ret; }void xfrm4_local_error (struct sk_buff *skb,u32 mtu)
net/xfrm/xfrm_output.c +3 -1
@@ -604,9 +604,11 @@ int xfrm_output_resume(struct sock *sk, struct sk_buff *skb, int err)if (!skb_dst (skb)->xfrm)return dst_output (net, sk, skb); +rcu_read_lock (); err =nf_hook (skb_dst (skb)->ops->family,NF_INET_POST_ROUTING , net, sk, skb, -NULL ,skb_dst (skb)->dev, xfrm_output2); +NULL ,skb_dst_dev_rcu (skb), xfrm_output2); +rcu_read_unlock ();if (unlikely (err !=1 )) goto out; }
Signature · 1 line
2.43.0
Ido Schimmel
maintainerQuoted 28 lines · Hyunwoo Kim
drivers/net/vrf.c | 16 +++++++++++----- net/ipv4/ip_output.c | 27 +++++++++++++++++++-------- net/ipv4/raw.c | 4 +++- net/ipv4/xfrm4_output.c | 13 +++++++++---- net/xfrm/xfrm_output.c | 4 +++- 5 files changed, 45 insertions(+), 19 deletions(-)drivers/net/vrf.c +5
-2@@ -833,17 +833,23 @@ static int vrf_finish_output(struct net *net, struct sock *sk, struct sk_buff *sstatic int vrf_output (struct net *net,struct sock *sk,struct sk_buff *skb) { -struct net_device *dev =skb_dst (skb)->dev; +struct net_device *dev; +int ret; + +rcu_read_lock (); + dev =skb_dst_dev_rcu (skb);IP_UPD_PO_STATS (net,IPSTATS_MIB_OUT , skb->len); skb->dev = dev; skb->protocol =htons (ETH_P_IP ); - return NF_HOOK_COND(NFPROTO_IPV4, NF_INET_POST_ROUTING,
drivers/net/vrf.c static int vrf_finish_output(struct net *net, struct sock *sk, struct sk_buff *s · line 853
Quoted 10 lines
drivers/net/vrf.c +6
-3- net, sk, skb,NULL , dev, - vrf_finish_output, - !(IPCB(skb)->flags & IPSKB_REROUTED)); + ret =NF_HOOK_COND (NFPROTO_IPV4 ,NF_INET_POST_ROUTING , + net, sk, skb,NULL , dev, + vrf_finish_output, + !(IPCB (skb)->flags &IPSKB_REROUTED )); +rcu_read_unlock (); +return ret; }
Patch LGTM, thanks, but what about the IPv6 counterpart (vrf_output6()) and the other similar existing issues that Sashiko is flagging?
David Ahern
maintainerQuoted 28 lines · Ido Schimmel
Quoted 27 lines · Hyunwoo Kim
drivers/net/vrf.c | 16 +++++++++++----- net/ipv4/ip_output.c | 27 +++++++++++++++++++-------- net/ipv4/raw.c | 4 +++- net/ipv4/xfrm4_output.c | 13 +++++++++---- net/xfrm/xfrm_output.c | 4 +++- 5 files changed, 45 insertions(+), 19 deletions(-)drivers/net/vrf.c +5
-1@@ -833,17 +833,23 @@ static int vrf_finish_output(struct net *net, struct sock *sk, struct sk_buff *sstatic int vrf_output (struct net *net,struct sock *sk,struct sk_buff *skb) { -struct net_device *dev =skb_dst (skb)->dev; +struct net_device *dev; +int ret; + +rcu_read_lock (); + dev =skb_dst_dev_rcu (skb);IP_UPD_PO_STATS (net,IPSTATS_MIB_OUT , skb->len); skb->dev = dev; skb->protocol =htons (ETH_P_IP );
Quoted 14 lines
drivers/net/vrf.c static int vrf_finish_output(struct net *net, struct sock *sk, struct sk_buff *s · line 853
Quoted 11 lines
drivers/net/vrf.c +6
-4-return NF_HOOK_COND (NFPROTO_IPV4 ,NF_INET_POST_ROUTING , - net, sk, skb,NULL , dev, - vrf_finish_output, - !(IPCB (skb)->flags &IPSKB_REROUTED )); + ret =NF_HOOK_COND (NFPROTO_IPV4 ,NF_INET_POST_ROUTING , + net, sk, skb,NULL , dev, + vrf_finish_output, + !(IPCB (skb)->flags &IPSKB_REROUTED )); +rcu_read_unlock (); +return ret; }Patch LGTM, thanks, but what about the IPv6 counterpart (
vrf_output6()) and the other similar existing issues that Sashiko is flagging?
Common locking at a higher level than dst->output seems like a better approach. This function is called under rcu_read_lock for some paths.
Quoted 29 lines · David Ahern
Quoted 28 lines · Ido Schimmel
Quoted 27 lines · Hyunwoo Kim
drivers/net/vrf.c | 16 +++++++++++----- net/ipv4/ip_output.c | 27 +++++++++++++++++++-------- net/ipv4/raw.c | 4 +++- net/ipv4/xfrm4_output.c | 13 +++++++++---- net/xfrm/xfrm_output.c | 4 +++- 5 files changed, 45 insertions(+), 19 deletions(-)drivers/net/vrf.c +5
-1@@ -833,17 +833,23 @@ static int vrf_finish_output(struct net *net, struct sock *sk, struct sk_buff *sstatic int vrf_output (struct net *net,struct sock *sk,struct sk_buff *skb) { -struct net_device *dev =skb_dst (skb)->dev; +struct net_device *dev; +int ret; + +rcu_read_lock (); + dev =skb_dst_dev_rcu (skb);IP_UPD_PO_STATS (net,IPSTATS_MIB_OUT , skb->len); skb->dev = dev; skb->protocol =htons (ETH_P_IP );
Quoted 13 lines
Quoted 5 lines
Quoted 5 lines
drivers/net/vrf.c +1
-4-return NF_HOOK_COND (NFPROTO_IPV4 ,NF_INET_POST_ROUTING , - net, sk, skb,NULL , dev, - vrf_finish_output, - !(IPCB (skb)->flags &IPSKB_REROUTED )); + ret = NF_HOOK_COND(NFPROTO_IPV4, NF_INET_POST_ROUTING,Quoted 8 lines
drivers/net/vrf.c static int vrf_finish_output(struct net *net, struct sock *sk, struct sk_buff *s · line 853
Quoted 6 lines
drivers/net/vrf.c +5
-0+ net, sk, skb,NULL , dev, + vrf_finish_output, + !(IPCB(skb)->flags & IPSKB_REROUTED)); +rcu_read_unlock (); +return ret; }Patch LGTM, thanks, but what about the IPv6 counterpart (
vrf_output6())
I left ipv6 out because its callers (ip6_send_skb, ip6_xmit, etc.) already hold rcu. Though some niche paths may not. (maybe rxe_send?)
Quoted 1 line
Quoted 1 line
and the other similar existing issues that Sashiko is flagging?
I couldn't find Sashiko's review. Could you share a url?
Quoted 3 lines
Common locking at a higher level than
dst->outputseems like a better approach. This function is called underrcu_read_lockfor some paths.
Adding rcu to ip_local_out would protect all functions reached via dst_output in one place; raw_send_hdrinc and xfrm_output_resume still need their own patches. What do you think of this diff?
As a follow-up, the rcu in ip_output etc. then becomes redundant and should be removed.
Best regards,
Hyunwoo Kim
---
net/ipv4/ip_output.c +2 -0
@@ -126,9 +126,11 @@ int ip_local_out(struct net *net, struct sock *sk, struct sk_buff *skb) {int err; +rcu_read_lock (); err =__ip_local_out (net, sk, skb);if (likely (err ==1 )) err =dst_output (net, sk, skb); +rcu_read_unlock ();return err; }
net/ipv4/raw.c +3 -1
@@ -410,9 +410,11 @@ static int raw_send_hdrinc(struct sock *sk, struct flowi4 *fl4, skb_transport_header(skb))->type); } +rcu_read_lock (); err =NF_HOOK (NFPROTO_IPV4 ,NF_INET_LOCAL_OUT , - net, sk, skb,NULL , rt->dst.dev, + net, sk, skb,NULL ,skb_dst_dev_rcu (skb), dst_output); +rcu_read_unlock ();if (err >0 ) err =net_xmit_errno (err);if (err)
net/xfrm/xfrm_output.c +7 -3
@@ -594,6 +594,7 @@ int xfrm_output_resume(struct sock *sk, struct sk_buff *skb, int err) {struct net *net =xs_net (skb_dst (skb)->xfrm); +rcu_read_lock ();while (likely ((err =xfrm_output_one (skb, err)) ==0 )) {nf_reset_ct (skb);
@@ -601,12 +602,14 @@ int xfrm_output_resume(struct sock *sk, struct sk_buff *skb, int err)if (unlikely (err !=1 )) goto out; -if (!skb_dst (skb)->xfrm) -return dst_output (net, sk, skb); +if (!skb_dst (skb)->xfrm) { + err =dst_output (net, sk, skb); + goto out; + } err =nf_hook (skb_dst (skb)->ops->family,NF_INET_POST_ROUTING , net, sk, skb, -NULL ,skb_dst (skb)->dev, xfrm_output2); +NULL ,skb_dst_dev_rcu (skb), xfrm_output2);if (unlikely (err !=1 )) goto out; }
@@ -615,6 +618,7 @@ int xfrm_output_resume(struct sock *sk, struct sk_buff *skb, int err) err =0 ; out: +rcu_read_unlock ();return err; }EXPORT_SYMBOL_GPL (xfrm_output_resume);
David Ahern
maintainerQuoted 132 lines · Hyunwoo Kim
Quoted 29 lines · David Ahern
Quoted 28 lines · Ido Schimmel
Quoted 27 lines · Hyunwoo Kim
drivers/net/vrf.c | 16 +++++++++++----- net/ipv4/ip_output.c | 27 +++++++++++++++++++-------- net/ipv4/raw.c | 4 +++- net/ipv4/xfrm4_output.c | 13 +++++++++---- net/xfrm/xfrm_output.c | 4 +++- 5 files changed, 45 insertions(+), 19 deletions(-)drivers/net/vrf.c +5
-1@@ -833,17 +833,23 @@ static int vrf_finish_output(struct net *net, struct sock *sk, struct sk_buff *sstatic int vrf_output (struct net *net,struct sock *sk,struct sk_buff *skb) { -struct net_device *dev =skb_dst (skb)->dev; +struct net_device *dev; +int ret; + +rcu_read_lock (); + dev =skb_dst_dev_rcu (skb);IP_UPD_PO_STATS (net,IPSTATS_MIB_OUT , skb->len); skb->dev = dev; skb->protocol =htons (ETH_P_IP );Quoted 13 lines
Quoted 5 lines
Quoted 5 lines
drivers/net/vrf.c +1
-4-return NF_HOOK_COND (NFPROTO_IPV4 ,NF_INET_POST_ROUTING , - net, sk, skb,NULL , dev, - vrf_finish_output, - !(IPCB (skb)->flags &IPSKB_REROUTED )); + ret = NF_HOOK_COND(NFPROTO_IPV4, NF_INET_POST_ROUTING,Quoted 8 lines
drivers/net/vrf.c static int vrf_finish_output(struct net *net, struct sock *sk, struct sk_buff *s · line 853
Quoted 6 lines
drivers/net/vrf.c +5
-0+ net, sk, skb,NULL , dev, + vrf_finish_output, + !(IPCB(skb)->flags & IPSKB_REROUTED)); +rcu_read_unlock (); +return ret; }Patch LGTM, thanks, but what about the IPv6 counterpart (
vrf_output6())I left ipv6 out because its callers (
ip6_send_skb,ip6_xmit, etc.) already hold rcu. Though some niche paths may not. (mayberxe_send?)Quoted 1 line
Quoted 1 line
and the other similar existing issues that Sashiko is flagging?
I couldn't find Sashiko's review. Could you share a url?
Quoted 3 lines
Common locking at a higher level than
dst->outputseems like a better approach. This function is called underrcu_read_lockfor some paths.Adding rcu to
ip_local_outwould protect all functions reached viadst_outputin one place;raw_send_hdrincandxfrm_output_resumestill need their own patches. What do you think of this diff?As a follow-up, the rcu in
ip_outputetc. then becomes redundant and should be removed.Best regards,
Hyunwoo Kim---
net/ipv4/ip_output.c +2
-0@@ -126,9 +126,11 @@ int ip_local_out(struct net *net, struct sock *sk, struct sk_buff *skb) {int err; +rcu_read_lock (); err =__ip_local_out (net, sk, skb);if (likely (err ==1 )) err =dst_output (net, sk, skb); +rcu_read_unlock ();return err; }net/ipv4/raw.c +3
-1@@ -410,9 +410,11 @@ static int raw_send_hdrinc(struct sock *sk, struct flowi4 *fl4, skb_transport_header(skb))->type); } +rcu_read_lock (); err =NF_HOOK (NFPROTO_IPV4 ,NF_INET_LOCAL_OUT , - net, sk, skb,NULL , rt->dst.dev, + net, sk, skb,NULL ,skb_dst_dev_rcu (skb), dst_output); +rcu_read_unlock ();if (err >0 ) err =net_xmit_errno (err);if (err)net/xfrm/xfrm_output.c +6
-3@@ -594,6 +594,7 @@ int xfrm_output_resume(struct sock *sk, struct sk_buff *skb, int err) {struct net *net =xs_net (skb_dst (skb)->xfrm); +rcu_read_lock ();while (likely ((err =xfrm_output_one (skb, err)) ==0 )) {nf_reset_ct (skb);@@ -601,12 +602,14 @@ int xfrm_output_resume(struct sock *sk, struct sk_buff *skb, int err)if (unlikely (err !=1 )) goto out; -if (!skb_dst (skb)->xfrm) -return dst_output (net, sk, skb); +if (!skb_dst (skb)->xfrm) { + err =dst_output (net, sk, skb); + goto out; + } err =nf_hook (skb_dst (skb)->ops->family,NF_INET_POST_ROUTING , net, sk, skb, -NULL ,skb_dst (skb)->dev, xfrm_output2); +NULL ,skb_dst_dev_rcu (skb), xfrm_output2);if (unlikely (err !=1 )) goto out; }
net/xfrm/xfrm_output.c int xfrm_output_resume(struct sock *sk, struct sk_buff *skb, int err) · line 624
Quoted 7 lines
net/xfrm/xfrm_output.c +1
-0@@ -615,6 +618,7 @@ int xfrm_output_resume(struct sock *sk, struct sk_buff *skb, int err) err =0 ; out: +rcu_read_unlock ();return err; }EXPORT_SYMBOL_GPL (xfrm_output_resume);
I prefer this over locking in each of the output functions. IPv6 should be updated as well.
Herbert Xu
maintainernet/xfrm/xfrm_output.c int xfrm_output_resume(struct sock *sk, struct sk_buff *skb, int err) · line 597
Quoted 10 lines · Hyunwoo Kim
net/xfrm/xfrm_output.c +1
-0@@ -594,6 +594,7 @@ int xfrm_output_resume(struct sock *sk, struct sk_buff *skb, int err) {struct net *net =xs_net (skb_dst (skb)->xfrm); +rcu_read_lock ();
xfrm_output_resume is supposed to run with BH off so there is no need to acquire the RCU lock.
Please also check the other code paths that you chagned to ensure that they don't already run with BH off.
Cheers,
Signature · 3 lines
Email: Herbert Xu <[email protected]> Home Page: http://gondor.apana.org.au/~herbert/ PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txt
Herbert Xu
maintainerOn Thu, Jun 04, 2026 at 09:28:08AM +0800, Herbert Xu wrote:
.
Quoted 2 lines
xfrm_output_resumeis supposed to run with BH off so there is no need to acquire the RCU lock.
OK only the ones that actually come from crypto offload run with BH off. So perhaps add the RCU lock to the other callers of xfrm_output_resume (ones which are not really resumptions) instead.
Cheers,
Signature · 3 lines
Email: Herbert Xu <[email protected]> Home Page: http://gondor.apana.org.au/~herbert/ PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txt
Ido Schimmel
maintainerQuoted 1 line · Hyunwoo Kim
I couldn't find Sashiko's review. Could you share a url?