openvswitch: Fix CT limit teardown use-after-free
Subsystems (3)
A message that is not in the archive (sent off-list or not yet received): [email protected]
PATCH 1/1 openvswitch: Fix CT limit teardown use-after-free
From: Yuqi Xu <[email protected]>
Packet processing uses CT limit state under RCU, while netns teardown frees that state under ovs_mutex. The CT limit pointer was neither removed from readers nor protected by a grace period, allowing packet processing to dereference the freed state.
Replace the pointer before freeing the CT limit state. Wait for in-flight RCU readers before freeing its contents. Serialize CT limit netlink operations with teardown for the full lifetime of their state accesses.
--- net/openvswitch/conntrack.c | 79 +++++++++++++++++++++++-------------- net/openvswitch/datapath.h | 2 +- 2 files changed, 51 insertions(+), 30 deletions(-)
net/openvswitch/conntrack.c +50 -29
@@ -932,11 +932,15 @@ static int ovs_ct_check_limit(struct net *net,const struct sk_buff *skb, const struct ovs_conntrack_info *info) { + const struct ovs_ct_limit_info *ct_limit_info;struct ovs_net *ovs_net =net_generic (net, ovs_net_id); -const struct ovs_ct_limit_info *ct_limit_info = ovs_net->ct_limit_info;u32 per_zone_limit, connections;u32 conncount_key; + ct_limit_info =rcu_dereference (ovs_net->ct_limit_info); +if (!ct_limit_info) +return 0 ; + conncount_key = info->zone.id; per_zone_limit =ct_limit_get (ct_limit_info, info->zone.id);
@@ -1585,40 +1589,47 @@ static void __ovs_ct_free_action(struct ovs_conntrack_info *ct_info)#if IS_ENABLED (CONFIG_NETFILTER_CONNCOUNT )static int ovs_ct_limit_init (struct net *net,struct ovs_net *ovs_net) { +struct ovs_ct_limit_info *info;int i, err; - ovs_net->ct_limit_info =kmalloc_obj (*ovs_net->ct_limit_info); -if (!ovs_net->ct_limit_info) + info =kmalloc_obj (*info); +if (!info)return -ENOMEM ; - ovs_net->ct_limit_info->default_limit =OVS_CT_LIMIT_DEFAULT ; - ovs_net->ct_limit_info->limits = + info->default_limit =OVS_CT_LIMIT_DEFAULT ; + info->limits = kmalloc_objs(struct hlist_head, CT_LIMIT_HASH_BUCKETS); -if (!ovs_net->ct_limit_info->limits) { -kfree (ovs_net->ct_limit_info); +if (!info->limits) { +kfree (info);return -ENOMEM ; }for (i =0 ; i <CT_LIMIT_HASH_BUCKETS ; i++) -INIT_HLIST_HEAD (&ovs_net->ct_limit_info->limits[i]); +INIT_HLIST_HEAD (&info->limits[i]); - ovs_net->ct_limit_info->data =nf_conncount_init (net,sizeof (u32)); + info->data =nf_conncount_init (net,sizeof (u32)); -if (IS_ERR (ovs_net->ct_limit_info->data)) { - err =PTR_ERR (ovs_net->ct_limit_info->data); -kfree (ovs_net->ct_limit_info->limits); -kfree (ovs_net->ct_limit_info); +if (IS_ERR (info->data)) { + err =PTR_ERR (info->data); +kfree (info->limits); +kfree (info);pr_err ("openvswitch: failed to init nf_conncount %d\n" , err);return err; } +rcu_assign_pointer (ovs_net->ct_limit_info, info);return 0 ; }static void ovs_ct_limit_exit (struct net *net,struct ovs_net *ovs_net) { -const struct ovs_ct_limit_info *info = ovs_net->ct_limit_info; +const struct ovs_ct_limit_info *info;int i; + info =rcu_replace_pointer (ovs_net->ct_limit_info,NULL , +lockdep_ovsl_is_held ()); +/* ovs_ct_check_limit() accesses the info under the datapath RCU lock. */ +synchronize_rcu (); +nf_conncount_destroy (net, info->data);for (i =0 ; i <CT_LIMIT_HASH_BUCKETS ; ++i) {struct hlist_head *head = &info->limits[i];
@@ -1678,9 +1689,7 @@ static int ovs_ct_limit_set_zone_limit(struct nlattr *nla_zone_limit,while (rem >=sizeof (*zone_limit)) {if (unlikely (zone_limit->zone_id ==OVS_ZONE_LIMIT_DEFAULT_ZONE )) { -ovs_lock (); info->default_limit = zone_limit->limit; -ovs_unlock (); }else if (unlikely (!check_zone_id ( zone_limit->zone_id, &zone))) {OVS_NLERR (true,"zone id is out of range" );
@@ -1694,9 +1703,7 @@ static int ovs_ct_limit_set_zone_limit(struct nlattr *nla_zone_limit, ct_limit->zone = zone; ct_limit->limit = zone_limit->limit; -ovs_lock ();ct_limit_set (info, ct_limit); -ovs_unlock (); } rem -=NLA_ALIGN (sizeof (*zone_limit)); zone_limit = (struct ovs_zone_limit *)((u8 *)zone_limit +
@@ -1722,16 +1729,12 @@ static int ovs_ct_limit_del_zone_limit(struct nlattr *nla_zone_limit,while (rem >=sizeof (*zone_limit)) {if (unlikely (zone_limit->zone_id ==OVS_ZONE_LIMIT_DEFAULT_ZONE )) { -ovs_lock (); info->default_limit =OVS_CT_LIMIT_DEFAULT ; -ovs_unlock (); }else if (unlikely (!check_zone_id ( zone_limit->zone_id, &zone))) {OVS_NLERR (true,"zone id is out of range" ); }else { -ovs_lock ();ct_limit_del (info, zone); -ovs_unlock (); } rem -=NLA_ALIGN (sizeof (*zone_limit)); zone_limit = (struct ovs_zone_limit *)((u8 *)zone_limit +
@@ -1849,8 +1852,8 @@ static int ovs_ct_limit_cmd_set(struct sk_buff *skb, struct genl_info *info)struct nlattr **a = info->attrs;struct sk_buff *reply;struct ovs_header *ovs_reply_header; +struct ovs_ct_limit_info *ct_limit_info;struct ovs_net *ovs_net =net_generic (sock_net (skb->sk), ovs_net_id); -struct ovs_ct_limit_info *ct_limit_info = ovs_net->ct_limit_info;int err; reply = ovs_ct_limit_cmd_reply_start(info, OVS_CT_LIMIT_CMD_SET,
@@ -1863,16 +1866,22 @@ static int ovs_ct_limit_cmd_set(struct sk_buff *skb, struct genl_info *info) goto exit_err; } +ovs_lock (); + ct_limit_info =rcu_dereference_protected (ovs_net->ct_limit_info, +lockdep_ovsl_is_held ()); err =ovs_ct_limit_set_zone_limit (a[OVS_CT_LIMIT_ATTR_ZONE_LIMIT ], ct_limit_info);if (err) - goto exit_err; + goto exit_unlock;static_branch_enable (&ovs_ct_limit_enabled);genlmsg_end (reply, ovs_reply_header); +ovs_unlock ();return genlmsg_reply (reply, info); +exit_unlock: +ovs_unlock (); exit_err:nlmsg_free (reply);return err;
@@ -1883,8 +1892,8 @@ static int ovs_ct_limit_cmd_del(struct sk_buff *skb, struct genl_info *info)struct nlattr **a = info->attrs;struct sk_buff *reply;struct ovs_header *ovs_reply_header; +struct ovs_ct_limit_info *ct_limit_info;struct ovs_net *ovs_net =net_generic (sock_net (skb->sk), ovs_net_id); -struct ovs_ct_limit_info *ct_limit_info = ovs_net->ct_limit_info;int err; reply = ovs_ct_limit_cmd_reply_start(info, OVS_CT_LIMIT_CMD_DEL,
@@ -1897,14 +1906,20 @@ static int ovs_ct_limit_cmd_del(struct sk_buff *skb, struct genl_info *info) goto exit_err; } +ovs_lock (); + ct_limit_info =rcu_dereference_protected (ovs_net->ct_limit_info, +lockdep_ovsl_is_held ()); err =ovs_ct_limit_del_zone_limit (a[OVS_CT_LIMIT_ATTR_ZONE_LIMIT ], ct_limit_info);if (err) - goto exit_err; + goto exit_unlock;genlmsg_end (reply, ovs_reply_header); +ovs_unlock ();return genlmsg_reply (reply, info); +exit_unlock: +ovs_unlock (); exit_err:nlmsg_free (reply);return err;
@@ -1917,8 +1932,8 @@ static int ovs_ct_limit_cmd_get(struct sk_buff *skb, struct genl_info *info)struct sk_buff *reply;struct ovs_header *ovs_reply_header;struct net *net =sock_net (skb->sk); +struct ovs_ct_limit_info *ct_limit_info;struct ovs_net *ovs_net =net_generic (net, ovs_net_id); -struct ovs_ct_limit_info *ct_limit_info = ovs_net->ct_limit_info;int err; reply = ovs_ct_limit_cmd_reply_start(info, OVS_CT_LIMIT_CMD_GET,
@@ -1932,23 +1947,29 @@ static int ovs_ct_limit_cmd_get(struct sk_buff *skb, struct genl_info *info) goto exit_err; } +ovs_lock (); + ct_limit_info =rcu_dereference_protected (ovs_net->ct_limit_info, +lockdep_ovsl_is_held ());if (a[OVS_CT_LIMIT_ATTR_ZONE_LIMIT ]) { err =ovs_ct_limit_get_zone_limit ( net, a[OVS_CT_LIMIT_ATTR_ZONE_LIMIT ], ct_limit_info, reply);if (err) - goto exit_err; + goto exit_unlock; }else { err =ovs_ct_limit_get_all_zone_limit (net, ct_limit_info, reply);if (err) - goto exit_err; + goto exit_unlock; }nla_nest_end (reply, nla_reply);genlmsg_end (reply, ovs_reply_header); +ovs_unlock ();return genlmsg_reply (reply, info); +exit_unlock: +ovs_unlock (); exit_err:nlmsg_free (reply);return err;
net/openvswitch/datapath.h +1 -1
@@ -172,7 +172,7 @@ struct ovs_net {struct work_struct dp_notify_work;struct delayed_work masks_rebalance;#if IS_ENABLED (CONFIG_NETFILTER_CONNCOUNT ) -struct ovs_ct_limit_info *ct_limit_info; +struct ovs_ct_limit_info __rcu *ct_limit_info;#endif bool xt_label; };
Signature · 1 line
2.54.0
Quoted 7 lines · Ren Wei
From: Yuqi Xu <[email protected]>
Packet processing uses CT limit state under RCU, while netns teardown frees that state under
ovs_mutex. The CT limit pointer was neither removed from readers nor protected by a grace period, allowing packet processing to dereference the freed state.
Quoted 7 lines
Replace the pointer before freeing the CT limit state. Wait for in-flight RCU readers before freeing its contents. Serialize CT limit netlink operations with teardown for the full lifetime of their state accesses.
Fixes 11efd5cb04a1 ("openvswitch: Support conntrack zone limit")Reported-by Vega <[email protected]>
Is Vega a person?
Quoted 5 lines
Assisted-by Codex:GPT-5.4Co-developed-by Nan Li <[email protected]>Signed-off-by Nan Li <[email protected]>Signed-off-by Yuqi Xu <[email protected]>Reviewed-by Ren Wei <[email protected]>
Please take a look at
https://docs.kernel.org/process/submitting-patches.html#sign-your-work-the-developer-s-certificate-of-origin
and the sections that follow. What is listed here does not follow the rules.
net/openvswitch/conntrack.c static int ovs_ct_check_limit(struct net *net, · line 938
Quoted 9 lines
net/openvswitch/conntrack.c +1
-1@@ -932,11 +932,15 @@ static int ovs_ct_check_limit(struct net *net,const struct sk_buff *skb, const struct ovs_conntrack_info *info) { + const struct ovs_ct_limit_info *ct_limit_info;struct ovs_net *ovs_net =net_generic (net, ovs_net_id); -const struct ovs_ct_limit_info *ct_limit_info = ovs_net->ct_limit_info;u32 per_zone_limit, connections;u32 conncount_key;
Reverse Christmas tree. The lines should be sorted longest to shortest. Yes, it was already wrong, but you can actually fix it here.
Andrew
---
pw-bot: cr
Quoted 8 lines · Andrew Lunn
Quoted 7 lines · Ren Wei
From: Yuqi Xu <[email protected]>
Packet processing uses CT limit state under RCU, while netns teardown frees that state under
ovs_mutex. The CT limit pointer was neither removed from readers nor protected by a grace period, allowing packet processing to dereference the freed state.
Quoted 8 lines
Quoted 7 lines
Replace the pointer before freeing the CT limit state. Wait for in-flight RCU readers before freeing its contents. Serialize CT limit netlink operations with teardown for the full lifetime of their state accesses.
Fixes 11efd5cb04a1 ("openvswitch: Support conntrack zone limit")Reported-by Vega <[email protected]>Is Vega a person?
Hi Andrew,
Thank you very much for your review!
For context, we had previously understood that using the tool name in the Reported-by tag was acceptable, based on examples such as
Anthropic.
https://lore.kernel.org/all/[email protected]/
Of course, we’re happy to adjust it if a different format is preferred.
Quoted 25 lines
Quoted 5 lines
Assisted-by Codex:GPT-5.4Co-developed-by Nan Li <[email protected]>Signed-off-by Nan Li <[email protected]>Signed-off-by Yuqi Xu <[email protected]>Reviewed-by Ren Wei <[email protected]>Please take a look at
https://docs.kernel.org/process/submitting-patches.html#sign-your-work-the-developer-s-certificate-of-origin
and the sections that follow. What is listed here does not follow the rules.net/openvswitch/conntrack.c static int ovs_ct_check_limit(struct net *net, · line 938
Quoted 9 lines
net/openvswitch/conntrack.c +1
-1@@ -932,11 +932,15 @@ static int ovs_ct_check_limit(struct net *net,const struct sk_buff *skb, const struct ovs_conntrack_info *info) { + const struct ovs_ct_limit_info *ct_limit_info;struct ovs_net *ovs_net =net_generic (net, ovs_net_id); -const struct ovs_ct_limit_info *ct_limit_info = ovs_net->ct_limit_info;u32 per_zone_limit, connections;u32 conncount_key;Reverse Christmas tree. The lines should be sorted longest to shortest. Yes, it was already wrong, but you can actually fix it here.
Andrew---
pw-bot: cr
Quoted 21 lines · Yuan Tan
Quoted 8 lines · Andrew Lunn
Quoted 7 lines · Ren Wei
From: Yuqi Xu <[email protected]>
Packet processing uses CT limit state under RCU, while netns teardown frees that state under
ovs_mutex. The CT limit pointer was neither removed from readers nor protected by a grace period, allowing packet processing to dereference the freed state.Quoted 8 lines
Quoted 7 lines
Replace the pointer before freeing the CT limit state. Wait for in-flight RCU readers before freeing its contents. Serialize CT limit netlink operations with teardown for the full lifetime of their state accesses.
Fixes 11efd5cb04a1 ("openvswitch: Support conntrack zone limit")Reported-by Vega <[email protected]>Is Vega a person?
Hi Andrew,
Quoted 7 lines
Thank you very much for your review!
For context, we had previously understood that using the tool name in the Reported-by tag was acceptable, based on examples such asReported-by [email protected] and Reported-by:Anthropic.
https://docs.kernel.org/process/submitting-patches.html
The Reported-by tag gives credit to people who find bugs and report them and it hopefully inspires them to help us again in the future. The tag is intended for bugs; please do not use it to credit feature requests. The tag should be followed by a Closes: tag pointing to the report, unless the report is not available on the web.
If you believe this is out of date, please submit a patch with new text to this document.
I find it valuable being a person. It indicate somebody is bothered by the problem you are fixing. We see a lot theoretical bug fixes, which in practice nobody ever hit. I would prefer to spend my time reviewing real issues, not theoretical issues, and the Reported-by: is a quick indicator of this.
Quoted 9 lines
Quoted 9 lines
Quoted 5 lines
Assisted-by Codex:GPT-5.4Co-developed-by Nan Li <[email protected]>Signed-off-by Nan Li <[email protected]>Signed-off-by Yuqi Xu <[email protected]>Reviewed-by Ren Wei <[email protected]>Please take a look at
https://docs.kernel.org/process/submitting-patches.html#sign-your-work-the-developer-s-certificate-of-origin
and the sections that follow. What is listed here does not follow the rules.
After reading this document, did you spot the second issues?
Andrew
Aaron Conole
maintainerQuoted 39 lines · Andrew Lunn
Quoted 21 lines · Yuan Tan
Quoted 8 lines · Andrew Lunn
Quoted 7 lines · Ren Wei
From: Yuqi Xu <[email protected]>
Packet processing uses CT limit state under RCU, while netns teardown frees that state under
ovs_mutex. The CT limit pointer was neither removed from readers nor protected by a grace period, allowing packet processing to dereference the freed state.Quoted 8 lines
Quoted 7 lines
Replace the pointer before freeing the CT limit state. Wait for in-flight RCU readers before freeing its contents. Serialize CT limit netlink operations with teardown for the full lifetime of their state accesses.
Fixes 11efd5cb04a1 ("openvswitch: Support conntrack zone limit")Reported-by Vega <[email protected]>Is Vega a person?
Hi Andrew,
Quoted 7 lines
Thank you very much for your review!
For context, we had previously understood that using the tool name in the Reported-by tag was acceptable, based on examples such asReported-by [email protected] and Reported-by:Anthropic.
https://docs.kernel.org/process/submitting-patches.html
The Reported-by tag gives credit to people who find bugs and report them and it hopefully inspires them to help us again in the future. The tag is intended for bugs; please do not use it to credit feature requests. The tag should be followed by a Closes: tag pointing to the report, unless the report is not available on the web.
Quoted 2 lines
If you believe this is out of date, please submit a patch with new text to this document.
It is common practice to accept syzbot reports as well, which look like:
Reported-by: [email protected] Closes: https://syzkaller.appspot.com/bug?extid=36256deb69a588e9290e
(see commit 539dfcf69105d8d3d4d677b71de6e5ede2e6dfa0 for example).
Quoted 5 lines
I find it valuable being a person. It indicate somebody is bothered by the problem you are fixing. We see a lot theoretical bug fixes, which in practice nobody ever hit. I would prefer to spend my time reviewing real issues, not theoretical issues, and the Reported-by: is a quick indicator of this.
+1
In the case of syzbot reports, they are real actionable reports that we can look at and address (and I agree with your sentiment of wanting to focus on real issues). They include the 'Closes:' tag as well, and a reviewer can just visit the link and see the splat. If there is a report link that this Vega tool pushes, maybe that would be acceptable since it's the way syzbot works as well. It also makes sense to update the documentation to reflect how it is used currently.
As for the v2, it would help my review to include some kind of reproducer description - I usually try to experience OVS splats for myself.
Quoted 68 lines · Aaron Conole
Quoted 39 lines · Andrew Lunn
Quoted 21 lines · Yuan Tan
Quoted 8 lines · Andrew Lunn
Quoted 7 lines · Ren Wei
From: Yuqi Xu <[email protected]>
Packet processing uses CT limit state under RCU, while netns teardown frees that state under
ovs_mutex. The CT limit pointer was neither removed from readers nor protected by a grace period, allowing packet processing to dereference the freed state.Quoted 8 lines
Quoted 7 lines
Replace the pointer before freeing the CT limit state. Wait for in-flight RCU readers before freeing its contents. Serialize CT limit netlink operations with teardown for the full lifetime of their state accesses.
Fixes 11efd5cb04a1 ("openvswitch: Support conntrack zone limit")Reported-by Vega <[email protected]>Is Vega a person?
Hi Andrew,
Quoted 7 lines
Thank you very much for your review!
For context, we had previously understood that using the tool name in the Reported-by tag was acceptable, based on examples such asReported-by [email protected] and Reported-by:Anthropic.
https://docs.kernel.org/process/submitting-patches.html
The Reported-by tag gives credit to people who find bugs and report them and it hopefully inspires them to help us again in the future. The tag is intended for bugs; please do not use it to credit feature requests. The tag should be followed by a Closes: tag pointing to the report, unless the report is not available on the web.
Quoted 2 lines
If you believe this is out of date, please submit a patch with new text to this document.
It is common practice to accept syzbot reports as well, which look like:
Reported-by: [email protected] Closes: https://syzkaller.appspot.com/bug?extid=36256deb69a588e9290e(see commit
539dfcf69105d8d3d4d677b71de6e5ede2e6dfa0for example).Quoted 5 lines
I find it valuable being a person. It indicate somebody is bothered by the problem you are fixing. We see a lot theoretical bug fixes, which in practice nobody ever hit. I would prefer to spend my time reviewing real issues, not theoretical issues, and the Reported-by: is a quick indicator of this.
+1
In the case of syzbot reports, they are real actionable reports that we can look at and address (and I agree with your sentiment of wanting to focus on real issues). They include the 'Closes:' tag as well, and a reviewer can just visit the link and see the splat. If there is a report link that this Vega tool pushes, maybe that would be acceptable since it's the way syzbot works as well. It also makes sense to update the documentation to reflect how it is used currently.
Quoted 4 lines
As for the v2, it would help my review to include some kind of reproducer description - I usually try to experience OVS splats for myself.
Hi Andrew and Aaron,
Thank you very much for your feedback!
In fact, all of the bugs for which we have submitted patches have been reproduced with working PoCs. We also include detailed information about the bug, the PoC, and the QEMU crash log in the cover letter. Previously, however, we did not send the cover letters to the public mailing list due to security concerns.
For example, this particular bug is a use-after-free write that can be triggered from a user namespace. Based on our analysis, we believe it is likely exploitable for local privilege escalation.
Going forward, we will send all cover letters to the public mailing list. We apologize for any inconvenience caused by our previous approach.
In addition, we plan to launch a website (in approximately one week) containing the bug detail, PoCs and crash logs for all the bugs we have found, making them easier for maintainer to view, browse and search. We also plan to include bugs found by other tools, such as Sashiko.
Besides, I will send a patch to update the document.
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Assisted-by Codex:GPT-5.4Co-developed-by Nan Li <[email protected]>Signed-off-by Nan Li <[email protected]>Signed-off-by Yuqi Xu <[email protected]>Reviewed-by Ren Wei <[email protected]>
Quoted 8 lines
Quoted 4 lines
Quoted 4 lines
Please take a look at
https://docs.kernel.org/process/submitting-patches.html#sign-your-work-the-developer-s-certificate-of-origin
and the sections that follow. What is listed here does not follow the rules.After reading this document, did you spot the second issues?
Andrew
Hi Andrew,
I have read through the document several times, but I am still not sure what you mean by the second issue. Could you please clarify which specific issue you are referring to?
At the moment, all of our patches are reviewed by Ren Wei before they are sent out. Are you referring to that Ren Wei is not included with a Signed-off-by tag? We did consider adding one, although I was concerned that including too many tags might make the commit message unnecessarily cluttered.
Thank you for the clarification.
Best,
Yuan
Quoted 30 lines · Yuan Tan
Quoted 14 lines · Aaron Conole
Quoted 11 lines · Andrew Lunn
Quoted 10 lines · Yuan Tan
Quoted 8 lines · Andrew Lunn
Quoted 1 line · Ren Wei
From: Yuqi Xu <[email protected]>
Assisted-by Codex:GPT-5.4Co-developed-by Nan Li <[email protected]>Signed-off-by Nan Li <[email protected]>Signed-off-by Yuqi Xu <[email protected]>Reviewed-by Ren Wei <[email protected]>Quoted 8 lines
Quoted 4 lines
Quoted 4 lines
Please take a look at
https://docs.kernel.org/process/submitting-patches.html#sign-your-work-the-developer-s-certificate-of-origin
and the sections that follow. What is listed here does not follow the rules.After reading this document, did you spot the second issues?
Andrew
Hi Andrew,
I have read through the document several times, but I am still not sure what you mean by the second issue. Could you please clarify which specific issue you are referring to?
Quoted 5 lines
At the moment, all of our patches are reviewed by Ren Wei before they are sent out. Are you referring to that Ren Wei is not included with a Signed-off-by tag? We did consider adding one, although I was concerned that including too many tags might make the commit message unnecessarily cluttered.
This patch has been submitted by you. There needs to be a Signed-off-by from you last on the list.
Any further SoBs (Signed-off-by:’s) following the author’s SoB are from people handling and transporting the patch, but were not involved in its development. SoB chains should reflect the real route a patch took as it was propagated to the maintainers and ultimately to Linus, with the first SoB entry signalling primary authorship of a single author.
Andrew