nexthops: Fix 2 fundamental flaws with nexthop groups
Subsystems (3)
David Ahern
maintainerv2 cover nexthops: Fix 2 fundamental flaws with nexthop groups
Nik's torture tests have exposed 2 fundamental mistakes with the initial nexthop code for groups. First, the nexthops entries and num_nh in the nh_grp struct should not be modified once the struct is set under rcu. Doing so has major affects on the datapath seeing valid nexthop entries.
Second, the helpers in the header file were convenient for not repeating code, but they cause datapath walks to potentially see 2 different group structs after an rcu replace, disrupting a walk of the path objects. This second problem applies solely to IPv4 as I re-used too much of the existing code in walking legs of a multipath route.
Patches 1 is refactoring change to simplify the overhead of reviewing and understanding the change in patch 2 which fixes the update of nexthop groups when a compnent leg is removed.
Patches 3-5 address the second problem. Patch 3 inlines the multipath check such that the mpath lookup and subsequent calls all use the same nh_grp struct. Patches 4 and 5 fix datapath uses of fib_info_num_path with iterative calls to fib_info_nhc.
fib_info_num_path can be used in control plane path in a 'for loop' with subsequent fib_info_nhc calls to get each leg since the nh_grp struct is only changed while holding the rtnl; the combination can not be used in the data plane with external nexthops as it involves repeated dereferences of nh_grp struct which can change between calls.
Similarly, nexthop_is_multipath can be used for branching decisions in the datapath since the nexthop type can not be changed (a group can not be converted to standalone and vice versa).
Patch set developed in coordination with Nikolay Aleksandrov. He did a lot of work creating a good reproducer, discussing options to fix it and testing iterations.
I have adapted Nik's commands into additional tests in the nexthops selftest script which I will send against -next.
v2
- fixed whitespace errors
David Ahern (4):
nexthops: Move code from remove_nexthop_from_groups to
remove_nh_grp_entry
nexthop: Expand nexthop_is_multipath in a few places
ipv4: Refactor nhc evaluation in fib_table_lookup
ipv4: nexthop version of fib_info_nh_uses_dev
Nikolay Aleksandrov (1):
nexthops: don't modify published nexthop groups
include/net/ip_fib.h | 12 +++++ include/net/nexthop.h | 100 ++++++++++++++++++++++++++++++++------- net/ipv4/fib_frontend.c | 19 ++++---- net/ipv4/fib_trie.c | 51 ++++++++++++++------ net/ipv4/nexthop.c | 102 +++++++++++++++++++++++++--------------- 5 files changed, 205 insertions(+), 79 deletions(-)
Signature · 1 line
2.21.1 (Apple Git-122.3)
David Ahern
maintainerPATCH v2 1/5 nexthops: Move code from remove_nexthop_from_groups to remove_nh_grp_entry
From: David Ahern <[email protected]>
Move nh_grp dereference and check for removing nexthop group due to all members gone into remove_nh_grp_entry.
--- net/ipv4/nexthop.c | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-)
net/ipv4/nexthop.c +13 -14
@@ -694,17 +694,21 @@ static void nh_group_rebalance(struct nh_group *nhg) } } -static void remove_nh_grp_entry (struct nh_grp_entry *nhge, -struct nh_group *nhg, +static void remove_nh_grp_entry (struct net *net,struct nh_grp_entry *nhge,struct nl_info *nlinfo) { +struct nexthop *nhp = nhge->nh_parent;struct nexthop *nh = nhge->nh;struct nh_grp_entry *nhges; +struct nh_group *nhg;bool found = false;int i;WARN_ON (!nh); +list_del (&nhge->nh_list); + + nhg =rtnl_dereference (nhp->nh_grp); nhges = nhg->nh_entries;for (i =0 ; i < nhg->num_nh; ++i) {if (found) {
@@ -728,7 +732,11 @@ static void remove_nh_grp_entry(struct nh_grp_entry *nhge,nexthop_put (nh);if (nlinfo) -nexthop_notify (RTM_NEWNEXTHOP , nhge->nh_parent, nlinfo); +nexthop_notify (RTM_NEWNEXTHOP , nhp, nlinfo); + +/* if this group has no more entries then remove it */ +if (!nhg->num_nh) +remove_nexthop (net, nhp, nlinfo); } static void remove_nexthop_from_groups(struct net *net, struct nexthop *nh,
@@ -736,17 +744,8 @@ static void remove_nexthop_from_groups(struct net *net, struct nexthop *nh, {struct nh_grp_entry *nhge, *tmp; -list_for_each_entry_safe (nhge, tmp, &nh->grp_list, nh_list) { -struct nh_group *nhg; - -list_del (&nhge->nh_list); - nhg =rtnl_dereference (nhge->nh_parent->nh_grp); -remove_nh_grp_entry (nhge, nhg, nlinfo); - -/* if this group has no more entries then remove it */ -if (!nhg->num_nh) -remove_nexthop (net, nhge->nh_parent, nlinfo); - } +list_for_each_entry_safe (nhge, tmp, &nh->grp_list, nh_list) +remove_nh_grp_entry (net, nhge, nlinfo); }static void remove_nexthop_group (struct nexthop *nh,struct nl_info *nlinfo)
Signature · 1 line
2.21.1 (Apple Git-122.3)
David Ahern
maintainerPATCH v2 2/5 nexthops: don't modify published nexthop groups
In reply to David AhernFrom: Nikolay Aleksandrov <[email protected]>
We must avoid modifying published nexthop groups while they might be in use, otherwise we might see NULL ptr dereferences. In order to do that we allocate 2 nexthoup group structures upon nexthop creation and swap between them when we have to delete an entry. The reason is that we can't fail nexthop group removal, so we can't handle allocation failure thus we move the extra allocation on creation where we can safely fail and return ENOMEM.
--- include/net/nexthop.h | 1 + net/ipv4/nexthop.c | 91 +++++++++++++++++++++++++++---------------- 2 files changed, 59 insertions(+), 33 deletions(-)
include/net/nexthop.h +1 -0
@@ -70,6 +70,7 @@ struct nh_grp_entry { };struct nh_group { +struct nh_group *spare;/* spare group for removals */ u16 num_nh;bool mpath;bool has_v4;
net/ipv4/nexthop.c +58 -33
153 lines not on this page · raw message
Signature · 1 line
2.21.1 (Apple Git-122.3)
David Ahern
maintainerPATCH v2 3/5 nexthop: Expand nexthop_is_multipath in a few places
In reply to David AhernFrom: David Ahern <[email protected]>
I got too fancy consolidating checks on multipath type. The result is that path lookups can access 2 different nh_grp structs as exposed by Nik's torture tests. Expand nexthop_is_multipath within nexthop.h to avoid multiple, nh_grp dereferences and make decisions based on the consistent struct.
Only 2 places left using nexthop_is_multipath are within IPv6, both only check that the nexthop is a multipath for a branching decision which are acceptable.
--- include/net/nexthop.h | 41 +++++++++++++++++++++++++---------------- 1 file changed, 25 insertions(+), 16 deletions(-)
include/net/nexthop.h +25 -16
@@ -137,21 +137,20 @@ static inline unsigned int nexthop_num_path(const struct nexthop *nh) {unsigned int rc =1 ; -if (nexthop_is_multipath (nh)) { +if (nh->is_group) {struct nh_group *nh_grp; nh_grp =rcu_dereference_rtnl (nh->nh_grp); - rc = nh_grp->num_nh; +if (nh_grp->mpath) + rc = nh_grp->num_nh; }return rc; }static inline -struct nexthop *nexthop_mpath_select (const struct nexthop *nh,int nhsel) +struct nexthop *nexthop_mpath_select (const struct nh_group *nhg,int nhsel) { -const struct nh_group *nhg =rcu_dereference_rtnl (nh->nh_grp); -/* for_nexthops macros in fib_semantics.c grabs a pointer to * the nexthop before checking nhsel */
@@ -186,12 +185,14 @@ static inline bool nexthop_is_blackhole(const struct nexthop *nh) {const struct nh_info *nhi; -if (nexthop_is_multipath (nh)) { -if (nexthop_num_path (nh) >1 ) -return false; - nh =nexthop_mpath_select (nh,0 ); -if (!nh) +if (nh->is_group) { +struct nh_group *nh_grp; + + nh_grp =rcu_dereference_rtnl (nh->nh_grp); +if (nh_grp->num_nh >1 )return false; + + nh = nh_grp->nh_entries[0 ].nh; } nhi =rcu_dereference_rtnl (nh->nh_info);
@@ -217,10 +218,15 @@ struct fib_nh_common *nexthop_fib_nhc(struct nexthop *nh, int nhsel)BUILD_BUG_ON (offsetof(struct fib_nh , nh_common) !=0 );BUILD_BUG_ON (offsetof(struct fib6_nh , nh_common) !=0 ); -if (nexthop_is_multipath (nh)) { - nh =nexthop_mpath_select (nh, nhsel); -if (!nh) -return NULL ; +if (nh->is_group) { +struct nh_group *nh_grp; + + nh_grp =rcu_dereference_rtnl (nh->nh_grp); +if (nh_grp->mpath) { + nh =nexthop_mpath_select (nh_grp, nhsel); +if (!nh) +return NULL ; + } } nhi =rcu_dereference_rtnl (nh->nh_info);
@@ -264,8 +270,11 @@ static inline struct fib6_nh *nexthop_fib6_nh(struct nexthop *nh) {struct nh_info *nhi; -if (nexthop_is_multipath (nh)) { - nh =nexthop_mpath_select (nh,0 ); +if (nh->is_group) { +struct nh_group *nh_grp; + + nh_grp =rcu_dereference_rtnl (nh->nh_grp); + nh =nexthop_mpath_select (nh_grp,0 );if (!nh)return NULL ; }
Signature · 1 line
2.21.1 (Apple Git-122.3)
David Ahern
maintainerPATCH v2 4/5 ipv4: Refactor nhc evaluation in fib_table_lookup
In reply to David AhernFrom: David Ahern <[email protected]>
FIB lookups can return an entry that references an external nexthop. While walking the nexthop struct we do not want to make multiple calls into the nexthop code which can result in 2 different structs getting accessed - one returning the number of paths the rest of the loop seeing a different nh_grp struct. If the nexthop group shrunk, the result is an attempt to access a fib_nh_common that does not exist for the new nh_grp struct but did for the old one.
To fix that move the device evaluation code to a helper that can be used for inline fib_nh path as well as external nexthops.
Update the existing check for fi->nh in fib_table_lookup to call a new helper, nexthop_get_nhc_lookup, which walks the external nexthop with a single rcu dereference.
--- include/net/ip_fib.h | 2 ++ include/net/nexthop.h | 33 ++++++++++++++++++++++++++++ net/ipv4/fib_trie.c | 51 ++++++++++++++++++++++++++++++------------- 3 files changed, 71 insertions(+), 15 deletions(-)
include/net/ip_fib.h +2 -0
@@ -479,6 +479,8 @@ void fib_nh_common_release(struct fib_nh_common *nhc);void fib_alias_hw_flags_set (struct net *net,const struct fib_rt_info *fri);void fib_trie_init (void );struct fib_table *fib_trie_table (u32 id,struct fib_table *alias); +bool fib_lookup_good_nhc (const struct fib_nh_common *nhc,int fib_flags, +const struct flowi4 *flp);static inline void fib_combine_itag (u32 *itag,const struct fib_result *res) {
include/net/nexthop.h +33 -0
@@ -233,6 +233,39 @@ struct fib_nh_common *nexthop_fib_nhc(struct nexthop *nh, int nhsel)return &nhi->fib_nhc; } +/* called from fib_table_lookup with rcu_lock */ +static inline +struct fib_nh_common *nexthop_get_nhc_lookup (const struct nexthop *nh, +int fib_flags, +const struct flowi4 *flp, +int *nhsel) +{ +struct nh_info *nhi; + +if (nh->is_group) { +struct nh_group *nhg =rcu_dereference (nh->nh_grp); +int i; + +for (i =0 ; i < nhg->num_nh; i++) { +struct nexthop *nhe = nhg->nh_entries[i].nh; + + nhi =rcu_dereference (nhe->nh_info); +if (fib_lookup_good_nhc (&nhi->fib_nhc, fib_flags, flp)) { + *nhsel = i; +return &nhi->fib_nhc; + } + } + }else { + nhi =rcu_dereference (nh->nh_info); +if (fib_lookup_good_nhc (&nhi->fib_nhc, fib_flags, flp)) { + *nhsel =0 ; +return &nhi->fib_nhc; + } + } + +return NULL ; +} +static inline unsigned int fib_info_num_path (const struct fib_info *fi) {if (unlikely (fi->nh))
net/ipv4/fib_trie.c +36 -15
80 lines not on this page · raw message
Signature · 1 line
2.21.1 (Apple Git-122.3)
David Ahern
maintainerPATCH v2 5/5 ipv4: nexthop version of fib_info_nh_uses_dev
In reply to David AhernFrom: David Ahern <[email protected]>
Similar to the last path, need to fix fib_info_nh_uses_dev for external nexthops to avoid referencing multiple nh_grp structs. Move the device check in fib_info_nh_uses_dev to a helper and create a nexthop version that is called if the fib_info uses an external nexthop.
--- include/net/ip_fib.h | 10 ++++++++++ include/net/nexthop.h | 25 +++++++++++++++++++++++++ net/ipv4/fib_frontend.c | 19 ++++++++++--------- 3 files changed, 45 insertions(+), 9 deletions(-)
include/net/ip_fib.h +10 -0
@@ -447,6 +447,16 @@ static inline int fib_num_tclassid_users(struct net *net)#endif int fib_unmerge (struct net *net); +static inline bool nhc_l3mdev_matches_dev (const struct fib_nh_common *nhc, +const struct net_device *dev) +{ +if (nhc->nhc_dev == dev || +l3mdev_master_ifindex_rcu (nhc->nhc_dev) == dev->ifindex) +return true; + +return false; +} +/* Exported by fib_semantics.c */ int ip_fib_check_default (__be32 gw,struct net_device *dev);int fib_sync_down_dev (struct net_device *dev,unsigned long event,bool force);
include/net/nexthop.h +25 -0
@@ -266,6 +266,31 @@ struct fib_nh_common *nexthop_get_nhc_lookup(const struct nexthop *nh,return NULL ; } +static inline bool nexthop_uses_dev (const struct nexthop *nh, +const struct net_device *dev) +{ +struct nh_info *nhi; + +if (nh->is_group) { +struct nh_group *nhg =rcu_dereference (nh->nh_grp); +int i; + +for (i =0 ; i < nhg->num_nh; i++) { +struct nexthop *nhe = nhg->nh_entries[i].nh; + + nhi =rcu_dereference (nhe->nh_info); +if (nhc_l3mdev_matches_dev (&nhi->fib_nhc, dev)) +return true; + } + }else { + nhi =rcu_dereference (nh->nh_info); +if (nhc_l3mdev_matches_dev (&nhi->fib_nhc, dev)) +return true; + } + +return false; +} +static inline unsigned int fib_info_num_path (const struct fib_info *fi) {if (unlikely (fi->nh))
net/ipv4/fib_frontend.c +10 -9
@@ -309,17 +309,18 @@ bool fib_info_nh_uses_dev(struct fib_info *fi, const struct net_device *dev) {bool dev_match = false;#ifdef CONFIG_IP_ROUTE_MULTIPATH -int ret; +if (unlikely (fi->nh)) { + dev_match =nexthop_uses_dev (fi->nh, dev); + }else { +int ret; -for (ret =0 ; ret <fib_info_num_path (fi); ret++) { -const struct fib_nh_common *nhc =fib_info_nhc (fi, ret); +for (ret =0 ; ret <fib_info_num_path (fi); ret++) { +const struct fib_nh_common *nhc =fib_info_nhc (fi, ret); -if (nhc->nhc_dev == dev) { - dev_match = true; -break ; - }else if (l3mdev_master_ifindex_rcu (nhc->nhc_dev) == dev->ifindex) { - dev_match = true; -break ; +if (nhc_l3mdev_matches_dev (nhc, dev)) { + dev_match = true; +break ; + } } }#else
Signature · 1 line
2.21.1 (Apple Git-122.3)
David Miller
maintainerFrom: David Ahern <[email protected]>
Date: Tue, 26 May 2020 12:56:13 -0600
Quoted 2 lines
v2
- fixed whitespace errors
I was able to catch this in time before I pushed out v1, just FYI.