net: introduce per-netns netdevice notifiers and use them in mlxsw
Subsystems (3)
cover net: introduce per-netns netdevice notifiers and use them in mlxsw
From: Jiri Pirko <[email protected]>
Some drivers, like mlxsw, are not interested in notifications coming in for netdevices from other network namespaces. So introduce per-netns notifiers and allow to reduce overhead by listening only for notifications from the same netns.
This is also a preparation for upcoming patchset "devlink: allow devlink
instances to change network namespace". This resolves deadlock during
reload mlxsw into initial netns made possible by328fbe747ad4 ("net: Close race between {un, }register_netdevice_notifier() and setup_net()/cleanup_net()").
Jiri Pirko (3):
net: push loops and nb calls into helper functions
net: introduce per-netns netdevice notifiers
mlxsw: spectrum: Use per-netns netdevice notifier registration
.../net/ethernet/mellanox/mlxsw/spectrum.c | 9 +- include/linux/netdevice.h | 6 + include/net/net_namespace.h | 6 +- net/core/dev.c | 176 +++++++++++++++--- 4 files changed, 165 insertions(+), 32 deletions(-)
Signature · 1 line
2.21.0
PATCH 1/3 net: push loops and nb calls into helper functions
From: Jiri Pirko <[email protected]>
Push iterations over net namespaces and netdevices from register_netdevice_notifier() and unregister_netdevice_notifier() into helper functions. Along with that introduce continue_reverse macros to make the code a bit nicer allowing to get rid of "last" marks.
--- include/linux/netdevice.h | 3 ++ include/net/net_namespace.h | 3 +- net/core/dev.c | 89 +++++++++++++++++++++++++------------ 3 files changed, 66 insertions(+), 29 deletions(-)
include/linux/netdevice.h +3 -0
@@ -2564,6 +2564,9 @@ extern rwlock_t dev_base_lock; /* Device list lock */ list_for_each_entry_safe(d, n, &(net)->dev_base_head, dev_list)#define for_each_netdev_continue (net, d) \ list_for_each_entry_continue(d, &(net)->dev_base_head, dev_list) +#define for_each_netdev_continue_reverse (net, d) \ + list_for_each_entry_continue_reverse(d, &(net)->dev_base_head, \ + dev_list)#define for_each_netdev_continue_rcu (net, d) \ list_for_each_entry_continue_rcu(d, &(net)->dev_base_head, dev_list)#define for_each_netdev_in_bond_rcu (bond, slave) \
include/net/net_namespace.h +2 -1
@@ -317,7 +317,8 @@ static inline struct net *read_pnet(const possible_net_t *pnet)/* Protected by net_rwsem */ #define for_each_net (VAR ) \ list_for_each_entry(VAR, &net_namespace_list, list) - +#define for_each_net_continue_reverse (VAR ) \ + list_for_each_entry_continue_reverse(VAR, &net_namespace_list, list)#define for_each_net_rcu (VAR ) \ list_for_each_entry_rcu(VAR, &net_namespace_list, list)
net/core/dev.c +61 -28
114 lines not on this page · raw message
Signature · 1 line
2.21.0
PATCH 2/3 net: introduce per-netns netdevice notifiers
In reply to Jiri PirkoFrom: Jiri Pirko <[email protected]>
Often the code for example in drivers is interested in getting notifier call only from certain network namespace. In addition to the existing global netdevice notifier chain introduce per-netns chains and allow users to register to that. Eventually this would eliminate unnecessary overhead in case there are many netdevices in many network namespaces.
--- include/linux/netdevice.h | 3 ++ include/net/net_namespace.h | 3 ++ net/core/dev.c | 87 +++++++++++++++++++++++++++++++++++++ 3 files changed, 93 insertions(+)
include/linux/netdevice.h +3 -0
@@ -2494,6 +2494,9 @@ const char *netdev_cmd_to_name(enum netdev_cmd cmd);int register_netdevice_notifier (struct notifier_block *nb);int unregister_netdevice_notifier (struct notifier_block *nb); +int register_netdevice_notifier_net (struct net *net,struct notifier_block *nb); +int unregister_netdevice_notifier_net (struct net *net, +struct notifier_block *nb);struct netdev_notifier_info {struct net_device *dev;
include/net/net_namespace.h +3 -0
@@ -36,6 +36,7 @@#include <linux/ns_common.h> #include <linux/idr.h> #include <linux/skbuff.h> +#include <linux/notifier.h> struct user_namespace ;struct proc_dir_entry ;
@@ -96,6 +97,8 @@ struct net {struct list_head dev_base_head;struct hlist_head *dev_name_head;struct hlist_head *dev_index_head; +struct raw_notifier_head netdev_chain; +unsigned int dev_base_seq;/* protected by rtnl_mutex */ int ifindex;unsigned int dev_unreg_count;
net/core/dev.c +87 -0
106 lines not on this page · raw message
Signature · 1 line
2.21.0
Re: [patch net-next 2/3] net: introduce per-netns netdevice notifiers
net/core/dev.c EXPORT_SYMBOL(unregister_netdevice_notifier); · line 1863
Quoted 12 lines
static int call_netdevice_notifiers_info(unsigned long val,
net/core/dev.c +8
-0struct netdev_notifier_info *info) { +struct net *net =dev_net (info->dev); +int ret; +ASSERT_RTNL (); + +/* Run per-netns notifier block chain first, then run the global one. +* Hopefully, one day, the global one is going to be removed after +* all notifier block registrators get converted to be per-netns. +*/
Hi Jiri
Is that really going to happen? register_netdevice_notifier() is used in 130 files. Do you plan to spend the time to make it happen?
net/core/dev.c EXPORT_SYMBOL(unregister_netdevice_notifier); · line 1868
Quoted 5 lines
net/core/dev.c +3
-0+ ret =raw_notifier_call_chain (&net->netdev_chain, val, info); +if (ret &NOTIFY_STOP_MASK ) +return ret;return raw_notifier_call_chain (&netdev_chain, val, info); }
Humm. I wonder about NOTIFY_STOP_MASK here. These are two separate chains. Should one chain be able to stop the other chain? Are there other examples where NOTIFY_STOP_MASK crosses a chain boundary?
Andrew
Re: [patch net-next 2/3] net: introduce per-netns netdevice notifiers
Quoted 15 lines · Mon, Sep 30, 2019 at 03:38:24PM CEST, [email protected] wrote:
net/core/dev.c EXPORT_SYMBOL(unregister_netdevice_notifier); · line 1863
Quoted 12 lines
static int call_netdevice_notifiers_info(unsigned long val,
net/core/dev.c +8
-0struct netdev_notifier_info *info) { +struct net *net =dev_net (info->dev); +int ret; +ASSERT_RTNL (); + +/* Run per-netns notifier block chain first, then run the global one. +* Hopefully, one day, the global one is going to be removed after +* all notifier block registrators get converted to be per-netns. +*/ Hi Jiri
Quoted 2 lines
Is that really going to happen?
register_netdevice_notifier()is used in 130 files. Do you plan to spend the time to make it happen?
That's why I prepended the sentency with "Hopefully, one day"...
Quoted 9 lines
net/core/dev.c EXPORT_SYMBOL(unregister_netdevice_notifier); · line 1868
Quoted 6 lines
net/core/dev.c +3
-0+ ret =raw_notifier_call_chain (&net->netdev_chain, val, info); +if (ret &NOTIFY_STOP_MASK ) +return ret;return raw_notifier_call_chain (&netdev_chain, val, info); }Humm. I wonder about
NOTIFY_STOP_MASKhere. These are two separate chains. Should one chain be able to stop the other chain? Are there
Well if the failing item would be in the second chain, at the beginning of it, it would be stopped too. Does not matter where the stop happens, the point is that the whole processing stops. That is why I added the check here.
Quoted 1 line
other examples where
NOTIFY_STOP_MASKcrosses a chain boundary?
Not aware of it, no. Could you please describe what is wrong?
Quoted 2 lines
Andrew
Re: [patch net-next 2/3] net: introduce per-netns netdevice notifiers
Quoted 38 lines · Jiri Pirko
Quoted 15 lines · Mon, Sep 30, 2019 at 03:38:24PM CEST, [email protected] wrote:
net/core/dev.c EXPORT_SYMBOL(unregister_netdevice_notifier); · line 1863
Quoted 12 lines
static int call_netdevice_notifiers_info(unsigned long val,
net/core/dev.c +8
-0struct netdev_notifier_info *info) { +struct net *net =dev_net (info->dev); +int ret; +ASSERT_RTNL (); + +/* Run per-netns notifier block chain first, then run the global one. +* Hopefully, one day, the global one is going to be removed after +* all notifier block registrators get converted to be per-netns. +*/ Hi Jiri
Quoted 2 lines
Is that really going to happen?
register_netdevice_notifier()is used in 130 files. Do you plan to spend the time to make it happen?That's why I prepended the sentency with "Hopefully, one day"...
Quoted 9 lines
net/core/dev.c EXPORT_SYMBOL(unregister_netdevice_notifier); · line 1868
Quoted 6 lines
net/core/dev.c +3
-0+ ret =raw_notifier_call_chain (&net->netdev_chain, val, info); +if (ret &NOTIFY_STOP_MASK ) +return ret;return raw_notifier_call_chain (&netdev_chain, val, info); }Humm. I wonder about
NOTIFY_STOP_MASKhere. These are two separate chains. Should one chain be able to stop the other chain? Are thereWell if the failing item would be in the second chain, at the beginning of it, it would be stopped too. Does not matter where the stop happens, the point is that the whole processing stops. That is why I added the check here.
Quoted 3 lines
Quoted 1 line
other examples where
NOTIFY_STOP_MASKcrosses a chain boundary?Not aware of it, no. Could you please describe what is wrong?
You are expanding the meaning of NOTIFY_STOP_MASK. It now can stop some other chain. If this was one chain with a filter, i would not be asking. But this is two different chains, and one chain can stop another? At minimum, i think this needs to be reviewed by the core kernel people.
But i'm also wondering if you are solving the problem at the wrong level. Are there other notifier chains which would benefit from respecting name space boundaries? Would a better solution be to extend struct notifier_block with some sort of filter?
Do you have some performance numbers? Where are you getting your performance gains from? By the fact you are doing NOTIFY_STOP_MASK earlier, so preventing a long chain being walked? I notice notifer_block has a priority field. Did you try using that to put your notified earlier on the chain?
Andrew
Re: [patch net-next 2/3] net: introduce per-netns netdevice notifiers
Quoted 39 lines · Mon, Sep 30, 2019 at 05:33:43PM CEST, [email protected] wrote:
Quoted 38 lines · Jiri Pirko
Quoted 15 lines · Mon, Sep 30, 2019 at 03:38:24PM CEST, [email protected] wrote:
net/core/dev.c EXPORT_SYMBOL(unregister_netdevice_notifier); · line 1863
Quoted 12 lines
static int call_netdevice_notifiers_info(unsigned long val,
net/core/dev.c +8
-0struct netdev_notifier_info *info) { +struct net *net =dev_net (info->dev); +int ret; +ASSERT_RTNL (); + +/* Run per-netns notifier block chain first, then run the global one. +* Hopefully, one day, the global one is going to be removed after +* all notifier block registrators get converted to be per-netns. +*/ Hi Jiri
Quoted 2 lines
Is that really going to happen?
register_netdevice_notifier()is used in 130 files. Do you plan to spend the time to make it happen?That's why I prepended the sentency with "Hopefully, one day"...
Quoted 9 lines
net/core/dev.c EXPORT_SYMBOL(unregister_netdevice_notifier); · line 1868
Quoted 6 lines
net/core/dev.c +3
-0+ ret =raw_notifier_call_chain (&net->netdev_chain, val, info); +if (ret &NOTIFY_STOP_MASK ) +return ret;return raw_notifier_call_chain (&netdev_chain, val, info); }Humm. I wonder about
NOTIFY_STOP_MASKhere. These are two separate chains. Should one chain be able to stop the other chain? Are thereWell if the failing item would be in the second chain, at the beginning of it, it would be stopped too. Does not matter where the stop happens, the point is that the whole processing stops. That is why I added the check here.
Quoted 6 lines
Quoted 3 lines
Quoted 1 line
other examples where
NOTIFY_STOP_MASKcrosses a chain boundary?Not aware of it, no. Could you please describe what is wrong?
You are expanding the meaning of
NOTIFY_STOP_MASK. It now can stop some other chain. If this was one chain with a filter, i would not be
Well, it was originally a single chain, so the semantics stays intact. Again, it is not some other independent chain. It's just netns one and general one, both serve the same purpose.
Quoted 8 lines
asking. But this is two different chains, and one chain can stop another? At minimum, i think this needs to be reviewed by the core kernel people.
But i'm also wondering if you are solving the problem at the wrong level. Are there other notifier chains which would benefit from respecting name space boundaries? Would a better solution be to extend struct
notifier_blockwith some sort of filter?
I mentioned my primary motivation in the cover letter. What I want to avoid is need of taking &pernet_ops_rwsem during registration of tne notifier and avoid deadlock in my usecase.
Plus it seems very clear that if a notifier knows what netns is he interested in, he just registers in that particular netns chain. Having one fat generic chain with filters is basically what we have right now.
Quoted 6 lines
Do you have some performance numbers? Where are you getting your performance gains from? By the fact you are doing
NOTIFY_STOP_MASKearlier, so preventing a long chain being walked? I noticenotifer_blockhas a priority field. Did you try using that to put your notified earlier on the chain?
It is not about stopping the chain earlier, not at all. It is the fact that with many netdevices in many network namespaces you gat a lot of wasted calls to notifiers registators that does not care.
Quoted 2 lines
Andrew
David Miller
maintainerRe: [patch net-next 2/3] net: introduce per-netns netdevice notifiers
In reply to Jiri PirkoFrom: Jiri Pirko <[email protected]>
Date: Mon, 30 Sep 2019 10:15:10 +0200
Quoted 9 lines
From: Jiri Pirko <[email protected]>
Often the code for example in drivers is interested in getting notifier call only from certain network namespace. In addition to the existing global netdevice notifier chain introduce per-netns chains and allow users to register to that. Eventually this would eliminate unnecessary overhead in case there are many netdevices in many network namespaces.
Signed-off-by Jiri Pirko <[email protected]>
Ok, so there was a discussion about stop semantics.
Honestly, I think that's fine.
Stop means the operation cannot be performed and whoever is firing off the notifier will have to fail and undo the config change being attempted.
In that context, it doesn't matter who or where in the chain we trigger the stop.
Given all of that I am pretty sure this change is fine and I will add it to net-next. We can fix any actual semantic problems this might introduce as a follow-on.
PATCH 3/3 mlxsw: spectrum: Use per-netns netdevice notifier registration
In reply to Jiri PirkoFrom: Jiri Pirko <[email protected]>
The mlxsw_sp instance is not interested in events happening in other network namespaces. So use "_net" variants for netdevice notifier registration/unregistration and get only events which are happening in the net the instance is in.
--- drivers/net/ethernet/mellanox/mlxsw/spectrum.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-)
drivers/net/ethernet/mellanox/mlxsw/spectrum.c +6 -3
@@ -4864,7 +4864,8 @@ static int mlxsw_sp_init(struct mlxsw_core *mlxsw_core,* respin. */ mlxsw_sp->netdevice_nb.notifier_call = mlxsw_sp_netdevice_event; - err =register_netdevice_notifier (&mlxsw_sp->netdevice_nb); + err =register_netdevice_notifier_net (&init_net, + &mlxsw_sp->netdevice_nb);if (err) {dev_err (mlxsw_sp->bus_info->dev,"Failed to register netdev notifier\n" ); goto err_netdev_notifier;
@@ -4887,7 +4888,8 @@ static int mlxsw_sp_init(struct mlxsw_core *mlxsw_core, err_ports_create:mlxsw_sp_dpipe_fini (mlxsw_sp); err_dpipe_init: -unregister_netdevice_notifier (&mlxsw_sp->netdevice_nb); +unregister_netdevice_notifier_net (&init_net, + &mlxsw_sp->netdevice_nb); err_netdev_notifier:if (mlxsw_sp->clock) mlxsw_sp->ptp_ops->fini (mlxsw_sp->ptp_state);
@@ -4973,7 +4975,8 @@ static void mlxsw_sp_fini(struct mlxsw_core *mlxsw_core)mlxsw_sp_ports_remove (mlxsw_sp);mlxsw_sp_dpipe_fini (mlxsw_sp); -unregister_netdevice_notifier (&mlxsw_sp->netdevice_nb); +unregister_netdevice_notifier_net (&init_net, + &mlxsw_sp->netdevice_nb);if (mlxsw_sp->clock) { mlxsw_sp->ptp_ops->fini (mlxsw_sp->ptp_state); mlxsw_sp->ptp_ops->clock_fini (mlxsw_sp->clock);
Signature · 1 line
2.21.0
David Miller
maintainerFrom: Jiri Pirko <[email protected]>
Date: Mon, 30 Sep 2019 10:15:08 +0200
Quoted 11 lines
From: Jiri Pirko <[email protected]>
Some drivers, like mlxsw, are not interested in notifications coming in for netdevices from other network namespaces. So introduce per-netns notifiers and allow to reduce overhead by listening only for notifications from the same netns.
This is also a preparation for upcoming patchset "devlink: allow devlink
instances to change network namespace". This resolves deadlock during
reload mlxsw into initial netns made possible by328fbe747ad4("net: Close race between {un, }register_netdevice_notifier() and setup_net()/cleanup_net()").
Series applied.