net: change addr_list_lock back to static key
Subsystems (8)
PATCH 1/1 net: change addr_list_lock back to static key
The dynamic key update for addr_list_lock still causes troubles, for example the following race condition still exists:
CPU 0: CPU 1:
(RCU read lock) (RTNL lock)dev_mc_seq_show() netdev_update_lockdep_key()
-> lockdep_unregister_key()
-> netif_addr_lock_bh()
because lockdep doesn't provide an API to update it atomically. Therefore, we have to move it back to static keys and use subclass for nest locking like before.
In commit 1a33e10e4a95 ("net: partially revert dynamic lockdep key changes"), I already reverted most parts of commit ab92d68fc22f ("net: core: add generic lockdep keys").
This patch reverts the rest and also part of commit f3b0a18bb6cb ("net: remove unnecessary variables and callback"). After this patch, addr_list_lock changes back to using static keys and subclasses to satisfy lockdep. Thanks to dev->lower_level, we do not have to change back to ->ndo_get_lock_subclass().
And hopefully this reduces some syzbot lockdep noises too.
--- drivers/net/bonding/bond_main.c | 2 -- drivers/net/bonding/bond_options.c | 2 -- drivers/net/hamradio/bpqether.c | 2 ++ drivers/net/macsec.c | 5 ++++ drivers/net/macvlan.c | 13 ++++++-- drivers/net/vxlan.c | 4 +-- .../net/wireless/intersil/hostap/hostap_hw.c | 3 ++ include/linux/netdevice.h | 12 +++++--- net/8021q/vlan_dev.c | 8 +++-- net/batman-adv/soft-interface.c | 2 ++ net/bridge/br_device.c | 8 +++++ net/core/dev.c | 30 ++++++++++--------- net/core/dev_addr_lists.c | 12 ++++---- net/core/rtnetlink.c | 1 - net/dsa/master.c | 4 +++ net/netrom/af_netrom.c | 2 ++ net/rose/af_rose.c | 2 ++ 17 files changed, 76 insertions(+), 36 deletions(-)
drivers/net/bonding/bond_main.c +0 -2
@@ -3687,8 +3687,6 @@ static int bond_do_ioctl(struct net_device *bond_dev, struct ifreq *ifr, int cmdcase BOND_RELEASE_OLD :case SIOCBONDRELEASE : res =bond_release (bond_dev, slave_dev); -if (!res) -netdev_update_lockdep_key (slave_dev);break ;case BOND_SETHWADDR_OLD :case SIOCBONDSETHWADDR :
drivers/net/bonding/bond_options.c +0 -2
@@ -1398,8 +1398,6 @@ static int bond_option_slaves_set(struct bonding *bond,case '-' :slave_dbg (bond->dev, dev,"Releasing interface\n" ); ret =bond_release (bond->dev, dev); -if (!ret) -netdev_update_lockdep_key (dev);break ;default :
drivers/net/hamradio/bpqether.c +2 -0
@@ -113,6 +113,7 @@ static LIST_HEAD(bpq_devices);* off into a separate class since they always nest. */ static struct lock_class_key bpq_netdev_xmit_lock_key; +static struct lock_class_key bpq_netdev_addr_lock_key; static void bpq_set_lockdep_class_one(struct net_device *dev, struct netdev_queue *txq,
@@ -123,6 +124,7 @@ static void bpq_set_lockdep_class_one(struct net_device *dev,static void bpq_set_lockdep_class (struct net_device *dev) { +lockdep_set_class (&dev->addr_list_lock, &bpq_netdev_addr_lock_key);netdev_for_each_tx_queue (dev, bpq_set_lockdep_class_one,NULL ); }
drivers/net/macsec.c +5 -0
@@ -3999,6 +3999,8 @@ static int macsec_add_dev(struct net_device *dev, sci_t sci, u8 icv_len)return 0 ; } +static struct lock_class_key macsec_netdev_addr_lock_key; +static int macsec_newlink (struct net *net,struct net_device *dev,struct nlattr *tb[],struct nlattr *data[],struct netlink_ext_ack *extack)
@@ -4050,6 +4052,9 @@ static int macsec_newlink(struct net *net, struct net_device *dev,return err;netdev_lockdep_set_classes (dev); +lockdep_set_class_and_subclass (&dev->addr_list_lock, + &macsec_netdev_addr_lock_key, + dev->lower_level); err =netdev_upper_dev_link (real_dev, dev, extack);if (err <0 )
drivers/net/macvlan.c +11 -2
@@ -860,6 +860,8 @@ static int macvlan_do_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd)* "super class" of normal network devices; split their locks off into a * separate class since they always nest. */ +static struct lock_class_key macvlan_netdev_addr_lock_key; +#define ALWAYS_ON_OFFLOADS \ (NETIF_F_SG | NETIF_F_HW_CSUM | NETIF_F_GSO_SOFTWARE | \ NETIF_F_GSO_ROBUST | NETIF_F_GSO_ENCAP_ALL)
@@ -875,6 +877,14 @@ static int macvlan_do_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd)#define MACVLAN_STATE_MASK \ ((1<<__LINK_STATE_NOCARRIER) | (1<<__LINK_STATE_DORMANT)) +static void macvlan_set_lockdep_class (struct net_device *dev) +{ +netdev_lockdep_set_classes (dev); +lockdep_set_class_and_subclass (&dev->addr_list_lock, + &macvlan_netdev_addr_lock_key, + dev->lower_level); +} +static int macvlan_init (struct net_device *dev) {struct macvlan_dev *vlan =netdev_priv (dev);
@@ -892,8 +902,7 @@ static int macvlan_init(struct net_device *dev) dev->gso_max_size = lowerdev->gso_max_size; dev->gso_max_segs = lowerdev->gso_max_segs; dev->hard_header_len = lowerdev->hard_header_len; - -netdev_lockdep_set_classes (dev); +macvlan_set_lockdep_class (dev); vlan->pcpu_stats = netdev_alloc_pcpu_stats(struct vlan_pcpu_stats);if (!vlan->pcpu_stats)
drivers/net/vxlan.c +1 -3
@@ -4245,10 +4245,8 @@ static int vxlan_changelink(struct net_device *dev, struct nlattr *tb[],mod_timer (&vxlan->age_timer, jiffies);netdev_adjacent_change_commit (dst->remote_dev, lowerdev, dev); -if (lowerdev && lowerdev != dst->remote_dev) { +if (lowerdev && lowerdev != dst->remote_dev) dst->remote_dev = lowerdev; -netdev_update_lockdep_key (lowerdev); - }vxlan_config_apply (dev, &conf, lowerdev, vxlan->net, true);return 0 ; }
drivers/net/wireless/intersil/hostap/hostap_hw.c +3 -0
@@ -3048,6 +3048,7 @@ static void prism2_clear_set_tim_queue(local_info_t *local)* This is a natural nesting, which needs a split lock type. */ static struct lock_class_key hostap_netdev_xmit_lock_key; +static struct lock_class_key hostap_netdev_addr_lock_key; static void prism2_set_lockdep_class_one(struct net_device *dev, struct netdev_queue *txq,
@@ -3059,6 +3060,8 @@ static void prism2_set_lockdep_class_one(struct net_device *dev,static void prism2_set_lockdep_class (struct net_device *dev) { +lockdep_set_class (&dev->addr_list_lock, + &hostap_netdev_addr_lock_key);netdev_for_each_tx_queue (dev, prism2_set_lockdep_class_one,NULL ); }
include/linux/netdevice.h +8 -4
@@ -1821,8 +1821,6 @@ enum netdev_priv_flags {* for hardware timestamping * @sfp_bus: attached &struct sfp_bus structure. * -* @addr_list_lock_key: lockdep class annotating -* net_device->addr_list_lock spinlock * @qdisc_tx_busylock: lockdep class annotating Qdisc->busylock spinlock * @qdisc_running_key: lockdep class annotating Qdisc->running seqcount *
@@ -2125,7 +2123,6 @@ struct net_device {#endif struct phy_device *phydev;struct sfp_bus *sfp_bus; -struct lock_class_key addr_list_lock_key;struct lock_class_key *qdisc_tx_busylock;struct lock_class_key *qdisc_running_key;bool proto_down;
@@ -2217,10 +2214,13 @@ static inline void netdev_for_each_tx_queue(struct net_device *dev,static struct lock_class_key qdisc_tx_busylock_key; \static struct lock_class_key qdisc_running_key; \static struct lock_class_key qdisc_xmit_lock_key; \ +static struct lock_class_key dev_addr_list_lock_key; \unsigned int i; \ \ (dev)->qdisc_tx_busylock = &qdisc_tx_busylock_key; \ (dev)->qdisc_running_key = &qdisc_running_key; \ +lockdep_set_class (&(dev)->addr_list_lock, \ + &dev_addr_list_lock_key); \for (i =0 ; i < (dev)->num_tx_queues; i++) \lockdep_set_class (&(dev)->_tx[i]._xmit_lock, \ &qdisc_xmit_lock_key); \
@@ -3253,7 +3253,6 @@ static inline void netif_stop_queue(struct net_device *dev) }void netif_tx_stop_all_queues (struct net_device *dev); -void netdev_update_lockdep_key (struct net_device *dev);static inline bool netif_tx_queue_stopped (const struct netdev_queue *dev_queue) {
@@ -4239,6 +4238,11 @@ static inline void netif_addr_lock(struct net_device *dev)spin_lock (&dev->addr_list_lock); } +static inline void netif_addr_lock_nested (struct net_device *dev) +{ +spin_lock_nested (&dev->addr_list_lock, dev->lower_level); +} +static inline void netif_addr_lock_bh (struct net_device *dev) {spin_lock_bh (&dev->addr_list_lock);
net/8021q/vlan_dev.c +6 -2
@@ -494,6 +494,7 @@ static void vlan_dev_set_rx_mode(struct net_device *vlan_dev)* separate class since they always nest. */ static struct lock_class_key vlan_netdev_xmit_lock_key; +static struct lock_class_key vlan_netdev_addr_lock_key; static void vlan_dev_set_lockdep_one(struct net_device *dev, struct netdev_queue *txq,
@@ -502,8 +503,11 @@ static void vlan_dev_set_lockdep_one(struct net_device *dev,lockdep_set_class (&txq->_xmit_lock, &vlan_netdev_xmit_lock_key); } -static void vlan_dev_set_lockdep_class (struct net_device *dev) +static void vlan_dev_set_lockdep_class (struct net_device *dev,int subclass) { +lockdep_set_class_and_subclass (&dev->addr_list_lock, + &vlan_netdev_addr_lock_key, + subclass);netdev_for_each_tx_queue (dev, vlan_dev_set_lockdep_one,NULL ); }
@@ -597,7 +601,7 @@ static int vlan_dev_init(struct net_device *dev)SET_NETDEV_DEVTYPE (dev, &vlan_type); -vlan_dev_set_lockdep_class (dev); +vlan_dev_set_lockdep_class (dev, dev->lower_level); vlan->vlan_pcpu_stats = netdev_alloc_pcpu_stats(struct vlan_pcpu_stats);if (!vlan->vlan_pcpu_stats)
net/batman-adv/soft-interface.c +2 -0
@@ -745,6 +745,7 @@ static int batadv_interface_kill_vid(struct net_device *dev, __be16 proto,* separate class since they always nest. */ static struct lock_class_key batadv_netdev_xmit_lock_key; +static struct lock_class_key batadv_netdev_addr_lock_key;/** * batadv_set_lockdep_class_one() - Set lockdep class for a single tx queue
@@ -765,6 +766,7 @@ static void batadv_set_lockdep_class_one(struct net_device *dev,*/ static void batadv_set_lockdep_class (struct net_device *dev) { +lockdep_set_class (&dev->addr_list_lock, &batadv_netdev_addr_lock_key);netdev_for_each_tx_queue (dev, batadv_set_lockdep_class_one,NULL ); }
net/bridge/br_device.c +8 -0
@@ -105,6 +105,13 @@ netdev_tx_t br_dev_xmit(struct sk_buff *skb, struct net_device *dev)return NETDEV_TX_OK ; } +static struct lock_class_key bridge_netdev_addr_lock_key; + +static void br_set_lockdep_class (struct net_device *dev) +{ +lockdep_set_class (&dev->addr_list_lock, &bridge_netdev_addr_lock_key); +} +static int br_dev_init (struct net_device *dev) {struct net_bridge *br =netdev_priv (dev);
@@ -143,6 +150,7 @@ static int br_dev_init(struct net_device *dev)br_fdb_hash_fini (br); } +br_set_lockdep_class (dev);return err; }
net/core/dev.c +16 -14
@@ -439,6 +439,7 @@ static const char *const netdev_lock_name[] = { "_xmit_IEEE802154", "_xmit_VOID", "_xmit_NONE"};static struct lock_class_key netdev_xmit_lock_key[ARRAY_SIZE (netdev_lock_type)]; +static struct lock_class_key netdev_addr_lock_key[ARRAY_SIZE (netdev_lock_type)];static inline unsigned short netdev_lock_pos (unsigned short dev_type) {
@@ -460,11 +461,25 @@ static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,lockdep_set_class_and_name (lock, &netdev_xmit_lock_key[i], netdev_lock_name[i]); } + +static inline void netdev_set_addr_lockdep_class (struct net_device *dev) +{ +int i; + + i =netdev_lock_pos (dev->type); +lockdep_set_class_and_name (&dev->addr_list_lock, + &netdev_addr_lock_key[i], + netdev_lock_name[i]); +}#else static inline void netdev_set_xmit_lockdep_class (spinlock_t *lock,unsigned short dev_type) { } + +static inline void netdev_set_addr_lockdep_class (struct net_device *dev) +{ +}#endif /*******************************************************************************
@@ -9373,15 +9388,6 @@ void netif_tx_stop_all_queues(struct net_device *dev) }EXPORT_SYMBOL (netif_tx_stop_all_queues); -void netdev_update_lockdep_key (struct net_device *dev) -{ -lockdep_unregister_key (&dev->addr_list_lock_key); -lockdep_register_key (&dev->addr_list_lock_key); - -lockdep_set_class (&dev->addr_list_lock, &dev->addr_list_lock_key); -} -EXPORT_SYMBOL (netdev_update_lockdep_key); -/** * register_netdevice - register a network device * @dev: device to register
@@ -9420,7 +9426,7 @@ int register_netdevice(struct net_device *dev)return ret;spin_lock_init (&dev->addr_list_lock); -lockdep_set_class (&dev->addr_list_lock, &dev->addr_list_lock_key); +netdev_set_addr_lockdep_class (dev); ret =dev_get_valid_name (net, dev, dev->name);if (ret <0 )
@@ -9939,8 +9945,6 @@ struct net_device *alloc_netdev_mqs(int sizeof_priv, const char *name,dev_net_set (dev, &init_net); -lockdep_register_key (&dev->addr_list_lock_key); - dev->gso_max_size =GSO_MAX_SIZE ; dev->gso_max_segs =GSO_MAX_SEGS ; dev->upper_level =1 ;
@@ -10028,8 +10032,6 @@ void free_netdev(struct net_device *dev)free_percpu (dev->xdp_bulkq); dev->xdp_bulkq =NULL ; -lockdep_unregister_key (&dev->addr_list_lock_key); -/* Compatibility with error handling in drivers */ if (dev->reg_state ==NETREG_UNINITIALIZED ) {netdev_freemem (dev);
net/core/dev_addr_lists.c +6 -6
@@ -637,7 +637,7 @@ int dev_uc_sync(struct net_device *to, struct net_device *from)if (to->addr_len != from->addr_len)return -EINVAL ; -netif_addr_lock (to); +netif_addr_lock_nested (to); err =__hw_addr_sync (&to->uc, &from->uc, to->addr_len);if (!err)__dev_set_rx_mode (to);
@@ -667,7 +667,7 @@ int dev_uc_sync_multiple(struct net_device *to, struct net_device *from)if (to->addr_len != from->addr_len)return -EINVAL ; -netif_addr_lock (to); +netif_addr_lock_nested (to); err =__hw_addr_sync_multiple (&to->uc, &from->uc, to->addr_len);if (!err)__dev_set_rx_mode (to);
@@ -691,7 +691,7 @@ void dev_uc_unsync(struct net_device *to, struct net_device *from)return ;netif_addr_lock_bh (from); -netif_addr_lock (to); +netif_addr_lock_nested (to);__hw_addr_unsync (&to->uc, &from->uc, to->addr_len);__dev_set_rx_mode (to);netif_addr_unlock (to);
@@ -858,7 +858,7 @@ int dev_mc_sync(struct net_device *to, struct net_device *from)if (to->addr_len != from->addr_len)return -EINVAL ; -netif_addr_lock (to); +netif_addr_lock_nested (to); err =__hw_addr_sync (&to->mc, &from->mc, to->addr_len);if (!err)__dev_set_rx_mode (to);
@@ -888,7 +888,7 @@ int dev_mc_sync_multiple(struct net_device *to, struct net_device *from)if (to->addr_len != from->addr_len)return -EINVAL ; -netif_addr_lock (to); +netif_addr_lock_nested (to); err =__hw_addr_sync_multiple (&to->mc, &from->mc, to->addr_len);if (!err)__dev_set_rx_mode (to);
@@ -912,7 +912,7 @@ void dev_mc_unsync(struct net_device *to, struct net_device *from)return ;netif_addr_lock_bh (from); -netif_addr_lock (to); +netif_addr_lock_nested (to);__hw_addr_unsync (&to->mc, &from->mc, to->addr_len);__dev_set_rx_mode (to);netif_addr_unlock (to);
net/core/rtnetlink.c +0 -1
@@ -2462,7 +2462,6 @@ static int do_set_master(struct net_device *dev, int ifindex, err = ops->ndo_del_slave (upper_dev, dev);if (err)return err; -netdev_update_lockdep_key (dev); }else {return -EOPNOTSUPP ; }
net/dsa/master.c +4 -0
@@ -327,6 +327,8 @@ static void dsa_master_reset_mtu(struct net_device *dev)rtnl_unlock (); } +static struct lock_class_key dsa_master_addr_list_lock_key; +int dsa_master_setup (struct net_device *dev,struct dsa_port *cpu_dp) {int ret;
@@ -345,6 +347,8 @@ int dsa_master_setup(struct net_device *dev, struct dsa_port *cpu_dp)wmb (); dev->dsa_ptr = cpu_dp; +lockdep_set_class (&dev->addr_list_lock, + &dsa_master_addr_list_lock_key); ret =dsa_master_ethtool_setup (dev);if (ret)return ret;
net/netrom/af_netrom.c +2 -0
@@ -70,6 +70,7 @@ static const struct proto_ops nr_proto_ops;* separate class since they always nest. */ static struct lock_class_key nr_netdev_xmit_lock_key; +static struct lock_class_key nr_netdev_addr_lock_key; static void nr_set_lockdep_one(struct net_device *dev, struct netdev_queue *txq,
@@ -80,6 +81,7 @@ static void nr_set_lockdep_one(struct net_device *dev,static void nr_set_lockdep_key (struct net_device *dev) { +lockdep_set_class (&dev->addr_list_lock, &nr_netdev_addr_lock_key);netdev_for_each_tx_queue (dev, nr_set_lockdep_one,NULL ); }
net/rose/af_rose.c +2 -0
@@ -71,6 +71,7 @@ ax25_address rose_callsign;* separate class since they always nest. */ static struct lock_class_key rose_netdev_xmit_lock_key; +static struct lock_class_key rose_netdev_addr_lock_key; static void rose_set_lockdep_one(struct net_device *dev, struct netdev_queue *txq,
@@ -81,6 +82,7 @@ static void rose_set_lockdep_one(struct net_device *dev,static void rose_set_lockdep_key (struct net_device *dev) { +lockdep_set_class (&dev->addr_list_lock, &rose_netdev_addr_lock_key);netdev_for_each_tx_queue (dev, rose_set_lockdep_one,NULL ); }
Signature · 1 line
2.26.2
From: Cong Wang <[email protected]>
Date: Mon, 8 Jun 2020 14:53:01 -0700
Quoted 23 lines
The dynamic key update for
addr_list_lockstill causes troubles, for example the following race condition still exists:CPU 0: CPU 1:
(RCU read lock) (RTNL lock)dev_mc_seq_show()netdev_update_lockdep_key()-> lockdep_unregister_key()->
netif_addr_lock_bh()because lockdep doesn't provide an API to update it atomically. Therefore, we have to move it back to static keys and use subclass for nest locking like before.
In commit
1a33e10e4a95("net: partially revert dynamic lockdep key changes"), I already reverted most parts of commitab92d68fc22f("net: core: add generic lockdep keys").This patch reverts the rest and also part of commit
f3b0a18bb6cb("net: remove unnecessary variables and callback"). After this patch,addr_list_lockchanges back to using static keys and subclasses to satisfy lockdep. Thanks todev->lower_level, we do not have to change back to ->ndo_get_lock_subclass().
Quoted 6 lines
And hopefully this reduces some syzbot lockdep noises too.
Reported-by [email protected]Cc Taehee Yoo <[email protected]>Cc Dmitry Vyukov <[email protected]>Signed-off-by Cong Wang <[email protected]>
Ok, let's see how this goes.
Applied, thanks Cong.
Quoted 1 line · Cong Wang
Hi Cong,
Thank you for this work!
Quoted 7 lines
The dynamic key update for
addr_list_lockstill causes troubles, for example the following race condition still exists:CPU 0: CPU 1: (RCU read lock) (RTNL lock) dev_mc_seq_show() netdev_update_lockdep_key() -> lockdep_unregister_key()
Quoted 6 lines
->
netif_addr_lock_bh()because lockdep doesn't provide an API to update it atomically. Therefore, we have to move it back to static keys and use subclass for nest locking like before.
I'm sorry for the late reply.
I agree that using subclass mechanism to avoid too many lockdep keys. But the subclass mechanism is also not updated its subclass key automatically. So, if upper/lower relationship is changed, interface would have incorrect subclass key.
It eventually results in lockdep warning.
And, I think this patch doesn't contain bonding and team module part. So, an additional patch is needed.
Test commands:
ip link add bond0 type bond
ip link add vlan0 link bond0 type vlan id 1
ip link add bond1 type bond
ip link add vlan1 link bond1 type vlan id 2
ip link set bond0 up
ip link set vlan0 down
ip link set vlan0 master bond1
ip link set bond0 up
ip link set bond1 up
ip link set vlan0 up
ip link set vlan1 up
modprobe -rv 8021q
vlan0 vlan1 <-- subclass 1
| | bond0 bond1 <-- subclass 0
After master operation
vlan1 <-- subclass 1
|
bond1 <-- subclass 0
|
vlan0 <-- subclass 1
|
bond0 <-- subclass 0
vlan1 subclass should be 3 but it is still 1 because of a lack of subclass update routine.
Splat looks like:
[ 126.733736][ T1318] ============================================
[ 126.734882][ T1318] WARNING: possible recursive locking detected
[ 126.735993][ T1318] 5.7.0+ #577 Not tainted
[ 126.736814][ T1318] --------------------------------------------
[ 126.737979][ T1318] modprobe/1318 is trying to acquire lock:
[ 126.738922][ T1318] ffff88805f53c280
(&vlan_netdev_addr_lock_key/1){+...}-{2:2}, at:
dev_uc_sync_multiple+0xca/0x160
[ 126.740637][ T1318]
[ 126.740637][ T1318] but task is already holding lock:
[ 126.741834][ T1318] ffff888060ed1280
(&vlan_netdev_addr_lock_key/1){+...}-{2:2}, at:
dev_mc_unsync+0x98/0x170
[ 126.743442][ T1318]
[ 126.743442][ T1318] other info that might help us debug this:
[ 126.744585][ T1318] Possible unsafe locking scenario:
[ 126.744585][ T1318]
[ 126.747081][ T1318] CPU0
[ 126.747556][ T1318] ----
[ 126.748035][ T1318] lock(&vlan_netdev_addr_lock_key/1);
[ 126.748728][ T1318] lock(&vlan_netdev_addr_lock_key/1);
[ 126.749416][ T1318]
[ 126.749416][ T1318] *** DEADLOCK ***
[ 126.749416][ T1318]
[ 126.750449][ T1318] May be due to missing lock nesting notation
[ 126.750449][ T1318]
[ 126.751530][ T1318] 5 locks held by modprobe/1318:
[ 126.752152][ T1318] #0: ffffffffb6d013b8
(pernet_ops_rwsem){+.+.}-{3:3}, at: rtnl_link_unregister+0x88/0x220
[ 126.753427][ T1318] #1: ffffffffb6d0da30 (rtnl_mutex){+.+.}-{3:3},
at: rtnl_link_unregister+0xd6/0x220
[ 126.754609][ T1318] #2: ffff888060ed1280
(&vlan_netdev_addr_lock_key/1){+...}-{2:2}, at:
dev_mc_unsync+0x98/0x170
[ 126.756185][ T1318] #3: ffff88805c69a280
(&dev_addr_list_lock_key/2){+...}-{2:2}, at: dev_mc_unsync+0xdc/0x170
[ 126.758001][ T1318] #4: ffffffffb692b580
(rcu_read_lock){....}-{1:2}, at: bond_set_rx_mode+0x5/0x3b0 [bonding]
[ 126.759688][ T1318][ ... ]
Quoted 531 lines
In commit
1a33e10e4a95("net: partially revert dynamic lockdep key changes"), I already reverted most parts of commitab92d68fc22f("net: core: add generic lockdep keys").This patch reverts the rest and also part of commit
f3b0a18bb6cb("net: remove unnecessary variables and callback"). After this patch,addr_list_lockchanges back to using static keys and subclasses to satisfy lockdep. Thanks todev->lower_level, we do not have to change back to ->ndo_get_lock_subclass().And hopefully this reduces some syzbot lockdep noises too.
Reported-by [email protected]Cc Taehee Yoo <[email protected]>Cc Dmitry Vyukov <[email protected]>Signed-off-by Cong Wang <[email protected]>--- drivers/net/bonding/bond_main.c | 2 -- drivers/net/bonding/bond_options.c | 2 -- drivers/net/hamradio/bpqether.c | 2 ++ drivers/net/macsec.c | 5 ++++ drivers/net/macvlan.c | 13 ++++++-- drivers/net/vxlan.c | 4 +-- .../net/wireless/intersil/hostap/hostap_hw.c | 3 ++ include/linux/netdevice.h | 12 +++++--- net/8021q/vlan_dev.c | 8 +++-- net/batman-adv/soft-interface.c | 2 ++ net/bridge/br_device.c | 8 +++++ net/core/dev.c | 30 ++++++++++--------- net/core/dev_addr_lists.c | 12 ++++---- net/core/rtnetlink.c | 1 - net/dsa/master.c | 4 +++ net/netrom/af_netrom.c | 2 ++ net/rose/af_rose.c | 2 ++ 17 files changed, 76 insertions(+), 36 deletions(-)drivers/net/bonding/bond_main.c +0
-2@@ -3687,8 +3687,6 @@ static int bond_do_ioctl(struct net_device *bond_dev, struct ifreq *ifr, int cmdcase BOND_RELEASE_OLD :case SIOCBONDRELEASE : res =bond_release (bond_dev, slave_dev); -if (!res) -netdev_update_lockdep_key (slave_dev);break ;case BOND_SETHWADDR_OLD :case SIOCBONDSETHWADDR :drivers/net/bonding/bond_options.c +0
-2@@ -1398,8 +1398,6 @@ static int bond_option_slaves_set(struct bonding *bond,case '-' :slave_dbg (bond->dev, dev,"Releasing interface\n" ); ret =bond_release (bond->dev, dev); -if (!ret) -netdev_update_lockdep_key (dev);break ;default :drivers/net/hamradio/bpqether.c +2
-0@@ -113,6 +113,7 @@ static LIST_HEAD(bpq_devices);* off into a separate class since they always nest. */ static struct lock_class_key bpq_netdev_xmit_lock_key; +static struct lock_class_key bpq_netdev_addr_lock_key; static void bpq_set_lockdep_class_one(struct net_device *dev, struct netdev_queue *txq,@@ -123,6 +124,7 @@ static void bpq_set_lockdep_class_one(struct net_device *dev,static void bpq_set_lockdep_class (struct net_device *dev) { +lockdep_set_class (&dev->addr_list_lock, &bpq_netdev_addr_lock_key);netdev_for_each_tx_queue (dev, bpq_set_lockdep_class_one,NULL ); }drivers/net/macsec.c +5
-0@@ -3999,6 +3999,8 @@ static int macsec_add_dev(struct net_device *dev, sci_t sci, u8 icv_len)return 0 ; } +static struct lock_class_key macsec_netdev_addr_lock_key; +static int macsec_newlink (struct net *net,struct net_device *dev,struct nlattr *tb[],struct nlattr *data[],struct netlink_ext_ack *extack)@@ -4050,6 +4052,9 @@ static int macsec_newlink(struct net *net, struct net_device *dev,return err;netdev_lockdep_set_classes (dev); +lockdep_set_class_and_subclass (&dev->addr_list_lock, + &macsec_netdev_addr_lock_key, + dev->lower_level); err =netdev_upper_dev_link (real_dev, dev, extack);if (err <0 )drivers/net/macvlan.c +11
-2@@ -860,6 +860,8 @@ static int macvlan_do_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd)* "super class" of normal network devices; split their locks off into a * separate class since they always nest. */ +static struct lock_class_key macvlan_netdev_addr_lock_key; +#define ALWAYS_ON_OFFLOADS \ (NETIF_F_SG | NETIF_F_HW_CSUM | NETIF_F_GSO_SOFTWARE | \ NETIF_F_GSO_ROBUST | NETIF_F_GSO_ENCAP_ALL)@@ -875,6 +877,14 @@ static int macvlan_do_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd)#define MACVLAN_STATE_MASK \ ((1<<__LINK_STATE_NOCARRIER) | (1<<__LINK_STATE_DORMANT)) +static void macvlan_set_lockdep_class (struct net_device *dev) +{ +netdev_lockdep_set_classes (dev); +lockdep_set_class_and_subclass (&dev->addr_list_lock, + &macvlan_netdev_addr_lock_key, + dev->lower_level); +} +static int macvlan_init (struct net_device *dev) {struct macvlan_dev *vlan =netdev_priv (dev);@@ -892,8 +902,7 @@ static int macvlan_init(struct net_device *dev) dev->gso_max_size = lowerdev->gso_max_size; dev->gso_max_segs = lowerdev->gso_max_segs; dev->hard_header_len = lowerdev->hard_header_len; - -netdev_lockdep_set_classes (dev); +macvlan_set_lockdep_class (dev); vlan->pcpu_stats = netdev_alloc_pcpu_stats(struct vlan_pcpu_stats);if (!vlan->pcpu_stats)drivers/net/vxlan.c +1
-3@@ -4245,10 +4245,8 @@ static int vxlan_changelink(struct net_device *dev, struct nlattr *tb[],mod_timer (&vxlan->age_timer, jiffies);netdev_adjacent_change_commit (dst->remote_dev, lowerdev, dev); -if (lowerdev && lowerdev != dst->remote_dev) { +if (lowerdev && lowerdev != dst->remote_dev) dst->remote_dev = lowerdev; -netdev_update_lockdep_key (lowerdev); - }vxlan_config_apply (dev, &conf, lowerdev, vxlan->net, true);return 0 ; }drivers/net/wireless/intersil/hostap/hostap_hw.c +3
-0@@ -3048,6 +3048,7 @@ static void prism2_clear_set_tim_queue(local_info_t *local)* This is a natural nesting, which needs a split lock type. */ static struct lock_class_key hostap_netdev_xmit_lock_key; +static struct lock_class_key hostap_netdev_addr_lock_key; static void prism2_set_lockdep_class_one(struct net_device *dev, struct netdev_queue *txq,@@ -3059,6 +3060,8 @@ static void prism2_set_lockdep_class_one(struct net_device *dev,static void prism2_set_lockdep_class (struct net_device *dev) { +lockdep_set_class (&dev->addr_list_lock, + &hostap_netdev_addr_lock_key);netdev_for_each_tx_queue (dev, prism2_set_lockdep_class_one,NULL ); }include/linux/netdevice.h +8
-4@@ -1821,8 +1821,6 @@ enum netdev_priv_flags {* for hardware timestamping * @sfp_bus: attached &struct sfp_bus structure. * -* @addr_list_lock_key: lockdep class annotating -* net_device->addr_list_lock spinlock * @qdisc_tx_busylock: lockdep class annotating Qdisc->busylock spinlock * @qdisc_running_key: lockdep class annotating Qdisc->running seqcount * @@ -2125,7 +2123,6 @@ struct net_device {#endif struct phy_device *phydev;struct sfp_bus *sfp_bus; -struct lock_class_key addr_list_lock_key;struct lock_class_key *qdisc_tx_busylock;struct lock_class_key *qdisc_running_key;bool proto_down;@@ -2217,10 +2214,13 @@ static inline void netdev_for_each_tx_queue(struct net_device *dev,static struct lock_class_key qdisc_tx_busylock_key; \static struct lock_class_key qdisc_running_key; \static struct lock_class_key qdisc_xmit_lock_key; \ +static struct lock_class_key dev_addr_list_lock_key; \unsigned int i; \ \ (dev)->qdisc_tx_busylock = &qdisc_tx_busylock_key; \ (dev)->qdisc_running_key = &qdisc_running_key; \ +lockdep_set_class (&(dev)->addr_list_lock, \ + &dev_addr_list_lock_key); \for (i =0 ; i < (dev)->num_tx_queues; i++) \lockdep_set_class (&(dev)->_tx[i]._xmit_lock, \ &qdisc_xmit_lock_key); \@@ -3253,7 +3253,6 @@ static inline void netif_stop_queue(struct net_device *dev) }void netif_tx_stop_all_queues (struct net_device *dev); -void netdev_update_lockdep_key (struct net_device *dev);static inline bool netif_tx_queue_stopped (const struct netdev_queue *dev_queue) {@@ -4239,6 +4238,11 @@ static inline void netif_addr_lock(struct net_device *dev)spin_lock (&dev->addr_list_lock); } +static inline void netif_addr_lock_nested (struct net_device *dev) +{ +spin_lock_nested (&dev->addr_list_lock, dev->lower_level); +} +static inline void netif_addr_lock_bh (struct net_device *dev) {spin_lock_bh (&dev->addr_list_lock);net/8021q/vlan_dev.c +6
-2@@ -494,6 +494,7 @@ static void vlan_dev_set_rx_mode(struct net_device *vlan_dev)* separate class since they always nest. */ static struct lock_class_key vlan_netdev_xmit_lock_key; +static struct lock_class_key vlan_netdev_addr_lock_key; static void vlan_dev_set_lockdep_one(struct net_device *dev, struct netdev_queue *txq,@@ -502,8 +503,11 @@ static void vlan_dev_set_lockdep_one(struct net_device *dev,lockdep_set_class (&txq->_xmit_lock, &vlan_netdev_xmit_lock_key); } -static void vlan_dev_set_lockdep_class (struct net_device *dev) +static void vlan_dev_set_lockdep_class (struct net_device *dev,int subclass) { +lockdep_set_class_and_subclass (&dev->addr_list_lock, + &vlan_netdev_addr_lock_key, + subclass);netdev_for_each_tx_queue (dev, vlan_dev_set_lockdep_one,NULL ); }@@ -597,7 +601,7 @@ static int vlan_dev_init(struct net_device *dev)SET_NETDEV_DEVTYPE (dev, &vlan_type); -vlan_dev_set_lockdep_class (dev); +vlan_dev_set_lockdep_class (dev, dev->lower_level); vlan->vlan_pcpu_stats = netdev_alloc_pcpu_stats(struct vlan_pcpu_stats);if (!vlan->vlan_pcpu_stats)net/batman-adv/soft-interface.c +2
-0@@ -745,6 +745,7 @@ static int batadv_interface_kill_vid(struct net_device *dev, __be16 proto,* separate class since they always nest. */ static struct lock_class_key batadv_netdev_xmit_lock_key; +static struct lock_class_key batadv_netdev_addr_lock_key;/** * batadv_set_lockdep_class_one() - Set lockdep class for a single tx queue @@ -765,6 +766,7 @@ static void batadv_set_lockdep_class_one(struct net_device *dev,*/ static void batadv_set_lockdep_class (struct net_device *dev) { +lockdep_set_class (&dev->addr_list_lock, &batadv_netdev_addr_lock_key);netdev_for_each_tx_queue (dev, batadv_set_lockdep_class_one,NULL ); }net/bridge/br_device.c +8
-0@@ -105,6 +105,13 @@ netdev_tx_t br_dev_xmit(struct sk_buff *skb, struct net_device *dev)return NETDEV_TX_OK ; } +static struct lock_class_key bridge_netdev_addr_lock_key; + +static void br_set_lockdep_class (struct net_device *dev) +{ +lockdep_set_class (&dev->addr_list_lock, &bridge_netdev_addr_lock_key); +} +static int br_dev_init (struct net_device *dev) {struct net_bridge *br =netdev_priv (dev);@@ -143,6 +150,7 @@ static int br_dev_init(struct net_device *dev)br_fdb_hash_fini (br); } +br_set_lockdep_class (dev);return err; }net/core/dev.c +16
-14@@ -439,6 +439,7 @@ static const char *const netdev_lock_name[] = { "_xmit_IEEE802154", "_xmit_VOID", "_xmit_NONE"};static struct lock_class_key netdev_xmit_lock_key[ARRAY_SIZE (netdev_lock_type)]; +static struct lock_class_key netdev_addr_lock_key[ARRAY_SIZE (netdev_lock_type)];static inline unsigned short netdev_lock_pos (unsigned short dev_type) {@@ -460,11 +461,25 @@ static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,lockdep_set_class_and_name (lock, &netdev_xmit_lock_key[i], netdev_lock_name[i]); } + +static inline void netdev_set_addr_lockdep_class (struct net_device *dev) +{ +int i; + + i =netdev_lock_pos (dev->type); +lockdep_set_class_and_name (&dev->addr_list_lock, + &netdev_addr_lock_key[i], + netdev_lock_name[i]); +}#else static inline void netdev_set_xmit_lockdep_class (spinlock_t *lock,unsigned short dev_type) { } + +static inline void netdev_set_addr_lockdep_class (struct net_device *dev) +{ +}#endif /******************************************************************************* @@ -9373,15 +9388,6 @@ void netif_tx_stop_all_queues(struct net_device *dev) }EXPORT_SYMBOL (netif_tx_stop_all_queues); -void netdev_update_lockdep_key (struct net_device *dev) -{ -lockdep_unregister_key (&dev->addr_list_lock_key); -lockdep_register_key (&dev->addr_list_lock_key); - -lockdep_set_class (&dev->addr_list_lock, &dev->addr_list_lock_key); -} -EXPORT_SYMBOL (netdev_update_lockdep_key); -/** * register_netdevice - register a network device * @dev: device to register @@ -9420,7 +9426,7 @@ int register_netdevice(struct net_device *dev)return ret;spin_lock_init (&dev->addr_list_lock); -lockdep_set_class (&dev->addr_list_lock, &dev->addr_list_lock_key); +netdev_set_addr_lockdep_class (dev); ret =dev_get_valid_name (net, dev, dev->name);if (ret <0 )@@ -9939,8 +9945,6 @@ struct net_device *alloc_netdev_mqs(int sizeof_priv, const char *name,dev_net_set (dev, &init_net); -lockdep_register_key (&dev->addr_list_lock_key); - dev->gso_max_size =GSO_MAX_SIZE ; dev->gso_max_segs =GSO_MAX_SEGS ; dev->upper_level =1 ;@@ -10028,8 +10032,6 @@ void free_netdev(struct net_device *dev)free_percpu (dev->xdp_bulkq); dev->xdp_bulkq =NULL ; -lockdep_unregister_key (&dev->addr_list_lock_key); -/* Compatibility with error handling in drivers */ if (dev->reg_state ==NETREG_UNINITIALIZED ) {netdev_freemem (dev);net/core/dev_addr_lists.c +6
-6@@ -637,7 +637,7 @@ int dev_uc_sync(struct net_device *to, struct net_device *from)if (to->addr_len != from->addr_len)return -EINVAL ; -netif_addr_lock (to); +netif_addr_lock_nested (to); err =__hw_addr_sync (&to->uc, &from->uc, to->addr_len);if (!err)__dev_set_rx_mode (to);@@ -667,7 +667,7 @@ int dev_uc_sync_multiple(struct net_device *to, struct net_device *from)if (to->addr_len != from->addr_len)return -EINVAL ; -netif_addr_lock (to); +netif_addr_lock_nested (to); err =__hw_addr_sync_multiple (&to->uc, &from->uc, to->addr_len);if (!err)__dev_set_rx_mode (to);@@ -691,7 +691,7 @@ void dev_uc_unsync(struct net_device *to, struct net_device *from)return ;netif_addr_lock_bh (from); -netif_addr_lock (to); +netif_addr_lock_nested (to);__hw_addr_unsync (&to->uc, &from->uc, to->addr_len);__dev_set_rx_mode (to);netif_addr_unlock (to);@@ -858,7 +858,7 @@ int dev_mc_sync(struct net_device *to, struct net_device *from)if (to->addr_len != from->addr_len)return -EINVAL ; -netif_addr_lock (to); +netif_addr_lock_nested (to); err =__hw_addr_sync (&to->mc, &from->mc, to->addr_len);if (!err)__dev_set_rx_mode (to);@@ -888,7 +888,7 @@ int dev_mc_sync_multiple(struct net_device *to, struct net_device *from)if (to->addr_len != from->addr_len)return -EINVAL ; -netif_addr_lock (to); +netif_addr_lock_nested (to); err =__hw_addr_sync_multiple (&to->mc, &from->mc, to->addr_len);if (!err)__dev_set_rx_mode (to);@@ -912,7 +912,7 @@ void dev_mc_unsync(struct net_device *to, struct net_device *from)return ;netif_addr_lock_bh (from); -netif_addr_lock (to); +netif_addr_lock_nested (to);__hw_addr_unsync (&to->mc, &from->mc, to->addr_len);__dev_set_rx_mode (to);netif_addr_unlock (to);net/core/rtnetlink.c +0
-1@@ -2462,7 +2462,6 @@ static int do_set_master(struct net_device *dev, int ifindex, err = ops->ndo_del_slave (upper_dev, dev);if (err)return err; -netdev_update_lockdep_key (dev); }else {return -EOPNOTSUPP ; }net/dsa/master.c +4
-0@@ -327,6 +327,8 @@ static void dsa_master_reset_mtu(struct net_device *dev)rtnl_unlock (); } +static struct lock_class_key dsa_master_addr_list_lock_key; +int dsa_master_setup (struct net_device *dev,struct dsa_port *cpu_dp) {int ret;@@ -345,6 +347,8 @@ int dsa_master_setup(struct net_device *dev, struct dsa_port *cpu_dp)wmb (); dev->dsa_ptr = cpu_dp; +lockdep_set_class (&dev->addr_list_lock, + &dsa_master_addr_list_lock_key); ret =dsa_master_ethtool_setup (dev);if (ret)return ret;net/netrom/af_netrom.c +2
-0@@ -70,6 +70,7 @@ static const struct proto_ops nr_proto_ops;* separate class since they always nest. */ static struct lock_class_key nr_netdev_xmit_lock_key; +static struct lock_class_key nr_netdev_addr_lock_key; static void nr_set_lockdep_one(struct net_device *dev, struct netdev_queue *txq,@@ -80,6 +81,7 @@ static void nr_set_lockdep_one(struct net_device *dev,static void nr_set_lockdep_key (struct net_device *dev) { +lockdep_set_class (&dev->addr_list_lock, &nr_netdev_addr_lock_key);netdev_for_each_tx_queue (dev, nr_set_lockdep_one,NULL ); }net/rose/af_rose.c +2
-1@@ -71,6 +71,7 @@ ax25_address rose_callsign;* separate class since they always nest. */ static struct lock_class_key rose_netdev_xmit_lock_key; +static struct lock_class_key rose_netdev_addr_lock_key; static void rose_set_lockdep_one(struct net_device *dev, struct netdev_queue *txq,@@ -81,6 +82,7 @@ static void rose_set_lockdep_one(struct net_device *dev,static void rose_set_lockdep_key (struct net_device *dev) { +lockdep_set_class (&dev->addr_list_lock, &rose_netdev_addr_lock_key);netdev_for_each_tx_queue (dev, rose_set_lockdep_one,NULL ); } --2.26.2
Quoted 16 lines · Taehee Yoo
Quoted 1 line · Cong Wang
Hi Cong,
Thank you for this work!Quoted 9 lines
The dynamic key update for
addr_list_lockstill causes troubles, for example the following race condition still exists:CPU 0: CPU 1: (RCU read lock) (RTNL lock) dev_mc_seq_show() netdev_update_lockdep_key() -> lockdep_unregister_key()->
netif_addr_lock_bh()
Quoted 7 lines
Quoted 4 lines
because lockdep doesn't provide an API to update it atomically. Therefore, we have to move it back to static keys and use subclass for nest locking like before.
I'm sorry for the late reply.
I agree that using subclass mechanism to avoid too many lockdep keys.
Avoiding too many lockdep keys is not the real goal of my patch, its main purpose is to fix a race condition shown above. Just FYI.
Quoted 4 lines
But the subclass mechanism is also not updated its subclass key automatically. So, if upper/lower relationship is changed, interface would have incorrect subclass key.
It eventually results in lockdep warning.
So dev->lower_level is not updated accordingly? I just blindly trust dev->lower_level, as you use it in other places too.
Quoted 2 lines
And, I think this patch doesn't contain bonding and team module part. So, an additional patch is needed.
Hmm, dev->lower_level is generic, so is addr_list_lock.
Again, I just assume you already update dev->lower_level each time the topology changes. I added some printk() to verify it too for my simple bond over bond case. So, I can't immediately see what is wrong with dev->lower_level here. Do you mind to be more specific? Or I misunderstand your point?
Thanks!
Quoted 1 line · Cong Wang
Hi Cong :)
Quoted 22 lines · Taehee Yoo
Quoted 21 lines · Cong Wang
Quoted 1 line
Hi Cong,
Thank you for this work!Quoted 13 lines
The dynamic key update for
addr_list_lockstill causes troubles, for example the following race condition still exists:CPU 0: CPU 1: (RCU read lock) (RTNL lock) dev_mc_seq_show() netdev_update_lockdep_key() -> lockdep_unregister_key()->
netif_addr_lock_bh()because lockdep doesn't provide an API to update it atomically. Therefore, we have to move it back to static keys and use subclass for nest locking like before.
Quoted 6 lines
Quoted 2 lines
I'm sorry for the late reply.
I agree that using subclass mechanism to avoid too many lockdep keys.Avoiding too many lockdep keys is not the real goal of my patch, its main purpose is to fix a race condition shown above. Just FYI.
Thank you for notifying me.
Quoted 12 lines
Quoted 5 lines
But the subclass mechanism is also not updated its subclass key automatically. So, if upper/lower relationship is changed, interface would have incorrect subclass key.
It eventually results in lockdep warning.So
dev->lower_levelis not updated accordingly? I just blindly trustdev->lower_level, as you use it in other places too.Quoted 2 lines
And, I think this patch doesn't contain bonding and team module part. So, an additional patch is needed.
Quoted 12 lines
Hmm,
dev->lower_levelis generic, so isaddr_list_lock.Again, I just assume you already update
dev->lower_leveleach time the topology changes. I added someprintk()to verify it too for my simple bond over bond case. So, I can't immediately see what is wrong withdev->lower_levelhere. Do you mind to be more specific? Or I misunderstand your point?Quoted 3 lines
Quoted 3 lines
net/8021q/vlan_dev.c +3
-0+lockdep_set_class_and_subclass (&dev->addr_list_lock, + &vlan_netdev_addr_lock_key, + subclass);
In this patch, lockdep_set_class_and_subclass() is used.
As far as I know, this function initializes lockdep key and subclass value with a given variable.
A dev->lower_level variable is used as a subclass value in this patch. When dev->lower_level value is changed, the subclass value of this lockdep key is not changed automatically.
If this value has to be changed, additional function is needed.
Quoted 1 line
Quoted 1 line
Quoted 1 line
netif_addr_lock_bh(from);
In this function, internally spin_lock_bh() is used and this function might use an 'initialized subclass value' not a current dev->lower_level. At this point, I think the lockdep splat might occur.
+static inline void netif_addr_lock_nested(struct net_device *dev) +{
+ spin_lock_nested(&dev->addr_list_lock, dev->lower_level);
+}
In this patch, you used netif_addr_lock_nested() too. These two subclass values could be different.
But I'm not sure whether using spin_lock_nested with two different subclass values are the right way or not.
If I misunderstood the lockdep and this logic, please let me know!
Thanks :)
Quoted 14 lines · Taehee Yoo
Quoted 1 line · Cong Wang
Hi Cong :)
Quoted 7 lines · Taehee Yoo
Quoted 6 lines · Cong Wang
net/8021q/vlan_dev.c static void vlan_dev_set_lockdep_one(struct net_device *dev, · line 510
Quoted 4 lines
net/8021q/vlan_dev.c +3
-0+lockdep_set_class_and_subclass (&dev->addr_list_lock, + &vlan_netdev_addr_lock_key, + subclass);
Quoted 7 lines
In this patch,
lockdep_set_class_and_subclass()is used.
As far as I know, this function initializes lockdep key and subclass value with a given variable.
Adev->lower_levelvariable is used as a subclass value in this patch. Whendev->lower_levelvalue is changed, the subclass value of this lockdep key is not changed automatically.
If this value has to be changed, additional function is needed.
Hmm, but we pass a dynamic subclass to spin_lock_nested().
So I guess I should just remove all the lockdep_set_class_and_subclass() and leave subclass to 0?
Quoted 10 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
netif_addr_lock_bh(from);In this function, internally
spin_lock_bh()is used and this function might use an 'initialized subclass value' not a currentdev->lower_level. At this point, I think the lockdep splat might occur.include/linux/netdevice.h +4
-0+static inline void netif_addr_lock_nested (struct net_device *dev) +{ +spin_lock_nested (&dev->addr_list_lock, dev->lower_level); +}
Quoted 4 lines
In this patch, you used
netif_addr_lock_nested()too. These two subclass values could be different.
But I'm not sure whether usingspin_lock_nestedwith two different subclass values are the right way or not.
Yeah, as long as dev->lower_level is different, it should be different subclass. I assume dev->lower_level is automatically adjusted whenever the topology changes, like the vlan over bond case above.
Thanks.
Quoted 1 line · Cong Wang
Hi Cong,
Quoted 23 lines · Taehee Yoo
Quoted 14 lines · Cong Wang
Quoted 1 line
Hi Cong :)
Quoted 7 lines · Taehee Yoo
Quoted 6 lines · Cong Wang
net/8021q/vlan_dev.c static void vlan_dev_set_lockdep_one(struct net_device *dev, · line 510
Quoted 4 lines
net/8021q/vlan_dev.c +3
-0+lockdep_set_class_and_subclass (&dev->addr_list_lock, + &vlan_netdev_addr_lock_key, + subclass);Quoted 7 lines
In this patch,
lockdep_set_class_and_subclass()is used.
As far as I know, this function initializes lockdep key and subclass value with a given variable.
Adev->lower_levelvariable is used as a subclass value in this patch. Whendev->lower_levelvalue is changed, the subclass value of this lockdep key is not changed automatically.
If this value has to be changed, additional function is needed.
Quoted 5 lines
Hmm, but we pass a dynamic subclass to
spin_lock_nested().So I guess I should just remove all the
lockdep_set_class_and_subclass()and leave subclass to 0?
I agree with that.
And, do you have any plan to replace netif_addr_lock_bh() with netif_addr_lock_nested()?
(Of course, it needs BH handling code)
I'm not sure but I think it would be needed.
Quoted 6 lines
Quoted 6 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
netif_addr_lock_bh(from);In this function, internally
spin_lock_bh()is used and this function might use an 'initialized subclass value' not a currentdev->lower_level. At this point, I think the lockdep splat might occur.
Quoted 13 lines
Quoted 8 lines
include/linux/netdevice.h +4
-0+static inline void netif_addr_lock_nested (struct net_device *dev) +{ +spin_lock_nested (&dev->addr_list_lock, dev->lower_level); +}In this patch, you used
netif_addr_lock_nested()too. These two subclass values could be different.
But I'm not sure whether usingspin_lock_nestedwith two different subclass values are the right way or not.Yeah, as long as
dev->lower_levelis different, it should be different subclass. I assumedev->lower_levelis automatically adjusted whenever the topology changes, like the vlan over bond case above.
Yes, you're right.
Thanks!
Taehee Yoo
Quoted 6 lines · Taehee Yoo
I agree with that.
And, do you have any plan to replacenetif_addr_lock_bh()withnetif_addr_lock_nested()?
(Of course, it needs BH handling code)
I'm not sure but I think it would be needed.
Yeah, I agree it's needed. I have a patch now and will send it out after some testing.
Thanks.
Vladimir Oltean
maintainerHi Cong, Taehee,
Quoted 41 lines · Cong Wang
The dynamic key update for
addr_list_lockstill causes troubles, for example the following race condition still exists:CPU 0: CPU 1: (RCU read lock) (RTNL lock) dev_mc_seq_show() netdev_update_lockdep_key() -> lockdep_unregister_key()->
netif_addr_lock_bh()because lockdep doesn't provide an API to update it atomically. Therefore, we have to move it back to static keys and use subclass for nest locking like before.
In commit
1a33e10e4a95("net: partially revert dynamic lockdep key changes"), I already reverted most parts of commitab92d68fc22f("net: core: add generic lockdep keys").This patch reverts the rest and also part of commit
f3b0a18bb6cb("net: remove unnecessary variables and callback"). After this patch,addr_list_lockchanges back to using static keys and subclasses to satisfy lockdep. Thanks todev->lower_level, we do not have to change back to ->ndo_get_lock_subclass().And hopefully this reduces some syzbot lockdep noises too.
Reported-by [email protected]Cc Taehee Yoo <[email protected]>Cc Dmitry Vyukov <[email protected]>Signed-off-by Cong Wang <[email protected]>--- drivers/net/bonding/bond_main.c | 2 -- drivers/net/bonding/bond_options.c | 2 -- drivers/net/hamradio/bpqether.c | 2 ++ drivers/net/macsec.c | 5 ++++ drivers/net/macvlan.c | 13 ++++++-- drivers/net/vxlan.c | 4 +-- .../net/wireless/intersil/hostap/hostap_hw.c | 3 ++ include/linux/netdevice.h | 12 +++++--- net/8021q/vlan_dev.c | 8 +++-- net/batman-adv/soft-interface.c | 2 ++
Quoted 9 lines
net/bridge/br_device.c | 8 +++++ net/core/dev.c | 30 ++++++++++--------- net/core/dev_addr_lists.c | 12 ++++---- net/core/rtnetlink.c | 1 - net/dsa/master.c | 4 +++ net/netrom/af_netrom.c | 2 ++ net/rose/af_rose.c | 2 ++ 17 files changed, 76 insertions(+), 36 deletions(-)
It's me with the stacked DSA devices again:
[ 11.424642] ============================================
[ 11.429967] WARNING: possible recursive locking detected
[ 11.435295] 5.8.0-rc1-00133-g923e4b5032dd-dirty #208 Not tainted
[ 11.441319] --------------------------------------------
[ 11.446646] dhcpcd/323 is trying to acquire lock:
[ 11.451362] ffff000066dd4268
(&dsa_master_addr_list_lock_key/1){+...}-{2:2}, at:
dev_mc_sync+0x44/0x90
[ 11.460713]
[ 11.460713] but task is already holding lock:
[ 11.466561] ffff00006608c268
(&dsa_master_addr_list_lock_key/1){+...}-{2:2}, at:
dev_mc_sync+0x44/0x90
[ 11.475905]
[ 11.475905] other info that might help us debug this:
[ 11.482450] Possible unsafe locking scenario:
[ 11.482450]
[ 11.488386] CPU0
[ 11.490833] ----
[ 11.493280] lock(&dsa_master_addr_list_lock_key/1);
[ 11.498347] lock(&dsa_master_addr_list_lock_key/1);
[ 11.503413]
[ 11.503413] *** DEADLOCK ***
[ 11.503413]
[ 11.509349] May be due to missing lock nesting notation
[ 11.509349]
[ 11.516158] 3 locks held by dhcpcd/323:
[ 11.520001] #0: ffffdbd1381dda18 (rtnl_mutex){+.+.}-{3:3}, at:
rtnl_lock+0x24/0x30
[ 11.527688] #1: ffff00006614b268 (_xmit_ETHER){+...}-{2:2}, at:
dev_set_rx_mode+0x28/0x48
[ 11.535987] #2: ffff00006608c268
(&dsa_master_addr_list_lock_key/1){+...}-{2:2}, at:
dev_mc_sync+0x44/0x90
[ 11.545766]
[ 11.545766] stack backtrace:
[ 11.564098] Call trace:
[ 11.566549] dump_backtrace+0x0/0x1e0
[ 11.570220] show_stack+0x20/0x30
[ 11.573544] dump_stack+0xec/0x158
[ 11.576955] __lock_acquire+0xca0/0x2398
[ 11.580886] lock_acquire+0xe8/0x440
[ 11.584469] _raw_spin_lock_nested+0x64/0x90
[ 11.588749] dev_mc_sync+0x44/0x90
[ 11.592159] dsa_slave_set_rx_mode+0x34/0x50
[ 11.596438] __dev_set_rx_mode+0x60/0xa0
[ 11.600369] dev_mc_sync+0x84/0x90
[ 11.603778] dsa_slave_set_rx_mode+0x34/0x50
[ 11.608057] __dev_set_rx_mode+0x60/0xa0
[ 11.611989] dev_set_rx_mode+0x30/0x48
[ 11.615745] __dev_open+0x10c/0x180
[ 11.619240] __dev_change_flags+0x170/0x1c8
[ 11.623432] dev_change_flags+0x2c/0x70
[ 11.627279] devinet_ioctl+0x774/0x878
[ 11.631036] inet_ioctl+0x348/0x3b0
[ 11.634532] sock_do_ioctl+0x50/0x310
[ 11.638202] sock_ioctl+0x1f8/0x580
[ 11.641698] ksys_ioctl+0xb0/0xf0
[ 11.645019] __arm64_sys_ioctl+0x28/0x38
[ 11.648951] el0_svc_common.constprop.0+0x7c/0x180
[ 11.653753] do_el0_svc+0x2c/0x98
[ 11.657075] el0_sync_handler+0x9c/0x1b8
[ 11.661005] el0_sync+0x158/0x180Could you please share some suggestions?
Thanks,
-Vladimir
Quoted 2 lines · Vladimir Oltean
It's me with the stacked DSA devices again:
It looks like DSA never uses netdev API to link master device with slave devices? If so, their dev->lower_level are always 1, therefore triggers this warning.
I think it should call one of these netdev_upper_dev_link() API's when creating a slave device.
Thanks.
Quoted 12 lines · Cong Wang
Quoted 2 lines · Vladimir Oltean
It's me with the stacked DSA devices again:
It looks like DSA never uses netdev API to link master device with slave devices? If so, their
dev->lower_levelare always 1, therefore triggers this warning.I think it should call one of these
netdev_upper_dev_link()API's when creating a slave device.
I don't know whether DSA is too special to use the API, but something like this should work:
net/dsa/slave.c +6 -0
@@ -1807,6 +1807,11 @@ int dsa_slave_create(struct dsa_port *port) ret, slave_dev->name); goto out_phy; } + ret =netdev_upper_dev_link (slave_dev, master,NULL ); +if (ret) { +unregister_netdevice (slave_dev); + goto out_phy; + }return 0 ;
@@ -1832,6 +1837,7 @@ void dsa_slave_destroy(struct net_device *slave_dev)netif_carrier_off (slave_dev);rtnl_lock ();phylink_disconnect_phy (dp->pl); +netdev_upper_dev_unlink (slave_dev, dp->master);rtnl_unlock ();dsa_slave_notify (slave_dev,DSA_PORT_UNREGISTER );
Vladimir Oltean
maintainerQuoted 35 lines · Cong Wang
Quoted 12 lines · Cong Wang
Quoted 2 lines · Vladimir Oltean
It's me with the stacked DSA devices again:
It looks like DSA never uses netdev API to link master device with slave devices? If so, their
dev->lower_levelare always 1, therefore triggers this warning.I think it should call one of these
netdev_upper_dev_link()API's when creating a slave device.I don't know whether DSA is too special to use the API, but something like this should work:
net/dsa/slave.c +5
-0@@ -1807,6 +1807,11 @@ int dsa_slave_create(struct dsa_port *port) ret, slave_dev->name); goto out_phy; } + ret =netdev_upper_dev_link (slave_dev, master,NULL ); +if (ret) { +unregister_netdevice (slave_dev); + goto out_phy; + }return 0 ;
net/dsa/slave.c void dsa_slave_destroy(struct net_device *slave_dev) · line 1843
Quoted 7 lines
net/dsa/slave.c +1
-0@@ -1832,6 +1837,7 @@ void dsa_slave_destroy(struct net_device *slave_dev)netif_carrier_off (slave_dev);rtnl_lock ();phylink_disconnect_phy (dp->pl); +netdev_upper_dev_unlink (slave_dev, dp->master);rtnl_unlock ();dsa_slave_notify (slave_dev,DSA_PORT_UNREGISTER );
Thanks. This is a good approximation of what needed to be done:
- netdev_upper_dev_link needs to be under rtnl,
- "dp->master" should be "dsa_slave_to_master(slave_dev)" since it's actually a union if you look at struct dsa_port).
- And, most importantly, I think the hierarchy should be reversed: a (virtual) DSA switch port net device (slave) should be an upper of the (real) DSA master (the host port). Think of it like this: a DSA switch is a sort of port multiplier for a host port, based on a frame header. But, it works!
Do you mind if I submit your modified patch to "net"? What would be an adequate Fixes: tag?
Cheers,
-Vladimir
Quoted 43 lines · Vladimir Oltean
Quoted 41 lines · Cong Wang
Quoted 12 lines · Cong Wang
Quoted 2 lines · Vladimir Oltean
It's me with the stacked DSA devices again:
It looks like DSA never uses netdev API to link master device with slave devices? If so, their
dev->lower_levelare always 1, therefore triggers this warning.I think it should call one of these
netdev_upper_dev_link()API's when creating a slave device.I don't know whether DSA is too special to use the API, but something like this should work:
net/dsa/slave.c +6
-0@@ -1807,6 +1807,11 @@ int dsa_slave_create(struct dsa_port *port) ret, slave_dev->name); goto out_phy; } + ret =netdev_upper_dev_link (slave_dev, master,NULL ); +if (ret) { +unregister_netdevice (slave_dev); + goto out_phy; + }return 0 ;@@ -1832,6 +1837,7 @@ void dsa_slave_destroy(struct net_device *slave_dev)netif_carrier_off (slave_dev);rtnl_lock ();phylink_disconnect_phy (dp->pl); +netdev_upper_dev_unlink (slave_dev, dp->master);rtnl_unlock ();
Quoted 11 lines
net/dsa/slave.c void dsa_slave_destroy(struct net_device *slave_dev) · line 1843
Quoted 1 line
net/dsa/slave.c +0
-0dsa_slave_notify (slave_dev,DSA_PORT_UNREGISTER );Thanks. This is a good approximation of what needed to be done:
-netdev_upper_dev_linkneeds to be under rtnl,
- "dp->master" should be "dsa_slave_to_master(slave_dev)" since it's actually a union if you look at structdsa_port).
- And, most importantly, I think the hierarchy should be reversed: a (virtual) DSA switch port net device (slave) should be an upper of the (real) DSA master (the host port). Think of it like this: a DSA switch is a sort of port multiplier for a host port, based on a frame header. But, it works!
Please feel free to make any changes you need and submit it by yourself, as you know DSA better than me and I do not even have a DSA testing environment.
Quoted 3 lines
Do you mind if I submit your modified patch to "net"? What would be an adequate Fixes: tag?
If it is merely to fix the lockdep warning, my commit 845e0ebb4408d447 is the right one to blame.
Thanks.
Andrew Lunn
maintainerQuoted 49 lines · Vladimir Oltean
Quoted 35 lines · Cong Wang
Quoted 12 lines · Cong Wang
Quoted 2 lines · Vladimir Oltean
It's me with the stacked DSA devices again:
It looks like DSA never uses netdev API to link master device with slave devices? If so, their
dev->lower_levelare always 1, therefore triggers this warning.I think it should call one of these
netdev_upper_dev_link()API's when creating a slave device.I don't know whether DSA is too special to use the API, but something like this should work:
net/dsa/slave.c +5
-0@@ -1807,6 +1807,11 @@ int dsa_slave_create(struct dsa_port *port) ret, slave_dev->name); goto out_phy; } + ret =netdev_upper_dev_link (slave_dev, master,NULL ); +if (ret) { +unregister_netdevice (slave_dev); + goto out_phy; + }return 0 ;net/dsa/slave.c void dsa_slave_destroy(struct net_device *slave_dev) · line 1843
Quoted 7 lines
net/dsa/slave.c +1
-0@@ -1832,6 +1837,7 @@ void dsa_slave_destroy(struct net_device *slave_dev)netif_carrier_off (slave_dev);rtnl_lock ();phylink_disconnect_phy (dp->pl); +netdev_upper_dev_unlink (slave_dev, dp->master);rtnl_unlock ();dsa_slave_notify (slave_dev,DSA_PORT_UNREGISTER );Thanks. This is a good approximation of what needed to be done:
-netdev_upper_dev_linkneeds to be under rtnl,
- "dp->master" should be "dsa_slave_to_master(slave_dev)" since it's actually a union if you look at structdsa_port).
Quoted 5 lines
- And, most importantly, I think the hierarchy should be reversed: a (virtual) DSA switch port net device (slave) should be an upper of the (real) DSA master (the host port). Think of it like this: a DSA switch is a sort of port multiplier for a host port, based on a frame header. But, it works!
Hi Vladimir
So you are suggesting this?
Quoted 1 line
Quoted 1 line
+ ret =
netdev_upper_dev_link(master, slave_dev, NULL);
Andrew
Vladimir Oltean
maintainerQuoted 57 lines · Andrew Lunn
Quoted 49 lines · Vladimir Oltean
Quoted 35 lines · Cong Wang
Quoted 12 lines · Cong Wang
Quoted 2 lines · Vladimir Oltean
It's me with the stacked DSA devices again:
It looks like DSA never uses netdev API to link master device with slave devices? If so, their
dev->lower_levelare always 1, therefore triggers this warning.I think it should call one of these
netdev_upper_dev_link()API's when creating a slave device.I don't know whether DSA is too special to use the API, but something like this should work:
net/dsa/slave.c +5
-0@@ -1807,6 +1807,11 @@ int dsa_slave_create(struct dsa_port *port) ret, slave_dev->name); goto out_phy; } + ret =netdev_upper_dev_link (slave_dev, master,NULL ); +if (ret) { +unregister_netdevice (slave_dev); + goto out_phy; + }return 0 ;net/dsa/slave.c void dsa_slave_destroy(struct net_device *slave_dev) · line 1843
Quoted 7 lines
net/dsa/slave.c +1
-0@@ -1832,6 +1837,7 @@ void dsa_slave_destroy(struct net_device *slave_dev)netif_carrier_off (slave_dev);rtnl_lock ();phylink_disconnect_phy (dp->pl); +netdev_upper_dev_unlink (slave_dev, dp->master);rtnl_unlock ();dsa_slave_notify (slave_dev,DSA_PORT_UNREGISTER );Thanks. This is a good approximation of what needed to be done:
-netdev_upper_dev_linkneeds to be under rtnl,
- "dp->master" should be "dsa_slave_to_master(slave_dev)" since it's actually a union if you look at structdsa_port).Quoted 5 lines
- And, most importantly, I think the hierarchy should be reversed: a (virtual) DSA switch port net device (slave) should be an upper of the (real) DSA master (the host port). Think of it like this: a DSA switch is a sort of port multiplier for a host port, based on a frame header. But, it works!
Quoted 7 lines
Hi Vladimir
So you are suggesting this?
Quoted 1 line
Quoted 1 line
+ ret =
netdev_upper_dev_link(master, slave_dev, NULL);Andrew
Yes, basically this:
net/dsa/slave.c +9 -0
@@ -1807,6 +1807,13 @@ int dsa_slave_create(struct dsa_port *port) ret, slave_dev->name); goto out_phy; } +rtnl_lock (); + ret =netdev_upper_dev_link (master, slave_dev,NULL ); +rtnl_unlock (); +if (ret) { +unregister_netdevice (slave_dev); + goto out_phy; + }return 0 ;
@@ -1826,12 +1833,14 @@ int dsa_slave_create(struct dsa_port *port)void dsa_slave_destroy (struct net_device *slave_dev) { +struct net_device *master =dsa_slave_to_master (slave_dev);struct dsa_port *dp =dsa_slave_to_port (slave_dev);struct dsa_slave_priv *p =netdev_priv (slave_dev);netif_carrier_off (slave_dev);rtnl_lock ();phylink_disconnect_phy (dp->pl); +netdev_upper_dev_unlink (master, slave_dev);rtnl_unlock ();dsa_slave_notify (slave_dev,DSA_PORT_UNREGISTER );
Do you see a problem with it?
Thanks,
-Vladimir
Andrew Lunn
maintainerQuoted 44 lines
Quoted 7 lines
Hi Vladimir
So you are suggesting this?
Quoted 1 line
Quoted 1 line
+ ret =
netdev_upper_dev_link(master, slave_dev, NULL);Andrew
Yes, basically this:
net/dsa/slave.c +9
-0@@ -1807,6 +1807,13 @@ int dsa_slave_create(struct dsa_port *port) ret, slave_dev->name); goto out_phy; } +rtnl_lock (); + ret =netdev_upper_dev_link (master, slave_dev,NULL ); +rtnl_unlock (); +if (ret) { +unregister_netdevice (slave_dev); + goto out_phy; + }return 0 ;@@ -1826,12 +1833,14 @@ int dsa_slave_create(struct dsa_port *port)void dsa_slave_destroy (struct net_device *slave_dev) { +struct net_device *master =dsa_slave_to_master (slave_dev);struct dsa_port *dp =dsa_slave_to_port (slave_dev);struct dsa_slave_priv *p =netdev_priv (slave_dev);netif_carrier_off (slave_dev);rtnl_lock ();phylink_disconnect_phy (dp->pl); +netdev_upper_dev_unlink (master, slave_dev);rtnl_unlock ();dsa_slave_notify (slave_dev,DSA_PORT_UNREGISTER );
Quoted 1 line
Do you see a problem with it?
I was initially not sure you could do this. But it looks like you can have N : M relationships between uppers and lowers. I suppose it does make sense. You can have multiple VLAN uppers to one base device. You can have multiple lowers to one bond device, etc.
I wonder what 'side effects' there are for declaring this linkage. It is not something i've looked into before, since we never used it. So i don't see a problem with this, other than i don't know what problems we might run into :-)
Andrew
Vladimir Oltean
maintainerQuoted 52 lines · Andrew Lunn
Quoted 45 lines
Quoted 8 lines
Hi Vladimir
So you are suggesting this?
Quoted 1 line
Quoted 1 line
+ ret =
netdev_upper_dev_link(master, slave_dev, NULL);Andrew
Yes, basically this:
net/dsa/slave.c +9
-0@@ -1807,6 +1807,13 @@ int dsa_slave_create(struct dsa_port *port) ret, slave_dev->name); goto out_phy; } +rtnl_lock (); + ret =netdev_upper_dev_link (master, slave_dev,NULL ); +rtnl_unlock (); +if (ret) { +unregister_netdevice (slave_dev); + goto out_phy; + }return 0 ;@@ -1826,12 +1833,14 @@ int dsa_slave_create(struct dsa_port *port)void dsa_slave_destroy (struct net_device *slave_dev) { +struct net_device *master =dsa_slave_to_master (slave_dev);struct dsa_port *dp =dsa_slave_to_port (slave_dev);struct dsa_slave_priv *p =netdev_priv (slave_dev);netif_carrier_off (slave_dev);rtnl_lock ();phylink_disconnect_phy (dp->pl); +netdev_upper_dev_unlink (master, slave_dev);rtnl_unlock ();dsa_slave_notify (slave_dev,DSA_PORT_UNREGISTER );Quoted 1 line
Do you see a problem with it?
I was initially not sure you could do this. But it looks like you can have N : M relationships between uppers and lowers. I suppose it does make sense. You can have multiple VLAN uppers to one base device. You can have multiple lowers to one bond device, etc.
Quoted 7 lines
I wonder what 'side effects' there are for declaring this linkage. It is not something i've looked into before, since we never used it. So i don't see a problem with this, other than i don't know what problems we might run into :-)
Andrew
It was surprising to me as well, since I was used to the bridge model (a port can have only one bridge master). But it looks like, that is the difference between netdev_upper_dev_link and netdev_master_upper_dev_link. This uses the former, and the bridge layer uses the latter.
So I guess it is ok.
Thanks,
-Vladimir