iavf: fix reset task deadlock
Subsystems (2)
v9 cover iavf: fix reset task deadlock
From: Dawid Wesierski <[email protected]>
Changing the way we handle resets so that the callback operating under the RTNL lock will wait for the reset to finish, the rtnl_lock sensitive functions in reset flow will schedule the netdev update for later. This will eliminate circular dependency with the critical lock.
Marcin Szycik (4):
iavf: Wait for reset in callbacks which trigger it
iavf: Don't lock rtnl_lock twice in reset
Revert "iavf: Detach device during reset task"
Revert "iavf: Do not restart Tx queues after reset task failure"
drivers/net/ethernet/intel/iavf/iavf.h | 3 + .../net/ethernet/intel/iavf/iavf_ethtool.c | 31 +++-- drivers/net/ethernet/intel/iavf/iavf_main.c | 118 +++++++++++++----- .../net/ethernet/intel/iavf/iavf_virtchnl.c | 1 + 4 files changed, 105 insertions(+), 48 deletions(-)
Signature · 6 lines
2.31.1 _______________________________________________ Intel-wired-lan mailing list [email protected] https://lists.osuosl.org/mailman/listinfo/intel-wired-lan
PATCH v9 1/4 iavf: Wait for reset in callbacks which trigger it
From: Marcin Szycik <[email protected]>
There was a fail when trying to add the interface to bonding right after changing the MTU on the interface. It was caused by bonding interface unable to open the interface due to interface being in __RESETTING state because of MTU change.
Add new reset_waitqueue to indicate that reset has finished. Add waiting for reset to finish in .
Add waiting for reset to finish in callbacks which trigger hw reset:iavf_set_priv_flags(), iavf_change_mtu() and iavf_set_ringparam(). We choose 5 sec for the timeout waiting time because the same interval is used in oot where waiting
Add a function iavf_wait_for_reset() to reuse waiting for reset code and use it also in iavf_set_channels(), which already waits for reset. We don't use error handling in iavf_set_channels() as this could cause the device to be in incorrect state if the reset was scheduled but hit timeout or the waitng function was interrupted by a signal.
We use a 5000ms timeout period because on Hyper-V based systems, this operation takes around 3000-4000ms. In normal circumstances, it doesn't take more than 500ms to complete.
---v1->v3: changes were done internally
v4: fixed SOB's
v5: changed the way we wake up the reset_waitqueue to make sure that reset is woken up v6->v9: no changes
--- drivers/net/ethernet/intel/iavf/iavf.h | 2 + .../net/ethernet/intel/iavf/iavf_ethtool.c | 31 ++++++----- drivers/net/ethernet/intel/iavf/iavf_main.c | 51 ++++++++++++++++++- .../net/ethernet/intel/iavf/iavf_virtchnl.c | 1 + 4 files changed, 68 insertions(+), 17 deletions(-)
drivers/net/ethernet/intel/iavf/iavf.h +2 -0
@@ -257,6 +257,7 @@ struct iavf_adapter {struct work_struct adminq_task;struct delayed_work client_task;wait_queue_head_t down_waitqueue; +wait_queue_head_t reset_waitqueue;wait_queue_head_t vc_waitqueue;struct iavf_q_vector *q_vectors;struct list_head vlan_filter_list;
@@ -591,5 +592,6 @@ void iavf_add_adv_rss_cfg(struct iavf_adapter *adapter);void iavf_del_adv_rss_cfg (struct iavf_adapter *adapter);struct iavf_mac_filter *iavf_add_filter (struct iavf_adapter *adapter,const u8 *macaddr); +int iavf_wait_for_reset (struct iavf_adapter *adapter);int iavf_lock_timeout (struct mutex *lock,unsigned int msecs);#endif /* _IAVF_H_ */
drivers/net/ethernet/intel/iavf/iavf_ethtool.c +15 -16
@@ -484,6 +484,7 @@ static int iavf_set_priv_flags(struct net_device *netdev, u32 flags) {struct iavf_adapter *adapter =netdev_priv (netdev);u32 orig_flags, new_flags, changed_flags; +int ret =0 ;u32 i; orig_flags =READ_ONCE (adapter->flags);
@@ -533,10 +534,13 @@ static int iavf_set_priv_flags(struct net_device *netdev, u32 flags)if (netif_running (netdev)) { adapter->flags |=IAVF_FLAG_RESET_NEEDED ;queue_work (adapter->wq, &adapter->reset_task); + ret =iavf_wait_for_reset (adapter); +if (ret) +netdev_warn (netdev,"Changing private flags timeout or interrupted waiting for reset" ); } } -return 0 ; +return ret; }/**
@@ -627,6 +631,7 @@ static int iavf_set_ringparam(struct net_device *netdev, {struct iavf_adapter *adapter =netdev_priv (netdev);u32 new_rx_count, new_tx_count; +int ret =0 ;if ((ring->rx_mini_pending) || (ring->rx_jumbo_pending))return -EINVAL ;
@@ -673,9 +678,12 @@ static int iavf_set_ringparam(struct net_device *netdev,if (netif_running (netdev)) { adapter->flags |=IAVF_FLAG_RESET_NEEDED ;queue_work (adapter->wq, &adapter->reset_task); + ret =iavf_wait_for_reset (adapter); +if (ret) +netdev_warn (netdev,"Changing ring parameters timeout or interrupted waiting for reset" ); } -return 0 ; +return ret; }/**
@@ -1830,7 +1838,7 @@ static int iavf_set_channels(struct net_device *netdev, {struct iavf_adapter *adapter =netdev_priv (netdev);u32 num_req = ch->combined_count; -int i; +int ret =0 ;if ((adapter->vf_res->vf_cap_flags &VIRTCHNL_VF_OFFLOAD_ADQ ) && adapter->num_tc) {
@@ -1854,20 +1862,11 @@ static int iavf_set_channels(struct net_device *netdev, adapter->flags |=IAVF_FLAG_REINIT_ITR_NEEDED ;iavf_schedule_reset (adapter); -/* wait for the reset is done */ -for (i =0 ; i <IAVF_RESET_WAIT_COMPLETE_COUNT ; i++) { -msleep (IAVF_RESET_WAIT_MS ); -if (adapter->flags &IAVF_FLAG_RESET_PENDING ) -continue ; -break ; - } -if (i ==IAVF_RESET_WAIT_COMPLETE_COUNT ) { - adapter->flags &= ~IAVF_FLAG_REINIT_ITR_NEEDED ; - adapter->num_req_queues =0 ; -return -EOPNOTSUPP ; - } + ret =iavf_wait_for_reset (adapter); +if (ret) +netdev_warn (netdev,"Changing channel count timeout or interrupted waiting for reset" ); -return 0 ; +return ret; }/**
drivers/net/ethernet/intel/iavf/iavf_main.c +50 -1
83 lines not on this page · raw message
drivers/net/ethernet/intel/iavf/iavf_virtchnl.c +1 -0
@@ -2285,6 +2285,7 @@ void iavf_virtchnl_completion(struct iavf_adapter *adapter,case VIRTCHNL_OP_ENABLE_QUEUES :/* enable transmits */ iavf_irq_enable (adapter, true); +wake_up (&adapter->reset_waitqueue); adapter->flags &= ~IAVF_FLAG_QUEUES_DISABLED ;break ;case VIRTCHNL_OP_DISABLE_QUEUES :
Signature · 6 lines
2.31.1 _______________________________________________ Intel-wired-lan mailing list [email protected] https://lists.osuosl.org/mailman/listinfo/intel-wired-lan
PATCH v9 2/4 iavf: Don't lock rtnl_lock twice in reset
In reply to Mateusz PalczewskiFrom: Marcin Szycik <[email protected]>
Some ndo/ethtool callbacks are called under rtnl_lock.
If such a callback triggers a reset, the reset task might try to take the rtnl_lock again, causing a deadlock.
When the driver is unable to obtain the rtnl_lock in the reset flow, the reset task does not attempt to double-lock rtnl_lock and avoids the deadlock by scheduling the netdev update.
Iavf_delayed_set_interrupt_capability is scheduled on the global wq we don't schedule it on the iavf workqueue to avoid deadlock. (iavf_workqueue is ordered and designed to support only one active task)
e.g:
-A iavf schedules reset
-B another iavf reset asks for RTNL_LOCK
-A iavf schedules the iavf_delayed_set_interrupt_capability. -A reset ends
-A iavf_delayed_set_interrupt_capability asks for RTNL_LOCK -B obtains the RTNL_LOCK waits for iavf_delayed_set_interrupt_capability to free the working queue
-A waits for RTNL_LOCK
Before this patch, a deadlock could be caused by e.g.:
echo 1 > /sys/class/net/$PF1/device/sriov_numvfs
while :; do
ip l s $VF1 up
ethtool --set-channels $VF1 combined 8
ip l s $VF1 down
ip l s $VF1 up
ethtool --set-channels $VF1 combined 16
ip l s $VF1 down
done
---v1->v3: changes were done internally
v4: added space before open parenthesis '(', fixed code indent
v5: changed the way the scheduled function updates the netdev from { trylock -> reschedule }
design to one that just takes and wait for rtnl_lock lock. Introduced 30ms delay in scheduling to account for scheduling the resets in quick succession.
v6: added a guard to iavf_delayed_set_interrupt_capability function to prevent updating an unregistered netdev.
Removed the delay from the scheduling and moved the rtnl to obtain the number of queues after acquiring the lock
v7: made it compatible with net-queue, no changes in logicv8->v9: no changes
--- drivers/net/ethernet/intel/iavf/iavf.h | 1 + drivers/net/ethernet/intel/iavf/iavf_main.c | 38 ++++++++++++++++++--- 2 files changed, 34 insertions(+), 5 deletions(-)
drivers/net/ethernet/intel/iavf/iavf.h +1 -0
@@ -255,6 +255,7 @@ struct iavf_adapter {struct workqueue_struct *wq;struct work_struct reset_task;struct work_struct adminq_task; +struct work_struct set_interrupt_capability;struct delayed_work client_task;wait_queue_head_t down_waitqueue;wait_queue_head_t reset_waitqueue;
drivers/net/ethernet/intel/iavf/iavf_main.c +33 -5
@@ -1707,11 +1707,38 @@ static int iavf_set_interrupt_capability(struct iavf_adapter *adapter) err =iavf_acquire_msix_vectors (adapter, v_budget); out: -netif_set_real_num_rx_queues (adapter->netdev, pairs); -netif_set_real_num_tx_queues (adapter->netdev, pairs); +if (rtnl_trylock ()) { +netif_set_real_num_rx_queues (adapter->netdev, pairs); +netif_set_real_num_tx_queues (adapter->netdev, pairs); +rtnl_unlock (); + }else { +schedule_work (&adapter->set_interrupt_capability); + } +return err; } +/** +* iavf_delayed_set_interrupt_capability - schedule the update of the netdev +* @work: pointer to work_struct containing our data +**/ +static void iavf_delayed_set_interrupt_capability (struct work_struct *work) +{ +struct iavf_adapter *adapter; +int pairs; + +rtnl_lock (); + adapter = container_of(work, struct iavf_adapter, + set_interrupt_capability); + pairs = adapter->num_active_queues; + +if (adapter->netdev_registered) { +netif_set_real_num_rx_queues (adapter->netdev, pairs); +netif_set_real_num_tx_queues (adapter->netdev, pairs); + } +rtnl_unlock (); +} +/** * iavf_config_rss_aq - Configure RSS keys and lut by using AQ commands * @adapter: board private structure
@@ -1926,10 +1953,8 @@ int iavf_init_interrupt_scheme(struct iavf_adapter *adapter) "Unable to allocate memory for queues\n"); goto err_alloc_queues; } - -rtnl_lock (); err =iavf_set_interrupt_capability (adapter); -rtnl_unlock (); +if (err) {dev_err (&adapter->pdev->dev,"Unable to setup interrupt capabilities\n" );
@@ -4980,6 +5005,8 @@ static int iavf_probe(struct pci_dev *pdev, const struct pci_device_id *ent)INIT_WORK (&adapter->reset_task, iavf_reset_task);INIT_WORK (&adapter->adminq_task, iavf_adminq_task); +INIT_WORK (&adapter->set_interrupt_capability, + iavf_delayed_set_interrupt_capability);INIT_DELAYED_WORK (&adapter->watchdog_task, iavf_watchdog_task);INIT_DELAYED_WORK (&adapter->client_task, iavf_client_task); queue_delayed_work(adapter->wq, &adapter->watchdog_task,
@@ -5153,6 +5180,7 @@ static void iavf_remove(struct pci_dev *pdev)cancel_work_sync (&adapter->reset_task);cancel_delayed_work_sync (&adapter->watchdog_task);cancel_work_sync (&adapter->adminq_task); +cancel_work_sync (&adapter->set_interrupt_capability);cancel_delayed_work_sync (&adapter->client_task); adapter->aq_required =0 ;
Signature · 6 lines
2.31.1 _______________________________________________ Intel-wired-lan mailing list [email protected] https://lists.osuosl.org/mailman/listinfo/intel-wired-lan
PATCH v9 3/4 Revert "iavf: Detach device during reset task"
In reply to Mateusz PalczewskiFrom: Marcin Szycik <[email protected]>
This reverts commit aa626da947e9cd30c4cf727493903e1adbb2c0a0.
Detaching device during reset was not fully fixing the rtnl locking issue, as there could be a situation where callback was already in progress before detaching netdev.
Furthermore, detaching netdevice causes TX timeouts if traffic is running. To reproduce:
ip netns exec ns1 iperf3 -c $PEER_IP -t 600 --logfile /dev/null & while :; do
for i in 200 7000 400 5000 300 3000 ; do
ip netns exec ns1 ip link set $VF1 mtu $i sleep 2
done
sleep 10
done
Currently, callbacks such as iavf_change_mtu() wait for the reset. If the reset fails to acquire the rtnl_lock, they schedule the netdev update for later while continuing the reset flow. Operations like MTU changes are performed under the rtnl_lock. Therefore, when the operation finishes, another callback that uses rtnl_lock can start.
---v1->v7: no changes
v8: changed commit msg
v9: no changes
--- drivers/net/ethernet/intel/iavf/iavf_main.c | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-)
drivers/net/ethernet/intel/iavf/iavf_main.c +2 -11
@@ -3016,11 +3016,6 @@ static void iavf_reset_task(struct work_struct *work)int i =0 , err;bool running; -/* Detach interface to avoid subsequent NDO callbacks */ -rtnl_lock (); -netif_device_detach (netdev); -rtnl_unlock (); -/* When device is being removed it doesn't make sense to run the reset * task, just return in such a case. */
@@ -3028,7 +3023,7 @@ static void iavf_reset_task(struct work_struct *work)if (adapter->state != __IAVF_REMOVE)queue_work (adapter->wq, &adapter->reset_task); - goto reset_finish; +return ; }while (!mutex_trylock (&adapter->client_lock))
@@ -3231,7 +3226,7 @@ static void iavf_reset_task(struct work_struct *work)mutex_unlock (&adapter->client_lock);mutex_unlock (&adapter->crit_lock); - goto reset_finish; +return ; reset_err:if (running) {set_bit (__IAVF_VSI_DOWN, adapter->vsi.state);
@@ -3252,10 +3247,6 @@ static void iavf_reset_task(struct work_struct *work) }dev_err (&adapter->pdev->dev,"failed to allocate resources during reinit\n" ); -reset_finish: -rtnl_lock (); -netif_device_attach (netdev); -rtnl_unlock (); }/**
Signature · 6 lines
2.31.1 _______________________________________________ Intel-wired-lan mailing list [email protected] https://lists.osuosl.org/mailman/listinfo/intel-wired-lan
PATCH v9 4/4 Revert "iavf: Do not restart Tx queues after reset task failure"
In reply to Mateusz PalczewskiFrom: Marcin Szycik <[email protected]>
This reverts commit 08f1c147b7265245d67321585c68a27e990e0c4b.
Netdev is no longer being detached during reset, so this fix can be reverted. We leave the removal of "hacky" IFF_UP flag update.
---v1->v8: no changes
v9: we leave the removal of "hacky" IFF_UP update that was implemented in this patch
--- drivers/net/ethernet/intel/iavf/iavf_main.c | 15 --------------- 1 file changed, 15 deletions(-)
drivers/net/ethernet/intel/iavf/iavf_main.c +0 -15
@@ -3081,11 +3081,6 @@ static void iavf_reset_task(struct work_struct *work)iavf_disable_vf (adapter);mutex_unlock (&adapter->client_lock);mutex_unlock (&adapter->crit_lock); -if (netif_running (netdev)) { -rtnl_lock (); -dev_close (netdev); -rtnl_unlock (); - }return ;/* Do not attempt to reinit. It's dead, Jim. */ }
@@ -3236,16 +3231,6 @@ static void iavf_reset_task(struct work_struct *work)mutex_unlock (&adapter->client_lock);mutex_unlock (&adapter->crit_lock); - -if (netif_running (netdev)) { -/* Close device to ensure that Tx queues will not be started -* during netif_device_attach() at the end of the reset task. -*/ -rtnl_lock (); -dev_close (netdev); -rtnl_unlock (); - } -dev_err (&adapter->pdev->dev,"failed to allocate resources during reinit\n" ); }
Signature · 6 lines
2.31.1 _______________________________________________ Intel-wired-lan mailing list [email protected] https://lists.osuosl.org/mailman/listinfo/intel-wired-lan