ibmvnic: assorted bug fixes
v3 cover ibmvnic: assorted bug fixes
Assorted fixes for ibmvnic originated from "[PATCH net 00/15] ibmvnic:
assorted bug fixes" sent by Lijun Pan.
v3 Changes as suggested by Jakub Kicinski:
- Add a space between variable declaration and code in patch 3/9. Checkpatch does not catch this.
- Unwrapped FIXES lines in patch 9/9.
- Removed all extra line between Fixes and Signed-off-by lines in all patches.
V2 Changes as suggested by Jakub Kicinski:
- Added "Fixes" to each patch.
- Remove "ibmvnic: process HMC disable command" from the series. Submitting it separately to net-next.
- Squash V1 "ibmvnic: remove free_all_rwi function" into
ibmvnic: stop free_all_rwi on failed reset.
Dany Madden (7):
ibmvnic: handle inconsistent login with reset
ibmvnic: stop free_all_rwi on failed reset
ibmvnic: avoid memset null scrq msgs
ibmvnic: restore adapter state on failed reset
ibmvnic: send_login should check for crq errors
ibmvnic: no reset timeout for 5 seconds after reset
ibmvnic: reduce wait for completion time
Sukadev Bhattiprolu (2):
ibmvnic: delay next reset if hard reset fails
ibmvnic: track pending login
drivers/net/ethernet/ibm/ibmvnic.c | 168 ++++++++++++++++++----------- drivers/net/ethernet/ibm/ibmvnic.h | 3 + 2 files changed, 106 insertions(+), 65 deletions(-)
Signature · 1 line
2.26.2
PATCH v3 3/9 ibmvnic: avoid memset null scrq msgs
scrq->msgs could be NULL during device reset, causing Linux to crash. So, check before memset scrq->msgs.
--- drivers/net/ethernet/ibm/ibmvnic.c | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-)
drivers/net/ethernet/ibm/ibmvnic.c +15 -4
@@ -2845,15 +2845,26 @@ static int reset_one_sub_crq_queue(struct ibmvnic_adapter *adapter, {int rc; +if (!scrq) { +netdev_dbg (adapter->netdev, +"Invalid scrq reset. irq (%d) or msgs (%p).\n" , + scrq->irq, scrq->msgs); +return -EINVAL ; + } +if (scrq->irq) {free_irq (scrq->irq, scrq);irq_dispose_mapping (scrq->irq); scrq->irq =0 ; } - -memset (scrq->msgs,0 ,4 *PAGE_SIZE ); -atomic_set (&scrq->used,0 ); - scrq->cur =0 ; +if (scrq->msgs) { +memset (scrq->msgs,0 ,4 *PAGE_SIZE ); +atomic_set (&scrq->used,0 ); + scrq->cur =0 ; + }else { +netdev_dbg (adapter->netdev,"Invalid scrq reset\n" ); +return -EINVAL ; + } rc =h_reg_sub_crq (adapter->vdev->unit_address, scrq->msg_token,4 *PAGE_SIZE , &scrq->crq_num, &scrq->hw_irq);
Signature · 1 line
2.26.2
PATCH v3 5/9 ibmvnic: delay next reset if hard reset fails
In reply to Dany MaddenFrom: Sukadev Bhattiprolu <[email protected]>
If auto-priority failover is enabled, the backing device needs time to settle if hard resetting fails for any reason. Add a delay of 60 seconds before retrying the hard-reset.
--- drivers/net/ethernet/ibm/ibmvnic.c | 8 ++++++++ 1 file changed, 8 insertions(+)
drivers/net/ethernet/ibm/ibmvnic.c +8 -0
@@ -2242,6 +2242,14 @@ static void __ibmvnic_reset(struct work_struct *work) rc =do_hard_reset (adapter, rwi, reset_state);rtnl_unlock (); } +if (rc) { +/* give backing device time to settle down */ +netdev_dbg (adapter->netdev, +"[S:%d] Hard reset failed, waiting 60 secs\n" , + adapter->state); +set_current_state (TASK_UNINTERRUPTIBLE ); +schedule_timeout (60 *HZ ); + } }else if (!(rwi->reset_reason ==VNIC_RESET_FATAL && adapter->from_passive_init)) { rc =do_reset (adapter, rwi, reset_state);
Signature · 1 line
2.26.2
PATCH v3 7/9 ibmvnic: send_login should check for crq errors
In reply to Dany Maddensend_login() does not check for the result of ibmvnic_send_crq() of the login request. This results in the driver needlessly retrying the login 10 times even when CRQ is no longer active. Check the return code and give up in case of errors in sending the CRQ.
The only time we want to retry is if we get a PARITALSUCCESS response from the partner.
--- drivers/net/ethernet/ibm/ibmvnic.c | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-)
drivers/net/ethernet/ibm/ibmvnic.c +12 -6
@@ -850,10 +850,8 @@ static int ibmvnic_login(struct net_device *netdev) adapter->init_done_rc =0 ;reinit_completion (&adapter->init_done); rc =send_login (adapter); -if (rc) { -netdev_warn (netdev,"Unable to login\n" ); +if (rc)return rc; - }if (!wait_for_completion_timeout (&adapter->init_done, timeout)) {
@@ -3727,15 +3725,16 @@ static int send_login(struct ibmvnic_adapter *adapter)struct ibmvnic_login_rsp_buffer *login_rsp_buffer;struct ibmvnic_login_buffer *login_buffer;struct device *dev = &adapter->vdev->dev; +struct vnic_login_client_data *vlcd;dma_addr_t rsp_buffer_token;dma_addr_t buffer_token;size_t rsp_buffer_size;union ibmvnic_crq crq; +int client_data_len;size_t buffer_size;__be64 *tx_list_p;__be64 *rx_list_p; -int client_data_len; -struct vnic_login_client_data *vlcd; +int rc;int i;if (!adapter->tx_scrq || !adapter->rx_scrq) {
@@ -3841,16 +3840,23 @@ static int send_login(struct ibmvnic_adapter *adapter) crq.login.len =cpu_to_be32 (buffer_size); adapter->login_pending = true; -ibmvnic_send_crq (adapter, &crq); + rc =ibmvnic_send_crq (adapter, &crq); +if (rc) { + adapter->login_pending = false; +netdev_err (adapter->netdev,"Failed to send login, rc=%d\n" , rc); + goto buf_rsp_map_failed; + }return 0 ; buf_rsp_map_failed:kfree (login_rsp_buffer); + adapter->login_rsp_buf =NULL ; buf_rsp_alloc_failed:dma_unmap_single (dev, buffer_token, buffer_size,DMA_TO_DEVICE ); buf_map_failed:kfree (login_buffer); + adapter->login_buf =NULL ; buf_alloc_failed:return -1 ; }
Signature · 1 line
2.26.2
PATCH v3 6/9 ibmvnic: track pending login
In reply to Dany MaddenFrom: Sukadev Bhattiprolu <[email protected]>
If after ibmvnic sends a LOGIN it gets a FAILOVER, it is possible that the worker thread will start reset process and free the login response buffer before it gets a (now stale) LOGIN_RSP. The ibmvnic tasklet will then try to access the login response buffer and crash.
Have ibmvnic track pending logins and discard any stale login responses.
--- drivers/net/ethernet/ibm/ibmvnic.c | 17 +++++++++++++++++ drivers/net/ethernet/ibm/ibmvnic.h | 1 + 2 files changed, 18 insertions(+)
drivers/net/ethernet/ibm/ibmvnic.c +17 -0
@@ -3839,6 +3839,8 @@ static int send_login(struct ibmvnic_adapter *adapter) crq.login.cmd =LOGIN ; crq.login.ioba =cpu_to_be32 (buffer_token); crq.login.len =cpu_to_be32 (buffer_size); + + adapter->login_pending = true;ibmvnic_send_crq (adapter, &crq);return 0 ;
@@ -4391,6 +4393,15 @@ static int handle_login_rsp(union ibmvnic_crq *login_rsp_crq,u64 *size_array;int i; +/* CHECK: Test/set of login_pending does not need to be atomic +* because only ibmvnic_tasklet tests/clears this. +*/ +if (!adapter->login_pending) { +netdev_warn (netdev,"Ignoring unexpected login response\n" ); +return 0 ; + } + adapter->login_pending = false; +dma_unmap_single (dev, adapter->login_buf_token, adapter->login_buf_sz,DMA_TO_DEVICE ); dma_unmap_single(dev, adapter->login_rsp_buf_token,
@@ -4762,6 +4773,11 @@ static void ibmvnic_handle_crq(union ibmvnic_crq *crq,case IBMVNIC_CRQ_INIT :dev_info (dev,"Partner initialized\n" ); adapter->from_passive_init = true; +/* Discard any stale login responses from prev reset. +* CHECK: should we clear even on INIT_COMPLETE? +*/ + adapter->login_pending = false; +if (!completion_done (&adapter->init_done)) {complete (&adapter->init_done); adapter->init_done_rc = -EIO ;
@@ -5191,6 +5207,7 @@ static int ibmvnic_probe(struct vio_dev *dev, const struct vio_device_id *id)dev_set_drvdata (&dev->dev, netdev); adapter->vdev = dev; adapter->netdev = netdev; + adapter->login_pending = false;ether_addr_copy (adapter->mac_addr, mac_addr_p);ether_addr_copy (netdev->dev_addr, adapter->mac_addr);
drivers/net/ethernet/ibm/ibmvnic.h +1 -0
@@ -1087,6 +1087,7 @@ struct ibmvnic_adapter {struct delayed_work ibmvnic_delayed_reset;unsigned long resetting;bool napi_enabled, from_passive_init; +bool login_pending;bool failover_pending;bool force_reset_recovery;
Signature · 1 line
2.26.2
PATCH v3 9/9 ibmvnic: reduce wait for completion time
In reply to Dany MaddenReduce the wait time for Command Response Queue response from 30 seconds to 20 seconds, as recommended by VIOS and Power Hypervisor teams.
--- drivers/net/ethernet/ibm/ibmvnic.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-)
drivers/net/ethernet/ibm/ibmvnic.c +3 -3
@@ -834,7 +834,7 @@ static void release_napi(struct ibmvnic_adapter *adapter)static int ibmvnic_login (struct net_device *netdev) {struct ibmvnic_adapter *adapter =netdev_priv (netdev); -unsigned long timeout =msecs_to_jiffies (30000 ); +unsigned long timeout =msecs_to_jiffies (20000 );int retry_count =0 ;int retries =10 ;bool retry;
@@ -938,7 +938,7 @@ static void release_resources(struct ibmvnic_adapter *adapter)static int set_link_state (struct ibmvnic_adapter *adapter,u8 link_state) {struct net_device *netdev = adapter->netdev; -unsigned long timeout =msecs_to_jiffies (30000 ); +unsigned long timeout =msecs_to_jiffies (20000 );union ibmvnic_crq crq;bool resend;int rc;
@@ -5125,7 +5125,7 @@ static int init_crq_queue(struct ibmvnic_adapter *adapter)static int ibmvnic_reset_init (struct ibmvnic_adapter *adapter,bool reset) {struct device *dev = &adapter->vdev->dev; -unsigned long timeout =msecs_to_jiffies (30000 ); +unsigned long timeout =msecs_to_jiffies (20000 );u64 old_num_rx_queues, old_num_tx_queues;int rc;
Signature · 1 line
2.26.2
PATCH v3 8/9 ibmvnic: no reset timeout for 5 seconds after reset
In reply to Dany MaddenReset timeout is going off right after adapter reset. This patch ensures that timeout is scheduled if it has been 5 seconds since the last reset. 5 seconds is the default watchdog timeout.
--- drivers/net/ethernet/ibm/ibmvnic.c | 11 +++++++++-- drivers/net/ethernet/ibm/ibmvnic.h | 2 ++ 2 files changed, 11 insertions(+), 2 deletions(-)
drivers/net/ethernet/ibm/ibmvnic.c +9 -2
@@ -2253,6 +2253,7 @@ static void __ibmvnic_reset(struct work_struct *work) rc =do_reset (adapter, rwi, reset_state); }kfree (rwi); + adapter->last_reset_time = jiffies;if (rc)netdev_dbg (adapter->netdev,"Reset failed, rc=%d\n" , rc);
@@ -2356,7 +2357,13 @@ static void ibmvnic_tx_timeout(struct net_device *dev, unsigned int txqueue) "Adapter is resetting, skip timeout reset\n");return ; } - +/* No queuing up reset until at least 5 seconds (default watchdog val) +* after last reset +*/ +if (time_before (jiffies, (adapter->last_reset_time + dev->watchdog_timeo))) { +netdev_dbg (dev,"Not yet time to tx timeout.\n" ); +return ; + }ibmvnic_reset (adapter,VNIC_RESET_TIMEOUT ); }
@@ -5277,7 +5284,7 @@ static int ibmvnic_probe(struct vio_dev *dev, const struct vio_device_id *id) adapter->state =VNIC_PROBED ; adapter->wait_for_reset = false; - + adapter->last_reset_time = jiffies;return 0 ; ibmvnic_register_fail:
drivers/net/ethernet/ibm/ibmvnic.h +2 -0
@@ -1088,6 +1088,8 @@ struct ibmvnic_adapter {unsigned long resetting;bool napi_enabled, from_passive_init;bool login_pending; +/* last device reset time */ +unsigned long last_reset_time;bool failover_pending;bool force_reset_recovery;
Signature · 1 line
2.26.2
PATCH v3 2/9 ibmvnic: stop free_all_rwi on failed reset
In reply to Dany MaddenWhen ibmvnic fails to reset, it breaks out of the reset loop and frees all of the remaining resets from the workqueue. Doing so prevents the adapter from recovering if no reset is scheduled after that. Instead, have the driver continue to process resets on the workqueue.
Remove the no longer need free_all_rwi().
--- drivers/net/ethernet/ibm/ibmvnic.c | 22 +++------------------- 1 file changed, 3 insertions(+), 19 deletions(-)
drivers/net/ethernet/ibm/ibmvnic.c +3 -19
@@ -2173,17 +2173,6 @@ static struct ibmvnic_rwi *get_next_rwi(struct ibmvnic_adapter *adapter)return rwi; } -static void free_all_rwi (struct ibmvnic_adapter *adapter) -{ -struct ibmvnic_rwi *rwi; - - rwi =get_next_rwi (adapter); -while (rwi) { -kfree (rwi); - rwi =get_next_rwi (adapter); - } -} -static void __ibmvnic_reset (struct work_struct *work) {struct ibmvnic_rwi *rwi;
@@ -2253,9 +2242,9 @@ static void __ibmvnic_reset(struct work_struct *work)else adapter->state = reset_state; rc =0 ; - }else if (rc && rc !=IBMVNIC_INIT_FAILED && - !adapter->force_reset_recovery) -break ; + } +if (rc) +netdev_dbg (adapter->netdev,"Reset failed, rc=%d\n" , rc); rwi =get_next_rwi (adapter);
@@ -2269,11 +2258,6 @@ static void __ibmvnic_reset(struct work_struct *work)complete (&adapter->reset_done); } -if (rc) { -netdev_dbg (adapter->netdev,"Reset failed\n" ); -free_all_rwi (adapter); - } -clear_bit_unlock (0 , &adapter->resetting); }
Signature · 1 line
2.26.2
PATCH v3 4/9 ibmvnic: restore adapter state on failed reset
In reply to Dany MaddenIn a failed reset, driver could end up in VNIC_PROBED or VNIC_CLOSED state and cannot recover in subsequent resets, leaving it offline. This patch restores the adapter state to reset_state, the original state when reset was called.
--- drivers/net/ethernet/ibm/ibmvnic.c | 67 ++++++++++++++++-------------- 1 file changed, 36 insertions(+), 31 deletions(-)
drivers/net/ethernet/ibm/ibmvnic.c +36 -31
@@ -1857,7 +1857,7 @@ static int do_change_param_reset(struct ibmvnic_adapter *adapter,if (reset_state ==VNIC_OPEN ) { rc =__ibmvnic_close (netdev);if (rc) -return rc; + goto out; }release_resources (adapter);
@@ -1875,24 +1875,25 @@ static int do_change_param_reset(struct ibmvnic_adapter *adapter, } rc =ibmvnic_reset_init (adapter, true); -if (rc) -return IBMVNIC_INIT_FAILED ; +if (rc) { + rc =IBMVNIC_INIT_FAILED ; + goto out; + }/* If the adapter was in PROBE state prior to the reset, * exit here. */ if (reset_state ==VNIC_PROBED ) -return 0 ; + goto out; rc =ibmvnic_login (netdev);if (rc) { - adapter->state = reset_state; -return rc; + goto out; } rc =init_resources (adapter);if (rc) -return rc; + goto out;ibmvnic_disable_irqs (adapter);
@@ -1902,8 +1903,10 @@ static int do_change_param_reset(struct ibmvnic_adapter *adapter,return 0 ; rc =__ibmvnic_open (netdev); -if (rc) -return IBMVNIC_OPEN_FAILED ; +if (rc) { + rc =IBMVNIC_OPEN_FAILED ; + goto out; + }/* refresh device's multicast list */ ibmvnic_set_multi (netdev);
@@ -1912,7 +1915,10 @@ static int do_change_param_reset(struct ibmvnic_adapter *adapter,for (i =0 ; i < adapter->req_rx_queues; i++)napi_schedule (&adapter->napi[i]); -return 0 ; +out: +if (rc) + adapter->state = reset_state; +return rc; }/**
@@ -2015,7 +2021,6 @@ static int do_reset(struct ibmvnic_adapter *adapter, rc =ibmvnic_login (netdev);if (rc) { - adapter->state = reset_state; goto out; }
@@ -2083,6 +2088,9 @@ static int do_reset(struct ibmvnic_adapter *adapter, rc =0 ; out: +/* restore the adapter state if reset failed */ +if (rc) + adapter->state = reset_state;rtnl_unlock ();return rc;
@@ -2115,43 +2123,46 @@ static int do_hard_reset(struct ibmvnic_adapter *adapter,if (rc) {netdev_err (adapter->netdev,"Couldn't initialize crq. rc=%d\n" , rc); -return rc; + goto out; } rc =ibmvnic_reset_init (adapter, false);if (rc) -return rc; + goto out;/* If the adapter was in PROBE state prior to the reset, * exit here. */ if (reset_state ==VNIC_PROBED ) -return 0 ; + goto out; rc =ibmvnic_login (netdev); -if (rc) { - adapter->state =VNIC_PROBED ; -return 0 ; - } +if (rc) + goto out; rc =init_resources (adapter);if (rc) -return rc; + goto out;ibmvnic_disable_irqs (adapter); adapter->state =VNIC_CLOSED ;if (reset_state ==VNIC_CLOSED ) -return 0 ; + goto out; rc =__ibmvnic_open (netdev); -if (rc) -return IBMVNIC_OPEN_FAILED ; +if (rc) { + rc =IBMVNIC_OPEN_FAILED ; + goto out; + }call_netdevice_notifiers (NETDEV_NOTIFY_PEERS , netdev);call_netdevice_notifiers (NETDEV_RESEND_IGMP , netdev); - -return 0 ; +out: +/* restore adapter state if reset failed */ +if (rc) + adapter->state = reset_state; +return rc; }static struct ibmvnic_rwi *get_next_rwi (struct ibmvnic_adapter *adapter)
@@ -2236,13 +2247,7 @@ static void __ibmvnic_reset(struct work_struct *work) rc =do_reset (adapter, rwi, reset_state); }kfree (rwi); -if (rc ==IBMVNIC_OPEN_FAILED ) { -if (list_empty (&adapter->rwi_list)) - adapter->state =VNIC_CLOSED ; -else - adapter->state = reset_state; - rc =0 ; - } +if (rc)netdev_dbg (adapter->netdev,"Reset failed, rc=%d\n" , rc);
Signature · 1 line
2.26.2
PATCH v3 1/9 ibmvnic: handle inconsistent login with reset
In reply to Dany MaddenInconsistent login with the vnicserver is causing the device to be removed. This does not give the device a chance to recover from error state. This patch schedules a FATAL reset instead to bring the adapter up.
--- drivers/net/ethernet/ibm/ibmvnic.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
drivers/net/ethernet/ibm/ibmvnic.c +1 -1
@@ -4412,7 +4412,7 @@ static int handle_login_rsp(union ibmvnic_crq *login_rsp_crq, adapter->req_rx_add_queues != be32_to_cpu(login_rsp->num_rxadd_subcrqs))) {dev_err (dev,"FATAL: Inconsistent login and login rsp\n" ); -ibmvnic_remove (adapter->vdev); +ibmvnic_reset (adapter,VNIC_RESET_FATAL );return -EIO ; } size_array = (u64 *)((u8 *)(adapter->login_rsp_buf) +
Signature · 1 line
2.26.2
Quoted 2 lines · On Wed, 25 Nov 2020 18:04:23 -0600 Dany Madden wrote:
Assorted fixes for ibmvnic originated from "[PATCH net 00/15] ibmvnic:
assorted bug fixes" sent by Lijun Pan.
Applied, thanks!