hisi_sas: Some error handling improvements
Subsystems (2)
cover hisi_sas: Some error handling improvements
This series contains a few error handling improvements, generally speed-ups:
- Put a limit on link resets retries
- Always reset controller for internal abort - it was only occurring as part of final host adapter reset in error handling process
- Speed up error handling when internal abort occurs
Luo Jiaxing (5):
scsi: hisi_sas: Put a limit of link reset retries
scsi: hisi_sas: Run I_T nexus resets in parallel for clear nexus reset
scsi: hisi_sas: Include HZ in timer macros
scsi: hisi_sas: Reset controller for internal abort timeout
scsi: hisi_sas: Speed up error handling when internal abort timeout
occurs
drivers/scsi/hisi_sas/hisi_sas.h | 7 +- drivers/scsi/hisi_sas/hisi_sas_main.c | 99 ++++++++++++++++++-------- drivers/scsi/hisi_sas/hisi_sas_v2_hw.c | 2 +- drivers/scsi/hisi_sas/hisi_sas_v3_hw.c | 2 +- 4 files changed, 78 insertions(+), 32 deletions(-)
Signature · 1 line
2.26.2
PATCH 1/5 scsi: hisi_sas: Put a limit of link reset retries
From: Luo Jiaxing <[email protected]>
If an OOB event is received but the phy still fails to come up, a link reset will be issued repeatedly at an interval of 20s until the phy comes up.
Set a limit for link reset issue retries to avoid printing the timeout message endlessly.
--- drivers/scsi/hisi_sas/hisi_sas.h | 1 + drivers/scsi/hisi_sas/hisi_sas_main.c | 15 +++++++++++++-- 2 files changed, 14 insertions(+), 2 deletions(-)
drivers/scsi/hisi_sas/hisi_sas.h +1 -0
@@ -185,6 +185,7 @@ struct hisi_sas_phy {enum sas_linkrate minimum_linkrate;enum sas_linkrate maximum_linkrate;int enable; +int wait_phyup_cnt;atomic_t down_cnt;/* Trace FIFO */
drivers/scsi/hisi_sas/hisi_sas_main.c +13 -2
@@ -857,6 +857,7 @@ static void hisi_sas_phyup_work(struct work_struct *work)struct asd_sas_phy *sas_phy = &phy->sas_phy;int phy_no = sas_phy->id; + phy->wait_phyup_cnt =0 ;if (phy->identify.target_port_protocols ==SAS_PROTOCOL_SSP ) hisi_hba->hw->sl_notify_ssp (hisi_hba, phy_no);hisi_sas_bytes_dmaed (hisi_hba, phy_no,GFP_KERNEL );
@@ -899,6 +900,8 @@ static void hisi_sas_wait_phyup_timedout(struct timer_list *t)hisi_sas_notify_phy_event (phy,HISI_PHYE_LINK_RESET ); } +#define HISI_SAS_WAIT_PHYUP_RETRIES 10 +void hisi_sas_phy_oob_ready (struct hisi_hba *hisi_hba,int phy_no) {struct hisi_sas_phy *phy = &hisi_hba->phy[phy_no];
@@ -909,8 +912,16 @@ void hisi_sas_phy_oob_ready(struct hisi_hba *hisi_hba, int phy_no)return ;if (!timer_pending (&phy->timer)) { - phy->timer.expires = jiffies +HISI_SAS_WAIT_PHYUP_TIMEOUT *HZ ; -add_timer (&phy->timer); +if (phy->wait_phyup_cnt <HISI_SAS_WAIT_PHYUP_RETRIES ) { + phy->wait_phyup_cnt++; + phy->timer.expires = jiffies + +HISI_SAS_WAIT_PHYUP_TIMEOUT *HZ ; +add_timer (&phy->timer); + }else { +dev_warn (dev,"phy%d failed to come up %d times, giving up\n" , + phy_no, phy->wait_phyup_cnt); + phy->wait_phyup_cnt =0 ; + } } }EXPORT_SYMBOL_GPL (hisi_sas_phy_oob_ready);
Signature · 1 line
2.26.2
PATCH 2/5 scsi: hisi_sas: Run I_T nexus resets in parallel for clear nexus reset
In reply to John GarryFrom: Luo Jiaxing <[email protected]>
For a clear nexus reset operation, the I_T nexus resets are executed serially for each device. For devices attached through an expander, this may take 2s per device; so, in total, could take a long time.
Reduce the total time by running the I_T nexus resets in parallel through async operations.
--- drivers/scsi/hisi_sas/hisi_sas.h | 1 + drivers/scsi/hisi_sas/hisi_sas_main.c | 23 +++++++++++++++++------ 2 files changed, 18 insertions(+), 6 deletions(-)
drivers/scsi/hisi_sas/hisi_sas.h +1 -0
@@ -8,6 +8,7 @@#define _HISI_SAS_H_#include <linux/acpi.h> +#include <linux/async.h> #include <linux/blk-mq.h> #include <linux/blk-mq-pci.h> #include <linux/clk.h>
drivers/scsi/hisi_sas/hisi_sas_main.c +17 -6
@@ -1886,12 +1886,24 @@ static int hisi_sas_lu_reset(struct domain_device *device, u8 *lun)return rc; } +static void hisi_sas_async_I_T_nexus_reset (void *data,async_cookie_t cookie) +{ +struct domain_device *device = data; +struct hisi_hba *hisi_hba =dev_to_hisi_hba (device); +int rc; + + rc =hisi_sas_debug_I_T_nexus_reset (device); +if (rc !=TMF_RESP_FUNC_COMPLETE ) +dev_info (hisi_hba->dev,"I_T_nexus reset fail for dev:%016llx rc=%d\n" , +SAS_ADDR (device->sas_addr), rc); +} +static int hisi_sas_clear_nexus_ha (struct sas_ha_struct *sas_ha) {struct hisi_hba *hisi_hba = sas_ha->lldd_ha; -struct device *dev = hisi_hba->dev;HISI_SAS_DECLARE_RST_WORK_ON_STACK (r); -int rc, i; +ASYNC_DOMAIN_EXCLUSIVE (async); +int i;queue_work (hisi_hba->wq, &r.work);wait_for_completion (r.completion);
@@ -1906,12 +1918,11 @@ static int hisi_sas_clear_nexus_ha(struct sas_ha_struct *sas_ha) dev_is_expander(device->dev_type))continue ; - rc =hisi_sas_debug_I_T_nexus_reset (device); -if (rc !=TMF_RESP_FUNC_COMPLETE ) -dev_info (dev,"clear nexus ha: for device[%d] rc=%d\n" , - sas_dev->device_id, rc); +async_schedule_domain (hisi_sas_async_I_T_nexus_reset, + device, &async); } +async_synchronize_full_domain (&async);hisi_sas_release_tasks (hisi_hba);return TMF_RESP_FUNC_COMPLETE ;
Signature · 1 line
2.26.2
PATCH 3/5 scsi: hisi_sas: Include HZ in timer macros
In reply to John GarryFrom: Luo Jiaxing <[email protected]>
Include HZ in timer macros to make the code more concise.
--- drivers/scsi/hisi_sas/hisi_sas.h | 4 ++-- drivers/scsi/hisi_sas/hisi_sas_main.c | 17 ++++++++++------- drivers/scsi/hisi_sas/hisi_sas_v2_hw.c | 2 +- drivers/scsi/hisi_sas/hisi_sas_v3_hw.c | 2 +- 4 files changed, 14 insertions(+), 11 deletions(-)
drivers/scsi/hisi_sas/hisi_sas.h +2 -2
@@ -91,8 +91,8 @@#define HISI_SAS_PROT_MASK (HISI_SAS_DIF_PROT_MASK | HISI_SAS_DIX_PROT_MASK) -#define HISI_SAS_WAIT_PHYUP_TIMEOUT 20 -#define CLEAR_ITCT_TIMEOUT 20 +#define HISI_SAS_WAIT_PHYUP_TIMEOUT (20 * HZ) +#define HISI_SAS_CLEAR_ITCT_TIMEOUT (20 * HZ)struct hisi_hba ;
drivers/scsi/hisi_sas/hisi_sas_main.c +10 -7
@@ -915,7 +915,7 @@ void hisi_sas_phy_oob_ready(struct hisi_hba *hisi_hba, int phy_no)if (phy->wait_phyup_cnt <HISI_SAS_WAIT_PHYUP_RETRIES ) { phy->wait_phyup_cnt++; phy->timer.expires = jiffies + -HISI_SAS_WAIT_PHYUP_TIMEOUT *HZ ; +HISI_SAS_WAIT_PHYUP_TIMEOUT ;add_timer (&phy->timer); }else { dev_warn(dev, "phy%d failed to come up %d times, giving up\n",
@@ -1193,9 +1193,9 @@ static void hisi_sas_tmf_timedout(struct timer_list *t)complete (&task->slow_task->completion); } -#define TASK_TIMEOUT 20 -#define TASK_RETRY 3 -#define INTERNAL_ABORT_TIMEOUT 6 +#define TASK_TIMEOUT (20 * HZ) +#define TASK_RETRY 3 +#define INTERNAL_ABORT_TIMEOUT (6 * HZ)static int hisi_sas_exec_internal_tmf_task (struct domain_device *device,void *parameter,u32 para_len,struct hisi_sas_tmf_task *tmf)
@@ -1223,7 +1223,7 @@ static int hisi_sas_exec_internal_tmf_task(struct domain_device *device, task->task_done = hisi_sas_task_done; task->slow_task->timer.function = hisi_sas_tmf_timedout; - task->slow_task->timer.expires = jiffies +TASK_TIMEOUT *HZ ; + task->slow_task->timer.expires = jiffies +TASK_TIMEOUT ;add_timer (&task->slow_task->timer); res =hisi_sas_task_exec (task,GFP_KERNEL ,1 , tmf);
@@ -1761,6 +1761,8 @@ static int hisi_sas_clear_aca(struct domain_device *device, u8 *lun)return rc; } +#define I_T_NEXUS_RESET_PHYUP_TIMEOUT (2 * HZ) +static int hisi_sas_debug_I_T_nexus_reset (struct domain_device *device) {struct sas_phy *local_phy =sas_get_local_phy (device);
@@ -1795,7 +1797,8 @@ static int hisi_sas_debug_I_T_nexus_reset(struct domain_device *device) sas_ha->sas_phy[local_phy->number];struct hisi_sas_phy *phy = container_of(sas_phy, struct hisi_sas_phy, sas_phy); -int ret =wait_for_completion_timeout (&phyreset,2 *HZ ); +int ret =wait_for_completion_timeout (&phyreset, +I_T_NEXUS_RESET_PHYUP_TIMEOUT );unsigned long flags;spin_lock_irqsave (&phy->lock, flags);
@@ -2079,7 +2082,7 @@ _hisi_sas_internal_task_abort(struct hisi_hba *hisi_hba, task->task_proto = device->tproto; task->task_done = hisi_sas_task_done; task->slow_task->timer.function = hisi_sas_tmf_timedout; - task->slow_task->timer.expires = jiffies +INTERNAL_ABORT_TIMEOUT *HZ ; + task->slow_task->timer.expires = jiffies +INTERNAL_ABORT_TIMEOUT ;add_timer (&task->slow_task->timer); res = hisi_sas_internal_abort_task_exec(hisi_hba, sas_dev->device_id,
drivers/scsi/hisi_sas/hisi_sas_v2_hw.c +1 -1
@@ -994,7 +994,7 @@ static int clear_itct_v2_hw(struct hisi_hba *hisi_hba, reg_val =ITCT_CLR_EN_MSK | (dev_id &ITCT_DEV_MSK );hisi_sas_write32 (hisi_hba,ITCT_CLR , reg_val);if (!wait_for_completion_timeout (sas_dev->completion, -CLEAR_ITCT_TIMEOUT *HZ )) { +HISI_SAS_CLEAR_ITCT_TIMEOUT )) {dev_warn (dev,"failed to clear ITCT\n" );return -ETIMEDOUT ; }
drivers/scsi/hisi_sas/hisi_sas_v3_hw.c +1 -1
@@ -843,7 +843,7 @@ static int clear_itct_v3_hw(struct hisi_hba *hisi_hba,hisi_sas_write32 (hisi_hba,ITCT_CLR , reg_val);if (!wait_for_completion_timeout (sas_dev->completion, -CLEAR_ITCT_TIMEOUT *HZ )) { +HISI_SAS_CLEAR_ITCT_TIMEOUT )) {dev_warn (dev,"failed to clear ITCT\n" );return -ETIMEDOUT ; }
Signature · 1 line
2.26.2
PATCH 4/5 scsi: hisi_sas: Reset controller for internal abort timeout
In reply to John GarryFrom: Luo Jiaxing <[email protected]>
If an internal task abort timeout occurs, the controller has developed a fault, and needs to be reset to be recovered. However if a timeout occurs during SCSI error handling, issuing a controller reset immediately may conflict with the error handling.
To handle internal abort in these two scenarios, only queue the reset when not in an error handling function. In the case of a timeout during error handling, do nothing and rely on the inevitable ha nexus reset to reset the controller.
--- drivers/scsi/hisi_sas/hisi_sas_main.c | 40 +++++++++++++++++---------- 1 file changed, 26 insertions(+), 14 deletions(-)
drivers/scsi/hisi_sas/hisi_sas_main.c +26 -14
@@ -15,7 +15,7 @@ static int hisi_sas_debug_issue_ssp_tmf(struct domain_device *device,static int hisi_sas_internal_task_abort (struct hisi_hba *hisi_hba,struct domain_device *device, -int abort_flag,int tag); +int abort_flag,int tag,bool rst_to_recover);static int hisi_sas_softreset_ata_disk (struct domain_device *device);static int hisi_sas_control_phy (struct asd_sas_phy *sas_phy,enum phy_func func,void *funcdata);
@@ -1074,7 +1074,7 @@ static void hisi_sas_dev_gone(struct domain_device *device)down (&hisi_hba->sem);if (!test_bit (HISI_SAS_RESET_BIT , &hisi_hba->flags)) {hisi_sas_internal_task_abort (hisi_hba, device, -HISI_SAS_INT_ABT_DEV ,0 ); +HISI_SAS_INT_ABT_DEV ,0 , true);hisi_sas_dereg_device (hisi_hba, device);
@@ -1516,7 +1516,8 @@ static void hisi_sas_terminate_stp_reject(struct hisi_hba *hisi_hba)continue ; rc =hisi_sas_internal_task_abort (hisi_hba, device, -HISI_SAS_INT_ABT_DEV ,0 ); +HISI_SAS_INT_ABT_DEV ,0 , + false);if (rc <0 )dev_err (dev,"STP reject: abort dev failed %d\n" , rc); }
@@ -1671,7 +1672,8 @@ static int hisi_sas_abort_task(struct sas_task *task) &tmf_task); rc2 =hisi_sas_internal_task_abort (hisi_hba, device, -HISI_SAS_INT_ABT_CMD , tag); +HISI_SAS_INT_ABT_CMD , tag, + false);if (rc2 <0 ) {dev_err (dev,"abort task: internal abort (%d)\n" , rc2);return TMF_RESP_FUNC_FAILED ;
@@ -1693,7 +1695,7 @@ static int hisi_sas_abort_task(struct sas_task *task)if (task->dev->dev_type ==SAS_SATA_DEV ) { rc =hisi_sas_internal_task_abort (hisi_hba, device,HISI_SAS_INT_ABT_DEV , -0 ); +0 , false);if (rc <0 ) {dev_err (dev,"abort task: internal abort failed\n" ); goto out;
@@ -1708,7 +1710,8 @@ static int hisi_sas_abort_task(struct sas_task *task)struct hisi_sas_cq *cq = &hisi_hba->cq[slot->dlvry_queue]; rc =hisi_sas_internal_task_abort (hisi_hba, device, -HISI_SAS_INT_ABT_CMD , tag); +HISI_SAS_INT_ABT_CMD , tag, + false);if (((rc <0 ) || (rc ==TMF_RESP_FUNC_FAILED )) && task->lldd_task) {/*
@@ -1734,7 +1737,7 @@ static int hisi_sas_abort_task_set(struct domain_device *device, u8 *lun)int rc; rc =hisi_sas_internal_task_abort (hisi_hba, device, -HISI_SAS_INT_ABT_DEV ,0 ); +HISI_SAS_INT_ABT_DEV ,0 , false);if (rc <0 ) {dev_err (dev,"abort task set: internal abort rc=%d\n" , rc);return TMF_RESP_FUNC_FAILED ;
@@ -1828,7 +1831,7 @@ static int hisi_sas_I_T_nexus_reset(struct domain_device *device)int rc; rc =hisi_sas_internal_task_abort (hisi_hba, device, -HISI_SAS_INT_ABT_DEV ,0 ); +HISI_SAS_INT_ABT_DEV ,0 , false);if (rc <0 ) {dev_err (dev,"I_T nexus reset: internal abort (%d)\n" , rc);return TMF_RESP_FUNC_FAILED ;
@@ -1858,7 +1861,7 @@ static int hisi_sas_lu_reset(struct domain_device *device, u8 *lun)/* Clear internal IO and then lu reset */ rc =hisi_sas_internal_task_abort (hisi_hba, device, -HISI_SAS_INT_ABT_DEV ,0 ); +HISI_SAS_INT_ABT_DEV ,0 , false);if (rc <0 ) {dev_err (dev,"lu_reset: internal abort failed\n" ); goto out;
@@ -2054,11 +2057,13 @@ hisi_sas_internal_abort_task_exec(struct hisi_hba *hisi_hba, int device_id,* @tag: tag of IO to be aborted (only relevant to single * IO mode) * @dq: delivery queue for this internal abort command +* @rst_to_recover: If rst_to_recover set, queue a controller +* reset if an internal abort times out. */ static int _hisi_sas_internal_task_abort (struct hisi_hba *hisi_hba,struct domain_device *device,int abort_flag, -int tag,struct hisi_sas_dq *dq) +int tag,struct hisi_sas_dq *dq,bool rst_to_recover) {struct sas_task *task;struct hisi_sas_device *sas_dev = device->lldd_dev;
@@ -2114,7 +2119,13 @@ _hisi_sas_internal_task_abort(struct hisi_hba *hisi_hba,synchronize_irq (cq->irq_no); slot->task =NULL ; } -dev_err (dev,"internal task abort: timeout and not done.\n" ); + +if (rst_to_recover) { +dev_err (dev,"internal task abort: timeout and not done. Queuing reset.\n" ); +queue_work (hisi_hba->wq, &hisi_hba->rst_work); + }else { +dev_err (dev,"internal task abort: timeout and not done.\n" ); + } res = -EIO ; goto exit;
@@ -2147,7 +2158,7 @@ _hisi_sas_internal_task_abort(struct hisi_hba *hisi_hba,static int hisi_sas_internal_task_abort (struct hisi_hba *hisi_hba,struct domain_device *device, -int abort_flag,int tag) +int abort_flag,int tag,bool rst_to_recover) {struct hisi_sas_slot *slot;struct device *dev = hisi_hba->dev;
@@ -2159,7 +2170,8 @@ hisi_sas_internal_task_abort(struct hisi_hba *hisi_hba, slot = &hisi_hba->slot_info[tag]; dq = &hisi_hba->dq[slot->dlvry_queue];return _hisi_sas_internal_task_abort (hisi_hba, device, - abort_flag, tag, dq); + abort_flag, tag, dq, + rst_to_recover);case HISI_SAS_INT_ABT_DEV :for (i =0 ; i < hisi_hba->cq_nvecs; i++) {struct hisi_sas_cq *cq = &hisi_hba->cq[i];
@@ -2170,7 +2182,7 @@ hisi_sas_internal_task_abort(struct hisi_hba *hisi_hba, dq = &hisi_hba->dq[i]; rc =_hisi_sas_internal_task_abort (hisi_hba, device, abort_flag, tag, - dq); + dq, rst_to_recover);if (rc)return rc; }
Signature · 1 line
2.26.2
PATCH 5/5 scsi: hisi_sas: Speed up error handling when internal abort timeout occurs
In reply to John GarryFrom: Luo Jiaxing <[email protected]>
If an internal task abort timeout occurs, the controller has developed a fault, and needs to be reset to be recovered.
When this occurs during error handling, the current policy is to allow error handling to continue, and the inevitable nexus ha reset will handle the required reset.
However various steps of error handling need to taken before this happens. These also involve some level of HW interaction, which will also fail with various timeouts.
Speed up this process by recording a HW fault bit for an internal abort timeout - when this is set, just automatically error any HW interaction, and essentially go straight to clear nexus ha (to reset the controller).
--- drivers/scsi/hisi_sas/hisi_sas.h | 1 + drivers/scsi/hisi_sas/hisi_sas_main.c | 6 ++++++ 2 files changed, 7 insertions(+)
drivers/scsi/hisi_sas/hisi_sas.h +1 -0
@@ -38,6 +38,7 @@#define HISI_SAS_RESET_BIT 0#define HISI_SAS_REJECT_CMD_BIT 1#define HISI_SAS_PM_BIT 2 +#define HISI_SAS_HW_FAULT_BIT 3#define HISI_SAS_MAX_COMMANDS (HISI_SAS_QUEUE_SLOTS)#define HISI_SAS_RESERVED_IPTT 96#define HISI_SAS_UNRESERVED_IPTT \
drivers/scsi/hisi_sas/hisi_sas_main.c +6 -0
@@ -1616,6 +1616,7 @@ static int hisi_sas_controller_reset(struct hisi_hba *hisi_hba) }hisi_sas_controller_reset_done (hisi_hba); +clear_bit (HISI_SAS_HW_FAULT_BIT , &hisi_hba->flags);dev_info (dev,"controller reset complete\n" );return 0 ;
@@ -2079,6 +2080,9 @@ _hisi_sas_internal_task_abort(struct hisi_hba *hisi_hba,if (!hisi_hba->hw->prep_abort)return TMF_RESP_FUNC_FAILED ; +if (test_bit (HISI_SAS_HW_FAULT_BIT , &hisi_hba->flags)) +return -EIO ; + task =sas_alloc_slow_task (GFP_KERNEL );if (!task)return -ENOMEM ;
@@ -2109,6 +2113,8 @@ _hisi_sas_internal_task_abort(struct hisi_hba *hisi_hba,if (!(task->task_state_flags &SAS_TASK_STATE_DONE )) {struct hisi_sas_slot *slot = task->lldd_task; +set_bit (HISI_SAS_HW_FAULT_BIT , &hisi_hba->flags); +if (slot) {struct hisi_sas_cq *cq = &hisi_hba->cq[slot->dlvry_queue];
Signature · 1 line
2.26.2
Martin K. Petersen
AppliedJohn,
Quoted 6 lines
This series contains a few error handling improvements, generally speed-ups:
- Put a limit on link resets retries
- Always reset controller for internal abort - it was only occurring as part of final host adapter reset in error handling process
- Speed up error handling when internal abort occurs
Applied to 5.14/scsi-staging, thanks!
Signature · 1 line
Martin K. Petersen Oracle Linux Engineering
Martin K. Petersen
AppliedQuoted 8 lines · John Garry
This series contains a few error handling improvements, generally speed-ups:
- Put a limit on link resets retries
- Always reset controller for internal abort - it was only occurring as part of final host adapter reset in error handling process
- Speed up error handling when internal abort occurs[...]
Applied to 5.14/scsi-queue, thanks!
[1/5] scsi: hisi_sas: Put a limit of link reset retries
https://git.kernel.org/mkp/scsi/c/366da0da1f5f [2/5] scsi: hisi_sas: Run I_T nexus resets in parallel for clear nexus reset https://git.kernel.org/mkp/scsi/c/0f757339919d [3/5] scsi: hisi_sas: Include HZ in timer macros https://git.kernel.org/mkp/scsi/c/2f12a499511f [4/5] scsi: hisi_sas: Reset controller for internal abort timeout https://git.kernel.org/mkp/scsi/c/63ece9eb3503 [5/5] scsi: hisi_sas: Speed up error handling when internal abort timeout occurs https://git.kernel.org/mkp/scsi/c/e8a4d0daaef6
Signature · 1 line
Martin K. Petersen Oracle Linux Engineering