UFS patches for kernel v5.17
Subsystems (2)
cover UFS patches for kernel v5.17
Hi Martin,
Please consider these UFS driver kernel patches for kernel v5.17.
Thanks,
Bart.
Bart Van Assche (11):
scsi: ufs: Rename a function argument
scsi: ufs: Remove is_rpmb_wlun()
scsi: ufs: Remove the sdev_rpmb member
scsi: ufs: Remove dead code
scsi: core: Add support for reserved tags
scsi: ufs: Rework ufshcd_change_queue_depth()
scsi: ufs: Fix a deadlock in the error handler
scsi: ufs: Improve SCSI abort handling further
scsi: ufs: Fix a kernel crash during shutdown
scsi: ufs: Optimize the command queueing code
scsi: ufs: Implement polling support
drivers/scsi/scsi_lib.c | 4 +- drivers/scsi/ufs/ufs-exynos.c | 4 +- drivers/scsi/ufs/ufshcd.c | 233 +++++++++++++++------------------- drivers/scsi/ufs/ufshcd.h | 5 +- include/scsi/scsi_host.h | 9 +- 5 files changed, 115 insertions(+), 140 deletions(-)
PATCH 1/11 scsi: ufs: Rename a function argument
The new name makes it clear what the meaning of the function argument is.
--- drivers/scsi/ufs/ufs-exynos.c | 4 ++-- drivers/scsi/ufs/ufshcd.h | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-)
drivers/scsi/ufs/ufs-exynos.c +2 -2
@@ -853,14 +853,14 @@ static int exynos_ufs_post_pwr_mode(struct ufs_hba *hba, }static void exynos_ufs_specify_nexus_t_xfer_req (struct ufs_hba *hba, -int tag,bool op) +int tag,bool is_scsi_cmd) {struct exynos_ufs *ufs =ufshcd_get_variant (hba);u32 type; type =hci_readl (ufs,HCI_UTRL_NEXUS_TYPE ); -if (op) +if (is_scsi_cmd)hci_writel (ufs, type | (1 << tag),HCI_UTRL_NEXUS_TYPE );else hci_writel (ufs, type & ~(1 << tag),HCI_UTRL_NEXUS_TYPE );
drivers/scsi/ufs/ufshcd.h +2 -1
@@ -338,7 +338,8 @@ struct ufs_hba_variant_ops {enum ufs_notify_change_status status, struct ufs_pa_layer_attr *, struct ufs_pa_layer_attr *); -void (*setup_xfer_req)(struct ufs_hba *,int ,bool ); +void (*setup_xfer_req)(struct ufs_hba *hba,int tag, +bool is_scsi_cmd);void (*setup_task_mgmt)(struct ufs_hba *,int ,u8 );void (*hibern8_notify)(struct ufs_hba *,enum uic_cmd_dme ,enum ufs_notify_change_status );
RE: [PATCH 01/11] scsi: ufs: Rename a function argument
Hello,
Quoted 8 lines
-----Original Message-----
From: Bart Van Assche <[email protected]>
Sent: Wednesday, November 10, 2021 9:45 AM
To: Martin K . Petersen <[email protected]>
Cc: [email protected]; Jaegeuk Kim <[email protected]>; Adrian Hunter <[email protected]>; Bart Van Assche <[email protected]>; James E.J. Bottomley <[email protected]>; Krzysztof Kozlowski <[email protected]>; Alim Akhtar
Quoted 11 lines
Bean Huo <[email protected]>; Avri Altman <[email protected]>; Kiwoong Kim <[email protected]>; Yue Hu <[email protected]>; Peter Wang <[email protected]>; Chanho Park <[email protected]>; Stanley Chu <[email protected]>; Can Guo <[email protected]>; Asutosh Das <[email protected]>; Keoseong Park <[email protected]>
Subject: [PATCH 01/11] scsi: ufs: Rename a function argumentThe new name makes it clear what the meaning of the function argument is.
Signed-off-by Bart Van Assche <[email protected]>
Great.
Best Regards,
Chanho Park
RE: [PATCH 01/11] scsi: ufs: Rename a function argument
In reply to Bart Van AsscheHi Bart,
Quoted 3 lines
The new name makes it clear what the meaning of the function argument is.
Signed-off-by Bart Van Assche <[email protected]>
Best Regards,
Keoseong Park
Quoted 41 lines
---
drivers/scsi/ufs/ufs-exynos.c| 4 ++--drivers/scsi/ufs/ufshcd.h| 3 ++-2 files changed, 4 insertions(+), 3 deletions(-)drivers/scsi/ufs/ufs-exynos.c +2
-2@@ -853,14 +853,14 @@ static int exynos_ufs_post_pwr_mode(struct ufs_hba *hba, }static void exynos_ufs_specify_nexus_t_xfer_req (struct ufs_hba *hba, -int tag,bool op) +int tag,bool is_scsi_cmd) {struct exynos_ufs *ufs =ufshcd_get_variant (hba);u32 type; type =hci_readl (ufs,HCI_UTRL_NEXUS_TYPE ); -if (op) +if (is_scsi_cmd)hci_writel (ufs, type | (1 << tag),HCI_UTRL_NEXUS_TYPE );else hci_writel (ufs, type & ~(1 << tag),HCI_UTRL_NEXUS_TYPE );drivers/scsi/ufs/ufshcd.h +2
-1@@ -338,7 +338,8 @@ struct ufs_hba_variant_ops {enum ufs_notify_change_status status, struct ufs_pa_layer_attr *, struct ufs_pa_layer_attr *); -void (*setup_xfer_req)(struct ufs_hba *,int ,bool ); +void (*setup_xfer_req)(struct ufs_hba *hba,int tag, +bool is_scsi_cmd);void (*setup_task_mgmt)(struct ufs_hba *,int ,u8 );void (*hibern8_notify)(struct ufs_hba *,enum uic_cmd_dme ,enum ufs_notify_change_status );
Re: [PATCH 01/11] scsi: ufs: Rename a function argument
In reply to Bart Van AsscheHi Bart
Quoted 5 lines · Bart Van Assche
The new name makes it clear what the meaning of the function argument is.
Signed-off-by Bart Van Assche <[email protected]>---
Thanks
Quoted 39 lines
drivers/scsi/ufs/ufs-exynos.c | 4 ++-- drivers/scsi/ufs/ufshcd.h | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-)drivers/scsi/ufs/ufs-exynos.c +2
-2@@ -853,14 +853,14 @@ static int exynos_ufs_post_pwr_mode(struct ufs_hba *hba, }static void exynos_ufs_specify_nexus_t_xfer_req (struct ufs_hba *hba, -int tag,bool op) +int tag,bool is_scsi_cmd) {struct exynos_ufs *ufs =ufshcd_get_variant (hba);u32 type; type =hci_readl (ufs,HCI_UTRL_NEXUS_TYPE ); -if (op) +if (is_scsi_cmd)hci_writel (ufs, type | (1 << tag),HCI_UTRL_NEXUS_TYPE );else hci_writel (ufs, type & ~(1 << tag),HCI_UTRL_NEXUS_TYPE );drivers/scsi/ufs/ufshcd.h +2
-1@@ -338,7 +338,8 @@ struct ufs_hba_variant_ops {enum ufs_notify_change_status status, struct ufs_pa_layer_attr *, struct ufs_pa_layer_attr *); -void (*setup_xfer_req)(struct ufs_hba *,int ,bool ); +void (*setup_xfer_req)(struct ufs_hba *hba,int tag, +bool is_scsi_cmd);void (*setup_task_mgmt)(struct ufs_hba *,int ,u8 );void (*hibern8_notify)(struct ufs_hba *,enum uic_cmd_dme ,enum ufs_notify_change_status );
Signature · 2 lines
Regards, Alim
PATCH 2/11 scsi: ufs: Remove is_rpmb_wlun()
In reply to Bart Van AsscheCommit edc0596cc04b ("scsi: ufs: core: Stop clearing UNIT ATTENTIONS") removed all callers of is_rpmb_wlun(). Hence also remove the function itself.
--- drivers/scsi/ufs/ufshcd.c | 5 ----- 1 file changed, 5 deletions(-)
drivers/scsi/ufs/ufshcd.c +0 -5
@@ -2650,11 +2650,6 @@ static inline u16 ufshcd_upiu_wlun_to_scsi_wlun(u8 upiu_wlun_id)return (upiu_wlun_id & ~UFS_UPIU_WLUN_ID ) |SCSI_W_LUN_BASE ; } -static inline bool is_rpmb_wlun (struct scsi_device *sdev) -{ -return sdev->lun ==ufshcd_upiu_wlun_to_scsi_wlun (UFS_UPIU_RPMB_WLUN ); -} -static inline bool is_device_wlun (struct scsi_device *sdev) { return sdev->lun ==
Re: [PATCH 02/11] scsi: ufs: Remove is_rpmb_wlun()
Quoted 10 lines · Bart Van Assche
Commit
edc0596cc04b("scsi: ufs: core: Stop clearing UNIT ATTENTIONS") removed all callers ofis_rpmb_wlun(). Hence also remove the function itself.Reported-by kernel test robot <[email protected]>Signed-off-by Bart Van Assche <[email protected]>--- drivers/scsi/ufs/ufshcd.c | 5 ----- 1 file changed, 5 deletions(-)
LGTM.
Quoted 17 lines
drivers/scsi/ufs/ufshcd.c +0
-5@@ -2650,11 +2650,6 @@ static inline u16 ufshcd_upiu_wlun_to_scsi_wlun(u8 upiu_wlun_id)return (upiu_wlun_id & ~UFS_UPIU_WLUN_ID ) |SCSI_W_LUN_BASE ; } -static inline bool is_rpmb_wlun (struct scsi_device *sdev) -{ -return sdev->lun ==ufshcd_upiu_wlun_to_scsi_wlun (UFS_UPIU_RPMB_WLUN ); -} -static inline bool is_device_wlun (struct scsi_device *sdev) { return sdev->lun ==
Signature · 2 lines
The Qualcomm Innovation Center, Inc. is a member of the Code Aurora Forum, Linux Foundation Collaborative Project
Re: [PATCH 02/11] scsi: ufs: Remove is_rpmb_wlun()
In reply to Bart Van AsscheHi Bart
Quoted 8 lines · Bart Van Assche
Commit
edc0596cc04b("scsi: ufs: core: Stop clearing UNIT ATTENTIONS") removed all callers ofis_rpmb_wlun(). Hence also remove the function itself.Reported-by kernel test robot <[email protected]>Signed-off-by Bart Van Assche <[email protected]>---
Quoted 19 lines
drivers/scsi/ufs/ufshcd.c | 5 ----- 1 file changed, 5 deletions(-)drivers/scsi/ufs/ufshcd.c +0
-5@@ -2650,11 +2650,6 @@ static inline u16 ufshcd_upiu_wlun_to_scsi_wlun(u8 upiu_wlun_id)return (upiu_wlun_id & ~UFS_UPIU_WLUN_ID ) |SCSI_W_LUN_BASE ; } -static inline bool is_rpmb_wlun (struct scsi_device *sdev) -{ -return sdev->lun ==ufshcd_upiu_wlun_to_scsi_wlun (UFS_UPIU_RPMB_WLUN ); -} -static inline bool is_device_wlun (struct scsi_device *sdev) { return sdev->lun ==
Signature · 2 lines
Regards, Alim
PATCH 3/11 scsi: ufs: Remove the sdev_rpmb member
In reply to Bart Van AsscheSince the sdev_rpmb member of struct ufs_hba is only used inside ufshcd_scsi_add_wlus(), convert it into a local variable.
--- drivers/scsi/ufs/ufshcd.c | 12 ++++++------ drivers/scsi/ufs/ufshcd.h | 1 - 2 files changed, 6 insertions(+), 7 deletions(-)
drivers/scsi/ufs/ufshcd.c +6 -6
@@ -7407,7 +7407,7 @@ static inline void ufshcd_blk_pm_runtime_init(struct scsi_device *sdev)static int ufshcd_scsi_add_wlus (struct ufs_hba *hba) {int ret =0 ; -struct scsi_device *sdev_boot; +struct scsi_device *sdev_boot, *sdev_rpmb; hba->sdev_ufs_device =__scsi_add_device (hba->host,0 ,0 ,ufshcd_upiu_wlun_to_scsi_wlun (UFS_UPIU_UFS_DEVICE_WLUN ),NULL );
@@ -7418,14 +7418,14 @@ static int ufshcd_scsi_add_wlus(struct ufs_hba *hba) }scsi_device_put (hba->sdev_ufs_device); - hba->sdev_rpmb =__scsi_add_device (hba->host,0 ,0 , + sdev_rpmb =__scsi_add_device (hba->host,0 ,0 ,ufshcd_upiu_wlun_to_scsi_wlun (UFS_UPIU_RPMB_WLUN ),NULL ); -if (IS_ERR (hba->sdev_rpmb)) { - ret =PTR_ERR (hba->sdev_rpmb); +if (IS_ERR (sdev_rpmb)) { + ret =PTR_ERR (sdev_rpmb); goto remove_sdev_ufs_device; } -ufshcd_blk_pm_runtime_init (hba->sdev_rpmb); -scsi_device_put (hba->sdev_rpmb); +ufshcd_blk_pm_runtime_init (sdev_rpmb); +scsi_device_put (sdev_rpmb); sdev_boot =__scsi_add_device (hba->host,0 ,0 ,ufshcd_upiu_wlun_to_scsi_wlun (UFS_UPIU_BOOT_WLUN ),NULL );
drivers/scsi/ufs/ufshcd.h +0 -1
@@ -809,7 +809,6 @@ struct ufs_hba {* "UFS device" W-LU. */ struct scsi_device *sdev_ufs_device; -struct scsi_device *sdev_rpmb;#ifdef CONFIG_SCSI_UFS_HWMON struct device *hwmon_device;
Re: [PATCH 03/11] scsi: ufs: Remove the sdev_rpmb member
Quoted 6 lines · Bart Van Assche
Since the
sdev_rpmbmember of structufs_hbais only used insideufshcd_scsi_add_wlus(), convert it into a local variable.Suggested-by Jaegeuk Kim <[email protected]>Signed-off-by Bart Van Assche <[email protected]>---
Quoted 50 lines
drivers/scsi/ufs/ufshcd.c | 12 ++++++------ drivers/scsi/ufs/ufshcd.h | 1 - 2 files changed, 6 insertions(+), 7 deletions(-)drivers/scsi/ufs/ufshcd.c +6
-6@@ -7407,7 +7407,7 @@ static inline void ufshcd_blk_pm_runtime_init(struct scsi_device *sdev)static int ufshcd_scsi_add_wlus (struct ufs_hba *hba) {int ret =0 ; -struct scsi_device *sdev_boot; +struct scsi_device *sdev_boot, *sdev_rpmb; hba->sdev_ufs_device =__scsi_add_device (hba->host,0 ,0 ,ufshcd_upiu_wlun_to_scsi_wlun (UFS_UPIU_UFS_DEVICE_WLUN ),NULL );@@ -7418,14 +7418,14 @@ static int ufshcd_scsi_add_wlus(struct ufs_hba *hba) }scsi_device_put (hba->sdev_ufs_device); - hba->sdev_rpmb =__scsi_add_device (hba->host,0 ,0 , + sdev_rpmb =__scsi_add_device (hba->host,0 ,0 ,ufshcd_upiu_wlun_to_scsi_wlun (UFS_UPIU_RPMB_WLUN ),NULL ); -if (IS_ERR (hba->sdev_rpmb)) { - ret =PTR_ERR (hba->sdev_rpmb); +if (IS_ERR (sdev_rpmb)) { + ret =PTR_ERR (sdev_rpmb); goto remove_sdev_ufs_device; } -ufshcd_blk_pm_runtime_init (hba->sdev_rpmb); -scsi_device_put (hba->sdev_rpmb); +ufshcd_blk_pm_runtime_init (sdev_rpmb); +scsi_device_put (sdev_rpmb); sdev_boot =__scsi_add_device (hba->host,0 ,0 ,ufshcd_upiu_wlun_to_scsi_wlun (UFS_UPIU_BOOT_WLUN ),NULL );drivers/scsi/ufs/ufshcd.h +0
-1@@ -809,7 +809,6 @@ struct ufs_hba {* "UFS device" W-LU. */ struct scsi_device *sdev_ufs_device; -struct scsi_device *sdev_rpmb;#ifdef CONFIG_SCSI_UFS_HWMON struct device *hwmon_device;
Signature · 2 lines
The Qualcomm Innovation Center, Inc. is a member of the Code Aurora Forum, Linux Foundation Collaborative Project
Re: [PATCH 03/11] scsi: ufs: Remove the sdev_rpmb member
In reply to Bart Van AsscheHi Bart
Quoted 7 lines · Bart Van Assche
Since the
sdev_rpmbmember of structufs_hbais only used insideufshcd_scsi_add_wlus(), convert it into a local variable.Suggested-by Jaegeuk Kim <[email protected]>Signed-off-by Bart Van Assche <[email protected]>---
Thanks
Quoted 49 lines
drivers/scsi/ufs/ufshcd.c | 12 ++++++------ drivers/scsi/ufs/ufshcd.h | 1 - 2 files changed, 6 insertions(+), 7 deletions(-)drivers/scsi/ufs/ufshcd.c +6
-6@@ -7407,7 +7407,7 @@ static inline void ufshcd_blk_pm_runtime_init(struct scsi_device *sdev)static int ufshcd_scsi_add_wlus (struct ufs_hba *hba) {int ret =0 ; -struct scsi_device *sdev_boot; +struct scsi_device *sdev_boot, *sdev_rpmb; hba->sdev_ufs_device =__scsi_add_device (hba->host,0 ,0 ,ufshcd_upiu_wlun_to_scsi_wlun (UFS_UPIU_UFS_DEVICE_WLUN ),NULL );@@ -7418,14 +7418,14 @@ static int ufshcd_scsi_add_wlus(struct ufs_hba *hba) }scsi_device_put (hba->sdev_ufs_device); - hba->sdev_rpmb =__scsi_add_device (hba->host,0 ,0 , + sdev_rpmb =__scsi_add_device (hba->host,0 ,0 ,ufshcd_upiu_wlun_to_scsi_wlun (UFS_UPIU_RPMB_WLUN ),NULL ); -if (IS_ERR (hba->sdev_rpmb)) { - ret =PTR_ERR (hba->sdev_rpmb); +if (IS_ERR (sdev_rpmb)) { + ret =PTR_ERR (sdev_rpmb); goto remove_sdev_ufs_device; } -ufshcd_blk_pm_runtime_init (hba->sdev_rpmb); -scsi_device_put (hba->sdev_rpmb); +ufshcd_blk_pm_runtime_init (sdev_rpmb); +scsi_device_put (sdev_rpmb); sdev_boot =__scsi_add_device (hba->host,0 ,0 ,ufshcd_upiu_wlun_to_scsi_wlun (UFS_UPIU_BOOT_WLUN ),NULL );drivers/scsi/ufs/ufshcd.h +0
-1@@ -809,7 +809,6 @@ struct ufs_hba {* "UFS device" W-LU. */ struct scsi_device *sdev_ufs_device; -struct scsi_device *sdev_rpmb;#ifdef CONFIG_SCSI_UFS_HWMON struct device *hwmon_device;
Signature · 2 lines
Regards, Alim
PATCH 4/11 scsi: ufs: Remove dead code
In reply to Bart Van AsscheCommit 7252a3603015 ("scsi: ufs: Avoid busy-waiting by eliminating tag conflicts") guarantees that 'tag' is not in use by any SCSI command. Remove the check that returns early if a conflict occurs.
--- drivers/scsi/ufs/ufshcd.c | 6 ------ 1 file changed, 6 deletions(-)
drivers/scsi/ufs/ufshcd.c +0 -6
@@ -6724,11 +6724,6 @@ static int ufshcd_issue_devman_upiu_cmd(struct ufs_hba *hba, tag = req->tag;WARN_ONCE (tag <0 ,"Invalid tag %d\n" , tag); -if (unlikely (test_bit (tag, &hba->outstanding_reqs))) { - err = -EBUSY ; - goto out; - } - lrbp = &hba->lrb[tag];WARN_ON (lrbp->cmd); lrbp->cmd =NULL ;
@@ -6796,7 +6791,6 @@ static int ufshcd_issue_devman_upiu_cmd(struct ufs_hba *hba,ufshcd_add_query_upiu_trace (hba, err ?UFS_QUERY_ERR :UFS_QUERY_COMP , (struct utp_upiu_req *)lrbp->ucd_rsp_ptr); -out:blk_put_request (req); out_unlock:up_read (&hba->clk_scaling_lock);
RE: [PATCH 04/11] scsi: ufs: Remove dead code
Quoted 6 lines
Commit
7252a3603015("scsi: ufs: Avoid busy-waiting by eliminating tag conflicts") guarantees that 'tag' is not in use by any SCSI command. Remove the check that returns early if a conflict occurs.Signed-off-by Bart Van Assche <[email protected]>
Re: [PATCH 04/11] scsi: ufs: Remove dead code
In reply to Bart Van AsscheQuoted 35 lines · Bart Van Assche
Commit
7252a3603015("scsi: ufs: Avoid busy-waiting by eliminating tag
conflicts") guarantees that 'tag' is not in use by any SCSI command. Remove the check that returns early if a conflict occurs.Signed-off-by Bart Van Assche <[email protected]>--- drivers/scsi/ufs/ufshcd.c | 6 ------ 1 file changed, 6 deletions(-)drivers/scsi/ufs/ufshcd.c +0
-5@@ -6724,11 +6724,6 @@ static int ufshcd_issue_devman_upiu_cmd(structufs_hba *hba, tag = req->tag;WARN_ONCE (tag <0 ,"Invalid tag %d\n" , tag); -if (unlikely (test_bit (tag, &hba->outstanding_reqs))) { - err = -EBUSY ; - goto out; - } - lrbp = &hba->lrb[tag];WARN_ON (lrbp->cmd);drivers/scsi/ufs/ufshcd.c +0
-1lrbp->cmd =NULL ;@@ -6796,7 +6791,6 @@ static int ufshcd_issue_devman_upiu_cmd(struct ufs_hba *hba,ufshcd_add_query_upiu_trace (hba, err ?UFS_QUERY_ERR :UFS_QUERY_COMP , (struct utp_upiu_req *)lrbp- >ucd_rsp_ptr); -out:
Quoted 1 line
blk_put_request(req);
Hi Bart,
Doesn't it need to be paired with blk_get_request()? Kind regards,
Bean
Quoted 2 lines
out_unlock:
up_read(&hba->clk_scaling_lock);
Re: [PATCH 04/11] scsi: ufs: Remove dead code
In reply to Bart Van AsscheQuoted 10 lines · Bart Van Assche
Commit
7252a3603015("scsi: ufs: Avoid busy-waiting by eliminating tagconflicts") guarantees that 'tag' is not in use by any SCSI command.
Remove the check that returns early if a conflict occurs.
Signed-off-by Bart Van Assche <[email protected]>
Ignore my previous email.
PATCH 5/11 scsi: core: Add support for reserved tags
In reply to Bart Van AsscheAllow SCSI LLDs to allocate reserved tags by passing the BLK_MQ_REQ_RESERVED flag to blk_mq_alloc_request().
--- drivers/scsi/scsi_lib.c | 4 +++- include/scsi/scsi_host.h | 9 ++++++++- 2 files changed, 11 insertions(+), 2 deletions(-)
drivers/scsi/scsi_lib.c +3 -1
@@ -1916,6 +1916,7 @@ int scsi_mq_setup_tags(struct Scsi_Host *shost) {unsigned int cmd_size, sgl_size;struct blk_mq_tag_set *tag_set = &shost->tag_set; +unsigned int reserved_tags = shost->hostt->reserved_tags; sgl_size = max_t(unsigned int, sizeof(struct scatterlist),scsi_mq_inline_sgl_size (shost));
@@ -1931,7 +1932,8 @@ int scsi_mq_setup_tags(struct Scsi_Host *shost) tag_set->ops = &scsi_mq_ops_no_commit; tag_set->nr_hw_queues = shost->nr_hw_queues ? :1 ; tag_set->nr_maps = shost->nr_maps ? :1 ; - tag_set->queue_depth = shost->can_queue; + tag_set->queue_depth = shost->can_queue + reserved_tags; + tag_set->reserved_tags = reserved_tags; tag_set->cmd_size = cmd_size; tag_set->numa_node =NUMA_NO_NODE ; tag_set->flags =BLK_MQ_F_SHOULD_MERGE ;
include/scsi/scsi_host.h +8 -1
@@ -367,10 +367,17 @@ struct scsi_host_template {/* * This determines if we will use a non-interrupt driven * or an interrupt driven scheme. It is set to the maximum number -* of simultaneous commands a single hw queue in HBA will accept. +* of simultaneous commands a single hw queue in HBA will accept. Does +* not include @reserved_tags. */ int can_queue; +/* +* Number of tags to reserve. A reserved tag can be allocated by passing +* the BLK_MQ_REQ_RESERVED flag to blk_get_request(). +*/ +unsigned reserved_tags; +/* * In many instances, especially where disconnect / reconnect are * supported, our host also has an ID on the SCSI bus. If this is
PATCH 6/11 scsi: ufs: Rework ufshcd_change_queue_depth()
In reply to Bart Van AsschePrepare for making sdev->host->can_queue less than hba->nutrs. This patch does not change any functionality.
--- drivers/scsi/ufs/ufshcd.c | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-)
drivers/scsi/ufs/ufshcd.c +1 -5
@@ -4955,11 +4955,7 @@ static int ufshcd_slave_alloc(struct scsi_device *sdev)*/ static int ufshcd_change_queue_depth (struct scsi_device *sdev,int depth) { -struct ufs_hba *hba =shost_priv (sdev->host); - -if (depth > hba->nutrs) - depth = hba->nutrs; -return scsi_change_queue_depth (sdev, depth); +return scsi_change_queue_depth (sdev,min (depth, sdev->host->can_queue)); }static void ufshcd_hpb_destroy (struct ufs_hba *hba,struct scsi_device *sdev)
RE: [PATCH 06/11] scsi: ufs: Rework ufshcd_change_queue_depth()
Quoted 3 lines
Prepare for making
sdev->host->can_queueless thanhba->nutrs. This patch does not change any functionality.
ufshcd_set_queue_depth() may also needs similar adjustments?
Thanks,
Avri
Quoted 24 lines
Signed-off-by Bart Van Assche <[email protected]>--- drivers/scsi/ufs/ufshcd.c | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-)drivers/scsi/ufs/ufshcd.c index renamed from drivers/scsi/ufs/ufshcd.c +0
-0
312e8a5b7733..8400d8e9a6f7100644drivers/scsi/ufs/ufshcd.c +2
-5@@ -4955,11 +4955,7 @@ static int ufshcd_slave_alloc(struct scsi_device*sdev) */ static int ufshcd_change_queue_depth (struct scsi_device *sdev,int depth) { -struct ufs_hba *hba =shost_priv (sdev->host); - -if (depth > hba->nutrs) - depth = hba->nutrs; -return scsi_change_queue_depth (sdev, depth); +return scsi_change_queue_depth (sdev,min (depth, + sdev->host->can_queue)); }static void ufshcd_hpb_destroy (struct ufs_hba *hba,struct scsi_device *sdev)
Re: [PATCH 06/11] scsi: ufs: Rework ufshcd_change_queue_depth()
Quoted 5 lines · Avri Altman
Quoted 3 lines
Prepare for making
sdev->host->can_queueless thanhba->nutrs. This patch does not change any functionality.
ufshcd_set_queue_depth()may also needs similar adjustments?
That's a great question. I will modify scsi_change_queue_depth() such that it limits the queue depth to host->can_queue.
Thanks,
Bart.
PATCH 7/11 scsi: ufs: Fix a deadlock in the error handler
In reply to Bart Van AsscheThe following deadlock has been observed on a test setup:
* All tags allocated.
* The SCSI error handler calls ufshcd_eh_host_reset_handler()
* ufshcd_eh_host_reset_handler() queues work that calls ufshcd_err_handler()
* ufshcd_err_handler() locks up as follows:
Workqueue: ufs_eh_wq_0 ufshcd_err_handler.cfi_jt Call trace: __switch_to+0x298/0x5d8 __schedule+0x6cc/0xa94 schedule+0x12c/0x298 blk_mq_get_tag+0x210/0x480 __blk_mq_alloc_request+0x1c8/0x284 blk_get_request+0x74/0x134 ufshcd_exec_dev_cmd+0x68/0x640 ufshcd_verify_dev_init+0x68/0x35c ufshcd_probe_hba+0x12c/0x1cb8 ufshcd_host_reset_and_restore+0x88/0x254 ufshcd_reset_and_restore+0xd0/0x354 ufshcd_err_handler+0x408/0xc58 process_one_work+0x24c/0x66c worker_thread+0x3e8/0xa4c kthread+0x150/0x1b4 ret_from_fork+0x10/0x30
Fix this lockup by making ufshcd_exec_dev_cmd() allocate a reserved request.
--- drivers/scsi/ufs/ufshcd.c | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-)
drivers/scsi/ufs/ufshcd.c +5 -9
@@ -2940,12 +2940,7 @@ static int ufshcd_exec_dev_cmd(struct ufs_hba *hba,down_read (&hba->clk_scaling_lock); -/* -* Get free slot, sleep if slots are unavailable. -* Even though we use wait_event() which sleeps indefinitely, -* the maximum wait time is bounded by SCSI request timeout. -*/ - req =blk_get_request (q,REQ_OP_DRV_OUT ,0 ); + req =blk_mq_alloc_request (q,REQ_OP_DRV_OUT ,BLK_MQ_REQ_RESERVED );if (IS_ERR (req)) { err =PTR_ERR (req); goto out_unlock;
@@ -8152,7 +8147,8 @@ static struct scsi_host_template ufshcd_driver_template = { .this_id = -1, .sg_tablesize = SG_ALL, .cmd_per_lun = UFSHCD_CMD_PER_LUN, - .can_queue = UFSHCD_CAN_QUEUE, + .can_queue = UFSHCD_CAN_QUEUE - 1, + .reserved_tags = 1, .max_segment_size = PRDT_DATA_BYTE_COUNT_MAX, .max_host_blocked = 1, .track_queue_depth = 1,
@@ -9513,8 +9509,8 @@ int ufshcd_init(struct ufs_hba *hba, void __iomem *mmio_base, unsigned int irq)/* Configure LRB */ ufshcd_host_memory_configure (hba); - host->can_queue = hba->nutrs; - host->cmd_per_lun = hba->nutrs; + host->can_queue = hba->nutrs -1 ; + host->cmd_per_lun = hba->nutrs -1 ; host->max_id =UFSHCD_MAX_ID ; host->max_lun =UFS_MAX_LUNS ; host->max_channel =UFSHCD_MAX_CHANNEL ;
Re: [PATCH 07/11] scsi: ufs: Fix a deadlock in the error handler
Hi Bart,
as pointed out last time: LLDDS have no business directy allocating tags. Please work with Hannes and John on the proper APIs, as metnioned last time as well.
Re: [PATCH 07/11] scsi: ufs: Fix a deadlock in the error handler
Quoted 3 lines · Christoph Hellwig
as pointed out last time: LLDDS have no business directy allocating tags. Please work with Hannes and John on the proper APIs, as metnioned last time as well.
Hi Christoph,
I will do this.
Thanks,
Bart.
RE: [PATCH 07/11] scsi: ufs: Fix a deadlock in the error handler
In reply to Bart Van AsscheQuoted 52 lines
The following deadlock has been observed on a test setup:
* All tags allocated.
* The SCSI error handler callsufshcd_eh_host_reset_handler()
*ufshcd_eh_host_reset_handler()queues work that callsufshcd_err_handler()
*ufshcd_err_handler()locks up as follows:Workqueue: ufs_eh_wq_0 ufshcd_err_handler.cfi_jt Call trace: __switch_to+0x298/0x5d8 __schedule+0x6cc/0xa94 schedule+0x12c/0x298 blk_mq_get_tag+0x210/0x480 __blk_mq_alloc_request+0x1c8/0x284 blk_get_request+0x74/0x134 ufshcd_exec_dev_cmd+0x68/0x640 ufshcd_verify_dev_init+0x68/0x35c ufshcd_probe_hba+0x12c/0x1cb8 ufshcd_host_reset_and_restore+0x88/0x254 ufshcd_reset_and_restore+0xd0/0x354 ufshcd_err_handler+0x408/0xc58 process_one_work+0x24c/0x66c worker_thread+0x3e8/0xa4c kthread+0x150/0x1b4 ret_from_fork+0x10/0x30Fix this lockup by making
ufshcd_exec_dev_cmd()allocate a reserved request.Signed-off-by Bart Van Assche <[email protected]>--- drivers/scsi/ufs/ufshcd.c | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-)drivers/scsi/ufs/ufshcd.c index renamed from drivers/scsi/ufs/ufshcd.c +0
-0
8400d8e9a6f7..8f5640647054100644drivers/scsi/ufs/ufshcd.c +2
-6@@ -2940,12 +2940,7 @@ static int ufshcd_exec_dev_cmd(struct ufs_hba *hba,down_read (&hba->clk_scaling_lock); -/* -* Get free slot, sleep if slots are unavailable. -* Even though we use wait_event() which sleeps indefinitely, -* the maximum wait time is bounded by SCSI request timeout. -*/ - req =blk_get_request (q,REQ_OP_DRV_OUT ,0 ); + req =blk_mq_alloc_request (q,REQ_OP_DRV_OUT , +BLK_MQ_REQ_RESERVED );if (IS_ERR (req)) { err =PTR_ERR (req); goto out_unlock;
drivers/scsi/ufs/ufshcd.c static struct scsi_host_template · line 8150
Quoted 4 lines
drivers/scsi/ufs/ufshcd.c +0
-0@@ -8152,7 +8147,8 @@ static struct scsi_host_template ufshcd_driver_template = { .this_id = -1, .sg_tablesize = SG_ALL, .cmd_per_lun = UFSHCD_CMD_PER_LUN,
UFSHCD_CMD_PER_LUN needs to be < 32 as well?
Thanks,
Avri
Quoted 18 lines
drivers/scsi/ufs/ufshcd.c +2
-1- .can_queue = UFSHCD_CAN_QUEUE, + .can_queue = UFSHCD_CAN_QUEUE - 1, + .reserved_tags = 1, .max_segment_size = PRDT_DATA_BYTE_COUNT_MAX, .max_host_blocked = 1,.track_queue_depth = 1,
drivers/scsi/ufs/ufshcd.c +2
-2@@ -9513,8 +9509,8 @@ int ufshcd_init(struct ufs_hba *hba, void __iomem *mmio_base, unsigned int irq)/* Configure LRB */ ufshcd_host_memory_configure (hba ); - host->can_queue = hba->nutrs; - host->cmd_per_lun = hba->nutrs; + host->can_queue = hba->nutrs -1 ; + host->cmd_per_lun = hba->nutrs -1 ; host->max_id =UFSHCD_MAX_ID ; host->max_lun =UFS_MAX_LUNS ;host->max_channel = UFSHCD_MAX_CHANNEL;
Re: [PATCH 07/11] scsi: ufs: Fix a deadlock in the error handler
Quoted 7 lines · Avri Altman
drivers/scsi/ufs/ufshcd.c static struct scsi_host_template · line 8150
Quoted 5 lines
drivers/scsi/ufs/ufshcd.c +0
-0@@ -8152,7 +8147,8 @@ static struct scsi_host_template ufshcd_driver_template = { .this_id = -1, .sg_tablesize = SG_ALL, .cmd_per_lun = UFSHCD_CMD_PER_LUN,
UFSHCD_CMD_PER_LUNneeds to be < 32 as well?
I will make that change although that change should not affect the behavior of the UFS driver since the SCSI core limits cmd_per_lun to host->can_queue.
Thanks,
Bart.
PATCH 8/11 scsi: ufs: Improve SCSI abort handling further
In reply to Bart Van AsscheMake sure that aborted commands are completed once by clearing the corresponding tag bit from hba->outstanding_reqs. This patch is a follow-up for commit cd892096c940 ("scsi: ufs: core: Improve SCSI abort handling").
--- drivers/scsi/ufs/ufshcd.c | 9 +++++++++ 1 file changed, 9 insertions(+)
drivers/scsi/ufs/ufshcd.c +9 -0
@@ -7090,6 +7090,15 @@ static int ufshcd_abort(struct scsi_cmnd *cmd) goto release; } +/* +* ufshcd_try_to_abort_task() cleared the 'tag' bit in the doorbell +* register. Clear the corresponding bit from outstanding_reqs to +* prevent early completion. +*/ +spin_lock_irqsave (&hba->outstanding_lock, flags); +__clear_bit (tag, &hba->outstanding_reqs); +spin_unlock_irqrestore (&hba->outstanding_lock, flags); + lrbp->cmd =NULL ; err =SUCCESS ;
Re: [PATCH 08/11] scsi: ufs: Improve SCSI abort handling further
Quoted 17 lines · Bart Van Assche
Make sure that aborted commands are completed once by clearing the corresponding tag bit from
hba->outstanding_reqs. This patch is a follow-up for commitcd892096c940("scsi: ufs: core: Improve SCSI abort handling").Fixes 7a3e97b0dc4b ("[SCSI] ufshcd: UFS Host controller driver")Signed-off-by Bart Van Assche <[email protected]>--- drivers/scsi/ufs/ufshcd.c | 9 +++++++++ 1 file changed, 9 insertions(+)drivers/scsi/ufs/ufshcd.c +0
-0@@ -7090,6 +7090,15 @@ static int ufshcd_abort(struct scsi_cmnd *cmd) goto release;
drivers/scsi/ufs/ufshcd.c static int ufshcd_abort(struct scsi_cmnd *cmd) · line 7100
Quoted 10 lines
drivers/scsi/ufs/ufshcd.c +8
-0} +/* +* ufshcd_try_to_abort_task() cleared the 'tag' bit in the doorbell +* register. Clear the corresponding bit from outstanding_reqs to +* prevent early completion. +*/ +spin_lock_irqsave (&hba->outstanding_lock, flags); +__clear_bit (tag, &hba->outstanding_reqs); +spin_unlock_irqrestore (&hba->outstanding_lock, flags);
Seems like something ufshcd_clear_cmd() should be doing instead?
Quoted 5 lines
drivers/scsi/ufs/ufshcd.c +1
-0+ lrbp->cmd =NULL ; err =SUCCESS ;
Re: [PATCH 08/11] scsi: ufs: Improve SCSI abort handling further
Quoted 20 lines · Adrian Hunter
Quoted 19 lines · Bart Van Assche
Make sure that aborted commands are completed once by clearing the corresponding tag bit from
hba->outstanding_reqs. This patch is a follow-up for commitcd892096c940("scsi: ufs: core: Improve SCSI abort handling").Fixes 7a3e97b0dc4b ("[SCSI] ufshcd: UFS Host controller driver")Signed-off-by Bart Van Assche <[email protected]>--- drivers/scsi/ufs/ufshcd.c | 9 +++++++++ 1 file changed, 9 insertions(+)drivers/scsi/ufs/ufshcd.c +0
-0@@ -7090,6 +7090,15 @@ static int ufshcd_abort(struct scsi_cmnd *cmd) goto release; }
Quoted 10 lines
drivers/scsi/ufs/ufshcd.c static int ufshcd_abort(struct scsi_cmnd *cmd) · line 7100
Quoted 8 lines
drivers/scsi/ufs/ufshcd.c +8
-0+/* +* ufshcd_try_to_abort_task() cleared the 'tag' bit in the doorbell +* register. Clear the corresponding bit from outstanding_reqs to +* prevent early completion. +*/ +spin_lock_irqsave (&hba->outstanding_lock, flags); +__clear_bit (tag, &hba->outstanding_reqs); +spin_unlock_irqrestore (&hba->outstanding_lock, flags);Seems like something
ufshcd_clear_cmd()should be doing instead?
Hi Adrian,
I'm concerned that would break ufshcd_eh_device_reset_handler() since that reset handler retries SCSI commands by calling __ufshcd_transfer_req_compl() after having called ufshcd_clear_cmd().
Thanks,
Bart.
Re: [PATCH 08/11] scsi: ufs: Improve SCSI abort handling further
Quoted 31 lines · Bart Van Assche
Quoted 21 lines · Adrian Hunter
Quoted 20 lines · Bart Van Assche
Make sure that aborted commands are completed once by clearing the corresponding tag bit from
hba->outstanding_reqs. This patch is a follow-up for commitcd892096c940("scsi: ufs: core: Improve SCSI abort handling").Fixes 7a3e97b0dc4b ("[SCSI] ufshcd: UFS Host controller driver")Signed-off-by Bart Van Assche <[email protected]>---
drivers/scsi/ufs/ufshcd.c| 9 +++++++++1 file changed, 9 insertions(+)drivers/scsi/ufs/ufshcd.c +0
-0@@ -7090,6 +7090,15 @@ static int ufshcd_abort(struct scsi_cmnd *cmd) goto release; }+ /*
+ *ufshcd_try_to_abort_task()cleared the 'tag' bit in the doorbellQuoted 8 lines
Quoted 6 lines
+ * register. Clear the corresponding bit from
outstanding_reqsto + * prevent early completion.
+ */
+spin_lock_irqsave(&hba->outstanding_lock, flags);
+__clear_bit(tag, &hba->outstanding_reqs);
+spin_unlock_irqrestore(&hba->outstanding_lock, flags);Seems like something
ufshcd_clear_cmd()should be doing instead?
Quoted 3 lines
Hi Adrian,
I'm concerned that would break
ufshcd_eh_device_reset_handler()since that reset handler retries SCSI commands by calling__ufshcd_transfer_req_compl()after having calledufshcd_clear_cmd().
Whenever an outstanding_reqs bit transitions 1 -> 0, then __ufshcd_transfer_req_compl() must be called.
In all cases, the correct logic must have the effect of:
spin_lock_irqsave(&hba->outstanding_lock, flags); bit = __test_and_clear_bit(tag, &hba->outstanding_reqs); spin_unlock_irqrestore(&hba->outstanding_lock, flags); if (bit) __ufshcd_transfer_req_compl(hba, 1UL << tag);
To put it another way, how else does the driver know whether or not __ufshcd_transfer_req_compl() has been called already.
As a separate issue, in ufshcd_abort() there is:
/* If command is already aborted/completed, return FAILED. */
if (!(test_bit(tag, &hba->outstanding_reqs))) {
dev_err(hba->dev,
"%s: cmd at tag %d already completed, outstanding=0x%lx, doorbell=0x%x\n",
__func__, tag, hba->outstanding_reqs, reg);
goto release;
}which seems wrong. FAILED should only be returned to escalate the error handling, so if the slot has already successfully been cleared, that is SUCCESS. scsi_times_out() has already blocked the scsi_done() path (by setting SCMD_STATE_COMPLETE), so any use after free must be being caused by SCSI not the ufs driver.
path
Re: [PATCH 08/11] scsi: ufs: Improve SCSI abort handling further
Quoted 9 lines · Adrian Hunter
Quoted 6 lines · Bart Van Assche
Quoted 1 line · Adrian Hunter
Seems like something
ufshcd_clear_cmd()should be doing instead?I'm concerned that would break
ufshcd_eh_device_reset_handler()since that reset handler retries SCSI commands by calling__ufshcd_transfer_req_compl()after having calledufshcd_clear_cmd().Whenever an
outstanding_reqsbit transitions 1 -> 0, then__ufshcd_transfer_req_compl()must be called.
I will look further into this.
Quoted 10 lines
As a separate issue, in
ufshcd_abort()there is:/* If command is already aborted/completed, return FAILED. */ if (!(test_bit(tag, &hba->outstanding_reqs))) { dev_err(hba->dev, "%s: cmd at tag %d already completed, outstanding=0x%lx, doorbell=0x%x\n", __func__, tag, hba->outstanding_reqs, reg); goto release; }
Quoted 5 lines
which seems wrong. FAILED should only be returned to escalate the error handling, so if the slot has already successfully been cleared, that is SUCCESS.
scsi_times_out()has already blocked thescsi_done()path (by settingSCMD_STATE_COMPLETE), so any use after free must be being caused by SCSI not the ufs driver.
scmd_eh_abort_handler() would trigger a use-after-free if ufshcd_abort() would return SUCCESS for completed commands. Hence the choice for the return value FAILED for completed commands.
BTW, can this code path ever be reached since scsi_done() sets the SCMD_STATE_COMPLETE bit before it calls blk_mq_complete_request() and since scsi_times_out() tests that bit before it schedules a call of ufshcd_abort()?
Thanks,
Bart.
Re: [PATCH 08/11] scsi: ufs: Improve SCSI abort handling further
Quoted 29 lines · Bart Van Assche
Quoted 9 lines · Adrian Hunter
Quoted 6 lines · Bart Van Assche
Quoted 1 line · Adrian Hunter
Seems like something
ufshcd_clear_cmd()should be doing instead?I'm concerned that would break
ufshcd_eh_device_reset_handler()since that reset handler retries SCSI commands by calling__ufshcd_transfer_req_compl()after having calledufshcd_clear_cmd().Whenever an
outstanding_reqsbit transitions 1 -> 0, then__ufshcd_transfer_req_compl()must be called.I will look further into this.
Quoted 10 lines
As a separate issue, in
ufshcd_abort()there is:/* If command is already aborted/completed, return FAILED. */
if (!(test_bit(tag,&hba->outstanding_reqs))) {
dev_err(hba->dev,
"%s: cmd at tag %d already completed, outstanding=0x%lx, doorbell=0x%x\n",__func__, tag,hba->outstanding_reqs, reg);
goto release;
}Quoted 5 lines
which seems wrong. FAILED should only be returned to escalate the error handling, so if the slot has already successfully been cleared, that is SUCCESS.
scsi_times_out()has already blocked thescsi_done()path (by settingSCMD_STATE_COMPLETE), so any use after free must be being caused by SCSI not the ufs driver.
Quoted 3 lines
scmd_eh_abort_handler()would trigger a use-after-free ifufshcd_abort()would return SUCCESS for completed commands. Hence the choice for the return value FAILED for completed commands.BTW, can this code path ever be reached since
scsi_done()sets theSCMD_STATE_COMPLETEbit before it callsblk_mq_complete_request()and sincescsi_times_out()tests that bit before it schedules a call ofufshcd_abort()?
Racing with timing out, the UFS interrupt handler can clear hba->outstanding_reqs bit and call __ufshcd_transfer_req_compl(), but the request will not be freed if scsi_times_out() wins the race. So there should be no use-after-free unless SCSI error handling itself has a bug.
One perhaps unrelated issue in scsi_times_out():
if (test_and_set_bit(SCMD_STATE_COMPLETE, &scmd->state)) return BLK_EH_RESET_TIMER;
Shouldn't that return BLK_EH_DONE not BLK_EH_RESET_TIMER, since the request has been through blk_mq_complete_request() ?
Re: [PATCH 08/11] scsi: ufs: Improve SCSI abort handling further
Quoted 6 lines · Adrian Hunter
One perhaps unrelated issue in
scsi_times_out():if (test_and_set_bit(SCMD_STATE_COMPLETE, &scmd->state)) return BLK_EH_RESET_TIMER;Shouldn't that return
BLK_EH_DONEnotBLK_EH_RESET_TIMER, since the request has been throughblk_mq_complete_request()?
I think so. I will submit a patch.
Bart.
Re: [PATCH 08/11] scsi: ufs: Improve SCSI abort handling further
In reply to Bart Van AsscheHi Bart,
Quoted 18 lines · Bart Van Assche
Make sure that aborted commands are completed once by clearing the corresponding tag bit from
hba->outstanding_reqs. This patch is a follow-up for commitcd892096c940("scsi: ufs: core: Improve SCSI abort handling").Fixes 7a3e97b0dc4b ("[SCSI] ufshcd: UFS Host controller driver")Signed-off-by Bart Van Assche <[email protected]>--- drivers/scsi/ufs/ufshcd.c | 9 +++++++++ 1 file changed, 9 insertions(+)drivers/scsi/ufs/ufshcd.c +0
-0@@ -7090,6 +7090,15 @@ static int ufshcd_abort(struct scsi_cmnd *cmd) goto release; }
drivers/scsi/ufs/ufshcd.c static int ufshcd_abort(struct scsi_cmnd *cmd) · line 7101
Quoted 10 lines
drivers/scsi/ufs/ufshcd.c +9
-0+/* +* ufshcd_try_to_abort_task() cleared the 'tag' bit in the doorbell +* register. Clear the corresponding bit from outstanding_reqs to +* prevent early completion. +*/ +spin_lock_irqsave (&hba->outstanding_lock, flags); +__clear_bit (tag, &hba->outstanding_reqs); +spin_unlock_irqrestore (&hba->outstanding_lock, flags); +
should we also call unmap?
scsi_dma_unmap(cmd);
Quoted 3 lines
drivers/scsi/ufs/ufshcd.c +0
-0lrbp->cmd =NULL ; err =SUCCESS ;
Re: [PATCH 08/11] scsi: ufs: Improve SCSI abort handling further
Quoted 33 lines · Peter Wang
Hi Bart,
Quoted 19 lines · Bart Van Assche
Make sure that aborted commands are completed once by clearing the corresponding tag bit from
hba->outstanding_reqs. This patch is a follow-up for commitcd892096c940("scsi: ufs: core: Improve SCSI abort handling").Fixes 7a3e97b0dc4b ("[SCSI] ufshcd: UFS Host controller driver")Signed-off-by Bart Van Assche <[email protected]>---
drivers/scsi/ufs/ufshcd.c| 9 +++++++++1 file changed, 9 insertions(+)drivers/scsi/ufs/ufshcd.c +0
-0@@ -7090,6 +7090,15 @@ static int ufshcd_abort(struct scsi_cmnd *cmd) goto release; }+ /*
Quoted 8 lines
+ *
ufshcd_try_to_abort_task()cleared the 'tag' bit in the doorbell + * register. Clear the corresponding bit fromoutstanding_reqsto + * prevent early completion.
+ */
+spin_lock_irqsave(&hba->outstanding_lock, flags);
+__clear_bit(tag, &hba->outstanding_reqs);
+spin_unlock_irqrestore(&hba->outstanding_lock, flags); +should we also call unmap?
Quoted 2 lines
scsi_dma_unmap(cmd);
Hi Bart,
Should we add unmap?
Thanks.
Peter
Quoted 3 lines
Quoted 3 lines
lrbp->cmd= NULL;
err = SUCCESS;
Re: [PATCH 08/11] scsi: ufs: Improve SCSI abort handling further
Quoted 1 line · Peter Wang
Should we add unmap?
Hi Peter,
I will add DMA unmapping code in the abort handler.
Thanks,
Bart.
Re: [PATCH 08/11] scsi: ufs: Improve SCSI abort handling further
Quoted 6 lines · Bart Van Assche
Quoted 1 line · Peter Wang
Should we add unmap?
Hi Peter,
I will add DMA unmapping code in the abort handler.
I would note that __ufshcd_transfer_req_compl() does that, as well as providing a matching ufshcd_release() for the ufshcd_hold() in ufshcd_queuecommand(), so do consider __ufshcd_transfer_req_compl().
Using __ufshcd_transfer_req_compl() seems consistent with the error handler which uses __ufshcd_transfer_req_compl() via ufshcd_complete_requests(), which will pick up all the requests that the error handler has just aborted via ufshcd_try_to_abort_task(). Also ufshcd_host_reset_and_restore() uses __ufshcd_transfer_req_compl() via ufshcd_complete_requests(), which will pick up anything still in outstanding_reqs because the doorbell has become zero at that point.
Re: [PATCH 08/11] scsi: ufs: Improve SCSI abort handling further
Quoted 12 lines · Adrian Hunter
Quoted 6 lines · Bart Van Assche
Quoted 1 line · Peter Wang
Should we add unmap?
Hi Peter,
I will add DMA unmapping code in the abort handler.
I would note that
__ufshcd_transfer_req_compl()does that, as well as providing a matchingufshcd_release()for theufshcd_hold()inufshcd_queuecommand(), so do consider__ufshcd_transfer_req_compl().
Quoted 7 lines
Using
__ufshcd_transfer_req_compl()seems consistent with the error handler which uses__ufshcd_transfer_req_compl()viaufshcd_complete_requests(), which will pick up all the requests that the error handler has just aborted viaufshcd_try_to_abort_task(). Alsoufshcd_host_reset_and_restore()uses__ufshcd_transfer_req_compl()viaufshcd_complete_requests(), which will pick up anything still inoutstanding_reqsbecause the doorbell has become zero at that point.
Hi Adrian,
Although I agree with minimizing code duplication, I'm not sure that __ufshcd_transfer_req_compl() should be called from inside ufshcd_abort(). __ufshcd_transfer_req_compl() also calls ufshcd_update_monitor() and ufshcd_add_command_trace(hba, index, UFS_CMD_COMP). Neither function should be called when aborting a command.
Thanks,
Bart.
Re: [PATCH 08/11] scsi: ufs: Improve SCSI abort handling further
Quoted 21 lines · Bart Van Assche
Quoted 12 lines · Adrian Hunter
Quoted 6 lines · Bart Van Assche
Quoted 1 line · Peter Wang
Should we add unmap?
Hi Peter,
I will add DMA unmapping code in the abort handler.
I would note that
__ufshcd_transfer_req_compl()does that, as well as providing a matchingufshcd_release()for theufshcd_hold()inufshcd_queuecommand(), so do consider__ufshcd_transfer_req_compl().Quoted 7 lines
Using
__ufshcd_transfer_req_compl()seems consistent with the error handler which uses__ufshcd_transfer_req_compl()viaufshcd_complete_requests(), which will pick up all the requests that the error handler has just aborted via ufshcd_try_to_abort_task(). Alsoufshcd_host_reset_and_restore()uses__ufshcd_transfer_req_compl()viaufshcd_complete_requests(), which will pick up anything still inoutstanding_reqsbecause the doorbell has become zero at that point.
Quoted 3 lines
Hi Adrian,
Although I agree with minimizing code duplication, I'm not sure that
__ufshcd_transfer_req_compl()should be called from insideufshcd_abort().__ufshcd_transfer_req_compl()also callsufshcd_update_monitor()andufshcd_add_command_trace(hba, index, UFS_CMD_COMP). Neither function should be called when aborting a command.
But we should trace the abort I would have thought - seems like a separate issue though.
PATCH 9/11 scsi: ufs: Fix a kernel crash during shutdown
In reply to Bart Van AsscheFix the following kernel crash:
Unable to handle kernel paging request at virtual address ffffffc91e735000 Call trace: __queue_work+0x26c/0x624 queue_work_on+0x6c/0xf0 ufshcd_hold+0x12c/0x210 __ufshcd_wl_suspend+0xc0/0x400 ufshcd_wl_shutdown+0xb8/0xcc device_shutdown+0x184/0x224 kernel_restart+0x4c/0x124 __arm64_sys_reboot+0x194/0x264 el0_svc_common+0xc8/0x1d4 do_el0_svc+0x30/0x8c el0_svc+0x20/0x30 el0_sync_handler+0x84/0xe4 el0_sync+0x1bc/0x1c0
Fix this crash by ungating the clock before destroying the work queue on which clock gating work is queued.
--- drivers/scsi/ufs/ufshcd.c | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-)
drivers/scsi/ufs/ufshcd.c +10 -5
@@ -1666,7 +1666,8 @@ int ufshcd_hold(struct ufs_hba *hba, bool async)bool flush_result;unsigned long flags; -if (!ufshcd_is_clkgating_allowed (hba)) +if (!ufshcd_is_clkgating_allowed (hba) || + !hba->clk_gating.is_initialized) goto out;spin_lock_irqsave (hba->host->host_lock, flags); hba->clk_gating.active_reqs++;
@@ -1826,7 +1827,7 @@ static void __ufshcd_release(struct ufs_hba *hba)if (hba->clk_gating.active_reqs || hba->clk_gating.is_suspended || hba->ufshcd_state !=UFSHCD_STATE_OPERATIONAL || - hba->outstanding_tasks || + hba->outstanding_tasks || !hba->clk_gating.is_initialized || hba->active_uic_cmd || hba->uic_async_done || hba->clk_gating.state ==CLKS_OFF )return ;
@@ -1961,11 +1962,15 @@ static void ufshcd_exit_clk_gating(struct ufs_hba *hba) {if (!hba->clk_gating.is_initialized)return ; +ufshcd_remove_clk_gating_sysfs (hba); -cancel_work_sync (&hba->clk_gating.ungate_work); -cancel_delayed_work_sync (&hba->clk_gating.gate_work); -destroy_workqueue (hba->clk_gating.clk_gating_workq); + +/* Ungate the clock if necessary. */ +ufshcd_hold (hba, false); hba->clk_gating.is_initialized = false; +ufshcd_release (hba); + +destroy_workqueue (hba->clk_gating.clk_gating_workq); }/* Must be called with host lock acquired */
RE: [PATCH 09/11] scsi: ufs: Fix a kernel crash during shutdown
Quoted 53 lines
Fix the following kernel crash:
Unable to handle kernel paging request at virtual address ffffffc91e735000 Call trace: __queue_work+0x26c/0x624 queue_work_on+0x6c/0xf0 ufshcd_hold+0x12c/0x210 __ufshcd_wl_suspend+0xc0/0x400 ufshcd_wl_shutdown+0xb8/0xcc device_shutdown+0x184/0x224 kernel_restart+0x4c/0x124 __arm64_sys_reboot+0x194/0x264 el0_svc_common+0xc8/0x1d4 do_el0_svc+0x30/0x8c el0_svc+0x20/0x30 el0_sync_handler+0x84/0xe4 el0_sync+0x1bc/0x1c0Fix this crash by ungating the clock before destroying the work queue on which clock gating work is queued.
Signed-off-by Bart Van Assche <[email protected]>--- drivers/scsi/ufs/ufshcd.c | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-)drivers/scsi/ufs/ufshcd.c index renamed from drivers/scsi/ufs/ufshcd.c +0
-0
1e15ed1f639f..13848e93cda8100644drivers/scsi/ufs/ufshcd.c +3
-2@@ -1666,7 +1666,8 @@ int ufshcd_hold(struct ufs_hba *hba, bool async)bool flush_result;unsigned long flags; -if (!ufshcd_is_clkgating_allowed (hba)) +if (!ufshcd_is_clkgating_allowed (hba) || + !hba->clk_gating.is_initialized) goto out;spin_lock_irqsave (hba->host->host_lock, flags); hba->clk_gating.active_reqs++;@@ -1826,7 +1827,7 @@ static void __ufshcd_release(struct ufs_hba *hba)if (hba->clk_gating.active_reqs || hba->clk_gating.is_suspended || hba->ufshcd_state !=UFSHCD_STATE_OPERATIONAL || - hba->outstanding_tasks || + hba->outstanding_tasks || !hba->clk_gating.is_initialized || hba->active_uic_cmd || hba->uic_async_done || hba->clk_gating.state ==CLKS_OFF )return ;@@ -1961,11 +1962,15 @@ static void ufshcd_exit_clk_gating(struct ufs_hba *hba) {if (!hba->clk_gating.is_initialized)return ;
drivers/scsi/ufs/ufshcd.c static void ufshcd_exit_clk_gating(struct ufs_hba · line 1971
Quoted 10 lines
drivers/scsi/ufs/ufshcd.c +5
-3+ufshcd_remove_clk_gating_sysfs (hba); -cancel_work_sync (&hba->clk_gating.ungate_work); -cancel_delayed_work_sync (&hba->clk_gating.gate_work); -destroy_workqueue (hba->clk_gating.clk_gating_workq); + +/* Ungate the clock if necessary. */ +ufshcd_hold (hba, false); hba->clk_gating.is_initialized = false; +ufshcd_release (hba);
Not sure that the symmetry in calling ufshcd_release() is meaningful here?
You don't really want to queue a new gate work, this is why you added !.is_initialized to release(), Or did I get it wrong?
Thanks,
Avri
Quoted 5 lines
drivers/scsi/ufs/ufshcd.c +2
-0+ +destroy_workqueue (hba->clk_gating.clk_gating_workq); }/* Must be called with host lock acquired */
Re: [PATCH 09/11] scsi: ufs: Fix a kernel crash during shutdown
Quoted 10 lines · Avri Altman
Quoted 10 lines
drivers/scsi/ufs/ufshcd.c +2
-3@@ -1961,11 +1962,15 @@ static void ufshcd_exit_clk_gating(struct ufs_hba *hba) {if (!hba->clk_gating.is_initialized)return ; +ufshcd_remove_clk_gating_sysfs (hba); -cancel_work_sync (&hba->clk_gating.ungate_work); -cancel_delayed_work_sync (&hba->clk_gating.gate_work); -destroy_workqueue (hba->clk_gating.clk_gating_workq); +
Quoted 8 lines
drivers/scsi/ufs/ufshcd.c static void ufshcd_exit_clk_gating(struct ufs_hba *hba) · line 1971
Quoted 4 lines
drivers/scsi/ufs/ufshcd.c +3
-0+/* Ungate the clock if necessary. */ +ufshcd_hold (hba, false); hba->clk_gating.is_initialized = false; +ufshcd_release (hba);Not sure that the symmetry in calling
ufshcd_release()is meaningful here?
You don't really want to queue a new gate work, this is why you added !.is_initialized torelease(), Or did I get it wrong?
Hi Avri,
The purpose of the above changes is as follows:
* Call ufshcd_hold() to ungate the clock and also to wait for any pending work.
* Change hba->clk_gating.is_initialized from true to false to make sure that the clock remains ungated and also that no new work is queued.
* Call ufshcd_release() to decrement hba->clk_gating.active_reqs.
Thanks,
Bart.
PATCH 10/11 scsi: ufs: Optimize the command queueing code
In reply to Bart Van AsscheRemove the clock scaling lock from ufshcd_queuecommand() since it is a performance bottleneck. As requested by Asutosh Das, change the behavior of ufshcd_clock_scaling_prepare() from waiting until all pending commands have finished into quiescing request queues. Insert a rcu_read_lock() / rcu_read_unlock() pair in ufshcd_queuecommand() and also in __ufshcd_issue_tm_cmd(). Use synchronize_rcu_expedited() to wait for ongoing command and TMF queueing.
--- drivers/scsi/ufs/ufshcd.c | 121 +++++++++++++------------------------- drivers/scsi/ufs/ufshcd.h | 1 + 2 files changed, 42 insertions(+), 80 deletions(-)
drivers/scsi/ufs/ufshcd.c +41 -80
@@ -1069,65 +1069,6 @@ static bool ufshcd_is_devfreq_scaling_required(struct ufs_hba *hba,return false; } -static int ufshcd_wait_for_doorbell_clr (struct ufs_hba *hba, -u64 wait_timeout_us) -{ -unsigned long flags; -int ret =0 ; -u32 tm_doorbell; -u32 tr_doorbell; -bool timeout = false, do_last_check = false; -ktime_t start; - -ufshcd_hold (hba, false); -spin_lock_irqsave (hba->host->host_lock, flags); -/* -* Wait for all the outstanding tasks/transfer requests. -* Verify by checking the doorbell registers are clear. -*/ - start =ktime_get (); -do { -if (hba->ufshcd_state !=UFSHCD_STATE_OPERATIONAL ) { - ret = -EBUSY ; - goto out; - } - - tm_doorbell =ufshcd_readl (hba,REG_UTP_TASK_REQ_DOOR_BELL ); - tr_doorbell =ufshcd_readl (hba,REG_UTP_TRANSFER_REQ_DOOR_BELL ); -if (!tm_doorbell && !tr_doorbell) { - timeout = false; -break ; - }else if (do_last_check) { -break ; - } - -spin_unlock_irqrestore (hba->host->host_lock, flags); -schedule (); -if (ktime_to_us (ktime_sub (ktime_get (), start)) > - wait_timeout_us) { - timeout = true; -/* -* We might have scheduled out for long time so make -* sure to check if doorbells are cleared by this time -* or not. -*/ - do_last_check = true; - } -spin_lock_irqsave (hba->host->host_lock, flags); - }while (tm_doorbell || tr_doorbell); - -if (timeout) { -dev_err (hba->dev, -"%s: timedout waiting for doorbell to clear (tm=0x%x, tr=0x%x)\n" , - __func__, tm_doorbell, tr_doorbell); - ret = -EBUSY ; - } -out: -spin_unlock_irqrestore (hba->host->host_lock, flags); -ufshcd_release (hba); -return ret; -} -/** * ufshcd_scale_gear - scale up/down UFS gear * @hba: per adapter instance
@@ -1175,37 +1116,51 @@ static int ufshcd_scale_gear(struct ufs_hba *hba, bool scale_up)static int ufshcd_clock_scaling_prepare (struct ufs_hba *hba) { -#define DOORBELL_CLR_TOUT_US (1000 * 1000)/* 1 sec */ -int ret =0 ; +struct scsi_device *sdev; +/* -* make sure that there are no outstanding requests when -* clock scaling is in progress +* Make sure that no commands are being queued while clock scaling +* is in progress. +* +* Since ufshcd_exec_dev_cmd() and ufshcd_issue_devman_upiu_cmd() lock +* the clk_scaling_lock before calling blk_get_request(), lock +* clk_scaling_lock before freezing the request queues to prevent lock +* inversion. */ -ufshcd_scsi_block_requests (hba);down_write (&hba->clk_scaling_lock); - -if (!hba->clk_scaling.is_allowed || -ufshcd_wait_for_doorbell_clr (hba,DOORBELL_CLR_TOUT_US )) { - ret = -EBUSY ; +if (!hba->clk_scaling.is_allowed) {up_write (&hba->clk_scaling_lock); -ufshcd_scsi_unblock_requests (hba); - goto out; +return -EBUSY ; } - +blk_mq_quiesce_queue_nowait (hba->tmf_queue); +blk_mq_quiesce_queue_nowait (hba->cmd_queue); +shost_for_each_device (sdev, hba->host) +blk_mq_quiesce_queue_nowait (sdev->request_queue); +/* +* Calling synchronize_rcu_expedited() reduces the time required to +* quiesce request queues from milliseconds to microseconds. +* +* See also the rcu_read_lock() and rcu_read_unlock() calls in +* ufshcd_queuecommand() and also in __ufshcd_issue_tm_cmd(). +*/ +synchronize_rcu_expedited ();/* let's not get into low power until clock scaling is completed */ ufshcd_hold (hba, false); - -out: -return ret; +return 0 ; }static void ufshcd_clock_scaling_unprepare (struct ufs_hba *hba,bool writelock) { +struct scsi_device *sdev; + +shost_for_each_device (sdev, hba->host) +blk_mq_unquiesce_queue (sdev->request_queue); +blk_mq_unquiesce_queue (hba->cmd_queue); +blk_mq_unquiesce_queue (hba->tmf_queue);if (writelock)up_write (&hba->clk_scaling_lock);else up_read (&hba->clk_scaling_lock); -ufshcd_scsi_unblock_requests (hba);ufshcd_release (hba); }
@@ -2698,8 +2653,11 @@ static int ufshcd_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *cmd)WARN_ONCE (tag <0 ,"Invalid tag %d\n" , tag); -if (!down_read_trylock (&hba->clk_scaling_lock)) -return SCSI_MLQUEUE_HOST_BUSY ; +/* +* Allows ufshcd_clock_scaling_prepare() and also the UFS error handler +* to wait for prior ufshcd_queuecommand() calls. +*/ +rcu_read_lock ();switch (hba->ufshcd_state) {case UFSHCD_STATE_OPERATIONAL :
@@ -2780,7 +2738,7 @@ static int ufshcd_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *cmd)ufshcd_send_command (hba, tag); out: -up_read (&hba->clk_scaling_lock); +rcu_read_unlock ();if (ufs_trigger_eh ()) {unsigned long flags;
@@ -5977,8 +5935,7 @@ static void ufshcd_err_handling_prepare(struct ufs_hba *hba) }ufshcd_scsi_block_requests (hba);/* Drain ufshcd_queuecommand() */ -down_write (&hba->clk_scaling_lock); -up_write (&hba->clk_scaling_lock); +synchronize_rcu ();cancel_work_sync (&hba->eeh_work); }
@@ -6582,6 +6539,8 @@ static int __ufshcd_issue_tm_cmd(struct ufs_hba *hba, req->end_io_data = &wait;ufshcd_hold (hba, false); +rcu_read_lock (); +spin_lock_irqsave (host->host_lock, flags); task_tag = req->tag;
@@ -6600,6 +6559,8 @@ static int __ufshcd_issue_tm_cmd(struct ufs_hba *hba,spin_unlock_irqrestore (host->host_lock, flags); +rcu_read_unlock (); +ufshcd_add_tm_upiu_trace (hba, task_tag,UFS_TM_SEND );/* wait until the task management command is completed */
drivers/scsi/ufs/ufshcd.h +1 -0
@@ -778,6 +778,7 @@ struct ufs_hba_monitor {* @clk_list_head: UFS host controller clocks list node head * @pwr_info: holds current power mode * @max_pwr_info: keeps the device max valid pwm +* @clk_scaling_lock: used to serialize device commands and clock scaling * @desc_size: descriptor sizes reported by device * @urgent_bkops_lvl: keeps track of urgent bkops level for device * @is_urgent_bkops_lvl_checked: keeps track if the urgent bkops level for
Re: [PATCH 10/11] scsi: ufs: Optimize the command queueing code
Quoted 14 lines · Bart Van Assche
Remove the clock scaling lock from
ufshcd_queuecommand()since it is a performance bottleneck. As requested by Asutosh Das, change the behavior ofufshcd_clock_scaling_prepare()from waiting until all pending commands have finished into quiescing request queues. Insert arcu_read_lock()/rcu_read_unlock()pair inufshcd_queuecommand()and also in__ufshcd_issue_tm_cmd(). Usesynchronize_rcu_expedited()to wait for ongoing command and TMF queueing.Signed-off-by Bart Van Assche <[email protected]>--- drivers/scsi/ufs/ufshcd.c | 121 +++++++++++++------------------------- drivers/scsi/ufs/ufshcd.h | 1 + 2 files changed, 42 insertions(+), 80 deletions(-)
<SNIP>
drivers/scsi/ufs/ufshcd.c static int ufshcd_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *cmd) · line 2660
Quoted 11 lines
drivers/scsi/ufs/ufshcd.c +5
-2@@ -2698,8 +2653,11 @@ static int ufshcd_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *cmd)WARN_ONCE (tag <0 ,"Invalid tag %d\n" , tag); -if (!down_read_trylock (&hba->clk_scaling_lock)) -return SCSI_MLQUEUE_HOST_BUSY ; +/* +* Allows ufshcd_clock_scaling_prepare() and also the UFS error handler +* to wait for prior ufshcd_queuecommand() calls. +*/ +rcu_read_lock ();
The improvement to flush/drain ufshcd_queuecommand() via RCU should be a separate patch because it is not dependent on the other changes.
<SNIP>
Re: [PATCH 10/11] scsi: ufs: Optimize the command queueing code
Quoted 5 lines · Adrian Hunter
Quoted 4 lines · Bart Van Assche
drivers/scsi/ufs/ufshcd.c +0
-0@@ -2698,8 +2653,11 @@ static int ufshcd_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *cmd)WARN_ONCE (tag <0 ,"Invalid tag %d\n" , tag);
Quoted 10 lines
drivers/scsi/ufs/ufshcd.c static int ufshcd_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *cmd) · line 2660
Quoted 7 lines
drivers/scsi/ufs/ufshcd.c +5
-2-if (!down_read_trylock (&hba->clk_scaling_lock)) -return SCSI_MLQUEUE_HOST_BUSY ; +/* +* Allows ufshcd_clock_scaling_prepare() and also the UFS error handler +* to wait for prior ufshcd_queuecommand() calls. +*/ +rcu_read_lock ();The improvement to flush/drain
ufshcd_queuecommand()via RCU should be a separate patch because it is not dependent on the other changes.
I will split this patch into two patches.
Thanks,
Bart.
RE: [PATCH 10/11] scsi: ufs: Optimize the command queueing code
In reply to Adrian HunterQuoted 23 lines · Bart Van Assche
Quoted 14 lines
Remove the clock scaling lock from
ufshcd_queuecommand()since it is a performance bottleneck. As requested by Asutosh Das, change the behavior ofufshcd_clock_scaling_prepare()from waiting until all pending commands have finished into quiescing request queues. Insert arcu_read_lock()/rcu_read_unlock()pair inufshcd_queuecommand()and also in__ufshcd_issue_tm_cmd(). Usesynchronize_rcu_expedited()to wait for ongoing command and TMF queueing.Signed-off-by Bart Van Assche <[email protected]>--- drivers/scsi/ufs/ufshcd.c | 121 +++++++++++++------------------------- drivers/scsi/ufs/ufshcd.h | 1 + 2 files changed, 42 insertions(+), 80 deletions(-)<SNIP>
Quoted 5 lines
drivers/scsi/ufs/ufshcd.c +0
-0@@ -2698,8 +2653,11 @@ static int ufshcd_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *cmd)WARN_ONCE (tag <0 ,"Invalid tag %d\n" , tag);
Quoted 10 lines
drivers/scsi/ufs/ufshcd.c static int ufshcd_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *cmd) · line 2660
Quoted 7 lines
drivers/scsi/ufs/ufshcd.c +5
-2-if (!down_read_trylock (&hba->clk_scaling_lock)) -return SCSI_MLQUEUE_HOST_BUSY ; +/* +* Allows ufshcd_clock_scaling_prepare() and also the UFS error handler +* to wait for prior ufshcd_queuecommand() calls. +*/ +rcu_read_lock ();The improvement to flush/drain
ufshcd_queuecommand()via RCU should be a separate patch because it is not dependent on the other changes.
Ack on that.
Thanks,
Avri
Quoted 2 lines
<SNIP>
Re: [PATCH 10/11] scsi: ufs: Optimize the command queueing code
In reply to Bart Van AsscheHi Bart,
Quoted 3 lines · Bart Van Assche
Remove the clock scaling lock from
ufshcd_queuecommand()since it is a
performance bottleneck. As requested by Asutosh Das, change the behavior
ofufshcd_clock_scaling_prepare()from waiting until all pending > commands have finished into quiescing request queues. Insert a
Umm, I was suggesting the following in prepare():
* The requests in the queue should not be issued - as is done now by ufshcd_scsi_block_requests()
* The ongoing requests in DBR should be completed i.e. DBR = 0.
* Proceed with scaling
The below code is not waiting for the ongoing requests to complete i.e. There's no call to wait for DBR to be 0. I think waiting for DBR to be 0 is still needed.
Quoted 212 lines
rcu_read_lock()/rcu_read_unlock()pair inufshcd_queuecommand()and also in__ufshcd_issue_tm_cmd(). Usesynchronize_rcu_expedited()to wait for ongoing command and TMF queueing.Signed-off-by Bart Van Assche <[email protected]>--- drivers/scsi/ufs/ufshcd.c | 121 +++++++++++++------------------------- drivers/scsi/ufs/ufshcd.h | 1 + 2 files changed, 42 insertions(+), 80 deletions(-)drivers/scsi/ufs/ufshcd.c +41
-80@@ -1069,65 +1069,6 @@ static bool ufshcd_is_devfreq_scaling_required(struct ufs_hba *hba,return false; } -static int ufshcd_wait_for_doorbell_clr (struct ufs_hba *hba, -u64 wait_timeout_us) -{ -unsigned long flags; -int ret =0 ; -u32 tm_doorbell; -u32 tr_doorbell; -bool timeout = false, do_last_check = false; -ktime_t start; - -ufshcd_hold (hba, false); -spin_lock_irqsave (hba->host->host_lock, flags); -/* -* Wait for all the outstanding tasks/transfer requests. -* Verify by checking the doorbell registers are clear. -*/ - start =ktime_get (); -do { -if (hba->ufshcd_state !=UFSHCD_STATE_OPERATIONAL ) { - ret = -EBUSY ; - goto out; - } - - tm_doorbell =ufshcd_readl (hba,REG_UTP_TASK_REQ_DOOR_BELL ); - tr_doorbell =ufshcd_readl (hba,REG_UTP_TRANSFER_REQ_DOOR_BELL ); -if (!tm_doorbell && !tr_doorbell) { - timeout = false; -break ; - }else if (do_last_check) { -break ; - } - -spin_unlock_irqrestore (hba->host->host_lock, flags); -schedule (); -if (ktime_to_us (ktime_sub (ktime_get (), start)) > - wait_timeout_us) { - timeout = true; -/* -* We might have scheduled out for long time so make -* sure to check if doorbells are cleared by this time -* or not. -*/ - do_last_check = true; - } -spin_lock_irqsave (hba->host->host_lock, flags); - }while (tm_doorbell || tr_doorbell); - -if (timeout) { -dev_err (hba->dev, -"%s: timedout waiting for doorbell to clear (tm=0x%x, tr=0x%x)\n" , - __func__, tm_doorbell, tr_doorbell); - ret = -EBUSY ; - } -out: -spin_unlock_irqrestore (hba->host->host_lock, flags); -ufshcd_release (hba); -return ret; -} -/** * ufshcd_scale_gear - scale up/down UFS gear * @hba: per adapter instance @@ -1175,37 +1116,51 @@ static int ufshcd_scale_gear(struct ufs_hba *hba, bool scale_up)static int ufshcd_clock_scaling_prepare (struct ufs_hba *hba) { -#define DOORBELL_CLR_TOUT_US (1000 * 1000)/* 1 sec */ -int ret =0 ; +struct scsi_device *sdev; +/* -* make sure that there are no outstanding requests when -* clock scaling is in progress +* Make sure that no commands are being queued while clock scaling +* is in progress. +* +* Since ufshcd_exec_dev_cmd() and ufshcd_issue_devman_upiu_cmd() lock +* the clk_scaling_lock before calling blk_get_request(), lock +* clk_scaling_lock before freezing the request queues to prevent lock +* inversion. */ -ufshcd_scsi_block_requests (hba);down_write (&hba->clk_scaling_lock); - -if (!hba->clk_scaling.is_allowed || -ufshcd_wait_for_doorbell_clr (hba,DOORBELL_CLR_TOUT_US )) { - ret = -EBUSY ; +if (!hba->clk_scaling.is_allowed) {up_write (&hba->clk_scaling_lock); -ufshcd_scsi_unblock_requests (hba); - goto out; +return -EBUSY ; } - +blk_mq_quiesce_queue_nowait (hba->tmf_queue); +blk_mq_quiesce_queue_nowait (hba->cmd_queue); +shost_for_each_device (sdev, hba->host) +blk_mq_quiesce_queue_nowait (sdev->request_queue); +/* +* Calling synchronize_rcu_expedited() reduces the time required to +* quiesce request queues from milliseconds to microseconds. +* +* See also the rcu_read_lock() and rcu_read_unlock() calls in +* ufshcd_queuecommand() and also in __ufshcd_issue_tm_cmd(). +*/ +synchronize_rcu_expedited ();/* let's not get into low power until clock scaling is completed */ ufshcd_hold (hba, false); - -out: -return ret; +return 0 ; }static void ufshcd_clock_scaling_unprepare (struct ufs_hba *hba,bool writelock) { +struct scsi_device *sdev; + +shost_for_each_device (sdev, hba->host) +blk_mq_unquiesce_queue (sdev->request_queue); +blk_mq_unquiesce_queue (hba->cmd_queue); +blk_mq_unquiesce_queue (hba->tmf_queue);if (writelock)up_write (&hba->clk_scaling_lock);else up_read (&hba->clk_scaling_lock); -ufshcd_scsi_unblock_requests (hba);ufshcd_release (hba); }@@ -2698,8 +2653,11 @@ static int ufshcd_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *cmd)WARN_ONCE (tag <0 ,"Invalid tag %d\n" , tag); -if (!down_read_trylock (&hba->clk_scaling_lock)) -return SCSI_MLQUEUE_HOST_BUSY ; +/* +* Allows ufshcd_clock_scaling_prepare() and also the UFS error handler +* to wait for prior ufshcd_queuecommand() calls. +*/ +rcu_read_lock ();switch (hba->ufshcd_state) {case UFSHCD_STATE_OPERATIONAL :@@ -2780,7 +2738,7 @@ static int ufshcd_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *cmd)ufshcd_send_command (hba, tag); out: -up_read (&hba->clk_scaling_lock); +rcu_read_unlock ();if (ufs_trigger_eh ()) {unsigned long flags;@@ -5977,8 +5935,7 @@ static void ufshcd_err_handling_prepare(struct ufs_hba *hba) }ufshcd_scsi_block_requests (hba);/* Drain ufshcd_queuecommand() */ -down_write (&hba->clk_scaling_lock); -up_write (&hba->clk_scaling_lock); +synchronize_rcu ();cancel_work_sync (&hba->eeh_work); }@@ -6582,6 +6539,8 @@ static int __ufshcd_issue_tm_cmd(struct ufs_hba *hba, req->end_io_data = &wait;ufshcd_hold (hba, false); +rcu_read_lock (); +spin_lock_irqsave (host->host_lock, flags); task_tag = req->tag;@@ -6600,6 +6559,8 @@ static int __ufshcd_issue_tm_cmd(struct ufs_hba *hba,spin_unlock_irqrestore (host->host_lock, flags); +rcu_read_unlock (); +ufshcd_add_tm_upiu_trace (hba, task_tag,UFS_TM_SEND );/* wait until the task management command is completed */ drivers/scsi/ufs/ufshcd.h +1
-0@@ -778,6 +778,7 @@ struct ufs_hba_monitor {* @clk_list_head: UFS host controller clocks list node head * @pwr_info: holds current power mode * @max_pwr_info: keeps the device max valid pwm +* @clk_scaling_lock: used to serialize device commands and clock scaling * @desc_size: descriptor sizes reported by device * @urgent_bkops_lvl: keeps track of urgent bkops level for device * @is_urgent_bkops_lvl_checked: keeps track if the urgent bkops level for
Signature · 2 lines
The Qualcomm Innovation Center, Inc. is a member of the Code Aurora Forum, Linux Foundation Collaborative Project
PATCH 11/11 scsi: ufs: Implement polling support
In reply to Bart Van AsscheThe time spent in io_schedule() is significant when submitting direct I/O to a UFS device. Hence this patch that implements polling support. User space software can enable polling by passing the RWF_HIPRI flag to the preadv2() system call or the IORING_SETUP_IOPOLL flag to the io_uring interface.
--- drivers/scsi/ufs/ufshcd.c | 45 +++++++++++++++++++++++---------------- 1 file changed, 27 insertions(+), 18 deletions(-)
drivers/scsi/ufs/ufshcd.c +27 -18
@@ -5250,6 +5250,31 @@ static void __ufshcd_transfer_req_compl(struct ufs_hba *hba, } } +/* +* Returns > 0 if one or more commands have been completed or 0 if no +* requests have been completed. +*/ +static int ufshcd_poll (struct Scsi_Host *shost,unsigned int queue_num) +{ +struct ufs_hba *hba =shost_priv (shost); +unsigned long completed_reqs, flags; +u32 tr_doorbell; + +spin_lock_irqsave (&hba->outstanding_lock, flags); + tr_doorbell =ufshcd_readl (hba,REG_UTP_TRANSFER_REQ_DOOR_BELL ); + completed_reqs = ~tr_doorbell & hba->outstanding_reqs; +WARN_ONCE (completed_reqs & ~hba->outstanding_reqs, +"completed: %#lx; outstanding: %#lx\n" , completed_reqs, + hba->outstanding_reqs); + hba->outstanding_reqs &= ~completed_reqs; +spin_unlock_irqrestore (&hba->outstanding_lock, flags); + +if (completed_reqs) +__ufshcd_transfer_req_compl (hba, completed_reqs); + +return completed_reqs; +} +/** * ufshcd_transfer_req_compl - handle SCSI and query command completion * @hba: per adapter instance
@@ -5260,9 +5285,6 @@ static void __ufshcd_transfer_req_compl(struct ufs_hba *hba,*/ static irqreturn_t ufshcd_transfer_req_compl (struct ufs_hba *hba) { -unsigned long completed_reqs, flags; -u32 tr_doorbell; -/* Resetting interrupt aggregation counters first and reading the * DOOR_BELL afterward allows us to handle all the completed requests. * In order to prevent other interrupts starvation the DB is read once
@@ -5277,21 +5299,7 @@ static irqreturn_t ufshcd_transfer_req_compl(struct ufs_hba *hba)if (ufs_fail_completion ())return IRQ_HANDLED ; -spin_lock_irqsave (&hba->outstanding_lock, flags); - tr_doorbell =ufshcd_readl (hba,REG_UTP_TRANSFER_REQ_DOOR_BELL ); - completed_reqs = ~tr_doorbell & hba->outstanding_reqs; -WARN_ONCE (completed_reqs & ~hba->outstanding_reqs, -"completed: %#lx; outstanding: %#lx\n" , completed_reqs, - hba->outstanding_reqs); - hba->outstanding_reqs &= ~completed_reqs; -spin_unlock_irqrestore (&hba->outstanding_lock, flags); - -if (completed_reqs) { -__ufshcd_transfer_req_compl (hba, completed_reqs); -return IRQ_HANDLED ; - }else { -return IRQ_NONE ; - } +return ufshcd_poll (hba->host,0 ) ?IRQ_HANDLED :IRQ_NONE ; }int __ufshcd_write_ee_control (struct ufs_hba *hba,u32 ee_ctrl_mask)
@@ -8112,6 +8120,7 @@ static struct scsi_host_template ufshcd_driver_template = { .name = UFSHCD, .proc_name = UFSHCD, .queuecommand = ufshcd_queuecommand, + .mq_poll = ufshcd_poll, .slave_alloc = ufshcd_slave_alloc, .slave_configure = ufshcd_slave_configure, .slave_destroy = ufshcd_slave_destroy,
Re: [PATCH 11/11] scsi: ufs: Implement polling support
Quoted 5 lines · Bart Van Assche
The time spent in
io_schedule()is significant when submitting direct I/O to a UFS device. Hence this patch that implements polling support. User space software can enable polling by passing theRWF_HIPRIflag to thepreadv2()system call or theIORING_SETUP_IOPOLLflag to theio_uringinterface.
There have been some changes recently (i.e. in linux-stable now), "HIPRI" seems to be on the out, replaced by "POLLED". [I'm using poll_lld in my sg rewrite to refer to this type of polling, as "poll" is an overloaded term in the kernel].
REQ_HIPRI has become REQ_POLLED and blk_poll() is now bio_poll(). That said RWF_HIPRI is still in fs.h and there is no RWF_POLLED (yet).
LL_POLL or LOW_POLL would be more indicative of what is happening, rather than POLLED, IMO.
Not sure if the new bio_poll() code is working, I'm looking at an NULL pointer dereference at: RIP: 0010:bio_poll+0x17/0xe0
Quoted 83 lines
Signed-off-by Bart Van Assche <[email protected]>--- drivers/scsi/ufs/ufshcd.c | 45 +++++++++++++++++++++++---------------- 1 file changed, 27 insertions(+), 18 deletions(-)drivers/scsi/ufs/ufshcd.c +27
-18@@ -5250,6 +5250,31 @@ static void __ufshcd_transfer_req_compl(struct ufs_hba *hba, } } +/* +* Returns > 0 if one or more commands have been completed or 0 if no +* requests have been completed. +*/ +static int ufshcd_poll (struct Scsi_Host *shost,unsigned int queue_num) +{ +struct ufs_hba *hba =shost_priv (shost); +unsigned long completed_reqs, flags; +u32 tr_doorbell; + +spin_lock_irqsave (&hba->outstanding_lock, flags); + tr_doorbell =ufshcd_readl (hba,REG_UTP_TRANSFER_REQ_DOOR_BELL ); + completed_reqs = ~tr_doorbell & hba->outstanding_reqs; +WARN_ONCE (completed_reqs & ~hba->outstanding_reqs, +"completed: %#lx; outstanding: %#lx\n" , completed_reqs, + hba->outstanding_reqs); + hba->outstanding_reqs &= ~completed_reqs; +spin_unlock_irqrestore (&hba->outstanding_lock, flags); + +if (completed_reqs) +__ufshcd_transfer_req_compl (hba, completed_reqs); + +return completed_reqs; +} +/** * ufshcd_transfer_req_compl - handle SCSI and query command completion * @hba: per adapter instance @@ -5260,9 +5285,6 @@ static void __ufshcd_transfer_req_compl(struct ufs_hba *hba,*/ static irqreturn_t ufshcd_transfer_req_compl (struct ufs_hba *hba) { -unsigned long completed_reqs, flags; -u32 tr_doorbell; -/* Resetting interrupt aggregation counters first and reading the * DOOR_BELL afterward allows us to handle all the completed requests. * In order to prevent other interrupts starvation the DB is read once @@ -5277,21 +5299,7 @@ static irqreturn_t ufshcd_transfer_req_compl(struct ufs_hba *hba)if (ufs_fail_completion ())return IRQ_HANDLED ; -spin_lock_irqsave (&hba->outstanding_lock, flags); - tr_doorbell =ufshcd_readl (hba,REG_UTP_TRANSFER_REQ_DOOR_BELL ); - completed_reqs = ~tr_doorbell & hba->outstanding_reqs; -WARN_ONCE (completed_reqs & ~hba->outstanding_reqs, -"completed: %#lx; outstanding: %#lx\n" , completed_reqs, - hba->outstanding_reqs); - hba->outstanding_reqs &= ~completed_reqs; -spin_unlock_irqrestore (&hba->outstanding_lock, flags); - -if (completed_reqs) { -__ufshcd_transfer_req_compl (hba, completed_reqs); -return IRQ_HANDLED ; - }else { -return IRQ_NONE ; - } +return ufshcd_poll (hba->host,0 ) ?IRQ_HANDLED :IRQ_NONE ; }int __ufshcd_write_ee_control (struct ufs_hba *hba,u32 ee_ctrl_mask)@@ -8112,6 +8120,7 @@ static struct scsi_host_template ufshcd_driver_template = { .name = UFSHCD, .proc_name = UFSHCD, .queuecommand = ufshcd_queuecommand, + .mq_poll = ufshcd_poll, .slave_alloc = ufshcd_slave_alloc, .slave_configure = ufshcd_slave_configure, .slave_destroy = ufshcd_slave_destroy,
Re: [PATCH 11/11] scsi: ufs: Implement polling support
Quoted 7 lines · Douglas Gilbert
Quoted 5 lines · Bart Van Assche
The time spent in
io_schedule()is significant when submitting direct I/O to a UFS device. Hence this patch that implements polling support. User space software can enable polling by passing theRWF_HIPRIflag to thepreadv2()system call or theIORING_SETUP_IOPOLLflag to theio_uringinterface.
Quoted 7 lines
There have been some changes recently (i.e. in linux-stable now), "HIPRI" seems to be on the out, replaced by "POLLED". [I'm using
poll_lldin my sg rewrite to refer to this type of polling, as "poll" is an overloaded term in the kernel].
REQ_HIPRIhas becomeREQ_POLLEDandblk_poll()is nowbio_poll(). That saidRWF_HIPRIis still infs.hand there is noRWF_POLLED(yet).
Hi Doug,
My reference to RWF_HIPRI in the patch description refers to the flag defined in the uapi headers. As far as I know that flag is still there:
$ PAGER= git grep define.RWF_HIPRI include/uapi include/uapi/linux/fs.h:#define RWF_HIPRI ((__force __kernel_rwf_t)0x00000001)
Thanks,
Bart.
RE: [PATCH 11/11] scsi: ufs: Implement polling support
In reply to Bart Van AsscheQuoted 6 lines
The time spent in
io_schedule()is significant when submitting direct I/O to a UFS device. Hence this patch that implements polling support. User space software can enable polling by passing theRWF_HIPRIflag to thepreadv2()system call or theIORING_SETUP_IOPOLLflag to theio_uringinterface.
Do you expect that this will fix, or at least to some extent, the queue drainage that is so noticeable in direct benchmarks? Can you share some early estimations?
Also I think this should be a separate patch as well.
Quoted 79 lines
Signed-off-by Bart Van Assche <[email protected]>--- drivers/scsi/ufs/ufshcd.c | 45 +++++++++++++++++++++++---------------- 1 file changed, 27 insertions(+), 18 deletions(-)drivers/scsi/ufs/ufshcd.c index renamed from drivers/scsi/ufs/ufshcd.c +0
-0
36df89e8a575..70f128f12445100644drivers/scsi/ufs/ufshcd.c +25
-0@@ -5250,6 +5250,31 @@ static void __ufshcd_transfer_req_compl(struct ufs_hba *hba, } } +/* +* Returns > 0 if one or more commands have been completed or 0 if no +* requests have been completed. +*/ +static int ufshcd_poll (struct Scsi_Host *shost,unsigned int queue_num) +{ +struct ufs_hba *hba =shost_priv (shost); +unsigned long completed_reqs, flags; +u32 tr_doorbell; + +spin_lock_irqsave (&hba->outstanding_lock, flags); + tr_doorbell =ufshcd_readl (hba,REG_UTP_TRANSFER_REQ_DOOR_BELL ); + completed_reqs = ~tr_doorbell & hba->outstanding_reqs; +WARN_ONCE (completed_reqs & ~hba->outstanding_reqs, +"completed: %#lx; outstanding: %#lx\n" , completed_reqs, + hba->outstanding_reqs); + hba->outstanding_reqs &= ~completed_reqs; +spin_unlock_irqrestore (&hba->outstanding_lock, flags); + +if (completed_reqs) +__ufshcd_transfer_req_compl (hba, completed_reqs); + +return completed_reqs; +} +/** * ufshcd_transfer_req_compl - handle SCSI and query command completion * @hba: per adapter instance
drivers/scsi/ufs/ufshcd.c +0
-3@@ -5260,9 +5285,6 @@ static void __ufshcd_transfer_req_compl(structufs_hba *hba, */ static irqreturn_t ufshcd_transfer_req_compl (struct ufs_hba *hba) { -unsigned long completed_reqs, flags; -u32 tr_doorbell; -/* Resetting interrupt aggregation counters first and reading the * DOOR_BELL afterward allows us to handle all the completed requests. * In order to prevent other interrupts starvation the DB is read once @@ - 5277,21 +5299,7 @@ static
irqreturn_tufshcd_transfer_req_compl(struct ufs_hba *hba)if (ufs_fail_completion()) return IRQ_HANDLED;drivers/scsi/ufs/ufshcd.c +1
-15-spin_lock_irqsave (&hba->outstanding_lock, flags); - tr_doorbell =ufshcd_readl (hba,REG_UTP_TRANSFER_REQ_DOOR_BELL ); - completed_reqs = ~tr_doorbell & hba->outstanding_reqs; -WARN_ONCE (completed_reqs & ~hba->outstanding_reqs, -"completed: %#lx; outstanding: %#lx\n" , completed_reqs, - hba->outstanding_reqs); - hba->outstanding_reqs &= ~completed_reqs; -spin_unlock_irqrestore (&hba->outstanding_lock, flags); - -if (completed_reqs) { -__ufshcd_transfer_req_compl (hba, completed_reqs); -return IRQ_HANDLED ; - }else { -return IRQ_NONE ; - } +return ufshcd_poll (hba->host,0 ) ?IRQ_HANDLED :IRQ_NONE ; } int __ufshcd_write_ee_control(struct ufs_hba *hba, u32 ee_ctrl_mask) @@ -8112,6 +8120,7 @@ static struct
scsi_host_templateufshcd_driver_template= {
Quoted 4 lines
.name = UFSHCD,
.proc_name = UFSHCD,
.queuecommand = ufshcd_queuecommand,
+ .mq_poll = ufshcd_poll,Did you consider to use some form blk_mq_tagset_busy_iter, And return nutrs - busy?
Thanks,
Avri
Quoted 3 lines
.slave_alloc = ufshcd_slave_alloc,
.slave_configure = ufshcd_slave_configure,
.slave_destroy = ufshcd_slave_destroy,Re: [PATCH 11/11] scsi: ufs: Implement polling support
Quoted 1 line · Avri Altman
Also I think this should be a separate patch as well.
I see an 8x IOPS improvement on my test setup (2736 IOPS with interrupt based completions; 22000 IOPS when using polling).
Quoted 3 lines
Quoted 1 line
+ .mq_poll =
ufshcd_poll,Did you consider to use some form
blk_mq_tagset_busy_iter, And return nutrs - busy?
Hmm ... wouldn't it be racy to use blk_mq_tagset_busy_iter() inside ufshcd_poll() since multiple threads may be polling at the same time?
Thanks,
Bart.