Skip to contentSkip to conversation

aacraid: reply queue mapping to CPUs based of IRQ affinity

PATCH Superseded Sagar Biradar 3 participants 3 messages May 16, 2023 – May 17, 2023

Subsystems (3)

v3 of 4 Superseded Latest is v4 →

lore
Share & tools
lore
#0 lore raw

PATCH v3 1/1 aacraid: reply queue mapping to CPUs based of IRQ affinity

Fix the IO hang that arises because of MSIx vector not having a mapped online CPU upon receiving completion.

The SCSI cmds take the blk_mq route, which is setup during the init. The reserved cmds fetch the vector_no from mq_map after the init is complete and before the init, they use 0 - as per the norm.

---
 drivers/scsi/aacraid/aacraid.h  |  1 +
 drivers/scsi/aacraid/comminit.c |  2 +-
 drivers/scsi/aacraid/commsup.c  |  6 +++++-
 drivers/scsi/aacraid/linit.c    | 14 ++++++++++++++
 drivers/scsi/aacraid/src.c      | 25 +++++++++++++++++++++++--
 5 files changed, 44 insertions(+), 4 deletions(-)
drivers/scsi/aacraid/aacraid.h +1 -0
index 5e115e8b2ba4..7c6efde75da6 100644
@@ -1678,6 +1678,7 @@ struct aac_dev
 	u32			handle_pci_error;
 	bool			init_reset;
 	u8			soft_reset_support;
+	u8			use_map_queue;
 };
 
 #define aac_adapter_interrupt(dev) \
drivers/scsi/aacraid/comminit.c +1 -1
index bd99c5492b7d..53924912417e 100644
@@ -630,6 +630,7 @@ struct aac_dev *aac_init_adapter(struct aac_dev *dev)
 
 	if (aac_is_src(dev))
 		aac_define_int_mode(dev);
+
 	/*
 	 *	Ok now init the communication subsystem
 	 */
@@ -657,4 +658,3 @@ struct aac_dev *aac_init_adapter(struct aac_dev *dev)
 
 	return dev;
 }
-
drivers/scsi/aacraid/commsup.c +5 -1
index deb32c9f4b3e..3f062e4013ab 100644
@@ -223,8 +223,12 @@ int aac_fib_setup(struct aac_dev * dev)
 struct fib *aac_fib_alloc_tag(struct aac_dev *dev, struct scsi_cmnd *scmd)
 {
 	struct fib *fibptr;
+	u32 blk_tag;
+	int i;
 
-	fibptr = &dev->fibs[scsi_cmd_to_rq(scmd)->tag];
+	blk_tag = blk_mq_unique_tag(scsi_cmd_to_rq(scmd));
+	i = blk_mq_unique_tag_to_tag(blk_tag);
+	fibptr = &dev->fibs[i];
 	/*
 	 *	Null out fields that depend on being zero at the start of
 	 *	each I/O
drivers/scsi/aacraid/linit.c +14 -0
index 5ba5c18b77b4..fa53a9b3341b 100644
@@ -34,6 +34,7 @@
 #include <linux/delay.h>
 #include <linux/kthread.h>
 #include <linux/msdos_partition.h>
+#include <linux/blk-mq-pci.h>
 
 #include <scsi/scsi.h>
 #include <scsi/scsi_cmnd.h>
@@ -505,6 +506,15 @@ static int aac_slave_configure(struct scsi_device *sdev)
 	return 0;
 }
 
+static void aac_map_queues(struct Scsi_Host *shost)
+{
+	struct aac_dev *aac = (struct aac_dev *)shost->hostdata;
+
+	blk_mq_pci_map_queues(&shost->tag_set.map[HCTX_TYPE_DEFAULT],
+				aac->pdev, 0);
+	aac->use_map_queue = true;
+}
+
 /**
  *	aac_change_queue_depth		-	alter queue depths
  *	@sdev:	SCSI device we are considering
@@ -1489,6 +1499,7 @@ static struct scsi_host_template aac_driver_template = {
 	.bios_param			= aac_biosparm,
 	.shost_groups			= aac_host_groups,
 	.slave_configure		= aac_slave_configure,
+	.map_queues			= aac_map_queues,
 	.change_queue_depth		= aac_change_queue_depth,
 	.sdev_groups			= aac_dev_groups,
 	.eh_abort_handler		= aac_eh_abort,
@@ -1776,6 +1787,8 @@ static int aac_probe_one(struct pci_dev *pdev, const struct pci_device_id *id)
 	shost->max_lun = AAC_MAX_LUN;
 
 	pci_set_drvdata(pdev, shost);
+	shost->nr_hw_queues = aac->max_msix;
+	shost->host_tagset = 1;
 
 	error = scsi_add_host(shost, &pdev->dev);
 	if (error)
@@ -1908,6 +1921,7 @@ static void aac_remove_one(struct pci_dev *pdev)
 	struct aac_dev *aac = (struct aac_dev *)shost->hostdata;
 
 	aac_cancel_rescan_worker(aac);
+	aac->use_map_queue = false;
 	scsi_remove_host(shost);
 
 	__aac_shutdown(aac);
drivers/scsi/aacraid/src.c +23 -2
index 11ef58204e96..61949f374188 100644
@@ -493,6 +493,10 @@ static int aac_src_deliver_message(struct fib *fib)
 #endif
 
 	u16 vector_no;
+	struct scsi_cmnd *scmd;
+	u32 blk_tag;
+	struct Scsi_Host *shost = dev->scsi_host_ptr;
+	struct blk_mq_queue_map *qmap;
 
 	atomic_inc(&q->numpending);
 
@@ -505,8 +509,25 @@ static int aac_src_deliver_message(struct fib *fib)
 		if ((dev->comm_interface == AAC_COMM_MESSAGE_TYPE3)
 			&& dev->sa_firmware)
 			vector_no = aac_get_vector(dev);
-		else
-			vector_no = fib->vector_no;
+		else {
+			if (!fib->vector_no || !fib->callback_data) {
+				if (shost && dev->use_map_queue) {
+					qmap = &shost->tag_set.map[HCTX_TYPE_DEFAULT];
+					vector_no = qmap->mq_map[raw_smp_processor_id()];
+				}
+				/*
+				 *	We hardcode the vector_no for
+				 *	reserved commands as a valid shost is
+				 *	absent during the init
+				 */
+				else
+					vector_no = 0;
+			} else {
+				scmd = (struct scsi_cmnd *)fib->callback_data;
+				blk_tag = blk_mq_unique_tag(scsi_cmd_to_rq(scmd));
+				vector_no = blk_mq_unique_tag_to_hwq(blk_tag);
+			}
+		}
 
 		if (native_hba) {
 			if (fib->flags & FIB_CONTEXT_FLAG_NATIVE_HBA_TMF) {
Signature · 1 line
2.29.0
#1 lore raw
FYI: kernel test robot notices the stable kernel rule is not satisfied.

Hi,

Thanks for your patch.

FYI: kernel test robot notices the stable kernel rule is not satisfied.

Rule: 'Cc: [email protected]' or 'commit <sha1> upstream.'
Subject: [PATCH v3] aacraid: reply queue mapping to CPUs based of IRQ affinity

The check is based on https://www.kernel.org/doc/html/latest/process/stable-kernel-rules.html

Signature · 2 lines
0-DAY CI Kernel Test Service
https://github.com/intel/lkp-tests
Reviewed-by on v3 1/1
#2 lore raw
In reply to Sagar Biradar
Quoted 9 lines · Sagar Biradar

Fix the IO hang that arises because of MSIx vector not having a mapped online CPU upon receiving completion.

The SCSI cmds take the blk_mq route, which is setup during the init. The reserved cmds fetch the vector_no from mq_map after the init is complete and before the init, they use 0 - as per the norm.

this looks ok apart from minor issues, below, so fwiw:

Quoted 25 lines
---
  drivers/scsi/aacraid/aacraid.h  |  1 +
  drivers/scsi/aacraid/comminit.c |  2 +-
  drivers/scsi/aacraid/commsup.c  |  6 +++++-
  drivers/scsi/aacraid/linit.c    | 14 ++++++++++++++
  drivers/scsi/aacraid/src.c      | 25 +++++++++++++++++++++++--
  5 files changed, 44 insertions(+), 4 deletions(-)
drivers/scsi/aacraid/aacraid.h +1 -0
index 5e115e8b2ba4..7c6efde75da6 100644
@@ -1678,6 +1678,7 @@ struct aac_dev
  	u32			handle_pci_error;
  	bool			init_reset;
  	u8			soft_reset_support;
+	u8			use_map_queue;
  };
 
  #define aac_adapter_interrupt(dev) \
drivers/scsi/aacraid/comminit.c +0 -0
index bd99c5492b7d..53924912417e 100644

drivers/scsi/aacraid/comminit.c struct aac_dev *aac_init_adapter(struct aac_dev *dev) · line 633

Quoted 4 lines
drivers/scsi/aacraid/comminit.c +1 -0
@@ -630,6 +630,7 @@ struct aac_dev *aac_init_adapter(struct aac_dev *dev)
 
  	if (aac_is_src(dev))
  		aac_define_int_mode(dev);
+

stray new line

Quoted 31 lines
drivers/scsi/aacraid/comminit.c +0 -1
  	/*
  	 *	Ok now init the communication subsystem
  	 */
@@ -657,4 +658,3 @@ struct aac_dev *aac_init_adapter(struct aac_dev *dev)
 
  	return dev;
  }
-
drivers/scsi/aacraid/commsup.c +5 -1
index deb32c9f4b3e..3f062e4013ab 100644
@@ -223,8 +223,12 @@ int aac_fib_setup(struct aac_dev * dev)
  struct fib *aac_fib_alloc_tag(struct aac_dev *dev, struct scsi_cmnd *scmd)
  {
  	struct fib *fibptr;
+	u32 blk_tag;
+	int i;
 
-	fibptr = &dev->fibs[scsi_cmd_to_rq(scmd)->tag];
+	blk_tag = blk_mq_unique_tag(scsi_cmd_to_rq(scmd));
+	i = blk_mq_unique_tag_to_tag(blk_tag);
+	fibptr = &dev->fibs[i];
  	/*
  	 *	Null out fields that depend on being zero at the start of
  	 *	each I/O
drivers/scsi/aacraid/linit.c +0 -0
index 5ba5c18b77b4..fa53a9b3341b 100644

drivers/scsi/aacraid/linit.c line 37

Quoted 4 lines
drivers/scsi/aacraid/linit.c +1 -0
@@ -34,6 +34,7 @@
  #include <linux/delay.h>
  #include <linux/kthread.h>
  #include <linux/msdos_partition.h>
+#include <linux/blk-mq-pci.h>

generally alphabetic ordering is preferred

drivers/scsi/aacraid/linit.c static int aac_slave_configure(struct scsi_device *sdev) · line 511

Quoted 10 lines
drivers/scsi/aacraid/linit.c +3 -0
  #include <scsi/scsi.h>
  #include <scsi/scsi_cmnd.h>
@@ -505,6 +506,15 @@ static int aac_slave_configure(struct scsi_device *sdev)
  	return 0;
  }
 
+static void aac_map_queues(struct Scsi_Host *shost)
+{
+	struct aac_dev *aac = (struct aac_dev *)shost->hostdata;

this pattern happens a lot in the driver, so I suggest a separate change for a helper to do this

Quoted 55 lines
drivers/scsi/aacraid/linit.c +10 -0
+
+	blk_mq_pci_map_queues(&shost->tag_set.map[HCTX_TYPE_DEFAULT],
+				aac->pdev, 0);
+	aac->use_map_queue = true;
+}
+
  /**
   *	aac_change_queue_depth		-	alter queue depths
   *	@sdev:	SCSI device we are considering
@@ -1489,6 +1499,7 @@ static struct scsi_host_template aac_driver_template = {
  	.bios_param			= aac_biosparm,
  	.shost_groups			= aac_host_groups,
  	.slave_configure		= aac_slave_configure,
+	.map_queues			= aac_map_queues,
  	.change_queue_depth		= aac_change_queue_depth,
  	.sdev_groups			= aac_dev_groups,
  	.eh_abort_handler		= aac_eh_abort,
@@ -1776,6 +1787,8 @@ static int aac_probe_one(struct pci_dev *pdev, const struct pci_device_id *id)
  	shost->max_lun = AAC_MAX_LUN;
 
  	pci_set_drvdata(pdev, shost);
+	shost->nr_hw_queues = aac->max_msix;
+	shost->host_tagset = 1;
 
  	error = scsi_add_host(shost, &pdev->dev);
  	if (error)
@@ -1908,6 +1921,7 @@ static void aac_remove_one(struct pci_dev *pdev)
  	struct aac_dev *aac = (struct aac_dev *)shost->hostdata;
 
  	aac_cancel_rescan_worker(aac);
+	aac->use_map_queue = false;
  	scsi_remove_host(shost);
 
  	__aac_shutdown(aac);
drivers/scsi/aacraid/src.c +4 -2
index 11ef58204e96..61949f374188 100644
@@ -493,6 +493,10 @@ static int aac_src_deliver_message(struct fib *fib)
  #endif
 
  	u16 vector_no;
+	struct scsi_cmnd *scmd;
+	u32 blk_tag;
+	struct Scsi_Host *shost = dev->scsi_host_ptr;
+	struct blk_mq_queue_map *qmap;
 
  	atomic_inc(&q->numpending);
 
@@ -505,8 +509,25 @@ static int aac_src_deliver_message(struct fib *fib)
  		if ((dev->comm_interface == AAC_COMM_MESSAGE_TYPE3)
  			&& dev->sa_firmware)
  			vector_no = aac_get_vector(dev);
-		else
-			vector_no = fib->vector_no;

drivers/scsi/aacraid/src.c static int aac_src_deliver_message(struct fib *fib) · line 521

Quoted 10 lines
drivers/scsi/aacraid/src.c +10 -0
+		else {
+			if (!fib->vector_no || !fib->callback_data) {
+				if (shost && dev->use_map_queue) {
+					qmap = &shost->tag_set.map[HCTX_TYPE_DEFAULT];
+					vector_no = qmap->mq_map[raw_smp_processor_id()];
+				}
+				/*
+				 *	We hardcode the vector_no for
+				 *	reserved commands as a valid shost is
+				 *	absent during the init

you prob have another way to check this without the need for dev->use_map_queue - that's just a guess

Quoted 12 lines
drivers/scsi/aacraid/src.c +9 -0
+				 */
+				else
+					vector_no = 0;
+			} else {
+				scmd = (struct scsi_cmnd *)fib->callback_data;
+				blk_tag = blk_mq_unique_tag(scsi_cmd_to_rq(scmd));
+				vector_no = blk_mq_unique_tag_to_hwq(blk_tag);
+			}
+		}
 
  		if (native_hba) {
  			if (fib->flags & FIB_CONTEXT_FLAG_NATIVE_HBA_TMF) {