Skip to contentSkip to conversation

AHCI: Optimize interrupt processing in multi-MSI mode

PATCH In review Alexander Gordeev 2 participants 9 messages Sep 29, 2014 – Oct 6, 2014

Subsystems (2)
lore
Share & tools
lore
#2 lore raw

PATCH v5 2/5 AHCI: Move ahci_host_activate() function to libahci.c

In reply to Alexander Gordeev

This update is a prerequisite for consolidation of AHCI host activation code within ahci_host_activate() function.

---
 drivers/ata/ahci.c    | 60 --------------------------------------------------
 drivers/ata/libahci.c | 61 +++++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 61 insertions(+), 60 deletions(-)
drivers/ata/ahci.c +0 -60
index fcda5b6..0b21604 100644
@@ -1226,66 +1226,6 @@ intx:
 	return 0;
 }
 
-/**
- *	ahci_host_activate - start AHCI host, request IRQs and register it
- *	@host: target ATA host
- *	@irq: base IRQ number to request
- *	@sht: scsi_host_template to use when registering the host
- *
- *	Similar to ata_host_activate, but requests IRQs according to AHCI-1.1
- *	when multiple MSIs were allocated. That is one MSI per port, starting
- *	from @irq.
- *
- *	LOCKING:
- *	Inherited from calling layer (may sleep).
- *
- *	RETURNS:
- *	0 on success, -errno otherwise.
- */
-int ahci_host_activate(struct ata_host *host, int irq,
-		       struct scsi_host_template *sht)
-{
-	int i, rc;
-
-	rc = ata_host_start(host);
-	if (rc)
-		return rc;
-
-	for (i = 0; i < host->n_ports; i++) {
-		struct ahci_port_priv *pp = host->ports[i]->private_data;
-
-		/* Do not receive interrupts sent by dummy ports */
-		if (!pp) {
-			disable_irq(irq + i);
-			continue;
-		}
-
-		rc = devm_request_threaded_irq(host->dev, irq + i,
-					       ahci_hw_interrupt,
-					       ahci_thread_fn, IRQF_SHARED,
-					       pp->irq_desc, host->ports[i]);
-		if (rc)
-			goto out_free_irqs;
-	}
-
-	for (i = 0; i < host->n_ports; i++)
-		ata_port_desc(host->ports[i], "irq %d", irq + i);
-
-	rc = ata_host_register(host, sht);
-	if (rc)
-		goto out_free_all_irqs;
-
-	return 0;
-
-out_free_all_irqs:
-	i = host->n_ports;
-out_free_irqs:
-	for (i--; i >= 0; i--)
-		devm_free_irq(host->dev, irq + i, host->ports[i]);
-
-	return rc;
-}
-
 static int ahci_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
 {
 	unsigned int board_id = ent->driver_data;
drivers/ata/libahci.c +61 -0

67 lines not on this page · raw message

Signature · 1 line
1.8.3.1
#3 lore raw

PATCH v5 3/5 AHCI: Move host activation code into ahci_host_activate()

In reply to Alexander Gordeev

Currently host activation done by calling either function ahci_host_activate() or ata_host_activate(). Consolidate the code by only calling ahci_host_activate() for all AHCI devices.

---
 drivers/ata/acard-ahci.c       |  3 +--
 drivers/ata/ahci.c             |  6 +----
 drivers/ata/ahci.h             |  3 ---
 drivers/ata/libahci.c          | 59 +++++++++++++++++++++++++-----------------
 drivers/ata/libahci_platform.c |  3 +--
 drivers/ata/sata_highbank.c    |  3 +--
 6 files changed, 39 insertions(+), 38 deletions(-)
drivers/ata/acard-ahci.c +1 -2
index 25d0ac3..c962886 100644
@@ -498,8 +498,7 @@ static int acard_ahci_init_one(struct pci_dev *pdev, const struct pci_device_id
 	acard_ahci_pci_print_info(host);
 
 	pci_set_master(pdev);
-	return ata_host_activate(host, pdev->irq, ahci_interrupt, IRQF_SHARED,
-				 &acard_ahci_sht);
+	return ahci_host_activate(host, pdev->irq, &acard_ahci_sht);
 }
 
 module_pci_driver(acard_ahci_pci_driver);
drivers/ata/ahci.c +1 -5
index 0b21604..d0ac38b 100644
@@ -1427,11 +1427,7 @@ static int ahci_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
 
 	pci_set_master(pdev);
 
-	if (hpriv->flags & AHCI_HFLAG_MULTI_MSI)
-		return ahci_host_activate(host, pdev->irq, &ahci_sht);
-
-	return ata_host_activate(host, pdev->irq, ahci_interrupt, IRQF_SHARED,
-				 &ahci_sht);
+	return ahci_host_activate(host, pdev->irq, &ahci_sht);
 }
 
 module_pci_driver(ahci_pci_driver);
drivers/ata/ahci.h +0 -3
index 31b4c44..6a22055 100644
@@ -388,9 +388,6 @@ int ahci_port_resume(struct ata_port *ap);
 void ahci_set_em_messages(struct ahci_host_priv *hpriv,
 			  struct ata_port_info *pi);
 int ahci_reset_em(struct ata_host *host);
-irqreturn_t ahci_interrupt(int irq, void *dev_instance);
-irqreturn_t ahci_hw_interrupt(int irq, void *dev_instance);
-irqreturn_t ahci_thread_fn(int irq, void *dev_instance);
 void ahci_print_info(struct ata_host *host, const char *scc_s);
 int ahci_host_activate(struct ata_host *host, int irq,
 		       struct scsi_host_template *sht);
drivers/ata/libahci.c +35 -24

102 lines not on this page · raw message

drivers/ata/libahci_platform.c +1 -2
index c7f787e..0b03f90 100644
@@ -493,8 +493,7 @@ int ahci_platform_init_host(struct platform_device *pdev,
 	ahci_init_controller(host);
 	ahci_print_info(host, "platform");
 
-	return ata_host_activate(host, irq, ahci_interrupt, IRQF_SHARED,
-				 &ahci_platform_sht);
+	return ahci_host_activate(host, irq, &ahci_platform_sht);
 }
 EXPORT_SYMBOL_GPL(ahci_platform_init_host);
 
drivers/ata/sata_highbank.c +1 -2
index da3bc27..ce2b99a 100644
@@ -568,8 +568,7 @@ static int ahci_highbank_probe(struct platform_device *pdev)
 	ahci_init_controller(host);
 	ahci_print_info(host, "platform");
 
-	rc = ata_host_activate(host, irq, ahci_interrupt, 0,
-					&ahci_highbank_platform_sht);
+	rc = ahci_host_activate(host, irq, &ahci_highbank_platform_sht);
 	if (rc)
 		goto err0;
 
Signature · 1 line
1.8.3.1
#4 lore raw

PATCH v5 4/5 AHCI: Make few function names more descriptive

In reply to Alexander Gordeev
---
 drivers/ata/libahci.c | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)
drivers/ata/libahci.c +8 -8
index 0080551..48175e5 100644
@@ -1789,7 +1789,7 @@ static void ahci_port_intr(struct ata_port *ap)
 	ahci_handle_port_interrupt(ap, port_mmio, status);
 }
 
-static irqreturn_t ahci_thread_fn(int irq, void *dev_instance)
+static irqreturn_t ahci_port_thread_fn(int irq, void *dev_instance)
 {
 	struct ata_port *ap = dev_instance;
 	struct ahci_port_priv *pp = ap->private_data;
@@ -1810,7 +1810,7 @@ static irqreturn_t ahci_thread_fn(int irq, void *dev_instance)
 	return IRQ_HANDLED;
 }
 
-static void ahci_hw_port_interrupt(struct ata_port *ap)
+static void ahci_update_intr_status(struct ata_port *ap)
 {
 	void __iomem *port_mmio = ahci_port_base(ap);
 	struct ahci_port_priv *pp = ap->private_data;
@@ -1822,7 +1822,7 @@ static void ahci_hw_port_interrupt(struct ata_port *ap)
 	pp->intr_status |= status;
 }
 
-static irqreturn_t ahci_hw_interrupt(int irq, void *dev_instance)
+static irqreturn_t ahci_multi_irqs_intr(int irq, void *dev_instance)
 {
 	struct ata_port *ap_this = dev_instance;
 	struct ahci_port_priv *pp = ap_this->private_data;
@@ -1858,7 +1858,7 @@ static irqreturn_t ahci_hw_interrupt(int irq, void *dev_instance)
 
 		ap = host->ports[i];
 		if (ap) {
-			ahci_hw_port_interrupt(ap);
+			ahci_update_intr_status(ap);
 			VPRINTK("port %u\n", i);
 		} else {
 			VPRINTK("port %u (no irq)\n", i);
@@ -1877,7 +1877,7 @@ static irqreturn_t ahci_hw_interrupt(int irq, void *dev_instance)
 	return IRQ_WAKE_THREAD;
 }
 
-static irqreturn_t ahci_interrupt(int irq, void *dev_instance)
+static irqreturn_t ahci_single_irq_intr(int irq, void *dev_instance)
 {
 	struct ata_host *host = dev_instance;
 	struct ahci_host_priv *hpriv;
@@ -2488,8 +2488,8 @@ static int ahci_host_activate_multi_irqs(struct ata_host *host, int irq,
 		}
 
 		rc = devm_request_threaded_irq(host->dev, irq + i,
-					       ahci_hw_interrupt,
-					       ahci_thread_fn, IRQF_SHARED,
+					       ahci_multi_irqs_intr,
+					       ahci_port_thread_fn, IRQF_SHARED,
 					       pp->irq_desc, host->ports[i]);
 		if (rc)
 			goto out_free_irqs;
@@ -2538,7 +2538,7 @@ int ahci_host_activate(struct ata_host *host, int irq,
 	if (hpriv->flags & AHCI_HFLAG_MULTI_MSI)
 		rc = ahci_host_activate_multi_irqs(host, irq, sht);
 	else
-		rc = ata_host_activate(host, irq, ahci_interrupt,
+		rc = ata_host_activate(host, irq, ahci_single_irq_intr,
 				       IRQF_SHARED, sht);
 	return rc;
 }
Signature · 1 line
1.8.3.1
#0 lore raw

v5 cover AHCI: Optimize interrupt processing in multi-MSI mode

Changes since v4:
- CONFIG_SATA_HIGHBANK platform build error fixed;
- code movement (from ahci.c to libahci.c) separated from other changes;

Changes since v3:
- series rebased on libata/for-3.18;
- single IRQ interrupt update removed, along with patches 4,5 "AHCI: Get rid of redundant arg to ahci_handle_port_interrupt()" "AHCI: Optimize single IRQ interrupt processing" removed;
- multi-MSI updated to skip zero value port status;

Changes since v2:
- single patch split in a series;
- benchmarking statistics reworded;
- HOST_IRQ_STAT reg optimization added (patch 6);

Alexander Gordeev (5):
AHCI: Pass SCSI host template as arg to ahci_host_activate()
AHCI: Move ahci_host_activate() function to libahci.c
AHCI: Move host activation code into ahci_host_activate()
AHCI: Make few function names more descriptive
AHCI: Do not read HOST_IRQ_STAT reg in multi-MSI mode

 drivers/ata/acard-ahci.c       |   3 +-
 drivers/ata/ahci.c             |  66 +-----------------
 drivers/ata/ahci.h             |   8 +--
 drivers/ata/libahci.c          | 149 +++++++++++++++++++++++------------------
 drivers/ata/libahci_platform.c |   3 +-
 drivers/ata/sata_highbank.c    |   3 +-
 6 files changed, 92 insertions(+), 140 deletions(-)
Signature · 1 line
1.8.3.1
#1 lore raw

PATCH v5 1/5 AHCI: Pass SCSI host template as arg to ahci_host_activate()

This update is a prerequisite for consolidation of AHCI host activation code within ahci_host_activate() function.

---
 drivers/ata/ahci.c | 10 +++++-----
 drivers/ata/ahci.h |  3 ++-
 2 files changed, 7 insertions(+), 6 deletions(-)
drivers/ata/ahci.c +5 -5
index f68a995..fcda5b6 100644
@@ -1230,8 +1230,7 @@ intx:
  *	ahci_host_activate - start AHCI host, request IRQs and register it
  *	@host: target ATA host
  *	@irq: base IRQ number to request
- *	@irq_handler: irq_handler used when requesting IRQs
- *	@irq_flags: irq_flags used when requesting IRQs
+ *	@sht: scsi_host_template to use when registering the host
  *
  *	Similar to ata_host_activate, but requests IRQs according to AHCI-1.1
  *	when multiple MSIs were allocated. That is one MSI per port, starting
@@ -1243,7 +1242,8 @@ intx:
  *	RETURNS:
  *	0 on success, -errno otherwise.
  */
-int ahci_host_activate(struct ata_host *host, int irq)
+int ahci_host_activate(struct ata_host *host, int irq,
+		       struct scsi_host_template *sht)
 {
 	int i, rc;
 
@@ -1271,7 +1271,7 @@ int ahci_host_activate(struct ata_host *host, int irq)
 	for (i = 0; i < host->n_ports; i++)
 		ata_port_desc(host->ports[i], "irq %d", irq + i);
 
-	rc = ata_host_register(host, &ahci_sht);
+	rc = ata_host_register(host, sht);
 	if (rc)
 		goto out_free_all_irqs;
 
@@ -1488,7 +1488,7 @@ static int ahci_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
 	pci_set_master(pdev);
 
 	if (hpriv->flags & AHCI_HFLAG_MULTI_MSI)
-		return ahci_host_activate(host, pdev->irq);
+		return ahci_host_activate(host, pdev->irq, &ahci_sht);
 
 	return ata_host_activate(host, pdev->irq, ahci_interrupt, IRQF_SHARED,
 				 &ahci_sht);
drivers/ata/ahci.h +2 -1
index a074c73..31b4c44 100644
@@ -392,7 +392,8 @@ irqreturn_t ahci_interrupt(int irq, void *dev_instance);
 irqreturn_t ahci_hw_interrupt(int irq, void *dev_instance);
 irqreturn_t ahci_thread_fn(int irq, void *dev_instance);
 void ahci_print_info(struct ata_host *host, const char *scc_s);
-int ahci_host_activate(struct ata_host *host, int irq);
+int ahci_host_activate(struct ata_host *host, int irq,
+		       struct scsi_host_template *sht);
 void ahci_error_handler(struct ata_port *ap);
 
 static inline void __iomem *__ahci_port_base(struct ata_host *host,
Signature · 1 line
1.8.3.1
#5 lore raw

PATCH v5 5/5 AHCI: Do not read HOST_IRQ_STAT reg in multi-MSI mode

In reply to Alexander Gordeev

As described in AHCI v1.0 specification chapter 10.6.2.2 "Multiple MSI Based Messages" generation of interrupts is not controlled through the HOST_IRQ_STAT register.

Considering MMIO access is expensive remove unnecessary reading and writing of HOST_IRQ_STAT register.

Further, serializing access to the host data is no longer needed and the interrupt service routine can avoid competing on the host lock.

---
 drivers/ata/ahci.h    |  2 +-
 drivers/ata/libahci.c | 67 ++++++---------------------------------------------
 2 files changed, 9 insertions(+), 60 deletions(-)
drivers/ata/ahci.h +1 -1
index 6a22055..40f0e34 100644
@@ -304,7 +304,7 @@ struct ahci_port_priv {
 	unsigned int		ncq_saw_d2h:1;
 	unsigned int		ncq_saw_dmas:1;
 	unsigned int		ncq_saw_sdb:1;
-	u32			intr_status;	/* interrupts to handle */
+	atomic_t		intr_status;	/* interrupts to handle */
 	spinlock_t		lock;		/* protects parent ata_port */
 	u32 			intr_mask;	/* interrupts to enable */
 	bool			fbs_supported;	/* set iff FBS is supported */
drivers/ata/libahci.c +8 -59

89 lines not on this page · raw message

Signature · 1 line
1.8.3.1
#6 lore raw

[PATCH 6/5] AHCI: Optimize single IRQ interrupt processing

In reply to Alexander Gordeev

Split interrupt service routine into hardware context handler and threaded context handler. That allows to protect ports with individual locks rather than with a single host-wide lock and move port interrupts handling out of the hardware interrupt context.

Testing was done by transferring 8GB on two hard drives in parallel using command 'dd if=/dev/sd{a,b} of=/dev/null'. With lock_stat statistics I measured access times to ata_host::lock spinlock (since interrupt handler code is fully embraced with this lock). The average lock's holdtime decreased eight times while average waittime decreased two times.

Both before and after the change the transfer time is the same, while 'perf record -e cycles:k ...' shows 1%-4% CPU time spent in ahci_single_irq_intr() routine before the update and not even sampled/shown ahci_single_irq_intr() after the update.

---
 drivers/ata/libahci.c | 74 ++++++++++++++++++++++++++++++++++++++++++---------
 1 file changed, 61 insertions(+), 13 deletions(-)
drivers/ata/libahci.c +61 -13
index 97683e4..3ce3d23 100644
@@ -1778,15 +1778,16 @@ static void ahci_handle_port_interrupt(struct ata_port *ap,
 	}
 }
 
-static void ahci_port_intr(struct ata_port *ap)
+static void ahci_update_intr_status(struct ata_port *ap)
 {
 	void __iomem *port_mmio = ahci_port_base(ap);
+	struct ahci_port_priv *pp = ap->private_data;
 	u32 status;
 
 	status = readl(port_mmio + PORT_IRQ_STAT);
 	writel(status, port_mmio + PORT_IRQ_STAT);
 
-	ahci_handle_port_interrupt(ap, port_mmio, status);
+	atomic_or(status, &pp->intr_status);
 }
 
 static irqreturn_t ahci_port_thread_fn(int irq, void *dev_instance)
@@ -1807,6 +1808,34 @@ static irqreturn_t ahci_port_thread_fn(int irq, void *dev_instance)
 	return IRQ_HANDLED;
 }
 
+irqreturn_t ahci_thread_fn(int irq, void *dev_instance)
+{
+	struct ata_host *host = dev_instance;
+	struct ahci_host_priv *hpriv = host->private_data;
+	u32 irq_masked = hpriv->port_map;
+	unsigned int i;
+
+	for (i = 0; i < host->n_ports; i++) {
+		struct ata_port *ap;
+
+		if (!(irq_masked & (1 << i)))
+			continue;
+
+		ap = host->ports[i];
+		if (ap) {
+			ahci_port_thread_fn(irq, ap);
+			VPRINTK("port %u\n", i);
+		} else {
+			VPRINTK("port %u (no irq)\n", i);
+			if (ata_ratelimit())
+				dev_warn(host->dev,
+					 "interrupt on disabled port %u\n", i);
+		}
+	}
+
+	return IRQ_HANDLED;
+}
+
 static irqreturn_t ahci_multi_irqs_intr(int irq, void *dev_instance)
 {
 	struct ata_port *ap = dev_instance;
@@ -1856,7 +1885,7 @@ static irqreturn_t ahci_single_irq_intr(int irq, void *dev_instance)
 
 		ap = host->ports[i];
 		if (ap) {
-			ahci_port_intr(ap);
+			ahci_update_intr_status(ap);
 			VPRINTK("port %u\n", i);
 		} else {
 			VPRINTK("port %u (no irq)\n", i);
@@ -1883,7 +1912,7 @@ static irqreturn_t ahci_single_irq_intr(int irq, void *dev_instance)
 
 	VPRINTK("EXIT\n");
 
-	return IRQ_RETVAL(handled);
+	return handled ? IRQ_WAKE_THREAD : IRQ_NONE;
 }
 
 unsigned int ahci_qc_issue(struct ata_queued_cmd *qc)
@@ -2295,13 +2324,8 @@ static int ahci_port_start(struct ata_port *ap)
 	 */
 	pp->intr_mask = DEF_PORT_IRQ;
 
-	/*
-	 * Switch to per-port locking in case each port has its own MSI vector.
-	 */
-	if ((hpriv->flags & AHCI_HFLAG_MULTI_MSI)) {
-		spin_lock_init(&pp->lock);
-		ap->lock = &pp->lock;
-	}
+	spin_lock_init(&pp->lock);
+	ap->lock = &pp->lock;
 
 	ap->private_data = pp;
 
@@ -2462,6 +2486,31 @@ out_free_irqs:
 	return rc;
 }
 
+static int ahci_host_activate_single_irq(struct ata_host *host, int irq,
+					 struct scsi_host_template *sht)
+{
+	int i, rc;
+
+	rc = ata_host_start(host);
+	if (rc)
+		return rc;
+
+	rc = devm_request_threaded_irq(host->dev, irq, ahci_single_irq_intr,
+				       ahci_thread_fn, IRQF_SHARED,
+				       dev_driver_string(host->dev), host);
+	if (rc)
+		return rc;
+
+	for (i = 0; i < host->n_ports; i++)
+		ata_port_desc(host->ports[i], "irq %d", irq);
+
+	rc = ata_host_register(host, sht);
+	if (rc)
+		devm_free_irq(host->dev, irq, host);
+
+	return rc;
+}
+
 /**
  *	ahci_host_activate - start AHCI host, request IRQs and register it
  *	@host: target ATA host
@@ -2487,8 +2536,7 @@ int ahci_host_activate(struct ata_host *host, int irq,
 	if (hpriv->flags & AHCI_HFLAG_MULTI_MSI)
 		rc = ahci_host_activate_multi_irqs(host, irq, sht);
 	else
-		rc = ata_host_activate(host, irq, ahci_single_irq_intr,
-				       IRQF_SHARED, sht);
+		rc = ahci_host_activate_single_irq(host, irq, sht);
 	return rc;
 }
 EXPORT_SYMBOL_GPL(ahci_host_activate);
Signature · 6 lines
1.8.3.1

-- 
Regards,
Alexander Gordeev
[email protected]
#7 lore raw

[PATCH 7/5] AHCI: Do not acquire ata_host::lock from single IRQ handler

In reply to Alexander Gordeev

There is no need to acquire ata_host::lock spinlock from hardware context single IRQ interrupt handler since the handler does not access host data that could be altered by concurrent processors.

---
 drivers/ata/libahci.c | 4 ----
 1 file changed, 4 deletions(-)
drivers/ata/libahci.c +0 -4
index 3ce3d23..5eb61c9 100644
@@ -1875,8 +1875,6 @@ static irqreturn_t ahci_single_irq_intr(int irq, void *dev_instance)
 
 	irq_masked = irq_stat & hpriv->port_map;
 
-	spin_lock(&host->lock);
-
 	for (i = 0; i < host->n_ports; i++) {
 		struct ata_port *ap;
 
@@ -1908,8 +1906,6 @@ static irqreturn_t ahci_single_irq_intr(int irq, void *dev_instance)
 	 */
 	writel(irq_stat, mmio + HOST_IRQ_STAT);
 
-	spin_unlock(&host->lock);
-
 	VPRINTK("EXIT\n");
 
 	return handled ? IRQ_WAKE_THREAD : IRQ_NONE;
Signature · 6 lines
1.8.3.1

-- 
Regards,
Alexander Gordeev
[email protected]