Skip to contentSkip to conversation

ocxl: Cleanup AFU interrupt allocation

PATCH Applied Frederic Barrat 5 participants 10 messages Apr 3, 2020 – Sep 9, 2020

Subsystems (4)
lore
Share & tools
lore
#0 lore raw

v2 cover ocxl: Cleanup AFU interrupt allocation

Short series to cleanup AFU interrupt allocation for opencapi. Current code was using its own allocation service, calling opal directly to get the trigger page. This is not needed and we can use xive to achieve the same thing. The only caveat is that the trigger page address is only valid after the interrupt has been mapped, but that is not a problem with the way the code is using it. No functional change.

Changelog:
v2:
Fix error path in patch 1
All other patches unchanged

Frederic Barrat (4):
scsi: cxlflash: Access interrupt trigger page from xive directly
ocxl: Access interrupt trigger page from xive directly
ocxl: Don't return trigger page when allocating an interrupt
ocxl: Remove custom service to allocate interrupts

 arch/powerpc/include/asm/pnv-ocxl.h   |  3 ---
 arch/powerpc/platforms/powernv/ocxl.c | 30 ---------------------------
 drivers/misc/ocxl/Kconfig             |  2 +-
 drivers/misc/ocxl/afu_irq.c           | 12 ++++++-----
 drivers/misc/ocxl/link.c              | 15 +++++++-------
 drivers/scsi/cxlflash/ocxl_hw.c       | 21 ++++++++-----------
 drivers/scsi/cxlflash/ocxl_hw.h       |  1 -
 include/misc/ocxl.h                   | 10 ++-------
 8 files changed, 26 insertions(+), 68 deletions(-)
Signature · 1 line
2.25.1
#1 lore raw

PATCH v2 1/4 scsi: cxlflash: Access interrupt trigger page from xive directly

xive is already mapping the trigger page in kernel space and it can be accessed through standard APIs, so let's reuse it and simplify the code.

---
Changelog:
v2: Define rc = -ENXIO on the error path (Matt)

 drivers/scsi/cxlflash/ocxl_hw.c | 18 ++++++++----------
 drivers/scsi/cxlflash/ocxl_hw.h |  1 -
 2 files changed, 8 insertions(+), 11 deletions(-)
drivers/scsi/cxlflash/ocxl_hw.c +8 -10
index 7018cd802569..d6eec434a607 100644
@@ -15,7 +15,8 @@
 #include <linux/pseudo_fs.h>
 #include <linux/poll.h>
 #include <linux/sched/signal.h>
-
+#include <linux/interrupt.h>
+#include <asm/xive.h>
 #include <misc/ocxl.h>
 
 #include <uapi/misc/cxl.h>
@@ -180,7 +181,7 @@ static int afu_map_irq(u64 flags, struct ocxlflash_context *ctx, int num,
 	struct ocxl_hw_afu *afu = ctx->hw_afu;
 	struct device *dev = afu->dev;
 	struct ocxlflash_irqs *irq;
-	void __iomem *vtrig;
+	struct xive_irq_data *xd;
 	u32 virq;
 	int rc = 0;
 
@@ -204,15 +205,15 @@ static int afu_map_irq(u64 flags, struct ocxlflash_context *ctx, int num,
 		goto err1;
 	}
 
-	vtrig = ioremap(irq->ptrig, PAGE_SIZE);
-	if (unlikely(!vtrig)) {
-		dev_err(dev, "%s: Trigger page mapping failed\n", __func__);
-		rc = -ENOMEM;
+	xd = irq_get_handler_data(virq);
+	if (unlikely(!xd)) {
+		dev_err(dev, "%s: Can't get interrupt data\n", __func__);
+		rc = -ENXIO;
 		goto err2;
 	}
 
 	irq->virq = virq;
-	irq->vtrig = vtrig;
+	irq->vtrig = xd->trig_mmio;
 out:
 	return rc;
 err2:
@@ -259,8 +260,6 @@ static void afu_unmap_irq(u64 flags, struct ocxlflash_context *ctx, int num,
 	}
 
 	irq = &ctx->irqs[num];
-	if (irq->vtrig)
-		iounmap(irq->vtrig);
 
 	if (irq_find_mapping(NULL, irq->hwirq)) {
 		free_irq(irq->virq, cookie);
@@ -648,7 +647,6 @@ static int alloc_afu_irqs(struct ocxlflash_context *ctx, int num)
 		}
 
 		irqs[i].hwirq = hwirq;
-		irqs[i].ptrig = addr;
 	}
 
 	ctx->irqs = irqs;
drivers/scsi/cxlflash/ocxl_hw.h +0 -1
index fc6ad4f985de..f2fe88816bea 100644
@@ -13,7 +13,6 @@
 struct ocxlflash_irqs {
 	int hwirq;
 	u32 virq;
-	u64 ptrig;
 	void __iomem *vtrig;
 };
 
Signature · 1 line
2.25.1
Acked-by on v2 1/4
#2 lore raw

Re: [PATCH v2 1/4] scsi: cxlflash: Access interrupt trigger page from xive directly

Quoted 4 lines · Frederic Barrat

xive is already mapping the trigger page in kernel space and it can be accessed through standard APIs, so let's reuse it and simplify the code.

Looks good!

Quoted 85 lines

---
Changelog:
v2: Define rc = -ENXIO on the error path (Matt)

 drivers/scsi/cxlflash/ocxl_hw.c | 18 ++++++++----------
 drivers/scsi/cxlflash/ocxl_hw.h |  1 -
 2 files changed, 8 insertions(+), 11 deletions(-)
drivers/scsi/cxlflash/ocxl_hw.c +8 -10
index 7018cd802569..d6eec434a607 100644
@@ -15,7 +15,8 @@
 #include <linux/pseudo_fs.h>
 #include <linux/poll.h>
 #include <linux/sched/signal.h>
-
+#include <linux/interrupt.h>
+#include <asm/xive.h>
 #include <misc/ocxl.h>
 
 #include <uapi/misc/cxl.h>
@@ -180,7 +181,7 @@ static int afu_map_irq(u64 flags, struct ocxlflash_context *ctx, int num,
 	struct ocxl_hw_afu *afu = ctx->hw_afu;
 	struct device *dev = afu->dev;
 	struct ocxlflash_irqs *irq;
-	void __iomem *vtrig;
+	struct xive_irq_data *xd;
 	u32 virq;
 	int rc = 0;
 
@@ -204,15 +205,15 @@ static int afu_map_irq(u64 flags, struct ocxlflash_context *ctx, int num,
 		goto err1;
 	}
 
-	vtrig = ioremap(irq->ptrig, PAGE_SIZE);
-	if (unlikely(!vtrig)) {
-		dev_err(dev, "%s: Trigger page mapping failed\n", __func__);
-		rc = -ENOMEM;
+	xd = irq_get_handler_data(virq);
+	if (unlikely(!xd)) {
+		dev_err(dev, "%s: Can't get interrupt data\n", __func__);
+		rc = -ENXIO;
 		goto err2;
 	}
 
 	irq->virq = virq;
-	irq->vtrig = vtrig;
+	irq->vtrig = xd->trig_mmio;
 out:
 	return rc;
 err2:
@@ -259,8 +260,6 @@ static void afu_unmap_irq(u64 flags, struct ocxlflash_context *ctx, int num,
 	}
 
 	irq = &ctx->irqs[num];
-	if (irq->vtrig)
-		iounmap(irq->vtrig);
 
 	if (irq_find_mapping(NULL, irq->hwirq)) {
 		free_irq(irq->virq, cookie);
@@ -648,7 +647,6 @@ static int alloc_afu_irqs(struct ocxlflash_context *ctx, int num)
 		}
 
 		irqs[i].hwirq = hwirq;
-		irqs[i].ptrig = addr;
 	}
 
 	ctx->irqs = irqs;
drivers/scsi/cxlflash/ocxl_hw.h +0 -1
index fc6ad4f985de..f2fe88816bea 100644
@@ -13,7 +13,6 @@
 struct ocxlflash_irqs {
 	int hwirq;
 	u32 virq;
-	u64 ptrig;
 	void __iomem *vtrig;
 };
 
Signature · 1 line
2.25.1
Reviewed-by on v2 1/4
#3 lore raw

Re: [PATCH v2 1/4] scsi: cxlflash: Access interrupt trigger page from xive directly

In reply to Frederic Barrat
Quoted 4 lines · Frederic Barrat

xive is already mapping the trigger page in kernel space and it can be accessed through standard APIs, so let's reuse it and simplify the code.

Thanks,

C.

Quoted 83 lines

---
Changelog:
v2: Define rc = -ENXIO on the error path (Matt)

 drivers/scsi/cxlflash/ocxl_hw.c | 18 ++++++++----------
 drivers/scsi/cxlflash/ocxl_hw.h |  1 -
 2 files changed, 8 insertions(+), 11 deletions(-)
drivers/scsi/cxlflash/ocxl_hw.c +8 -10
index 7018cd802569..d6eec434a607 100644
@@ -15,7 +15,8 @@
 #include <linux/pseudo_fs.h>
 #include <linux/poll.h>
 #include <linux/sched/signal.h>
-
+#include <linux/interrupt.h>
+#include <asm/xive.h>
 #include <misc/ocxl.h>
 
 #include <uapi/misc/cxl.h>
@@ -180,7 +181,7 @@ static int afu_map_irq(u64 flags, struct ocxlflash_context *ctx, int num,
 	struct ocxl_hw_afu *afu = ctx->hw_afu;
 	struct device *dev = afu->dev;
 	struct ocxlflash_irqs *irq;
-	void __iomem *vtrig;
+	struct xive_irq_data *xd;
 	u32 virq;
 	int rc = 0;
 
@@ -204,15 +205,15 @@ static int afu_map_irq(u64 flags, struct ocxlflash_context *ctx, int num,
 		goto err1;
 	}
 
-	vtrig = ioremap(irq->ptrig, PAGE_SIZE);
-	if (unlikely(!vtrig)) {
-		dev_err(dev, "%s: Trigger page mapping failed\n", __func__);
-		rc = -ENOMEM;
+	xd = irq_get_handler_data(virq);
+	if (unlikely(!xd)) {
+		dev_err(dev, "%s: Can't get interrupt data\n", __func__);
+		rc = -ENXIO;
 		goto err2;
 	}
 
 	irq->virq = virq;
-	irq->vtrig = vtrig;
+	irq->vtrig = xd->trig_mmio;
 out:
 	return rc;
 err2:
@@ -259,8 +260,6 @@ static void afu_unmap_irq(u64 flags, struct ocxlflash_context *ctx, int num,
 	}
 
 	irq = &ctx->irqs[num];
-	if (irq->vtrig)
-		iounmap(irq->vtrig);
 
 	if (irq_find_mapping(NULL, irq->hwirq)) {
 		free_irq(irq->virq, cookie);
@@ -648,7 +647,6 @@ static int alloc_afu_irqs(struct ocxlflash_context *ctx, int num)
 		}
 
 		irqs[i].hwirq = hwirq;
-		irqs[i].ptrig = addr;
 	}
 
 	ctx->irqs = irqs;
drivers/scsi/cxlflash/ocxl_hw.h +0 -1
index fc6ad4f985de..f2fe88816bea 100644
@@ -13,7 +13,6 @@
 struct ocxlflash_irqs {
 	int hwirq;
 	u32 virq;
-	u64 ptrig;
 	void __iomem *vtrig;
 };
 
#4 lore raw

PATCH v2 2/4 ocxl: Access interrupt trigger page from xive directly

In reply to Frederic Barrat

We can access the trigger page through standard APIs so let's use it and avoid saving it when allocating the interrupt. It will also allow to simplify allocation in a later patch.

---
 drivers/misc/ocxl/afu_irq.c | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)
drivers/misc/ocxl/afu_irq.c +6 -2
index 70f8f1c3929d..b30ec0ef7be7 100644
@@ -2,6 +2,7 @@
 // Copyright 2017 IBM Corp.
 #include <linux/interrupt.h>
 #include <asm/pnv-ocxl.h>
+#include <asm/xive.h>
 #include "ocxl_internal.h"
 #include "trace.h"
 
@@ -196,13 +197,16 @@ void ocxl_afu_irq_free_all(struct ocxl_context *ctx)
 
 u64 ocxl_afu_irq_get_addr(struct ocxl_context *ctx, int irq_id)
 {
+	struct xive_irq_data *xd;
 	struct afu_irq *irq;
 	u64 addr = 0;
 
 	mutex_lock(&ctx->irq_lock);
 	irq = idr_find(&ctx->irq_idr, irq_id);
-	if (irq)
-		addr = irq->trigger_page;
+	if (irq) {
+		xd = irq_get_handler_data(irq->virq);
+		addr = xd ? xd->trig_page : 0;
+	}
 	mutex_unlock(&ctx->irq_lock);
 	return addr;
 }
Signature · 1 line
2.25.1
#5 lore raw

PATCH v2 3/4 ocxl: Don't return trigger page when allocating an interrupt

In reply to Frederic Barrat

Existing users of ocxl_link_irq_alloc() have been converted to obtain the trigger page of an interrupt through xive directly, we therefore have no need to return the trigger page when allocating an interrupt.

It also allows ocxl to use the xive native interface to allocate interrupts, instead of its custom service.

---
 drivers/misc/ocxl/Kconfig       |  2 +-
 drivers/misc/ocxl/afu_irq.c     |  4 +---
 drivers/misc/ocxl/link.c        | 15 +++++++--------
 drivers/scsi/cxlflash/ocxl_hw.c |  3 +--
 include/misc/ocxl.h             | 10 ++--------
 5 files changed, 12 insertions(+), 22 deletions(-)
drivers/misc/ocxl/Kconfig +1 -1
index 2d2266c1439e..e65773f5cf59 100644
@@ -9,7 +9,7 @@ config OCXL_BASE
 
 config OCXL
 	tristate "OpenCAPI coherent accelerator support"
-	depends on PPC_POWERNV && PCI && EEH
+	depends on PPC_POWERNV && PCI && EEH && PPC_XIVE_NATIVE
 	select OCXL_BASE
 	select HOTPLUG_PCI_POWERNV
 	default m
drivers/misc/ocxl/afu_irq.c +1 -3
index b30ec0ef7be7..ecdcfae025b7 100644
@@ -11,7 +11,6 @@ struct afu_irq {
 	int hw_irq;
 	unsigned int virq;
 	char *name;
-	u64 trigger_page;
 	irqreturn_t (*handler)(void *private);
 	void (*free_private)(void *private);
 	void *private;
@@ -125,8 +124,7 @@ int ocxl_afu_irq_alloc(struct ocxl_context *ctx, int *irq_id)
 		goto err_unlock;
 	}
 
-	rc = ocxl_link_irq_alloc(ctx->afu->fn->link, &irq->hw_irq,
-				&irq->trigger_page);
+	rc = ocxl_link_irq_alloc(ctx->afu->fn->link, &irq->hw_irq);
 	if (rc)
 		goto err_idr;
 
drivers/misc/ocxl/link.c +7 -8
index 58d111afd9f6..fd73d3bc0eb6 100644
@@ -6,6 +6,7 @@
 #include <linux/mmu_context.h>
 #include <asm/copro.h>
 #include <asm/pnv-ocxl.h>
+#include <asm/xive.h>
 #include <misc/ocxl.h>
 #include "ocxl_internal.h"
 #include "trace.h"
@@ -682,23 +683,21 @@ int ocxl_link_remove_pe(void *link_handle, int pasid)
 }
 EXPORT_SYMBOL_GPL(ocxl_link_remove_pe);
 
-int ocxl_link_irq_alloc(void *link_handle, int *hw_irq, u64 *trigger_addr)
+int ocxl_link_irq_alloc(void *link_handle, int *hw_irq)
 {
 	struct ocxl_link *link = (struct ocxl_link *) link_handle;
-	int rc, irq;
-	u64 addr;
+	int irq;
 
 	if (atomic_dec_if_positive(&link->irq_available) < 0)
 		return -ENOSPC;
 
-	rc = pnv_ocxl_alloc_xive_irq(&irq, &addr);
-	if (rc) {
+	irq = xive_native_alloc_irq();
+	if (!irq) {
 		atomic_inc(&link->irq_available);
-		return rc;
+		return -ENXIO;
 	}
 
 	*hw_irq = irq;
-	*trigger_addr = addr;
 	return 0;
 }
 EXPORT_SYMBOL_GPL(ocxl_link_irq_alloc);
@@ -707,7 +706,7 @@ void ocxl_link_free_irq(void *link_handle, int hw_irq)
 {
 	struct ocxl_link *link = (struct ocxl_link *) link_handle;
 
-	pnv_ocxl_free_xive_irq(hw_irq);
+	xive_native_free_irq(hw_irq);
 	atomic_inc(&link->irq_available);
 }
 EXPORT_SYMBOL_GPL(ocxl_link_free_irq);
drivers/scsi/cxlflash/ocxl_hw.c +1 -2
index d6eec434a607..e4e0d767b98e 100644
@@ -614,7 +614,6 @@ static int alloc_afu_irqs(struct ocxlflash_context *ctx, int num)
 	struct ocxl_hw_afu *afu = ctx->hw_afu;
 	struct device *dev = afu->dev;
 	struct ocxlflash_irqs *irqs;
-	u64 addr;
 	int rc = 0;
 	int hwirq;
 	int i;
@@ -639,7 +638,7 @@ static int alloc_afu_irqs(struct ocxlflash_context *ctx, int num)
 	}
 
 	for (i = 0; i < num; i++) {
-		rc = ocxl_link_irq_alloc(afu->link_token, &hwirq, &addr);
+		rc = ocxl_link_irq_alloc(afu->link_token, &hwirq);
 		if (unlikely(rc)) {
 			dev_err(dev, "%s: ocxl_link_irq_alloc failed rc=%d\n",
 				__func__, rc);
include/misc/ocxl.h +2 -8
index 06dd5839e438..a2868adec22f 100644
@@ -480,14 +480,8 @@ int ocxl_link_remove_pe(void *link_handle, int pasid);
  * Allocate an AFU interrupt associated to the link.
  *
  * 'hw_irq' is the hardware interrupt number
- * 'obj_handle' is the 64-bit object handle to be passed to the AFU to
- * trigger the interrupt.
- * On P9, 'obj_handle' is an address, which, if written, triggers the
- * interrupt. It is an MMIO address which needs to be remapped (one
- * page).
- */
-int ocxl_link_irq_alloc(void *link_handle, int *hw_irq,
-			u64 *obj_handle);
+ */
+int ocxl_link_irq_alloc(void *link_handle, int *hw_irq);
 
 /*
  * Free a previously allocated AFU interrupt
Signature · 1 line
2.25.1
Acked-by on v2 3/4
#6 lore raw

Re: [PATCH v2 3/4] ocxl: Don't return trigger page when allocating an interrupt

Quoted 10 lines · Frederic Barrat

Existing users of ocxl_link_irq_alloc() have been converted to obtain the trigger page of an interrupt through xive directly, we therefore have no need to return the trigger page when allocating an interrupt.

It also allows ocxl to use the xive native interface to allocate interrupts, instead of its custom service.

No obvious issues on a cursory review

Signature · 2 lines
Andrew Donnellan              OzLabs, ADL Canberra
[email protected]             IBM Australia Limited
#7 lore raw

PATCH v2 4/4 ocxl: Remove custom service to allocate interrupts

In reply to Frederic Barrat

We now allocate interrupts through xive directly.

---
 arch/powerpc/include/asm/pnv-ocxl.h   |  3 ---
 arch/powerpc/platforms/powernv/ocxl.c | 30 ---------------------------
 2 files changed, 33 deletions(-)
arch/powerpc/include/asm/pnv-ocxl.h +0 -3
index 7de82647e761..e90650328c9c 100644
@@ -30,7 +30,4 @@ extern int pnv_ocxl_spa_setup(struct pci_dev *dev, void *spa_mem, int PE_mask,
 extern void pnv_ocxl_spa_release(void *platform_data);
 extern int pnv_ocxl_spa_remove_pe_from_cache(void *platform_data, int pe_handle);
 
-extern int pnv_ocxl_alloc_xive_irq(u32 *irq, u64 *trigger_addr);
-extern void pnv_ocxl_free_xive_irq(u32 irq);
-
 #endif /* _ASM_PNV_OCXL_H */
arch/powerpc/platforms/powernv/ocxl.c +0 -30
index 8c65aacda9c8..ecdad219d704 100644
@@ -2,7 +2,6 @@
 // Copyright 2017 IBM Corp.
 #include <asm/pnv-ocxl.h>
 #include <asm/opal.h>
-#include <asm/xive.h>
 #include <misc/ocxl-config.h>
 #include "pci.h"
 
@@ -484,32 +483,3 @@ int pnv_ocxl_spa_remove_pe_from_cache(void *platform_data, int pe_handle)
 	return rc;
 }
 EXPORT_SYMBOL_GPL(pnv_ocxl_spa_remove_pe_from_cache);
-
-int pnv_ocxl_alloc_xive_irq(u32 *irq, u64 *trigger_addr)
-{
-	__be64 flags, trigger_page;
-	s64 rc;
-	u32 hwirq;
-
-	hwirq = xive_native_alloc_irq();
-	if (!hwirq)
-		return -ENOENT;
-
-	rc = opal_xive_get_irq_info(hwirq, &flags, NULL, &trigger_page, NULL,
-				NULL);
-	if (rc || !trigger_page) {
-		xive_native_free_irq(hwirq);
-		return -ENOENT;
-	}
-	*irq = hwirq;
-	*trigger_addr = be64_to_cpu(trigger_page);
-	return 0;
-
-}
-EXPORT_SYMBOL_GPL(pnv_ocxl_alloc_xive_irq);
-
-void pnv_ocxl_free_xive_irq(u32 irq)
-{
-	xive_native_free_irq(irq);
-}
-EXPORT_SYMBOL_GPL(pnv_ocxl_free_xive_irq);
Signature · 1 line
2.25.1
Acked-by on v2 4/4
#8 lore raw

Re: [PATCH v2 4/4] ocxl: Remove custom service to allocate interrupts

Quoted 6 lines · Frederic Barrat

We now allocate interrupts through xive directly.

---

No obvious issues on a cursory review

Signature · 2 lines
Andrew Donnellan              OzLabs, ADL Canberra
[email protected]             IBM Australia Limited
#9 lore raw
In reply to Frederic Barrat
Quoted 9 lines · Frederic Barrat

Short series to cleanup AFU interrupt allocation for opencapi. Current code was using its own allocation service, calling opal directly to get the trigger page. This is not needed and we can use xive to achieve the same thing. The only caveat is that the trigger page address is only valid after the interrupt has been mapped, but that is not a problem with the way the code is using it. No functional change.

[...]

Applied to powerpc/next.

[1/4] scsi: cxlflash: Access interrupt trigger page from xive directly

      https://git.kernel.org/powerpc/c/1e89da5ef9c28c673e86048c89ef9495618d987d
[2/4] ocxl: Access interrupt trigger page from xive directly
      https://git.kernel.org/powerpc/c/ad857d47df6a1adc9798558701dd5426643b859f
[3/4] ocxl: Don't return trigger page when allocating an interrupt
      https://git.kernel.org/powerpc/c/dde6f18a8779dcd88d9fd5d6336032fee7e07fcd
[4/4] ocxl: Remove custom service to allocate interrupts
      https://git.kernel.org/powerpc/c/374f6178f3483dcad151fc14b2fad92ed6652f07

cheers