Skip to contentSkip to conversation

cdx: add MSI support for CDX bus

PATCH Superseded Nipun Gupta 2 participants 5 messages May 18, 2023 – Jun 26, 2023

Subsystems (3)

v2 of 3 Superseded Latest is v3 →

lore
Share & tools
lore

Nipun Gupta

author maintainer
#0 lore raw

PATCH v2 1/1 cdx: add MSI support for CDX bus

Add CDX-MSI domain per CDX controller with gic-its domain as a parent, to support MSI for CDX devices. CDX devices allocate MSIs from the CDX domain. Also, introduce APIs to alloc and free IRQs for CDX domain.

In CDX subsystem firmware is a controller for all devices and their configuration. CDX bus controller sends all the write_msi_msg commands to firmware running on RPU and the firmware interfaces with actual devices to pass this information to devices

Since, CDX controller is the only way to communicate with the Firmware for MSI write info, CDX domain per controller required in contrast to having a CDX domain per device.

---

Changes v1->v2:
- fixed scenario where msi write was called asyncronously in an atomic context, by using irq_chip_(un)lock, and using sync MCDI API for write MSI message.
- fixed broken Signed-off-by chain.

 drivers/cdx/Kconfig                     |   1 +
 drivers/cdx/Makefile                    |   2 +-
 drivers/cdx/cdx.c                       |   8 ++
 drivers/cdx/cdx.h                       |  10 ++
 drivers/cdx/cdx_msi.c                   | 183 ++++++++++++++++++++++++
 drivers/cdx/controller/cdx_controller.c |  23 +++
 drivers/cdx/controller/mc_cdx_pcol.h    |  58 ++++++++
 drivers/cdx/controller/mcdi_functions.c |  19 +++
 drivers/cdx/controller/mcdi_functions.h |  20 +++
 include/linux/cdx/cdx_bus.h             |  30 ++++
 kernel/irq/msi.c                        |   1 +
 11 files changed, 354 insertions(+), 1 deletion(-)
 create mode 100644 drivers/cdx/cdx_msi.c
drivers/cdx/Kconfig +1 -0
index a08958485e31..86df7ccb76bb 100644
@@ -8,6 +8,7 @@
 config CDX_BUS
 	bool "CDX Bus driver"
 	depends on OF && ARM64
+	select GENERIC_MSI_IRQ_DOMAIN
 	help
 	  Driver to enable Composable DMA Transfer(CDX) Bus. CDX bus
 	  exposes Fabric devices which uses composable DMA IP to the
drivers/cdx/Makefile +1 -1
index 0324e4914f6e..4bad79d1d188 100644
@@ -5,4 +5,4 @@
 # Copyright (C) 2022-2023, Advanced Micro Devices, Inc.
 #
 
-obj-$(CONFIG_CDX_BUS) += cdx.o controller/
+obj-$(CONFIG_CDX_BUS) += cdx.o cdx_msi.o controller/
drivers/cdx/cdx.c +8 -0
index 38511fd36325..37bf70077c70 100644
@@ -56,6 +56,7 @@
  */
 
 #include <linux/init.h>
+#include <linux/irqdomain.h>
 #include <linux/kernel.h>
 #include <linux/of_device.h>
 #include <linux/slab.h>
@@ -473,12 +474,19 @@ int cdx_device_add(struct cdx_dev_params *dev_params)
 	cdx_dev->dev.bus = &cdx_bus_type;
 	cdx_dev->dev.dma_mask = &cdx_dev->dma_mask;
 	cdx_dev->dev.release = cdx_device_release;
+	cdx_dev->msi_write_pending = false;
+	mutex_init(&cdx_dev->irqchip_lock);
 
 	/* Set Name */
 	dev_set_name(&cdx_dev->dev, "cdx-%02x:%02x",
 		     ((cdx->id << CDX_CONTROLLER_ID_SHIFT) | (cdx_dev->bus_num & CDX_BUS_NUM_MASK)),
 		     cdx_dev->dev_num);
 
+	if (cdx->msi_domain) {
+		cdx_dev->num_msi = dev_params->num_msi;
+		dev_set_msi_domain(&cdx_dev->dev, cdx->msi_domain);
+	}
+
 	ret = device_add(&cdx_dev->dev);
 	if (ret) {
 		dev_err(&cdx_dev->dev,
drivers/cdx/cdx.h +10 -0
index c436ac7ac86f..90e2da1e320d 100644
@@ -21,6 +21,7 @@
  * @res: array of MMIO region entries
  * @res_count: number of valid MMIO regions
  * @req_id: Requestor ID associated with CDX device
+ * @num_msi: Number of MSI's supported by the device
  */
 struct cdx_dev_params {
 	struct cdx_controller *cdx;
@@ -31,6 +32,7 @@ struct cdx_dev_params {
 	struct resource res[MAX_CDX_DEV_RESOURCES];
 	u8 res_count;
 	u32 req_id;
+	u32 num_msi;
 };
 
 /**
@@ -59,4 +61,12 @@ void cdx_unregister_controller(struct cdx_controller *cdx);
  */
 int cdx_device_add(struct cdx_dev_params *dev_params);
 
+/**
+ * cdx_msi_domain_init - Init the CDX bus MSI domain.
+ * @dev: Device of the CDX bus controller
+ *
+ * Return: CDX MSI domain, NULL on failure
+ */
+struct irq_domain *cdx_msi_domain_init(struct device *dev);
+
 #endif /* _CDX_H_ */
drivers/cdx/cdx_msi.c new file +183 -0

183 lines not on this page · raw message

drivers/cdx/controller/cdx_controller.c +23 -0
index dc52f95f8978..29e8e4a3b8c1 100644
@@ -8,6 +8,7 @@
 #include <linux/of_platform.h>
 #include <linux/slab.h>
 #include <linux/cdx/cdx_bus.h>
+#include <linux/irqdomain.h>
 
 #include "cdx_controller.h"
 #include "../cdx.h"
@@ -49,9 +50,20 @@ static int cdx_configure_device(struct cdx_controller *cdx,
 				u8 bus_num, u8 dev_num,
 				struct cdx_device_config *dev_config)
 {
+	u16 msi_index;
 	int ret = 0;
+	u32 data;
+	u64 addr;
 
 	switch (dev_config->type) {
+	case CDX_DEV_MSI_CONF:
+		msi_index = dev_config->msi.msi_index;
+		data = dev_config->msi.data;
+		addr = dev_config->msi.addr;
+
+		ret = cdx_mcdi_write_msi(cdx->priv, bus_num, dev_num,
+					 msi_index, addr, data);
+		break;
 	case CDX_DEV_RESET_CONF:
 		ret = cdx_mcdi_reset_device(cdx->priv, bus_num, dev_num);
 		break;
@@ -154,6 +166,14 @@ static int xlnx_cdx_probe(struct platform_device *pdev)
 	cdx->priv = cdx_mcdi;
 	cdx->ops = &cdx_ops;
 
+	/* Create MSI domain */
+	cdx->msi_domain = cdx_msi_domain_init(&pdev->dev);
+	if (!cdx->msi_domain) {
+		dev_err(&pdev->dev, "cdx_msi_domain_init() failed");
+		ret = -ENODEV;
+		goto cdx_msi_fail;
+	}
+
 	ret = cdx_setup_rpmsg(pdev);
 	if (ret) {
 		if (ret != -EPROBE_DEFER)
@@ -165,6 +185,8 @@ static int xlnx_cdx_probe(struct platform_device *pdev)
 	return 0;
 
 cdx_rpmsg_fail:
+	irq_domain_remove(cdx->msi_domain);
+cdx_msi_fail:
 	kfree(cdx);
 cdx_alloc_fail:
 	cdx_mcdi_finish(cdx_mcdi);
@@ -181,6 +203,7 @@ static int xlnx_cdx_remove(struct platform_device *pdev)
 
 	cdx_destroy_rpmsg(pdev);
 
+	irq_domain_remove(cdx->msi_domain);
 	kfree(cdx);
 
 	cdx_mcdi_finish(cdx_mcdi);
drivers/cdx/controller/mc_cdx_pcol.h +58 -0

64 lines not on this page · raw message

drivers/cdx/controller/mcdi_functions.c +19 -0
index 0158f26533dd..c26a0d94abf1 100644
@@ -120,10 +120,29 @@ int cdx_mcdi_get_dev_config(struct cdx_mcdi *cdx,
 
 	dev_params->vendor = MCDI_WORD(outbuf, CDX_BUS_GET_DEVICE_CONFIG_OUT_VENDOR_ID);
 	dev_params->device = MCDI_WORD(outbuf, CDX_BUS_GET_DEVICE_CONFIG_OUT_DEVICE_ID);
+	dev_params->num_msi = MCDI_DWORD(outbuf, CDX_BUS_GET_DEVICE_CONFIG_OUT_MSI_COUNT);
 
 	return 0;
 }
 
+int cdx_mcdi_write_msi(struct cdx_mcdi *cdx, u8 bus_num, u8 dev_num,
+		       u32 msi_vector, u64 msi_address, u32 msi_data)
+{
+	MCDI_DECLARE_BUF(inbuf, MC_CMD_CDX_DEVICE_WRITE_MSI_MSG_IN_LEN);
+	int ret;
+
+	MCDI_SET_DWORD(inbuf, CDX_DEVICE_WRITE_MSI_MSG_IN_BUS, bus_num);
+	MCDI_SET_DWORD(inbuf, CDX_DEVICE_WRITE_MSI_MSG_IN_DEVICE, dev_num);
+	MCDI_SET_DWORD(inbuf, CDX_DEVICE_WRITE_MSI_MSG_IN_MSI_VECTOR, msi_vector);
+	MCDI_SET_QWORD(inbuf, CDX_DEVICE_WRITE_MSI_MSG_IN_MSI_ADDRESS, msi_address);
+	MCDI_SET_DWORD(inbuf, CDX_DEVICE_WRITE_MSI_MSG_IN_MSI_DATA, msi_data);
+
+	ret = cdx_mcdi_rpc(cdx, MC_CMD_CDX_DEVICE_WRITE_MSI_MSG, inbuf, sizeof(inbuf),
+			   NULL, 0, NULL);
+
+	return ret;
+}
+
 int cdx_mcdi_reset_device(struct cdx_mcdi *cdx, u8 bus_num, u8 dev_num)
 {
 	MCDI_DECLARE_BUF(inbuf, MC_CMD_CDX_DEVICE_RESET_IN_LEN);
drivers/cdx/controller/mcdi_functions.h +20 -0
index 7440ace5539a..35a300727f34 100644
@@ -47,6 +47,26 @@ int cdx_mcdi_get_dev_config(struct cdx_mcdi *cdx,
 			    u8 bus_num, u8 dev_num,
 			    struct cdx_dev_params *dev_params);
 
+/**
+ * cdx_mcdi_write_msi - Write MSI configuration for CDX device
+ * @cdx: pointer to MCDI interface.
+ * @bus_num: Bus number.
+ * @dev_num: Device number.
+ * @msi_vector: Device-relative MSI vector number.
+ *	Must be < MSI_COUNT reported for the device.
+ * @msi_address: MSI address to be used by the hardware. Typically, on ARM
+ *	systems this address is translated by the IOMMU (if enabled) and
+ *	it is the responsibility of the entity managing the IOMMU (APU kernel)
+ *	to supply the correct IOVA here.
+ * @msi_data: MSI data to be used by the hardware. On versal-net, only the
+ *	lower 16-bits are used, the remaining bits are ignored and should be
+ *	set to zero.
+ *
+ * Return: 0 on success, <0 on failure
+ */
+int cdx_mcdi_write_msi(struct cdx_mcdi *cdx, u8 bus_num, u8 dev_num,
+		       u32 msi_vector, u64 msi_address, u32 msi_data);
+
 /**
  * cdx_mcdi_reset_device - Reset cdx device represented by bus_num:dev_num
  * @cdx: pointer to MCDI interface.
include/linux/cdx/cdx_bus.h +30 -0
index bead71b7bc73..c1b73cef346c 100644
@@ -21,11 +21,19 @@
 struct cdx_controller;
 
 enum {
+	CDX_DEV_MSI_CONF,
 	CDX_DEV_RESET_CONF,
 };
 
+struct cdx_msi_config {
+	u16 msi_index;
+	u32 data;
+	u64 addr;
+};
+
 struct cdx_device_config {
 	u8 type;
+	struct cdx_msi_config msi;
 };
 
 typedef int (*cdx_scan_cb)(struct cdx_controller *cdx);
@@ -62,12 +70,14 @@ struct cdx_ops {
  * struct cdx_controller: CDX controller object
  * @dev: Linux device associated with the CDX controller.
  * @priv: private data
+ * @msi_domain: MSI domain
  * @id: Controller ID
  * @ops: CDX controller ops
  */
 struct cdx_controller {
 	struct device *dev;
 	void *priv;
+	struct irq_domain *msi_domain;
 	u32 id;
 	struct cdx_ops *ops;
 };
@@ -86,9 +96,12 @@ struct cdx_controller {
  * @dma_mask: Default DMA mask
  * @flags: CDX device flags
  * @req_id: Requestor ID associated with CDX device
+ * @num_msi: Number of MSI's supported by the device
  * @driver_override: driver name to force a match; do not set directly,
  *                   because core frees it; use driver_set_override() to
  *                   set or clear it.
+ * @irqchip_lock: lock to synchronize irq/msi configuration
+ * @msi_write_pending: MSI write pending for this device
  */
 struct cdx_device {
 	struct device dev;
@@ -102,7 +115,10 @@ struct cdx_device {
 	u64 dma_mask;
 	u16 flags;
 	u32 req_id;
+	u32 num_msi;
 	const char *driver_override;
+	struct mutex irqchip_lock; /* Serialize write msi configuration */
+	bool msi_write_pending;
 };
 
 #define to_cdx_device(_dev) \
@@ -162,6 +178,20 @@ void cdx_driver_unregister(struct cdx_driver *cdx_driver);
 
 extern struct bus_type cdx_bus_type;
 
+/**
+ * cdx_msi_domain_alloc_irqs - Allocate MSI's for the CDX device
+ * @dev: device pointer
+ * @irq_count: Number of MSI's to be allocated
+ *
+ * Return: 0 for success, -errno on failure
+ */
+int cdx_msi_domain_alloc_irqs(struct device *dev, unsigned int irq_count);
+
+/**
+ * cdx_msi_domain_free_irqs - Free MSI's for CDX device
+ */
+#define cdx_msi_domain_free_irqs msi_domain_free_irqs_all
+
 /**
  * cdx_dev_reset - Reset CDX device
  * @dev: device pointer
kernel/irq/msi.c +1 -0
index 7a97bcb086bf..d9b793900931 100644
@@ -1637,6 +1637,7 @@ void msi_domain_free_irqs_all(struct device *dev, unsigned int domid)
 	msi_domain_free_irqs_all_locked(dev, domid);
 	msi_unlock_descs(dev);
 }
+EXPORT_SYMBOL_GPL(msi_domain_free_irqs_all);
 
 /**
  * msi_get_domain_info - Get the MSI interrupt domain info for @domain
Signature · 1 line
2.17.1
author maintainer
#1 lore raw
Quoted 21 lines · Nipun Gupta

Add CDX-MSI domain per CDX controller with gic-its domain as a parent, to support MSI for CDX devices. CDX devices allocate MSIs from the CDX domain. Also, introduce APIs to alloc and free IRQs for CDX domain.

In CDX subsystem firmware is a controller for all devices and their configuration. CDX bus controller sends all the write_msi_msg commands to firmware running on RPU and the firmware interfaces with actual devices to pass this information to devices

Since, CDX controller is the only way to communicate with the Firmware for MSI write info, CDX domain per controller required in contrast to having a CDX domain per device.

Quoted 7 lines

---

Changes v1->v2:
- fixed scenario where msi write was called asyncronously in an atomic context, by using irq_chip_(un)lock, and using sync MCDI API for write MSI message.
- fixed broken Signed-off-by chain.

Hi Thomas,

Did you had a chance to look at patch v2. Please let me know in case anything else is required to be updated.

Thanks,
Nipun

author maintainer
#2 lore raw
Quoted 32 lines · Nipun Gupta
Quoted 21 lines · Nipun Gupta

Add CDX-MSI domain per CDX controller with gic-its domain as a parent, to support MSI for CDX devices. CDX devices allocate MSIs from the CDX domain. Also, introduce APIs to alloc and free IRQs for CDX domain.

In CDX subsystem firmware is a controller for all devices and their configuration. CDX bus controller sends all the write_msi_msg commands to firmware running on RPU and the firmware interfaces with actual devices to pass this information to devices

Since, CDX controller is the only way to communicate with the Firmware for MSI write info, CDX domain per controller required in contrast to having a CDX domain per device.

Quoted 7 lines

---

Changes v1->v2:
- fixed scenario where msi write was called asyncronously in an atomic context, by using irq_chip_(un)lock, and using sync MCDI API for write MSI message.
- fixed broken Signed-off-by chain.

Quoted 3 lines

Hi Thomas,

Did you had a chance to look at patch v2. Please let me know in case anything else is required to be updated.

Hi Thomas,

A gentle reminder, could you please have a look into this updated version of the patch?

Thanks,
Nipun

#3 lore raw
Quoted 37 lines · Nipun Gupta
Quoted 34 lines · Nipun Gupta
Quoted 21 lines · Nipun Gupta

Add CDX-MSI domain per CDX controller with gic-its domain as a parent, to support MSI for CDX devices. CDX devices allocate MSIs from the CDX domain. Also, introduce APIs to alloc and free IRQs for CDX domain.

In CDX subsystem firmware is a controller for all devices and their configuration. CDX bus controller sends all the write_msi_msg commands to firmware running on RPU and the firmware interfaces with actual devices to pass this information to devices

Since, CDX controller is the only way to communicate with the Firmware for MSI write info, CDX domain per controller required in contrast to having a CDX domain per device.

Quoted 7 lines

---

Changes v1->v2:
- fixed scenario where msi write was called asyncronously in an atomic context, by using irq_chip_(un)lock, and using sync MCDI API for write MSI message.
- fixed broken Signed-off-by chain.

Hi Thomas,

Quoted 7 lines
Quoted 2 lines

Did you had a chance to look at patch v2. Please let me know in case anything else is required to be updated.

Hi Thomas,

A gentle reminder, could you please have a look into this updated version of the patch?

It's the middle of the merge window, nothing we can do now with new stuff. Perhaps resend it after 6.5-rc1 is out and you have rebased it against that release?

thanks,

greg k-h

author maintainer
#4 lore raw
Quoted 41 lines · Greg KH
Quoted 40 lines · Nipun Gupta
Quoted 36 lines · Nipun Gupta
Quoted 21 lines · Nipun Gupta

Add CDX-MSI domain per CDX controller with gic-its domain as a parent, to support MSI for CDX devices. CDX devices allocate MSIs from the CDX domain. Also, introduce APIs to alloc and free IRQs for CDX domain.

In CDX subsystem firmware is a controller for all devices and their configuration. CDX bus controller sends all the write_msi_msg commands to firmware running on RPU and the firmware interfaces with actual devices to pass this information to devices

Since, CDX controller is the only way to communicate with the Firmware for MSI write info, CDX domain per controller required in contrast to having a CDX domain per device.

Quoted 7 lines

---

Changes v1->v2:
- fixed scenario where msi write was called asyncronously in an atomic context, by using irq_chip_(un)lock, and using sync MCDI API for write MSI message.
- fixed broken Signed-off-by chain.

Hi Thomas,

Did you had a chance to look at patch v2. Please let me know in case anything else is required to be updated.

Quoted 8 lines
Quoted 4 lines

Hi Thomas,

A gentle reminder, could you please have a look into this updated version of the patch?

It's the middle of the merge window, nothing we can do now with new stuff. Perhaps resend it after 6.5-rc1 is out and you have rebased it against that release?

Sure, will rebase and resend after 6.5-rc1 is out.

Thanks,
Nipun

Quoted 4 lines

thanks,

greg k-h