Skip to contentSkip to conversation

powerpc/dma: Define map/unmap mmio resource callbacks

PATCH Superseded Max Gurtovoy 3 participants 8 messages Apr 16, 2020 – Apr 19, 2020

Subsystems (2)

v1 of 2 Superseded Latest is v2 →

lore
Share & tools
lore
#0 lore raw

PATCH 1/2 powerpc/dma: Define map/unmap mmio resource callbacks

Define the map_resource/unmap_resource callbacks for the dma_iommu_ops used by several powerpc platforms. The map_resource callback is called when trying to map a mmio resource through the dma_map_resource() driver API.

For now, the callback returns an invalid address for devices using translations, but will "direct" map the resource when in bypass mode. Previous behavior for dma_map_resource() was to always return an invalid address.

We also call an optional platform-specific controller op in case some setup is needed for the platform.

---
 arch/powerpc/include/asm/pci-bridge.h |  6 ++++++
 arch/powerpc/kernel/dma-iommu.c       | 35 +++++++++++++++++++++++++++++++++++
 2 files changed, 41 insertions(+)
arch/powerpc/include/asm/pci-bridge.h +6 -0
index 69f4cb3..d14b4ee 100644
@@ -44,6 +44,12 @@ struct pci_controller_ops {
 #endif
 
 	void		(*shutdown)(struct pci_controller *hose);
+	int		(*dma_map_resource)(struct pci_dev *pdev,
+					    phys_addr_t phys_addr, size_t size,
+					    enum dma_data_direction dir);
+	void		(*dma_unmap_resource)(struct pci_dev *pdev,
+					      dma_addr_t addr, size_t size,
+					      enum dma_data_direction dir);
 };
 
 /*
arch/powerpc/kernel/dma-iommu.c +35 -0
index e486d1d..dc53acc 100644
@@ -108,6 +108,39 @@ static void dma_iommu_unmap_sg(struct device *dev, struct scatterlist *sglist,
 		dma_direct_unmap_sg(dev, sglist, nelems, direction, attrs);
 }
 
+static dma_addr_t dma_iommu_map_resource(struct device *dev,
+					 phys_addr_t phys_addr, size_t size,
+					 enum dma_data_direction dir,
+					 unsigned long attrs)
+{
+	struct pci_dev *pdev = to_pci_dev(dev);
+	struct pci_controller *phb = pci_bus_to_host(pdev->bus);
+
+	if (dma_iommu_map_bypass(dev, attrs)) {
+		if (phb->controller_ops.dma_map_resource &&
+		    phb->controller_ops.dma_map_resource(pdev, phys_addr, size,
+							 dir))
+			return DMA_MAPPING_ERROR;
+		return dma_direct_map_resource(dev, phys_addr, size,
+					       dir, attrs);
+	}
+	return DMA_MAPPING_ERROR;
+}
+
+static void dma_iommu_unmap_resource(struct device *dev, dma_addr_t dma_handle,
+				     size_t size, enum dma_data_direction dir,
+				     unsigned long attrs)
+{
+	struct pci_dev *pdev = to_pci_dev(dev);
+	struct pci_controller *phb = pci_bus_to_host(pdev->bus);
+
+	if (dma_iommu_map_bypass(dev, attrs)) {
+		if (phb->controller_ops.dma_unmap_resource)
+			phb->controller_ops.dma_unmap_resource(pdev, dma_handle,
+							size, dir);
+	}
+}
+
 static bool dma_iommu_bypass_supported(struct device *dev, u64 mask)
 {
 	struct pci_dev *pdev = to_pci_dev(dev);
@@ -199,6 +232,8 @@ extern void dma_iommu_sync_sg_for_device(struct device *dev,
 	.free			= dma_iommu_free_coherent,
 	.map_sg			= dma_iommu_map_sg,
 	.unmap_sg		= dma_iommu_unmap_sg,
+	.map_resource		= dma_iommu_map_resource,
+	.unmap_resource		= dma_iommu_unmap_resource,
 	.dma_supported		= dma_iommu_dma_supported,
 	.map_page		= dma_iommu_map_page,
 	.unmap_page		= dma_iommu_unmap_page,
Signature · 1 line
1.8.3.1
#1 lore raw

PATCH 2/2 powerpc/powernv: Enable and setup PCI P2P

Implement the generic dma_map_resource callback on the PCI controller for powernv. This will enable PCI P2P on POWER9 architecture. It will allow catching a cross-PHB mmio mapping, which needs to be setup in hardware by calling opal. Both the initiator and target PHBs need to be configured, so we look for which PHB owns the mmio address being mapped.

[maxg: added CONFIG_PCI_P2PDMA wrappers]
---
 arch/powerpc/include/asm/opal.h            |   5 +-
 arch/powerpc/platforms/powernv/opal-call.c |   1 +
 arch/powerpc/platforms/powernv/pci-ioda.c  | 164 +++++++++++++++++++++++++++++
 arch/powerpc/platforms/powernv/pci.h       |   9 ++
 4 files changed, 178 insertions(+), 1 deletion(-)
arch/powerpc/include/asm/opal.h +4 -1
index 9986ac3..d1b35c4 100644
@@ -284,7 +284,10 @@ int64_t opal_xive_set_queue_state(uint64_t vp, uint32_t prio,
 				  uint32_t qtoggle,
 				  uint32_t qindex);
 int64_t opal_xive_get_vp_state(uint64_t vp, __be64 *out_w01);
-
+#ifdef CONFIG_PCI_P2PDMA
+int64_t opal_pci_set_p2p(uint64_t phb_init, uint64_t phb_target,
+			 uint64_t desc, uint16_t pe_number);
+#endif
 int64_t opal_imc_counters_init(uint32_t type, uint64_t address,
 							uint64_t cpu_pir);
 int64_t opal_imc_counters_start(uint32_t type, uint64_t cpu_pir);
arch/powerpc/platforms/powernv/opal-call.c +1 -0
index 5cd0f52..442d5445c 100644
@@ -273,6 +273,7 @@ int64_t name(int64_t a0, int64_t a1, int64_t a2, int64_t a3,	\
 OPAL_CALL(opal_imc_counters_init,		OPAL_IMC_COUNTERS_INIT);
 OPAL_CALL(opal_imc_counters_start,		OPAL_IMC_COUNTERS_START);
 OPAL_CALL(opal_imc_counters_stop,		OPAL_IMC_COUNTERS_STOP);
+OPAL_CALL(opal_pci_set_p2p,			OPAL_PCI_SET_P2P);
 OPAL_CALL(opal_get_powercap,			OPAL_GET_POWERCAP);
 OPAL_CALL(opal_set_powercap,			OPAL_SET_POWERCAP);
 OPAL_CALL(opal_get_power_shift_ratio,		OPAL_GET_POWER_SHIFT_RATIO);
arch/powerpc/platforms/powernv/pci-ioda.c +164 -0

176 lines not on this page · raw message

arch/powerpc/platforms/powernv/pci.h +9 -0
index d3bbdea..5f85d9c 100644
@@ -79,6 +79,10 @@ struct pnv_ioda_pe {
 	struct pnv_ioda_pe	*master;
 	struct list_head	slaves;
 
+#ifdef CONFIG_PCI_P2PDMA
+	/* PCI peer-to-peer*/
+	int			p2p_initiator_count;
+#endif
 	/* Link in list of PE#s */
 	struct list_head	list;
 };
@@ -168,6 +172,11 @@ struct pnv_phb {
 	/* PHB and hub diagnostics */
 	unsigned int		diag_data_size;
 	u8			*diag_data;
+
+#ifdef CONFIG_PCI_P2PDMA
+	/* PCI peer-to-peer*/
+	int			p2p_target_count;
+#endif
 };
 
 extern struct pci_ops pnv_pci_ops;
Signature · 1 line
1.8.3.1
#2 lore raw

Re: [PATCH 2/2] powerpc/powernv: Enable and setup PCI P2P

arch/powerpc/include/asm/opal.h int64_t opal_xive_set_queue_state(uint64_t vp, uint32_t prio, · line 290

Quoted 4 lines
arch/powerpc/include/asm/opal.h +4 -0
+#ifdef CONFIG_PCI_P2PDMA
+int64_t opal_pci_set_p2p(uint64_t phb_init, uint64_t phb_target,
+			 uint64_t desc, uint16_t pe_number);
+#endif

There should be no need for the ifdef here.

arch/powerpc/platforms/powernv/pci-ioda.c static void pnv_pci_ioda_dma_bus_setup(struct pci_bus *bus) · line 3742

Quoted 12 lines
arch/powerpc/platforms/powernv/pci-ioda.c +12 -0
+	/*
+	 * Configuring the initiator's PHB requires to adjust its
+	 * TVE#1 setting. Since the same device can be an initiator
+	 * several times for different target devices, we need to keep
+	 * a reference count to know when we can restore the default
+	 * bypass setting on its TVE#1 when disabling. Opal is not
+	 * tracking PE states, so we add a reference count on the PE
+	 * in linux.
+	 *
+	 * For the target, the configuration is per PHB, so we keep a
+	 * target reference count on the PHB.
+	 */

This could be shortened a bit by using up the whole 80 lines available in source files.

Quoted 29 lines
arch/powerpc/platforms/powernv/pci-ioda.c +29 -0
+	mutex_lock(&p2p_mutex);
+
+	if (desc & OPAL_PCI_P2P_ENABLE) {
+		/* always go to opal to validate the configuration */
+		rc = opal_pci_set_p2p(phb_init->opal_id, phb_target->opal_id,
+				      desc, pe_init->pe_number);
+
+		if (rc != OPAL_SUCCESS) {
+			rc = -EIO;
+			goto out;
+		}
+
+		pe_init->p2p_initiator_count++;
+		phb_target->p2p_target_count++;
+	} else {
+		if (!pe_init->p2p_initiator_count ||
+		    !phb_target->p2p_target_count) {
+			rc = -EINVAL;
+			goto out;
+		}
+
+		if (--pe_init->p2p_initiator_count == 0)
+			pnv_pci_ioda2_set_bypass(pe_init, true);
+
+		if (--phb_target->p2p_target_count == 0) {
+			rc = opal_pci_set_p2p(phb_init->opal_id,
+					      phb_target->opal_id, desc,
+					      pe_init->pe_number);
+			if (rc != OPAL_SUCCESS) {

arch/powerpc/platforms/powernv/pci-ioda.c static void pnv_pci_ioda_dma_bus_setup(struct pci_bus *bus) · line 3781

Quoted 10 lines
arch/powerpc/platforms/powernv/pci-ioda.c +10 -0
+				rc = -EIO;
+				goto out;
+			}
+		}
+	}
+	rc = 0;
+out:
+	mutex_unlock(&p2p_mutex);
+	return rc;
+}

The enable and disable path shares almost no code, why not split it into two functions?

arch/powerpc/platforms/powernv/pci-ioda.c static void pnv_pci_ioda_dma_bus_setup(struct pci_bus *bus) · line 3791

Quoted 9 lines
arch/powerpc/platforms/powernv/pci-ioda.c +9 -0
+static bool pnv_pci_controller_owns_addr(struct pci_controller *hose,
+					 phys_addr_t addr, size_t size)
+{
+	struct resource *r;
+	int i;
+
+	/*
+	 * it seems safe to assume the full range is under the same
+	 * PHB, so we can ignore the size

Capitalize the first character in a multi-line comment, and use up the whole 80 chars.

Review

Quoted 2 lines
(quoted diff) +2 -0
+	 */
+	for (i = 0; i < 3; i++) {

Replace the magic 3 with ARRAY_SIZE(hose->mem_resources) ?

arch/powerpc/platforms/powernv/pci-ioda.c static void pnv_pci_ioda_dma_bus_setup(struct pci_bus *bus) · line 3794

Quoted 1 line
arch/powerpc/platforms/powernv/pci-ioda.c +1 -0
+		r = &hose->mem_resources[i];

Move the r declaration here and initialize it on the same line.

arch/powerpc/platforms/powernv/pci-ioda.c static void pnv_pci_ioda_dma_bus_setup(struct pci_bus *bus) · line 3795

Quoted 1 line
arch/powerpc/platforms/powernv/pci-ioda.c +1 -0
+		if (r->flags && (addr >= r->start) && (addr < r->end))

No need for the inner braces.

Quoted 7 lines
arch/powerpc/platforms/powernv/pci-ioda.c +7 -0
+			return true;
+	}
+	return false;
+}
+
+/*
+ * find the phb owning a mmio address if not owned locally

arch/powerpc/platforms/powernv/pci-ioda.c static void pnv_pci_ioda_dma_bus_setup(struct pci_bus *bus) · line 3812

Quoted 10 lines
arch/powerpc/platforms/powernv/pci-ioda.c +10 -0
+ */
+static struct pnv_phb *pnv_pci_find_owning_phb(struct pci_dev *pdev,
+					       phys_addr_t addr, size_t size)
+{
+	struct pci_controller *hose = pci_bus_to_host(pdev->bus);
+	struct pnv_phb *phb;
+
+	/* fast path */
+	if (pnv_pci_controller_owns_addr(hose, addr, size))
+		return NULL;

Maybe open code the pci_bus_to_host(pdev->bus) call here, as we just overwrite host in the list iteration?

arch/powerpc/platforms/powernv/pci-ioda.c static void pnv_pci_ioda_dma_bus_setup(struct pci_bus *bus) · line 3815

Quoted 3 lines
arch/powerpc/platforms/powernv/pci-ioda.c +3 -0
+
+	list_for_each_entry(hose, &hose_list, list_node) {
+		phb = hose->private_data;

Move the ohb declaration here and initialize it on the same line.

Quoted 11 lines
arch/powerpc/platforms/powernv/pci-ioda.c +9 -0
+		if (phb->type != PNV_PHB_NPU_NVLINK &&
+		    phb->type != PNV_PHB_NPU_OCAPI) {
+			if (pnv_pci_controller_owns_addr(hose, addr, size))
+				return phb;
+		}
+	}
 
 
+	return NULL;
+}
+

arch/powerpc/platforms/powernv/pci-ioda.c static void pnv_pci_ioda_dma_bus_setup(struct pci_bus *bus) · line 3834

Quoted 10 lines
arch/powerpc/platforms/powernv/pci-ioda.c +10 -0
+static int pnv_pci_dma_map_resource(struct pci_dev *pdev,
+				    phys_addr_t phys_addr, size_t size,
+				    enum dma_data_direction dir)
+{
+	struct pnv_phb *target_phb;
+	int rc;
+	u64 desc;
+
+	target_phb = pnv_pci_find_owning_phb(pdev, phys_addr, size);
+	if (target_phb) {

Return early here for the !target_phb case?

arch/powerpc/platforms/powernv/pci-ioda.c static void pnv_pci_ioda_dma_bus_setup(struct pci_bus *bus) · line 3841

Quoted 7 lines
arch/powerpc/platforms/powernv/pci-ioda.c +7 -0
+		desc = OPAL_PCI_P2P_ENABLE;
+		if (dir == DMA_TO_DEVICE)
+			desc |= OPAL_PCI_P2P_STORE;
+		else if (dir == DMA_FROM_DEVICE)
+			desc |= OPAL_PCI_P2P_LOAD;
+		else if (dir == DMA_BIDIRECTIONAL)
+			desc |= OPAL_PCI_P2P_LOAD | OPAL_PCI_P2P_STORE;

Seems like this could be split into a little helper.

arch/powerpc/platforms/powernv/pci-ioda.c static void pnv_pci_ioda_dma_bus_setup(struct pci_bus *bus) · line 3847

Quoted 6 lines
arch/powerpc/platforms/powernv/pci-ioda.c +6 -0
+		rc = pnv_pci_ioda_set_p2p(pdev, target_phb, desc);
+		if (rc) {
+			dev_err(&pdev->dev, "Failed to setup PCI peer-to-peer for address %llx: %d\n",
+				phys_addr, rc);
+			return rc;
+		}

No need for this printk, the callers should already deal with mapping failures.

#3 lore raw

Re: [PATCH 2/2] powerpc/powernv: Enable and setup PCI P2P

Quoted 54 lines · Christoph Hellwig

arch/powerpc/include/asm/opal.h int64_t opal_xive_set_queue_state(uint64_t vp, uint32_t prio, · line 290

Quoted 4 lines
arch/powerpc/include/asm/opal.h +4 -0
+#ifdef CONFIG_PCI_P2PDMA
+int64_t opal_pci_set_p2p(uint64_t phb_init, uint64_t phb_target,
+			 uint64_t desc, uint16_t pe_number);
+#endif

There should be no need for the ifdef here.

arch/powerpc/platforms/powernv/pci-ioda.c static void pnv_pci_ioda_dma_bus_setup(struct pci_bus *bus) · line 3742

Quoted 12 lines
arch/powerpc/platforms/powernv/pci-ioda.c +12 -0
+	/*
+	 * Configuring the initiator's PHB requires to adjust its
+	 * TVE#1 setting. Since the same device can be an initiator
+	 * several times for different target devices, we need to keep
+	 * a reference count to know when we can restore the default
+	 * bypass setting on its TVE#1 when disabling. Opal is not
+	 * tracking PE states, so we add a reference count on the PE
+	 * in linux.
+	 *
+	 * For the target, the configuration is per PHB, so we keep a
+	 * target reference count on the PHB.
+	 */

This could be shortened a bit by using up the whole 80 lines available in source files.

Quoted 33 lines
arch/powerpc/platforms/powernv/pci-ioda.c +33 -0
+	mutex_lock(&p2p_mutex);
+
+	if (desc & OPAL_PCI_P2P_ENABLE) {
+		/* always go to opal to validate the configuration */
+		rc = opal_pci_set_p2p(phb_init->opal_id, phb_target->opal_id,
+				      desc, pe_init->pe_number);
+
+		if (rc != OPAL_SUCCESS) {
+			rc = -EIO;
+			goto out;
+		}
+
+		pe_init->p2p_initiator_count++;
+		phb_target->p2p_target_count++;
+	} else {
+		if (!pe_init->p2p_initiator_count ||
+		    !phb_target->p2p_target_count) {
+			rc = -EINVAL;
+			goto out;
+		}
+
+		if (--pe_init->p2p_initiator_count == 0)
+			pnv_pci_ioda2_set_bypass(pe_init, true);
+
+		if (--phb_target->p2p_target_count == 0) {
+			rc = opal_pci_set_p2p(phb_init->opal_id,
+					      phb_target->opal_id, desc,
+					      pe_init->pe_number);
+			if (rc != OPAL_SUCCESS) {
+				rc = -EIO;
+				goto out;
+			}
+		}
Quoted 8 lines

arch/powerpc/platforms/powernv/pci-ioda.c static void pnv_pci_ioda_dma_bus_setup(struct pci_bus *bus) · line 3781

Quoted 6 lines
arch/powerpc/platforms/powernv/pci-ioda.c +6 -0
+	}
+	rc = 0;
+out:
+	mutex_unlock(&p2p_mutex);
+	return rc;
+}

The enable and disable path shares almost no code, why not split it into two functions?

how about also changing the defines OPAL_PCI_P2P_* to an enum ?

/* PCI p2p operation descriptors */
enum opal_pci_p2p {

        OPAL_PCI_P2P_DISABLE    = 0,

        OPAL_PCI_P2P_ENABLE     = (1 << 0),
        OPAL_PCI_P2P_LOAD       = (1 << 1),
        OPAL_PCI_P2P_STORE      = (1 << 2),
};

Fred ?

Quoted 34 lines

arch/powerpc/platforms/powernv/pci-ioda.c static void pnv_pci_ioda_dma_bus_setup(struct pci_bus *bus) · line 3791

Quoted 9 lines
arch/powerpc/platforms/powernv/pci-ioda.c +9 -0
+static bool pnv_pci_controller_owns_addr(struct pci_controller *hose,
+					 phys_addr_t addr, size_t size)
+{
+	struct resource *r;
+	int i;
+
+	/*
+	 * it seems safe to assume the full range is under the same
+	 * PHB, so we can ignore the size

Capitalize the first character in a multi-line comment, and use up the whole 80 chars.

Review

Quoted 2 lines
(quoted diff) +2 -0
+	 */
+	for (i = 0; i < 3; i++) {

Replace the magic 3 with ARRAY_SIZE(hose->mem_resources) ?

arch/powerpc/platforms/powernv/pci-ioda.c static void pnv_pci_ioda_dma_bus_setup(struct pci_bus *bus) · line 3794

Quoted 1 line
arch/powerpc/platforms/powernv/pci-ioda.c +1 -0
+		r = &hose->mem_resources[i];

Move the r declaration here and initialize it on the same line.

arch/powerpc/platforms/powernv/pci-ioda.c static void pnv_pci_ioda_dma_bus_setup(struct pci_bus *bus) · line 3795

Quoted 1 line
arch/powerpc/platforms/powernv/pci-ioda.c +1 -0
+		if (r->flags && (addr >= r->start) && (addr < r->end))

No need for the inner braces.

Quoted 11 lines
arch/powerpc/platforms/powernv/pci-ioda.c +11 -0
+			return true;
+	}
+	return false;
+}
+
+/*
+ * find the phb owning a mmio address if not owned locally
+ */
+static struct pnv_phb *pnv_pci_find_owning_phb(struct pci_dev *pdev,
+					       phys_addr_t addr, size_t size)
+{
Quoted 8 lines

arch/powerpc/platforms/powernv/pci-ioda.c static void pnv_pci_ioda_dma_bus_setup(struct pci_bus *bus) · line 3812

Quoted 6 lines
arch/powerpc/platforms/powernv/pci-ioda.c +6 -0
+	struct pci_controller *hose = pci_bus_to_host(pdev->bus);
+	struct pnv_phb *phb;
+
+	/* fast path */
+	if (pnv_pci_controller_owns_addr(hose, addr, size))
+		return NULL;

Maybe open code the pci_bus_to_host(pdev->bus) call here, as we just overwrite host in the list iteration?

if this is more readable so no problem:

if (pnv_pci_controller_owns_addr(pdev->bus->sysdata, addr, size))

Quoted 28 lines

arch/powerpc/platforms/powernv/pci-ioda.c static void pnv_pci_ioda_dma_bus_setup(struct pci_bus *bus) · line 3815

Quoted 4 lines
arch/powerpc/platforms/powernv/pci-ioda.c +3 -0
+
+	list_for_each_entry(hose, &hose_list, list_node) {
+		phb = hose->private_data;

Move the ohb declaration here and initialize it on the same line.

Quoted 10 lines
arch/powerpc/platforms/powernv/pci-ioda.c +9 -0
+		if (phb->type != PNV_PHB_NPU_NVLINK &&
+		    phb->type != PNV_PHB_NPU_OCAPI) {
+			if (pnv_pci_controller_owns_addr(hose, addr, size))
+				return phb;
+		}
+	}
 
+	return NULL;
+}
+

arch/powerpc/platforms/powernv/pci-ioda.c static void pnv_pci_ioda_dma_bus_setup(struct pci_bus *bus) · line 3834

Quoted 10 lines
arch/powerpc/platforms/powernv/pci-ioda.c +10 -0
+static int pnv_pci_dma_map_resource(struct pci_dev *pdev,
+				    phys_addr_t phys_addr, size_t size,
+				    enum dma_data_direction dir)
+{
+	struct pnv_phb *target_phb;
+	int rc;
+	u64 desc;
+
+	target_phb = pnv_pci_find_owning_phb(pdev, phys_addr, size);
+	if (target_phb) {

Return early here for the !target_phb case?

Quoted 8 lines

arch/powerpc/platforms/powernv/pci-ioda.c static void pnv_pci_ioda_dma_bus_setup(struct pci_bus *bus) · line 3841

Quoted 7 lines
arch/powerpc/platforms/powernv/pci-ioda.c +7 -0
+		desc = OPAL_PCI_P2P_ENABLE;
+		if (dir == DMA_TO_DEVICE)
+			desc |= OPAL_PCI_P2P_STORE;
+		else if (dir == DMA_FROM_DEVICE)
+			desc |= OPAL_PCI_P2P_LOAD;
+		else if (dir == DMA_BIDIRECTIONAL)
+			desc |= OPAL_PCI_P2P_LOAD | OPAL_PCI_P2P_STORE;

Seems like this could be split into a little helper.

sure.

Quoted 10 lines

arch/powerpc/platforms/powernv/pci-ioda.c static void pnv_pci_ioda_dma_bus_setup(struct pci_bus *bus) · line 3847

Quoted 7 lines
arch/powerpc/platforms/powernv/pci-ioda.c +6 -0
+		rc = pnv_pci_ioda_set_p2p(pdev, target_phb, desc);
+		if (rc) {
+			dev_err(&pdev->dev, "Failed to setup PCI peer-to-peer for address %llx: %d\n",
+				phys_addr, rc);
+			return rc;
+		}

No need for this printk, the callers should already deal with mapping failures.

#4 lore raw

Re: [PATCH 2/2] powerpc/powernv: Enable and setup PCI P2P

Quoted 9 lines · Max Gurtovoy
Quoted 3 lines

The enable and disable path shares almost no code, why not split it into two functions?

how about also changing the defines OPAL_PCI_P2P_* to an enum ?

/* PCI p2p operation descriptors */
enum opal_pci_p2p {

Quoted 8 lines
         OPAL_PCI_P2P_DISABLE    = 0,

         OPAL_PCI_P2P_ENABLE     = (1 << 0),
         OPAL_PCI_P2P_LOAD       = (1 << 1),
         OPAL_PCI_P2P_STORE      = (1 << 2),
};

Fred ?

I'd rather you didn't. We try to keep the definitions in opal-api.h the same as the skiboot's opal-api.h since the skiboot version is canonical.

Also, generally patches to anything PowerNV related go through the powerpc tree rather than the pci tree even if they're PCI related. Can you make sure you have linuxppc-dev CCed when you post v2.

Oliver

#5 lore raw

Re: [PATCH 2/2] powerpc/powernv: Enable and setup PCI P2P

Quoted 15 lines · Oliver O'Halloran
Quoted 14 lines · Max Gurtovoy
Quoted 2 lines

The enable and disable path shares almost no code, why not split it into two functions?

how about also changing the defines OPAL_PCI_P2P_* to an enum ?

/* PCI p2p operation descriptors */
enum opal_pci_p2p {

          OPAL_PCI_P2P_DISABLE    = 0,

          OPAL_PCI_P2P_ENABLE     = (1 << 0),
          OPAL_PCI_P2P_LOAD       = (1 << 1),
          OPAL_PCI_P2P_STORE      = (1 << 2),
};
Quoted 8 lines
Quoted 1 line

Fred ?

I'd rather you didn't. We try to keep the definitions in opal-api.h the same as the skiboot's opal-api.h since the skiboot version is canonical.

Also, generally patches to anything PowerNV related go through the powerpc tree rather than the pci tree even if they're PCI related. Can you make sure you have linuxppc-dev CCed when you post v2.

Sure, no problem.

Quoted 2 lines

Oliver

#6 lore raw
In reply to Max Gurtovoy
Quoted 10 lines
arch/powerpc/kernel/dma-iommu.c +7 -0
 		dma_direct_unmap_sg(dev, sglist, nelems, direction, attrs);
 }
 
+static dma_addr_t dma_iommu_map_resource(struct device *dev,
+					 phys_addr_t phys_addr, size_t size,
+					 enum dma_data_direction dir,
+					 unsigned long attrs)
+{
+	struct pci_dev *pdev = to_pci_dev(dev);
+	struct pci_controller *phb = pci_bus_to_host(pdev->bus);

arch/powerpc/kernel/dma-iommu.c static void dma_iommu_unmap_sg(struct device *dev, struct scatterlist *sglist, · line 127

Quoted 10 lines
arch/powerpc/kernel/dma-iommu.c +10 -0
+
+	if (dma_iommu_map_bypass(dev, attrs)) {
+		if (phb->controller_ops.dma_map_resource &&
+		    phb->controller_ops.dma_map_resource(pdev, phys_addr, size,
+							 dir))
+			return DMA_MAPPING_ERROR;
+		return dma_direct_map_resource(dev, phys_addr, size,
+					       dir, attrs);
+	}
+	return DMA_MAPPING_ERROR;

Just a nitpick, but to the return errors early would be a much more logical flow. Also if the callback just decides if the mapping is possible in bypass mode it should have that in the name:

	struct pci_controller_ops *ops = &phb->controller_ops;

	if (!dma_iommu_map_bypass(dev, attrs) ||
	    !ops->dma_can_direct_map_resource ||
	    !ops->dma_can_direct_map_resource(pdev, phys_addr, size, dir)
		return DMA_MAPPING_ERROR;
	return dma_direct_map_resource(dev, phys_addr, size, dir, attrs);

arch/powerpc/kernel/dma-iommu.c static void dma_iommu_unmap_sg(struct device *dev, struct scatterlist *sglist, · line 141

Quoted 5 lines
arch/powerpc/kernel/dma-iommu.c +5 -0
+	if (dma_iommu_map_bypass(dev, attrs)) {
+		if (phb->controller_ops.dma_unmap_resource)
+			phb->controller_ops.dma_unmap_resource(pdev, dma_handle,
+							size, dir);
+	}

This can do a little early return as well, coupled with a WARN_ON_ONCE as we should never end up in the unmap path for a setup where the map didn't work.

#7 lore raw
Quoted 24 lines · Christoph Hellwig
Quoted 10 lines
  		dma_direct_unmap_sg(dev, sglist, nelems, direction, attrs);
  }
arch/powerpc/kernel/dma-iommu.c +7 -0
+static dma_addr_t dma_iommu_map_resource(struct device *dev,
+					 phys_addr_t phys_addr, size_t size,
+					 enum dma_data_direction dir,
+					 unsigned long attrs)
+{
+	struct pci_dev *pdev = to_pci_dev(dev);
+	struct pci_controller *phb = pci_bus_to_host(pdev->bus);

arch/powerpc/kernel/dma-iommu.c static void dma_iommu_unmap_sg(struct device *dev, struct scatterlist *sglist, · line 127

Quoted 10 lines
arch/powerpc/kernel/dma-iommu.c +10 -0
+
+	if (dma_iommu_map_bypass(dev, attrs)) {
+		if (phb->controller_ops.dma_map_resource &&
+		    phb->controller_ops.dma_map_resource(pdev, phys_addr, size,
+							 dir))
+			return DMA_MAPPING_ERROR;
+		return dma_direct_map_resource(dev, phys_addr, size,
+					       dir, attrs);
+	}
+	return DMA_MAPPING_ERROR;

Just a nitpick, but to the return errors early would be a much more logical flow. Also if the callback just decides if the mapping is possible in bypass mode it should have that in the name:

Quoted 7 lines
	struct pci_controller_ops *ops = &phb->controller_ops;

	if (!dma_iommu_map_bypass(dev, attrs) ||
	    !ops->dma_can_direct_map_resource ||
	    !ops->dma_can_direct_map_resource(pdev, phys_addr, size, dir)
		return DMA_MAPPING_ERROR;
	return dma_direct_map_resource(dev, phys_addr, size, dir, attrs);

maybe call it dma_direct_map_resource/dma_direct_unmap_resource for symmetry ?

do you mean (small logic change):

        if (!dma_iommu_map_bypass(dev, attrs) ||
            !ops->dma_direct_map_resource ||

            ops->dma_direct_map_resource(pdev, phys_addr, size, dir)) return DMA_MAPPING_ERROR;

        return dma_direct_map_resource(dev, phys_addr, size, dir, attrs);

Quoted 9 lines

arch/powerpc/kernel/dma-iommu.c static void dma_iommu_unmap_sg(struct device *dev, struct scatterlist *sglist, · line 141

Quoted 6 lines
arch/powerpc/kernel/dma-iommu.c +5 -0
+	if (dma_iommu_map_bypass(dev, attrs)) {
+		if (phb->controller_ops.dma_unmap_resource)
+			phb->controller_ops.dma_unmap_resource(pdev, dma_handle,
+							size, dir);
+	}

This can do a little early return as well, coupled with a WARN_ON_ONCE as we should never end up in the unmap path for a setup where the map didn't work.

how about:

        struct pci_controller_ops *ops = &phb->controller_ops;

        if (dma_iommu_map_bypass(dev, attrs) && ops->dma_direct_unmap_resource) ops->dma_direct_unmap_resource(pdev, dma_handle, size, dir);