Skip to contentSkip to conversation

PCI: dwc: Support multiple ATU memory regions

PATCH Applied Rob Herring 4 participants 4 messages Oct 26, 2020 – Nov 18, 2020

Subsystems (3)
lore
Share & tools
lore

Rob Herring

author reviewer
#0 lore raw

PATCH 1/1 PCI: dwc: Support multiple ATU memory regions

The current ATU setup only supports a single memory resource which isn't sufficient if there are also prefetchable memory regions. In order to support multiple memory regions, we need to move away from fixed ATU slots and rework the assignment. As there's always an ATU entry for config space, let's assign index 0 to config space. Then we assign memory resources to index 1 and up. Finally, if we have an I/O region and slots remaining, we assign the I/O region last. If there aren't remaining slots, we keep the same config and I/O space sharing.

Cc Vidya Sagar <[email protected]>
Cc Jingoo Han <[email protected]>
Cc Gustavo Pimentel <[email protected]>
Cc Lorenzo Pieralisi <[email protected]>
Cc Bjorn Helgaas <[email protected]>

---
For 5.11. This is based on the regression fix for 5.10 I sent[1].

Rob

[1] https://lore.kernel.org/linux-pci/[email protected]/

 .../pci/controller/dwc/pcie-designware-host.c | 54 +++++++++++--------
 drivers/pci/controller/dwc/pcie-designware.h  |  6 +--
 2 files changed, 34 insertions(+), 26 deletions(-)
drivers/pci/controller/dwc/pcie-designware-host.c +32 -22
index 44c2a6572199..a6ffab9b537e 100644
@@ -464,9 +464,7 @@ static void __iomem *dw_pcie_other_conf_map_bus(struct pci_bus *bus,
 		type = PCIE_ATU_TYPE_CFG1;
 
 
-	dw_pcie_prog_outbound_atu(pci, PCIE_ATU_REGION_INDEX1,
-				  type, pp->cfg0_base,
-				  busdev, pp->cfg0_size);
+	dw_pcie_prog_outbound_atu(pci, 0, type, pp->cfg0_base, busdev, pp->cfg0_size);
 
 	return pp->va_cfg0_base + where;
 }
@@ -480,9 +478,8 @@ static int dw_pcie_rd_other_conf(struct pci_bus *bus, unsigned int devfn,
 
 	ret = pci_generic_config_read(bus, devfn, where, size, val);
 
-	if (!ret && pci->num_viewport <= 2)
-		dw_pcie_prog_outbound_atu(pci, PCIE_ATU_REGION_INDEX1,
-					  PCIE_ATU_TYPE_IO, pp->io_base,
+	if (!ret && pci->io_cfg_atu_shared)
+		dw_pcie_prog_outbound_atu(pci, 0, PCIE_ATU_TYPE_IO, pp->io_base,
 					  pp->io_bus_addr, pp->io_size);
 
 	return ret;
@@ -497,9 +494,8 @@ static int dw_pcie_wr_other_conf(struct pci_bus *bus, unsigned int devfn,
 
 	ret = pci_generic_config_write(bus, devfn, where, size, val);
 
-	if (!ret && pci->num_viewport <= 2)
-		dw_pcie_prog_outbound_atu(pci, PCIE_ATU_REGION_INDEX1,
-					  PCIE_ATU_TYPE_IO, pp->io_base,
+	if (!ret && pci->io_cfg_atu_shared)
+		dw_pcie_prog_outbound_atu(pci, 0, PCIE_ATU_TYPE_IO, pp->io_base,
 					  pp->io_bus_addr, pp->io_size);
 
 	return ret;
@@ -586,21 +582,35 @@ void dw_pcie_setup_rc(struct pcie_port *pp)
 	 * ATU, so we should not program the ATU here.
 	 */
 	if (pp->bridge->child_ops == &dw_child_pcie_ops) {
-		struct resource_entry *tmp, *entry = NULL;
+		int atu_idx = 0;
+		struct resource_entry *entry;
 
 		/* Get last memory resource entry */
-		resource_list_for_each_entry(tmp, &pp->bridge->windows)
-			if (resource_type(tmp->res) == IORESOURCE_MEM)
-				entry = tmp;
-
-		dw_pcie_prog_outbound_atu(pci, PCIE_ATU_REGION_INDEX0,
-					  PCIE_ATU_TYPE_MEM, entry->res->start,
-					  entry->res->start - entry->offset,
-					  resource_size(entry->res));
-		if (pci->num_viewport > 2)
-			dw_pcie_prog_outbound_atu(pci, PCIE_ATU_REGION_INDEX2,
-						  PCIE_ATU_TYPE_IO, pp->io_base,
-						  pp->io_bus_addr, pp->io_size);
+		resource_list_for_each_entry(entry, &pp->bridge->windows) {
+			if (resource_type(entry->res) != IORESOURCE_MEM)
+				continue;
+
+			if (pci->num_viewport <= ++atu_idx)
+				break;
+
+			dw_pcie_prog_outbound_atu(pci, atu_idx,
+						  PCIE_ATU_TYPE_MEM, entry->res->start,
+						  entry->res->start - entry->offset,
+						  resource_size(entry->res));
+		}
+
+		if (pp->io_size) {
+			if (pci->num_viewport > ++atu_idx)
+				dw_pcie_prog_outbound_atu(pci, atu_idx,
+							  PCIE_ATU_TYPE_IO, pp->io_base,
+							  pp->io_bus_addr, pp->io_size);
+			else
+				pci->io_cfg_atu_shared = true;
+		}
+
+		if (pci->num_viewport <= atu_idx)
+			dev_warn(pci->dev, "Resources exceed number of ATU entries (%d)",
+				 pci->num_viewport);
 	}
 
 	dw_pcie_writel_dbi(pci, PCI_BASE_ADDRESS_0, 0);
drivers/pci/controller/dwc/pcie-designware.h +2 -4
index 9d2f511f13fa..ed19c34dd0fe 100644
@@ -80,9 +80,6 @@
 #define PCIE_ATU_VIEWPORT		0x900
 #define PCIE_ATU_REGION_INBOUND		BIT(31)
 #define PCIE_ATU_REGION_OUTBOUND	0
-#define PCIE_ATU_REGION_INDEX2		0x2
-#define PCIE_ATU_REGION_INDEX1		0x1
-#define PCIE_ATU_REGION_INDEX0		0x0
 #define PCIE_ATU_CR1			0x904
 #define PCIE_ATU_TYPE_MEM		0x0
 #define PCIE_ATU_TYPE_IO		0x2
@@ -266,7 +263,6 @@ struct dw_pcie {
 	/* Used when iatu_unroll_enabled is true */
 	void __iomem		*atu_base;
 	u32			num_viewport;
-	u8			iatu_unroll_enabled;
 	struct pcie_port	pp;
 	struct dw_pcie_ep	ep;
 	const struct dw_pcie_ops *ops;
@@ -274,6 +270,8 @@ struct dw_pcie {
 	int			num_lanes;
 	int			link_gen;
 	u8			n_fts[2];
+	bool			iatu_unroll_enabled: 1;
+	bool			io_cfg_atu_shared: 1;
 };
 
 #define to_dw_pcie_from_pp(port) container_of((port), struct dw_pcie, pp)
Signature · 1 line
2.25.1
Reviewed-by Tested-by on v1 1/1
#1 lore raw
Quoted 149 lines · Rob Herring

External email: Use caution opening links or attachments

The current ATU setup only supports a single memory resource which isn't sufficient if there are also prefetchable memory regions. In order to support multiple memory regions, we need to move away from fixed ATU slots and rework the assignment. As there's always an ATU entry for config space, let's assign index 0 to config space. Then we assign memory resources to index 1 and up. Finally, if we have an I/O region and slots remaining, we assign the I/O region last. If there aren't remaining slots, we keep the same config and I/O space sharing.

Cc Vidya Sagar <[email protected]>
Cc Jingoo Han <[email protected]>
Cc Gustavo Pimentel <[email protected]>
Cc Lorenzo Pieralisi <[email protected]>
Cc Bjorn Helgaas <[email protected]>

---
For 5.11. This is based on the regression fix for 5.10 I sent[1].

Rob

[1] https://lore.kernel.org/linux-pci/[email protected]/

  .../pci/controller/dwc/pcie-designware-host.c | 54 +++++++++++--------
  drivers/pci/controller/dwc/pcie-designware.h  |  6 +--
  2 files changed, 34 insertions(+), 26 deletions(-)
drivers/pci/controller/dwc/pcie-designware-host.c +32 -22
index 44c2a6572199..a6ffab9b537e 100644
@@ -464,9 +464,7 @@ static void __iomem *dw_pcie_other_conf_map_bus(struct pci_bus *bus,
                 type = PCIE_ATU_TYPE_CFG1;
 
 
-       dw_pcie_prog_outbound_atu(pci, PCIE_ATU_REGION_INDEX1,
-                                 type, pp->cfg0_base,
-                                 busdev, pp->cfg0_size);
+       dw_pcie_prog_outbound_atu(pci, 0, type, pp->cfg0_base, busdev, pp->cfg0_size);
 
         return pp->va_cfg0_base + where;
  }
@@ -480,9 +478,8 @@ static int dw_pcie_rd_other_conf(struct pci_bus *bus, unsigned int devfn,
 
         ret = pci_generic_config_read(bus, devfn, where, size, val);
 
-       if (!ret && pci->num_viewport <= 2)
-               dw_pcie_prog_outbound_atu(pci, PCIE_ATU_REGION_INDEX1,
-                                         PCIE_ATU_TYPE_IO, pp->io_base,
+       if (!ret && pci->io_cfg_atu_shared)
+               dw_pcie_prog_outbound_atu(pci, 0, PCIE_ATU_TYPE_IO, pp->io_base,
                                           pp->io_bus_addr, pp->io_size);
 
         return ret;
@@ -497,9 +494,8 @@ static int dw_pcie_wr_other_conf(struct pci_bus *bus, unsigned int devfn,
 
         ret = pci_generic_config_write(bus, devfn, where, size, val);
 
-       if (!ret && pci->num_viewport <= 2)
-               dw_pcie_prog_outbound_atu(pci, PCIE_ATU_REGION_INDEX1,
-                                         PCIE_ATU_TYPE_IO, pp->io_base,
+       if (!ret && pci->io_cfg_atu_shared)
+               dw_pcie_prog_outbound_atu(pci, 0, PCIE_ATU_TYPE_IO, pp->io_base,
                                           pp->io_bus_addr, pp->io_size);
 
         return ret;
@@ -586,21 +582,35 @@ void dw_pcie_setup_rc(struct pcie_port *pp)
          * ATU, so we should not program the ATU here.
          */
         if (pp->bridge->child_ops == &dw_child_pcie_ops) {
-               struct resource_entry *tmp, *entry = NULL;
+               int atu_idx = 0;
+               struct resource_entry *entry;
 
                 /* Get last memory resource entry */
-               resource_list_for_each_entry(tmp, &pp->bridge->windows)
-                       if (resource_type(tmp->res) == IORESOURCE_MEM)
-                               entry = tmp;
-
-               dw_pcie_prog_outbound_atu(pci, PCIE_ATU_REGION_INDEX0,
-                                         PCIE_ATU_TYPE_MEM, entry->res->start,
-                                         entry->res->start - entry->offset,
-                                         resource_size(entry->res));
-               if (pci->num_viewport > 2)
-                       dw_pcie_prog_outbound_atu(pci, PCIE_ATU_REGION_INDEX2,
-                                                 PCIE_ATU_TYPE_IO, pp->io_base,
-                                                 pp->io_bus_addr, pp->io_size);
+               resource_list_for_each_entry(entry, &pp->bridge->windows) {
+                       if (resource_type(entry->res) != IORESOURCE_MEM)
+                               continue;
+
+                       if (pci->num_viewport <= ++atu_idx)
+                               break;
+
+                       dw_pcie_prog_outbound_atu(pci, atu_idx,
+                                                 PCIE_ATU_TYPE_MEM, entry->res->start,
+                                                 entry->res->start - entry->offset,
+                                                 resource_size(entry->res));
+               }
+
+               if (pp->io_size) {
+                       if (pci->num_viewport > ++atu_idx)
+                               dw_pcie_prog_outbound_atu(pci, atu_idx,
+                                                         PCIE_ATU_TYPE_IO, pp->io_base,
+                                                         pp->io_bus_addr, pp->io_size);
+                       else
+                               pci->io_cfg_atu_shared = true;
+               }
+
+               if (pci->num_viewport <= atu_idx)
+                       dev_warn(pci->dev, "Resources exceed number of ATU entries (%d)",
+                                pci->num_viewport);
         }
 
         dw_pcie_writel_dbi(pci, PCI_BASE_ADDRESS_0, 0);
drivers/pci/controller/dwc/pcie-designware.h +2 -5
index 9d2f511f13fa..ed19c34dd0fe 100644
@@ -80,9 +80,6 @@
  #define PCIE_ATU_VIEWPORT              0x900
  #define PCIE_ATU_REGION_INBOUND                BIT(31)
  #define PCIE_ATU_REGION_OUTBOUND       0
-#define PCIE_ATU_REGION_INDEX2         0x2
-#define PCIE_ATU_REGION_INDEX1         0x1
-#define PCIE_ATU_REGION_INDEX0         0x0
  #define PCIE_ATU_CR1                   0x904
  #define PCIE_ATU_TYPE_MEM              0x0
  #define PCIE_ATU_TYPE_IO               0x2
@@ -266,7 +263,6 @@ struct dw_pcie {
         /* Used when iatu_unroll_enabled is true */
         void __iomem            *atu_base;
         u32                     num_viewport;
-       u8                      iatu_unroll_enabled;
         struct pcie_port        pp;
         struct dw_pcie_ep       ep;
         const struct dw_pcie_ops *ops;
@@ -274,6 +270,8 @@ struct dw_pcie {
         int                     num_lanes;
         int                     link_gen;
         u8                      n_fts[2];
+       bool                    iatu_unroll_enabled: 1;
+       bool                    io_cfg_atu_shared: 1;
  };
 
  #define to_dw_pcie_from_pp(port) container_of((port), struct dw_pcie, pp)
--
Quoted 2 lines

2.25.1

Reviewed and Verified it on Tegra194 and functionality is fine.

maintainer Acked-by on v1 1/1
#2 lore raw
In reply to Rob Herring
Quoted 10 lines · Rob Herring

The current ATU setup only supports a single memory resource which isn't sufficient if there are also prefetchable memory regions. In order to support multiple memory regions, we need to move away from fixed ATU slots and rework the assignment. As there's always an ATU entry for config space, let's assign index 0 to config space. Then we assign memory resources to index 1 and up. Finally, if we have an I/O region and slots remaining, we assign the I/O region last. If there aren't remaining slots, we keep the same config and I/O space sharing.

Quoted 2 lines
Cc Vidya Sagar <[email protected]>
Cc Jingoo Han <[email protected]>

Best regards,
Jingoo Han

Quoted 15 lines
Cc Gustavo Pimentel <[email protected]>
Cc Lorenzo Pieralisi <[email protected]>
Cc Bjorn Helgaas <[email protected]>

---
For 5.11. This is based on the regression fix for 5.10 I sent[1].

Rob

[1] https://lore.kernel.org/linux-pci/[email protected]/

.../pci/controller/dwc/pcie-designware-host.c | 54 +++++++++++--------

 drivers/pci/controller/dwc/pcie-designware.h  |  6 +--
 2 files changed, 34 insertions(+), 26 deletions(-)

[...]

#3 lore raw
In reply to Rob Herring
Quoted 8 lines · Rob Herring

The current ATU setup only supports a single memory resource which isn't sufficient if there are also prefetchable memory regions. In order to support multiple memory regions, we need to move away from fixed ATU slots and rework the assignment. As there's always an ATU entry for config space, let's assign index 0 to config space. Then we assign memory resources to index 1 and up. Finally, if we have an I/O region and slots remaining, we assign the I/O region last. If there aren't remaining slots, we keep the same config and I/O space sharing.

Applied to pci/dwc, thanks!

[1/1] PCI: dwc: Support multiple ATU memory regions

      https://git.kernel.org/lpieralisi/pci/c/9f9e59a480

Thanks,
Lorenzo