Skip to contentSkip to conversation

Add support for R-Car Gen4 PCI host controller

PATCH Superseded Mykyta Poturai 3 participants 13 messages Mar 11, 2025 – Mar 12, 2025

Subsystems (2)

v2 of 6 Superseded Latest is v6 →

lore
Share & tools
lore
#4 lore raw

PATCH v2 2/7 xen/arm: make pci_host_common_probe return the bridge

In reply to Mykyta Poturai

From: Oleksandr Andrushchenko <[email protected]>

Some of the PCI host bridges require additional processing during the probe phase. For that they need to access struct bridge of the probed host, so return pointer to the new bridge from pci_host_common_probe.

---
v1->v2:
* no change

---
 xen/arch/arm/include/asm/pci.h      |  8 +++++---
 xen/arch/arm/pci/pci-host-common.c  | 12 ++++++------
 xen/arch/arm/pci/pci-host-generic.c |  2 +-
 xen/arch/arm/pci/pci-host-zynqmp.c  |  2 +-
 4 files changed, 13 insertions(+), 11 deletions(-)
xen/arch/arm/include/asm/pci.h +5 -3
index 44344ac8c1..e1f63d75e3 100644
@@ -17,6 +17,8 @@
 
 #ifdef CONFIG_HAS_PCI
 
+#include <xen/err.h>
+
 #include <asm/p2m.h>
 
 #define pci_to_dev(pcidev) (&(pcidev)->arch.dev)
@@ -95,9 +97,9 @@ struct pci_ecam_ops {
 /* Default ECAM ops */
 extern const struct pci_ecam_ops pci_generic_ecam_ops;
 
-int pci_host_common_probe(struct dt_device_node *dev,
-                          const struct pci_ecam_ops *ops,
-                          size_t priv_sz);
+struct pci_host_bridge *pci_host_common_probe(struct dt_device_node *dev,
+                                              const struct pci_ecam_ops *ops,
+                                              size_t priv_sz);
 int pci_generic_config_read(struct pci_host_bridge *bridge, pci_sbdf_t sbdf,
                             uint32_t reg, uint32_t len, uint32_t *value);
 int pci_generic_config_write(struct pci_host_bridge *bridge, pci_sbdf_t sbdf,
xen/arch/arm/pci/pci-host-common.c +6 -6
index be7e6c3510..e4e05d1176 100644
@@ -208,9 +208,9 @@ static int pci_bus_find_domain_nr(struct dt_device_node *dev)
     return domain;
 }
 
-int pci_host_common_probe(struct dt_device_node *dev,
-                          const struct pci_ecam_ops *ops,
-                          size_t priv_sz)
+struct pci_host_bridge *pci_host_common_probe(struct dt_device_node *dev,
+                                              const struct pci_ecam_ops *ops,
+                                              size_t priv_sz)
 {
     struct pci_host_bridge *bridge;
     struct pci_config_window *cfg;
@@ -222,7 +222,7 @@ int pci_host_common_probe(struct dt_device_node *dev,
 
     bridge = pci_alloc_host_bridge();
     if ( !bridge )
-        return -ENOMEM;
+        return ERR_PTR(-ENOMEM);
 
     /* Parse and map our Configuration Space windows */
     cfg = gen_pci_init(dev, ops);
@@ -257,7 +257,7 @@ int pci_host_common_probe(struct dt_device_node *dev,
 
     pci_add_host_bridge(bridge);
 
-    return 0;
+    return bridge;
 
 err_priv:
     xfree(bridge->priv);
@@ -265,7 +265,7 @@ err_priv:
 err_exit:
     xfree(bridge);
 
-    return err;
+    return ERR_PTR(err);
 }
 
 /*
xen/arch/arm/pci/pci-host-generic.c +1 -1
index cc4bc70684..dde6a79a8e 100644
@@ -29,7 +29,7 @@ static const struct dt_device_match __initconstrel gen_pci_dt_match[] =
 static int __init pci_host_generic_probe(struct dt_device_node *dev,
                                          const void *data)
 {
-    return pci_host_common_probe(dev, &pci_generic_ecam_ops, 0);
+    return PTR_RET(pci_host_common_probe(dev, &pci_generic_ecam_ops, 0));
 }
 
 DT_DEVICE_START(pci_gen, "PCI HOST GENERIC", DEVICE_PCI_HOSTBRIDGE)
xen/arch/arm/pci/pci-host-zynqmp.c +1 -1
index 985a43c516..c796447ac4 100644
@@ -47,7 +47,7 @@ static const struct dt_device_match __initconstrel nwl_pcie_dt_match[] =
 static int __init pci_host_generic_probe(struct dt_device_node *dev,
                                          const void *data)
 {
-    return pci_host_common_probe(dev, &nwl_pcie_ops, 0);
+    return PTR_RET(pci_host_common_probe(dev, &nwl_pcie_ops, 0));
 }
 
 DT_DEVICE_START(pci_gen, "PCI HOST ZYNQMP", DEVICE_PCI_HOSTBRIDGE)
Signature · 1 line
2.34.1
#10 lore raw

PATCH v2 6/7 xen/arm: rcar4: add simple optimization to avoid ATU reprogramming

In reply to Mykyta Poturai

From: Volodymyr Babchuk <[email protected]>

There are high chances that there will be a number of a consecutive accesses to configuration space of one device. To speed things up, we can program ATU only during first access.

This is mostly beneficial taking into account the previous patch that adds 1ms delay after ATU reprogramming.

---
v1->v2:
* rebased

---
 xen/arch/arm/pci/pci-designware.c | 8 ++++++++
 1 file changed, 8 insertions(+)
xen/arch/arm/pci/pci-designware.c +8 -0
index def2c12d63..cec52cf81a 100644
@@ -272,6 +272,14 @@ static void dw_pcie_prog_outbound_atu(struct pci_host_bridge *pci, int index,
                                       int type, uint64_t cpu_addr,
                                       uint64_t pci_addr, uint64_t size)
 {
+    static uint64_t prev_addr = ~0;
+
+    /* Simple optimization to not-program ATU for every transaction */
+    if (prev_addr == pci_addr)
+        return;
+
+    prev_addr = pci_addr;
+
     __dw_pcie_prog_outbound_atu(pci, 0, index, type,
                                 cpu_addr, pci_addr, size);
 }
Signature · 1 line
2.34.1
#7 lore raw

PATCH v2 5/7 xen/arm: rcar4: add delay after programming ATU

In reply to Mykyta Poturai

From: Volodymyr Babchuk <[email protected]>

For some reason, we need a delay before accessing ATU region after we programmed it. Otherwise, we'll get erroneous TLP.

There is a code below, which should do this in proper way, by polling CTRL2 register, but according to documentation, hardware does not change this ATU_ENABLE bit at all.

---
v1->v2:
* rebased

---
 xen/arch/arm/pci/pci-designware.c | 5 +++++
 1 file changed, 5 insertions(+)
xen/arch/arm/pci/pci-designware.c +5 -0
index 6ab03cf9b0..def2c12d63 100644
@@ -194,6 +194,11 @@ static void dw_pcie_prog_outbound_atu_unroll(struct pci_host_bridge *pci,
     dw_pcie_writel_ob_unroll(pci, index, PCIE_ATU_UNR_REGION_CTRL2,
                              PCIE_ATU_ENABLE);
 
+    /*
+     * HACK: We need to delay there, because the next code does not
+     * work as expected on S4
+     */
+    mdelay(1);
     /*
      * Make sure ATU enable takes effect before any subsequent config
      * and I/O accesses.
Signature · 1 line
2.34.1
#6 lore raw

PATCH v2 4/7 xen/arm: add support for R-Car Gen4 PCI host controller

In reply to Mykyta Poturai

From: Oleksandr Andrushchenko <[email protected]>

Add support for Renesas R-Car Gen4 PCI host controller, specifically targeting the S4 and V4H SoCs. The implementation includes configuration read/write operations for both root and child buses. For accessing the child bus, iATU is used for address translation.

Code common to all DesignWare PCI host controllers is located in a separate file to allow for easy reuse in other DesignWare-based PCI host controllers.

---
v1->v2:
* move designware code in a separate file

---
 xen/arch/arm/pci/Makefile         |   2 +
 xen/arch/arm/pci/pci-designware.c | 409 ++++++++++++++++++++++++++++++
 xen/arch/arm/pci/pci-designware.h | 105 ++++++++
 xen/arch/arm/pci/pci-host-rcar4.c | 104 ++++++++
 4 files changed, 620 insertions(+)
 create mode 100644 xen/arch/arm/pci/pci-designware.c
 create mode 100644 xen/arch/arm/pci/pci-designware.h
 create mode 100644 xen/arch/arm/pci/pci-host-rcar4.c
xen/arch/arm/pci/Makefile +2 -0
index 1d045ade01..ca6135e282 100644
@@ -4,3 +4,5 @@ obj-y += pci-host-generic.o
 obj-y += pci-host-common.o
 obj-y += ecam.o
 obj-y += pci-host-zynqmp.o
+obj-y += pci-designware.o
+obj-y += pci-host-rcar4.o
xen/arch/arm/pci/pci-designware.c new file +409 -0

409 lines not on this page · raw message

xen/arch/arm/pci/pci-designware.h new file +105 -0

105 lines not on this page · raw message

xen/arch/arm/pci/pci-host-rcar4.c new file +104 -0

104 lines not on this page · raw message

Signature · 1 line
2.34.1
#0 lore raw

v2 cover Add support for R-Car Gen4 PCI host controller

This series adds support for R-Car Gen4 PCI host controller.

To fully support the controller, the following changes were made:
- Generic mechanism to support PCI child buses is added.
- Private data for PCI host bridge and means to access it are added.

The series also includes a workaround for proper ATU propramming and optimizations to lessen the performance impact of that workaround.

The series was tested as a part of the pci-passthrough patches[1] and build-tested standalone with enabled HAS_PCI and HAS_VPCI.

[1] https://github.com/Deedone/xen/tree/pci_passthrough_wip

v1->v2:
* see individual patches

Oleksandr Andrushchenko (4):
xen/arm: allow PCI host bridge to have private data xen/arm: make pci_host_common_probe return the bridge xen/arm: add support for PCI child bus xen/arm: add support for R-Car Gen4 PCI host controller

Volodymyr Babchuk (3):
xen/arm: rcar4: add delay after programming ATU xen/arm: rcar4: add simple optimization to avoid ATU reprogramming xen/arm: rcar4: program ATU to accesses to all functions

 xen/arch/arm/include/asm/pci.h      |  16 +-
 xen/arch/arm/pci/Makefile           |   2 +
 xen/arch/arm/pci/ecam.c             |  17 +-
 xen/arch/arm/pci/pci-access.c       |  37 ++-
 xen/arch/arm/pci/pci-designware.c   | 422 ++++++++++++++++++++++++++++
 xen/arch/arm/pci/pci-designware.h   | 105 +++++++
 xen/arch/arm/pci/pci-host-common.c  | 106 +++++--
 xen/arch/arm/pci/pci-host-generic.c |   2 +-
 xen/arch/arm/pci/pci-host-rcar4.c   | 104 +++++++
 xen/arch/arm/pci/pci-host-zynqmp.c  |   2 +-
 xen/arch/arm/vpci.c                 |  83 ++++--
 11 files changed, 846 insertions(+), 50 deletions(-)
 create mode 100644 xen/arch/arm/pci/pci-designware.c
 create mode 100644 xen/arch/arm/pci/pci-designware.h
 create mode 100644 xen/arch/arm/pci/pci-host-rcar4.c
Signature · 1 line
2.34.1
#5 lore raw

PATCH v2 3/7 xen/arm: add support for PCI child bus

From: Oleksandr Andrushchenko <[email protected]>

PCI host bridges often have different ways to access the root and child bus configuration spaces. One of the examples is Designware's host bridge and its multiple clones [1].

Linux kernel implements this by instantiating a child bus when device drivers provide not only the usual pci_ops to access ECAM space (this is the case for the generic host bridge), but also means to access the child bus which has a dedicated configuration space and own implementation for accessing the bus, e.g. child_ops.

For Xen it is not feasible to fully implement PCI bus infrastructure as Linux kernel does, but still child bus can be supported.

Add support for the PCI child bus which includes the following changes:
- introduce bus mapping functions depending on SBDF
- assign bus start and end for the child bus and re-configure the same for the parent (root) bus
- make pci_find_host_bridge be aware of multiple busses behind the same bridge
- update pci_host_bridge_mappings, so it also doesn't map to guest the memory spaces belonging to the child bus
- make pci_host_common_probe accept one more pci_ops structure for the child bus
- install MMIO handlers for the child bus
- re-work vpci_mmio_{write|read} with parent and child approach in mind

[1] https://elixir.bootlin.com/linux/v5.15/source/drivers/pci/controller/dwc

---
v1->v2:
* fixed compilation issues

---
 xen/arch/arm/include/asm/pci.h      | 14 ++++-
 xen/arch/arm/pci/ecam.c             | 17 ++++--
 xen/arch/arm/pci/pci-access.c       | 37 +++++++++++--
 xen/arch/arm/pci/pci-host-common.c  | 86 +++++++++++++++++++++++------
 xen/arch/arm/pci/pci-host-generic.c |  2 +-
 xen/arch/arm/pci/pci-host-zynqmp.c  |  2 +-
 xen/arch/arm/vpci.c                 | 83 ++++++++++++++++++++++------
 7 files changed, 192 insertions(+), 49 deletions(-)
xen/arch/arm/include/asm/pci.h +11 -3
index e1f63d75e3..0012c2ae9e 100644
@@ -69,6 +69,9 @@ struct pci_host_bridge {
     struct pci_config_window* cfg;   /* Pointer to the bridge config window */
     const struct pci_ops *ops;
     void *priv;                      /* Private data of the bridge. */
+    /* Child bus */
+    struct pci_config_window* child_cfg;
+    const struct pci_ops *child_ops;
 };
 
 struct pci_ops {
@@ -97,15 +100,20 @@ struct pci_ecam_ops {
 /* Default ECAM ops */
 extern const struct pci_ecam_ops pci_generic_ecam_ops;
 
-struct pci_host_bridge *pci_host_common_probe(struct dt_device_node *dev,
-                                              const struct pci_ecam_ops *ops,
-                                              size_t priv_sz);
+struct pci_host_bridge *
+pci_host_common_probe(struct dt_device_node *dev,
+                      const struct pci_ecam_ops *ops,
+                      const struct pci_ecam_ops *child_ops,
+                      size_t priv_sz);
 int pci_generic_config_read(struct pci_host_bridge *bridge, pci_sbdf_t sbdf,
                             uint32_t reg, uint32_t len, uint32_t *value);
 int pci_generic_config_write(struct pci_host_bridge *bridge, pci_sbdf_t sbdf,
                              uint32_t reg, uint32_t len, uint32_t value);
 void __iomem *pci_ecam_map_bus(struct pci_host_bridge *bridge,
                                pci_sbdf_t sbdf, uint32_t where);
+void __iomem *pci_ecam_map_bus_generic(const struct pci_config_window *cfg,
+                                       const struct pci_ecam_ops *ops,
+                                       pci_sbdf_t sbdf, uint32_t where);
 bool pci_ecam_need_p2m_hwdom_mapping(struct domain *d,
                                      struct pci_host_bridge *bridge,
                                      uint64_t addr);
xen/arch/arm/pci/ecam.c +12 -5
index 3987f96b01..5486881c5c 100644
@@ -20,12 +20,10 @@
 /*
  * Function to implement the pci_ops->map_bus method.
  */
-void __iomem *pci_ecam_map_bus(struct pci_host_bridge *bridge,
-                               pci_sbdf_t sbdf, uint32_t where)
+void __iomem *pci_ecam_map_bus_generic(const struct pci_config_window *cfg,
+                                       const struct pci_ecam_ops *ops,
+                                       pci_sbdf_t sbdf, uint32_t where)
 {
-    const struct pci_config_window *cfg = bridge->cfg;
-    const struct pci_ecam_ops *ops =
-        container_of(bridge->ops, const struct pci_ecam_ops, pci_ops);
     unsigned int devfn_shift = ops->bus_shift - 8;
     void __iomem *base;
     unsigned int busn = sbdf.bus;
@@ -39,6 +37,15 @@ void __iomem *pci_ecam_map_bus(struct pci_host_bridge *bridge,
     return base + (sbdf.devfn << devfn_shift) + where;
 }
 
+void __iomem *pci_ecam_map_bus(struct pci_host_bridge *bridge,
+                               pci_sbdf_t sbdf, uint32_t where)
+{
+    const struct pci_ecam_ops *ops =
+        container_of(bridge->ops, const struct pci_ecam_ops, pci_ops);
+
+    return pci_ecam_map_bus_generic(bridge->cfg, ops, sbdf, where);
+}
+
 bool __init pci_ecam_need_p2m_hwdom_mapping(struct domain *d,
                                             struct pci_host_bridge *bridge,
                                             uint64_t addr)
xen/arch/arm/pci/pci-access.c +31 -6
index 9f9aac43d7..4da607ac3f 100644
@@ -18,10 +18,31 @@
 #define INVALID_VALUE (~0U)
 #define PCI_ERR_VALUE(len) GENMASK(0, len * 8)
 
+static const struct pci_ops *get_ops(struct pci_host_bridge *bridge,
+                                     pci_sbdf_t sbdf)
+{
+    if ( bridge->child_ops )
+    {
+        struct pci_config_window* cfg = bridge->child_cfg;
+
+        if ( (sbdf.bus >= cfg->busn_start) && (sbdf.bus <= cfg->busn_end) )
+            return bridge->child_ops;
+    }
+    return bridge->ops;
+}
+
+static void __iomem *map_bus(struct pci_host_bridge *bridge, pci_sbdf_t sbdf,
+                            uint32_t reg)
+{
+    const struct pci_ops *ops = get_ops(bridge, sbdf);
+
+    return ops->map_bus(bridge, sbdf, reg);
+}
+
 int pci_generic_config_read(struct pci_host_bridge *bridge, pci_sbdf_t sbdf,
                             uint32_t reg, uint32_t len, uint32_t *value)
 {
-    void __iomem *addr = bridge->ops->map_bus(bridge, sbdf, reg);
+    void __iomem *addr = map_bus(bridge, sbdf, reg);
 
     if ( !addr )
     {
@@ -50,7 +71,7 @@ int pci_generic_config_read(struct pci_host_bridge *bridge, pci_sbdf_t sbdf,
 int pci_generic_config_write(struct pci_host_bridge *bridge, pci_sbdf_t sbdf,
                              uint32_t reg, uint32_t len, uint32_t value)
 {
-    void __iomem *addr = bridge->ops->map_bus(bridge, sbdf, reg);
+    void __iomem *addr = map_bus(bridge, sbdf, reg);
 
     if ( !addr )
         return -ENODEV;
@@ -78,14 +99,16 @@ static uint32_t pci_config_read(pci_sbdf_t sbdf, unsigned int reg,
 {
     uint32_t val = PCI_ERR_VALUE(len);
     struct pci_host_bridge *bridge = pci_find_host_bridge(sbdf.seg, sbdf.bus);
+    const struct pci_ops *ops;
 
     if ( unlikely(!bridge) )
         return val;
 
-    if ( unlikely(!bridge->ops->read) )
+    ops = get_ops(bridge, sbdf);
+    if ( unlikely(!ops->read) )
         return val;
 
-    bridge->ops->read(bridge, sbdf, reg, len, &val);
+    ops->read(bridge, sbdf, reg, len, &val);
 
     return val;
 }
@@ -94,14 +117,16 @@ static void pci_config_write(pci_sbdf_t sbdf, unsigned int reg,
                              unsigned int len, uint32_t val)
 {
     struct pci_host_bridge *bridge = pci_find_host_bridge(sbdf.seg, sbdf.bus);
+    const struct pci_ops *ops;
 
     if ( unlikely(!bridge) )
         return;
 
-    if ( unlikely(!bridge->ops->write) )
+    ops = get_ops(bridge, sbdf);
+    if ( unlikely(!ops->write) )
         return;
 
-    bridge->ops->write(bridge, sbdf, reg, len, val);
+    ops->write(bridge, sbdf, reg, len, val);
 }
 
 /*
xen/arch/arm/pci/pci-host-common.c +69 -17

142 lines not on this page · raw message

xen/arch/arm/pci/pci-host-generic.c +1 -1
index dde6a79a8e..08d94879b7 100644
@@ -29,7 +29,7 @@ static const struct dt_device_match __initconstrel gen_pci_dt_match[] =
 static int __init pci_host_generic_probe(struct dt_device_node *dev,
                                          const void *data)
 {
-    return PTR_RET(pci_host_common_probe(dev, &pci_generic_ecam_ops, 0));
+    return PTR_RET(pci_host_common_probe(dev, &pci_generic_ecam_ops, NULL, 0));
 }
 
 DT_DEVICE_START(pci_gen, "PCI HOST GENERIC", DEVICE_PCI_HOSTBRIDGE)
xen/arch/arm/pci/pci-host-zynqmp.c +1 -1
index c796447ac4..0b51ff6bd9 100644
@@ -47,7 +47,7 @@ static const struct dt_device_match __initconstrel nwl_pcie_dt_match[] =
 static int __init pci_host_generic_probe(struct dt_device_node *dev,
                                          const void *data)
 {
-    return PTR_RET(pci_host_common_probe(dev, &nwl_pcie_ops, 0));
+    return PTR_RET(pci_host_common_probe(dev, &nwl_pcie_ops, NULL, 0));
 }
 
 DT_DEVICE_START(pci_gen, "PCI HOST ZYNQMP", DEVICE_PCI_HOSTBRIDGE)
xen/arch/arm/vpci.c +67 -16

132 lines not on this page · raw message

Signature · 1 line
2.34.1
#12 lore raw

PATCH v2 7/7 xen/arm: rcar4: program ATU to accesses to all functions

In reply to Mykyta Poturai

From: Volodymyr Babchuk <[email protected]>

According to ATU documentation, bits [18:16] of accessed memory address correspond to a function number. This is somewhat similar to ECAM, but with huge holes between regions.

We can use this to minimize number of ATU re-programmings: configure ATU to access BDF with F=0 and adjust memory address with function number.

Taking into account the previous patch, that optimizes ATU reprogramming by skipping call to __dw_pcie_prog_outbound_atu() if we already configured pci_address, we can be sure that accesses to all functions of one device will not trigger ATU reprogramming at all.

---
v1->v2:
* rebased

---
 xen/arch/arm/pci/pci-designware.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
xen/arch/arm/pci/pci-designware.c +2 -2
index cec52cf81a..fec8c7296d 100644
@@ -297,7 +297,7 @@ void __iomem *dw_pcie_child_map_bus(struct pci_host_bridge *bridge,
     uint32_t busdev;
 
     busdev = PCIE_ATU_BUS(sbdf.bus) | PCIE_ATU_DEV(PCI_SLOT(sbdf.devfn)) |
-        PCIE_ATU_FUNC(PCI_FUNC(sbdf.devfn));
+        PCIE_ATU_FUNC(0);
 
     /* FIXME: Parent is the root bus, so use PCIE_ATU_TYPE_CFG0. */
     dw_pcie_prog_outbound_atu(bridge, PCIE_ATU_REGION_INDEX1,
@@ -305,7 +305,7 @@ void __iomem *dw_pcie_child_map_bus(struct pci_host_bridge *bridge,
                               bridge->child_cfg->phys_addr,
                               busdev, bridge->child_cfg->size);
 
-    return bridge->child_cfg->win + where;
+    return bridge->child_cfg->win + ((uint32_t)sbdf.fn << 16) + where;
 }
 
 int dw_pcie_child_config_read(struct pci_host_bridge *bridge,
Signature · 1 line
2.34.1
#1 lore raw

PATCH v2 1/7 xen/arm: allow PCI host bridge to have private data

In reply to Mykyta Poturai

From: Oleksandr Andrushchenko <[email protected]>

Some of the PCI host bridges require private data. Create a generic approach for that, so such bridges may request the private data to be allocated during initialization.

---
v1->v2:
* no change

---
 xen/arch/arm/include/asm/pci.h      |  4 +++-
 xen/arch/arm/pci/pci-host-common.c  | 18 +++++++++++++++++-
 xen/arch/arm/pci/pci-host-generic.c |  2 +-
 xen/arch/arm/pci/pci-host-zynqmp.c  |  2 +-
 4 files changed, 22 insertions(+), 4 deletions(-)
xen/arch/arm/include/asm/pci.h +3 -1
index 7f77226c9b..44344ac8c1 100644
@@ -66,6 +66,7 @@ struct pci_host_bridge {
     uint16_t segment;                /* Segment number */
     struct pci_config_window* cfg;   /* Pointer to the bridge config window */
     const struct pci_ops *ops;
+    void *priv;                      /* Private data of the bridge. */
 };
 
 struct pci_ops {
@@ -95,7 +96,8 @@ struct pci_ecam_ops {
 extern const struct pci_ecam_ops pci_generic_ecam_ops;
 
 int pci_host_common_probe(struct dt_device_node *dev,
-                          const struct pci_ecam_ops *ops);
+                          const struct pci_ecam_ops *ops,
+                          size_t priv_sz);
 int pci_generic_config_read(struct pci_host_bridge *bridge, pci_sbdf_t sbdf,
                             uint32_t reg, uint32_t len, uint32_t *value);
 int pci_generic_config_write(struct pci_host_bridge *bridge, pci_sbdf_t sbdf,
xen/arch/arm/pci/pci-host-common.c +17 -1
index c0faf0f436..be7e6c3510 100644
@@ -209,7 +209,8 @@ static int pci_bus_find_domain_nr(struct dt_device_node *dev)
 }
 
 int pci_host_common_probe(struct dt_device_node *dev,
-                          const struct pci_ecam_ops *ops)
+                          const struct pci_ecam_ops *ops,
+                          size_t priv_sz)
 {
     struct pci_host_bridge *bridge;
     struct pci_config_window *cfg;
@@ -241,11 +242,26 @@ int pci_host_common_probe(struct dt_device_node *dev,
         printk(XENLOG_ERR "Inconsistent \"linux,pci-domain\" property in DT\n");
         BUG();
     }
+
     bridge->segment = domain;
+
+    if ( priv_sz )
+    {
+        bridge->priv = xzalloc_bytes(priv_sz);
+        if ( !bridge->priv )
+        {
+            err = -ENOMEM;
+            goto err_priv;
+        }
+    }
+
     pci_add_host_bridge(bridge);
 
     return 0;
 
+err_priv:
+    xfree(bridge->priv);
+
 err_exit:
     xfree(bridge);
 
xen/arch/arm/pci/pci-host-generic.c +1 -1
index 46de6e43cc..cc4bc70684 100644
@@ -29,7 +29,7 @@ static const struct dt_device_match __initconstrel gen_pci_dt_match[] =
 static int __init pci_host_generic_probe(struct dt_device_node *dev,
                                          const void *data)
 {
-    return pci_host_common_probe(dev, &pci_generic_ecam_ops);
+    return pci_host_common_probe(dev, &pci_generic_ecam_ops, 0);
 }
 
 DT_DEVICE_START(pci_gen, "PCI HOST GENERIC", DEVICE_PCI_HOSTBRIDGE)
xen/arch/arm/pci/pci-host-zynqmp.c +1 -1
index 101edb8593..985a43c516 100644
@@ -47,7 +47,7 @@ static const struct dt_device_match __initconstrel nwl_pcie_dt_match[] =
 static int __init pci_host_generic_probe(struct dt_device_node *dev,
                                          const void *data)
 {
-    return pci_host_common_probe(dev, &nwl_pcie_ops);
+    return pci_host_common_probe(dev, &nwl_pcie_ops, 0);
 }
 
 DT_DEVICE_START(pci_gen, "PCI HOST ZYNQMP", DEVICE_PCI_HOSTBRIDGE)
Signature · 1 line
2.34.1
#2 lore raw

Re: [PATCH v2 1/7] xen/arm: allow PCI host bridge to have private data

Quoted 61 lines · Mykyta Poturai

From: Oleksandr Andrushchenko <[email protected]>

Some of the PCI host bridges require private data. Create a generic approach for that, so such bridges may request the private data to be allocated during initialization.

---
v1->v2:
* no change

---
  xen/arch/arm/include/asm/pci.h      |  4 +++-
  xen/arch/arm/pci/pci-host-common.c  | 18 +++++++++++++++++-
  xen/arch/arm/pci/pci-host-generic.c |  2 +-
  xen/arch/arm/pci/pci-host-zynqmp.c  |  2 +-
  4 files changed, 22 insertions(+), 4 deletions(-)
xen/arch/arm/include/asm/pci.h +3 -1
index 7f77226c9b..44344ac8c1 100644
@@ -66,6 +66,7 @@ struct pci_host_bridge {
      uint16_t segment;                /* Segment number */
      struct pci_config_window* cfg;   /* Pointer to the bridge config window */
      const struct pci_ops *ops;
+    void *priv;                      /* Private data of the bridge. */
  };
 
  struct pci_ops {
@@ -95,7 +96,8 @@ struct pci_ecam_ops {
  extern const struct pci_ecam_ops pci_generic_ecam_ops;
 
  int pci_host_common_probe(struct dt_device_node *dev,
-                          const struct pci_ecam_ops *ops);
+                          const struct pci_ecam_ops *ops,
+                          size_t priv_sz);
  int pci_generic_config_read(struct pci_host_bridge *bridge, pci_sbdf_t sbdf,
                              uint32_t reg, uint32_t len, uint32_t *value);
  int pci_generic_config_write(struct pci_host_bridge *bridge, pci_sbdf_t sbdf,
xen/arch/arm/pci/pci-host-common.c +4 -1
index c0faf0f436..be7e6c3510 100644
@@ -209,7 +209,8 @@ static int pci_bus_find_domain_nr(struct dt_device_node *dev)
  }
 
  int pci_host_common_probe(struct dt_device_node *dev,
-                          const struct pci_ecam_ops *ops)
+                          const struct pci_ecam_ops *ops,
+                          size_t priv_sz)
  {
      struct pci_host_bridge *bridge;
      struct pci_config_window *cfg;
@@ -241,11 +242,26 @@ int pci_host_common_probe(struct dt_device_node *dev,
          printk(XENLOG_ERR "Inconsistent \"linux,pci-domain\" property in DT\n");
          BUG();
      }
+
      bridge->segment = domain;
+

xen/arch/arm/pci/pci-host-common.c int pci_host_common_probe(struct dt_device_node *dev, · line 257

Quoted 10 lines
xen/arch/arm/pci/pci-host-common.c +10 -0
+    if ( priv_sz )
+    {
+        bridge->priv = xzalloc_bytes(priv_sz);
+        if ( !bridge->priv )
+        {
+            err = -ENOMEM;
+            goto err_priv;
+        }
+    }
+

I'd like to propose to move allocation into pci_alloc_host_bridge() to keep mallocs in one place and do it earlier, before proceeding with other initialization steps.

Also the pci_alloc_host_bridge() could be made static, seems.

xen/arch/arm/pci/pci-host-common.c int pci_host_common_probe(struct dt_device_node *dev, · line 267

Quoted 10 lines
xen/arch/arm/pci/pci-host-common.c +3 -0
      pci_add_host_bridge(bridge);
 
      return 0;
 
+err_priv:
+    xfree(bridge->priv);
+
  err_exit:
      xfree(bridge);
 

[...]

-grygorii

#3 lore raw

Re: [PATCH v2 1/7] xen/arm: allow PCI host bridge to have private data

Hi,

Quoted 69 lines · Grygorii Strashko
Quoted 66 lines · Mykyta Poturai

From: Oleksandr Andrushchenko <[email protected]>

Some of the PCI host bridges require private data. Create a generic approach for that, so such bridges may request the private data to be allocated during initialization.

---
v1->v2:
* no change
---
  xen/arch/arm/include/asm/pci.h      |  4 +++- xen/arch/arm/pci/pci-host-common.c  | 18 +++++++++++++++++- xen/arch/arm/pci/pci-host-generic.c |  2 +- xen/arch/arm/pci/pci-host-zynqmp.c  |  2 +-

  4 files changed, 22 insertions(+), 4 deletions(-)
xen/arch/arm/include/asm/pci.h +0 -0
index 7f77226c9b..44344ac8c1 100644
@@ -66,6 +66,7 @@ struct pci_host_bridge {

      uint16_t segment;                /* Segment number */
      struct pci_config_window* cfg;   /* Pointer to the bridge config window */ const struct pci_ops *ops;
+    void *priv;                      /* Private data of the bridge. */ };
  struct pci_ops {

xen/arch/arm/include/asm/pci.h +2 -1
@@ -95,7 +96,8 @@ struct pci_ecam_ops {
   extern const struct pci_ecam_ops pci_generic_ecam_ops;
   int pci_host_common_probe(struct dt_device_node *dev,
-                          const struct pci_ecam_ops *ops);
+                          const struct pci_ecam_ops *ops,
+                          size_t priv_sz);
   int pci_generic_config_read(struct pci_host_bridge *bridge, pci_sbdf_t sbdf,
                               uint32_t reg, uint32_t len, uint32_t *value);
   int pci_generic_config_write(struct pci_host_bridge *bridge, pci_sbdf_t sbdf,
xen/arch/arm/pci/pci-host-common.c +10 -1
index c0faf0f436..be7e6c3510 100644
@@ -209,7 +209,8 @@ static int pci_bus_find_domain_nr(struct 
 dt_device_node *dev)
   }
   int pci_host_common_probe(struct dt_device_node *dev,
-                          const struct pci_ecam_ops *ops)
+                          const struct pci_ecam_ops *ops,
+                          size_t priv_sz)
   {
       struct pci_host_bridge *bridge;
       struct pci_config_window *cfg;
@@ -241,11 +242,26 @@ int pci_host_common_probe(struct dt_device_node 
 *dev,
           printk(XENLOG_ERR "Inconsistent \"linux,pci-domain\" property in DT\n");
           BUG();
       }
+
       bridge->segment = domain;
+
+    if ( priv_sz )
+    {
+        bridge->priv = xzalloc_bytes(priv_sz);
+        if ( !bridge->priv )
+        {
+            err = -ENOMEM;
Quoted 8 lines

xen/arch/arm/pci/pci-host-common.c int pci_host_common_probe(struct dt_device_node *dev, · line 257

Quoted 4 lines
xen/arch/arm/pci/pci-host-common.c +4 -0
+            goto err_priv;
+        }
+    }
+

I'd like to propose to move allocation into pci_alloc_host_bridge() to keep mallocs in one place and do it earlier, before proceeding with other initialization steps.

In both cases, we need to pass the size of the structure in parameter and the structure is not initialized until later. I would rather prefer if the allocation is done by each PCI controller that needs it close to the initialization of "priv".

Cheers,

Signature · 1 line
Julien Grall
#11 lore raw

Re: [PATCH v2 6/7] xen/arm: rcar4: add simple optimization to avoid ATU reprogramming

In reply to Mykyta Poturai
Quoted 23 lines · Mykyta Poturai

From: Volodymyr Babchuk <[email protected]>

There are high chances that there will be a number of a consecutive accesses to configuration space of one device. To speed things up, we can program ATU only during first access.

This is mostly beneficial taking into account the previous patch that adds 1ms delay after ATU reprogramming.

---
v1->v2:
* rebased

---
  xen/arch/arm/pci/pci-designware.c | 8 ++++++++
  1 file changed, 8 insertions(+)
xen/arch/arm/pci/pci-designware.c +0 -0
index def2c12d63..cec52cf81a 100644

xen/arch/arm/pci/pci-designware.c static void dw_pcie_prog_outbound_atu(struct pci_host_bridge *pci, int index, · line 274

Quoted 3 lines
xen/arch/arm/pci/pci-designware.c +0 -0
@@ -272,6 +272,14 @@ static void dw_pcie_prog_outbound_atu(struct pci_host_bridge *pci, int index,
                                        int type, uint64_t cpu_addr,
                                        uint64_t pci_addr, uint64_t size)
  {

First, using (hiding) static var inside the function deserve big, fat comment on why and what.

xen/arch/arm/pci/pci-designware.c static void dw_pcie_prog_outbound_atu(struct pci_host_bridge *pci, int index, · line 275

Quoted 1 line
xen/arch/arm/pci/pci-designware.c +1 -0
+    static uint64_t prev_addr = ~0;

Second, from an experience, static vars in functions is source of potential problems, it's kinda land mines in code. For example, lets assume there are two pci_host_bridge instances in Xen and they are accessed concurrently.

Best to get rid of static var here - may be move in pci_host_bridge, like cached_pci_addr? not sure

xen/arch/arm/pci/pci-designware.c static void dw_pcie_prog_outbound_atu(struct pci_host_bridge *pci, int index, · line 285

Quoted 10 lines
xen/arch/arm/pci/pci-designware.c +7 -0
+
+    /* Simple optimization to not-program ATU for every transaction */
+    if (prev_addr == pci_addr)
+        return;
+
+    prev_addr = pci_addr;
+
      __dw_pcie_prog_outbound_atu(pci, 0, index, type,
                                  cpu_addr, pci_addr, size);
  }

BR,
-grygorii

#8 lore raw

Re: [PATCH v2 5/7] xen/arm: rcar4: add delay after programming ATU

In reply to Mykyta Poturai
Quoted 23 lines · Mykyta Poturai

From: Volodymyr Babchuk <[email protected]>

For some reason, we need a delay before accessing ATU region after we programmed it. Otherwise, we'll get erroneous TLP.

There is a code below, which should do this in proper way, by polling CTRL2 register, but according to documentation, hardware does not change this ATU_ENABLE bit at all.

---
v1->v2:
* rebased

---
  xen/arch/arm/pci/pci-designware.c | 5 +++++
  1 file changed, 5 insertions(+)
xen/arch/arm/pci/pci-designware.c +0 -0
index 6ab03cf9b0..def2c12d63 100644

xen/arch/arm/pci/pci-designware.c static void dw_pcie_prog_outbound_atu_unroll(struct pci_host_bridge *pci, · line 201

Quoted 8 lines
xen/arch/arm/pci/pci-designware.c +5 -0
@@ -194,6 +194,11 @@ static void dw_pcie_prog_outbound_atu_unroll(struct pci_host_bridge *pci,
      dw_pcie_writel_ob_unroll(pci, index, PCIE_ATU_UNR_REGION_CTRL2,
                               PCIE_ATU_ENABLE);
 
+    /*
+     * HACK: We need to delay there, because the next code does not
+     * work as expected on S4
+     */
+    mdelay(1);

It seems like a HACK to WA some platform issue, but in its current form it will affect all DW PCI compatible platforms.

So, it is required a proper solution for the affected platform to be found, or some sort of DW PCI "quirk"s processing code be introduced.

I'd recommend to drop this patch for now from this series.

xen/arch/arm/pci/pci-designware.c static void dw_pcie_prog_outbound_atu_unroll(struct pci_host_bridge *pci, · line 204

Quoted 3 lines
xen/arch/arm/pci/pci-designware.c +0 -0
      /*
       * Make sure ATU enable takes effect before any subsequent config
       * and I/O accesses.

BR,
-grygorii

#9 lore raw

Re: [PATCH v2 5/7] xen/arm: rcar4: add delay after programming ATU

Quoted 43 lines · Grygorii Strashko
Quoted 24 lines · Mykyta Poturai

From: Volodymyr Babchuk <[email protected]>

For some reason, we need a delay before accessing ATU region after we programmed it. Otherwise, we'll get erroneous TLP.

There is a code below, which should do this in proper way, by polling CTRL2 register, but according to documentation, hardware does not change this ATU_ENABLE bit at all.

---
v1->v2:
* rebased
---
  xen/arch/arm/pci/pci-designware.c | 5 +++++

  1 file changed, 5 insertions(+)
xen/arch/arm/pci/pci-designware.c +0 -0
index 6ab03cf9b0..def2c12d63 100644

xen/arch/arm/pci/pci-designware.c static void · line 201

Quoted 8 lines
xen/arch/arm/pci/pci-designware.c +5 -0
@@ -194,6 +194,11 @@ static void 
 dw_pcie_prog_outbound_atu_unroll(struct pci_host_bridge *pci,
       dw_pcie_writel_ob_unroll(pci, index, PCIE_ATU_UNR_REGION_CTRL2,
                                PCIE_ATU_ENABLE);
+    /*
+     * HACK: We need to delay there, because the next code does not
+     * work as expected on S4
+     */
+    mdelay(1);

It seems like a HACK to WA some platform issue, but in its current form it will affect all DW PCI compatible platforms.

So, it is required a proper solution for the affected platform to be found, or
some sort of DW PCI "quirk"s processing code be introduced.

Quoted 8 lines

I'd recommend to drop this patch for now from this series.

xen/arch/arm/pci/pci-designware.c static void dw_pcie_prog_outbound_atu_unroll(struct pci_host_bridge *pci, · line 204

Quoted 3 lines
xen/arch/arm/pci/pci-designware.c +0 -0
       /*
        * Make sure ATU enable takes effect before any subsequent config
        * and I/O accesses.

BR,
-grygorii

Hi Grygorii

After some further investigations I have retested this on V4H, and it seems to work fine without delays. Considering this and the issue with static variables and multiple PCI hosts I think I will drop ATU related workarounds until some proper quirks handling system is established.

Signature · 1 line
Mykyta