Skip to contentSkip to conversation

Add region id support for PCI

RFC PATCH In review Mykyta Poturai 5 participants 9 messages Apr 21 – Apr 24 Arm/RISC-V/Renesas architecture

lore
Share & tools
lore
#0 lore raw

cover Add region id support for PCI

On Renesas R-Car platforms, the Region ID (RGID) feature encodes an identifier into the upper bits of the physical address for every CPU memory access. This address manipulations can mostly be done via system and domain congirutaion, but for PCI bar it is not possible as some BARs are 32bit only.

This series introduces a platform hook that lets the PCI host bridge driver rewrite BAR addresses at vPCI init time, and provides the R-Car Gen4 implementation that adds the configured RGID into each address.

This is sent as RFC to get early feedback on the approach:
- Is a pci_ops callback the right place for this kind of address fixup?
- Is platform Kconfig a good place for RGID related options or is it better to put them in some other place.

Mykyta Poturai (2):
pci: Allow platforms to modify BAR adresses
plat/rcar: Add region id support for PCI

 xen/arch/arm/include/asm/pci.h    |  3 +++
 xen/arch/arm/include/asm/vpci.h   |  9 +++++++++
 xen/arch/arm/pci/pci-host-rcar4.c | 15 +++++++++++++++
 xen/arch/arm/platforms/Kconfig    | 21 +++++++++++++++++++++
 xen/arch/arm/vpci.c               | 12 ++++++++++++
 xen/arch/x86/include/asm/vpci.h   |  6 ++++++
 xen/drivers/vpci/header.c         |  2 ++
 7 files changed, 68 insertions(+)
Signature · 1 line
2.51.2
#1 lore raw

RFC PATCH 1/2 pci: Allow platforms to modify BAR adresses

This patch is a preparatory work for adding Region ID support on Renesas R-Car series boards. Add new host bridge op "fixup_bar" that allows platforms to modify BAR addresses before they are mapped.

Because x86 don't have support for PCI Host Bridge drivers, add another level of indirection in form of platform_pci_fixup_bar() function, that will call host bridge op on ARM and do nothing on x86.

---
 xen/arch/arm/include/asm/pci.h  |  3 +++
 xen/arch/arm/include/asm/vpci.h |  9 +++++++++
 xen/arch/arm/vpci.c             | 12 ++++++++++++
 xen/arch/x86/include/asm/vpci.h |  6 ++++++
 xen/drivers/vpci/header.c       |  2 ++
 5 files changed, 32 insertions(+)
xen/arch/arm/include/asm/pci.h +3 -0
index 7c3211823f..398a4eb746 100644
@@ -80,6 +80,9 @@ struct pci_ops {
     void (*init_bus_range)(struct dt_device_node *dev,
                            struct pci_host_bridge *bridge,
                            struct pci_config_window *cfg);
+    void (*fixup_bar)(struct pci_host_bridge *bridge,
+                      unsigned int bar_num,
+                      paddr_t *addr);
 };
 
 /*
xen/arch/arm/include/asm/vpci.h +9 -0
index 0cc6f5a105..f5c817a51c 100644
@@ -16,6 +16,10 @@ struct vpci_arch_msix_entry {
 
 int domain_vpci_init(struct domain *d);
 unsigned int domain_vpci_get_num_mmio_handlers(struct domain *d);
+
+void platform_pci_fixup_bar(const struct pci_dev *pdev, unsigned int bar_num,
+                            paddr_t *addr);
+
 #else
 static inline int domain_vpci_init(struct domain *d)
 {
@@ -26,6 +30,11 @@ static inline unsigned int domain_vpci_get_num_mmio_handlers(struct domain *d)
 {
     return 0;
 }
+
+static inline void platform_pci_fixup_bar(const struct pci_dev *pdev,
+                                          unsigned int bar_num,
+                                          paddr_t *addr)
+{}
 #endif /* CONFIG_HAS_VPCI */
 
 #endif /* ARM_VPCI_H */
xen/arch/arm/vpci.c +12 -0
index d41aa383a8..ec6efec22e 100644
@@ -189,6 +189,18 @@ unsigned int domain_vpci_get_num_mmio_handlers(struct domain *d)
     return 1;
 }
 
+void platform_pci_fixup_bar(const struct pci_dev *pdev,
+                                          unsigned int bar_num,
+                                          paddr_t *addr)
+{
+    struct pci_host_bridge *bridge = pci_find_host_bridge(pdev->sbdf.seg, pdev->sbdf.bus);
+
+    if ( bridge->ops->fixup_bar )
+    {
+        bridge->ops->fixup_bar(bridge, bar_num, addr);
+    }
+}
+
 /*
  * Local variables:
  * mode: C
xen/arch/x86/include/asm/vpci.h +6 -0
index c501ff1709..a05b70abbf 100644
@@ -16,6 +16,12 @@ struct vpci_arch_msix_entry {
     int pirq;
 };
 
+/* X86 does not require PCI BAR modifications */
+static inline void platform_pci_fixup_bar(const struct pci_dev *pdev,
+                                          unsigned int bar_num,
+                                          paddr_t *addr)
+{}
+
 #endif /* X86_VPCI_H */
 
 /*
xen/drivers/vpci/header.c +2 -0
index a760d8c32f..d89e43354c 100644
@@ -882,6 +882,8 @@ int vpci_init_header(struct pci_dev *pdev)
         bars[i].size = size;
         bars[i].prefetchable = val & PCI_BASE_ADDRESS_MEM_PREFETCH;
 
+        platform_pci_fixup_bar(pdev, i, &bars[i].addr);
+
         rc = vpci_add_register(pdev->vpci,
                                is_hwdom ? vpci_hw_read32 : guest_mem_bar_read,
                                is_hwdom ? bar_write : guest_mem_bar_write,
Signature · 1 line
2.51.2
#4 lore raw

RFC PATCH 2/2 plat/rcar: Add region id support for PCI

In reply to Mykyta Poturai

With Region ID enabled, all CPU memory accesses need to have rgid bits set in the physical address. This creates a problem for PCI BAR accesses, as it would require all BARs to be 64bit. Implement fixup_bar callback to add rgid bits to the address before mapping it to the guests.

Add Kconfig options to enable region id support and set the rgid value and physical address space size.

---
 xen/arch/arm/pci/pci-host-rcar4.c | 15 +++++++++++++++
 xen/arch/arm/platforms/Kconfig    | 21 +++++++++++++++++++++
 2 files changed, 36 insertions(+)
xen/arch/arm/pci/pci-host-rcar4.c +15 -0
index e1e8eb0ee1..50400c04c4 100644
@@ -43,6 +43,12 @@ struct rcar4_pcie_priv {
     DECLARE_BITMAP(osid_regs, NUM_OSID_REGS);
 };
 
+#define ULL(X) _AC(X, ULL)
+#define MADDR_RGID(a)    (ULL(a) << CONFIG_RCAR_PA_BITS)
+#define MADDR_PA_MASK    ((1ULL << CONFIG_RCAR_PA_BITS) - 1)
+
+#define MADDR_ENCODE_RGID(a)    (MADDR_RGID(CONFIG_RCAR_RGID) | (a))
+
 /*
  * PCI host bridges often have different ways to access the root and child
  * bus config spaces:
@@ -61,6 +67,14 @@ static int __init rcar4_child_cfg_reg_index(struct dt_device_node *np)
     return dt_property_match_string(np, "reg-names", "config");
 }
 
+static void rcar4_pcie_fixup_bar(struct pci_host_bridge *bridge,
+                                 unsigned int bar_num,
+                                 paddr_t *addr)
+{
+    if ( IS_ENABLED(CONFIG_RCAR_REGION_ID_SUPPORT) )
+        *addr = MADDR_ENCODE_RGID(*addr);
+}
+
 /* ECAM ops */
 static const struct pci_ecam_ops rcar4_pcie_ops = {
     .bus_shift  = 20,
@@ -71,6 +85,7 @@ static const struct pci_ecam_ops rcar4_pcie_ops = {
         .write                  = pci_generic_config_write,
         .need_p2m_hwdom_mapping = pci_ecam_need_p2m_hwdom_mapping,
         .init_bus_range         = pci_generic_init_bus_range,
+        .fixup_bar              = rcar4_pcie_fixup_bar,
     }
 };
 
xen/arch/arm/platforms/Kconfig +21 -0
index 888d0b85d5..db096952c8 100644
@@ -64,6 +64,27 @@ config NO_PLAT
 
 endchoice
 
+menu "RCar Region ID Support"
+    visible if RCAR4
+
+config RCAR_REGION_ID_SUPPORT
+	bool "Renesas Region ID support for R-Car Gen4 platforms" if EXPERT
+	depends on RCAR4
+	help
+	  Enable experimental Region ID support for R-Car Gen4 platforms
+
+config RCAR_RGID
+	int "Region ID encoded in physical address"
+	depends on RCAR_REGION_ID_SUPPORT
+	default 0
+
+config RCAR_PA_BITS
+	int "Physical address space size"
+	depends on RCAR_REGION_ID_SUPPORT
+	default 36
+
+endmenu
+
 config ALL64_PLAT
 	bool
 	default (ALL_PLAT && ARM_64)
Signature · 1 line
2.51.2
#5 lore raw

Re: [RFC PATCH 2/2] plat/rcar: Add region id support for PCI

Quoted 3 lines · Mykyta Poturai

With Region ID enabled, all CPU memory accesses need to have rgid bits set in the physical address. This creates a problem for PCI BAR accesses, as it would require all BARs to be 64bit.

I don't understand this. CPU accesses of addresses read / derived from BARs can be massaged in any way the OS likes. That doesn't require the BARs to be 64-bit. Are you trying to arrange for RGID to be transparent to guests?

Jan

#2 lore raw

Re: [RFC PATCH 1/2] pci: Allow platforms to modify BAR adresses

In reply to Mykyta Poturai
Quoted 22 lines · Mykyta Poturai

This patch is a preparatory work for adding Region ID support on Renesas R-Car series boards. Add new host bridge op "fixup_bar" that allows platforms to modify BAR addresses before they are mapped.

Because x86 don't have support for PCI Host Bridge drivers, add another level of indirection in form of platform_pci_fixup_bar() function, that will call host bridge op on ARM and do nothing on x86.

---
 xen/arch/arm/include/asm/pci.h  |  3 +++
 xen/arch/arm/include/asm/vpci.h |  9 +++++++++
 xen/arch/arm/vpci.c             | 12 ++++++++++++
 xen/arch/x86/include/asm/vpci.h |  6 ++++++
 xen/drivers/vpci/header.c       |  2 ++
 5 files changed, 32 insertions(+)
xen/arch/arm/include/asm/pci.h +0 -0
index 7c3211823f..398a4eb746 100644

xen/arch/arm/include/asm/pci.h struct pci_ops { · line 85

Quoted 6 lines
xen/arch/arm/include/asm/pci.h +3 -0
@@ -80,6 +80,9 @@ struct pci_ops {
     void (*init_bus_range)(struct dt_device_node *dev,
                            struct pci_host_bridge *bridge,
                            struct pci_config_window *cfg);
+    void (*fixup_bar)(struct pci_host_bridge *bridge,
+                      unsigned int bar_num,
+                      paddr_t *addr);

I'm not an Arm maintainer, but if such a hook - used only when a certain CONFIG_* is active, as per patch 2 - was introduced in common or x86 code, I'd ask for the hook to also be conditional. Doing so avoids accidental use without ...

Quoted 6 lines
xen/arch/arm/vpci.c +0 -0
@@ -189,6 +189,18 @@ unsigned int domain_vpci_get_num_mmio_handlers(struct domain *d)
     return 1;
 }
 

xen/arch/arm/vpci.c unsigned int domain_vpci_get_num_mmio_handlers(struct domain *d) · line 201

Quoted 10 lines
xen/arch/arm/vpci.c +10 -0
+void platform_pci_fixup_bar(const struct pci_dev *pdev,
+                                          unsigned int bar_num,
+                                          paddr_t *addr)
+{
+    struct pci_host_bridge *bridge = pci_find_host_bridge(pdev->sbdf.seg, pdev->sbdf.bus);
+
+    if ( bridge->ops->fixup_bar )
+    {
+        bridge->ops->fixup_bar(bridge, bar_num, addr);
+    }

... such a conditional around it.

Jan

#7 lore raw

Re: [RFC PATCH 2/2] plat/rcar: Add region id support for PCI

In reply to Mykyta Poturai

Hi Mykyta,

Quoted 16 lines · Mykyta Poturai
xen/arch/arm/platforms/Kconfig +8 -0
index 888d0b85d5..db096952c8 100644
@@ -64,6 +64,27 @@ config NO_PLAT
 
  endchoice
 
+menu "RCar Region ID Support"
+    visible if RCAR4
+
+config RCAR_REGION_ID_SUPPORT
+	bool "Renesas Region ID support for R-Car Gen4 platforms" if EXPERT
+	depends on RCAR4
+	help
+	  Enable experimental Region ID support for R-Car Gen4 platforms

xen/arch/arm/platforms/Kconfig config NO_PLAT · line 84

Quoted 10 lines
xen/arch/arm/platforms/Kconfig +10 -0
+
+config RCAR_RGID
+	int "Region ID encoded in physical address"
+	depends on RCAR_REGION_ID_SUPPORT
+	default 0
+
+config RCAR_PA_BITS
+	int "Physical address space size"
+	depends on RCAR_REGION_ID_SUPPORT
+	default 36

The 3 configs above implies that the value will change per-board. Is that correct? If so, this should be described in the firmware table so a single Xen binary can boot on all the platforms.

Cheers,

Signature · 1 line
Julien Grall
#6 lore raw

Re: [RFC PATCH 2/2] plat/rcar: Add region id support for PCI

In reply to Jan Beulich
Quoted 11 lines · Jan Beulich
Quoted 3 lines · Mykyta Poturai

With Region ID enabled, all CPU memory accesses need to have rgid bits set in the physical address. This creates a problem for PCI BAR accesses, as it would require all BARs to be 64bit.

I don't understand this. CPU accesses of addresses read / derived from BARs can be massaged in any way the OS likes. That doesn't require the BARs to be 64-bit. Are you trying to arrange for RGID to be transparent to guests?

Jan

Yes, making it transparent to guests is the goal.

Signature · 1 line
Mykyta
#8 lore raw

Re: [RFC PATCH 2/2] plat/rcar: Add region id support for PCI

In reply to Mykyta Poturai

On 4/21/26 10:57, Mykyta Poturai wrote:

Hello Mykyta

Quoted 5 lines

With Region ID enabled, all CPU memory accesses need to have rgid bits set in the physical address. This creates a problem for PCI BAR accesses, as it would require all BARs to be 64bit. Implement fixup_bar callback to add rgid bits to the address before mapping it to the guests.

For context. When Region ID support is enabled, the RGID value is encoded in the upper bits of physical addresses for both RAM and device MMIO in host device tree. From Xen's perspective, these are just regular physical addresses, and ideally Xen should not need to be RGID-aware at all. But I understand what this series is trying to solve.

It is worth mentioning that when Region ID support is enabled, there will be no 32-bit addresses in the host device tree anymore, since encoding the RGID effectively moves every address beyond the 4GB boundary (unless the RGID is 0). This might break any code in Xen that relies on 32-bit address space for some reason, e.g. HOST_ITS_WORKAROUND_32BIT_ADDR which requires 32-bit addresses for in-memory data structures (refer gicv3_its_enable_quirk_gen4()).

Quoted 15 lines

Add Kconfig options to enable region id support and set the rgid value and physical address space size.

---
  xen/arch/arm/pci/pci-host-rcar4.c | 15 +++++++++++++++
  xen/arch/arm/platforms/Kconfig    | 21 +++++++++++++++++++++
  2 files changed, 36 insertions(+)
xen/arch/arm/pci/pci-host-rcar4.c +0 -0
index e1e8eb0ee1..50400c04c4 100644

xen/arch/arm/pci/pci-host-rcar4.c struct rcar4_pcie_priv { · line 48

Quoted 6 lines
xen/arch/arm/pci/pci-host-rcar4.c +3 -0
@@ -43,6 +43,12 @@ struct rcar4_pcie_priv {
      DECLARE_BITMAP(osid_regs, NUM_OSID_REGS);
  };
 
+#define ULL(X) _AC(X, ULL)
+#define MADDR_RGID(a)    (ULL(a) << CONFIG_RCAR_PA_BITS)
+#define MADDR_PA_MASK    ((1ULL << CONFIG_RCAR_PA_BITS) - 1)

NIT: MADDR_PA_MASK is defined but never used ...

Quoted 45 lines
xen/arch/arm/pci/pci-host-rcar4.c +12 -0
+
+#define MADDR_ENCODE_RGID(a)    (MADDR_RGID(CONFIG_RCAR_RGID) | (a))
+
  /*
   * PCI host bridges often have different ways to access the root and child
   * bus config spaces:
@@ -61,6 +67,14 @@ static int __init rcar4_child_cfg_reg_index(struct dt_device_node *np)
      return dt_property_match_string(np, "reg-names", "config");
  }
 
+static void rcar4_pcie_fixup_bar(struct pci_host_bridge *bridge,
+                                 unsigned int bar_num,
+                                 paddr_t *addr)
+{
+    if ( IS_ENABLED(CONFIG_RCAR_REGION_ID_SUPPORT) )
+        *addr = MADDR_ENCODE_RGID(*addr);
+}
+
  /* ECAM ops */
  static const struct pci_ecam_ops rcar4_pcie_ops = {
      .bus_shift  = 20,
@@ -71,6 +85,7 @@ static const struct pci_ecam_ops rcar4_pcie_ops = {
          .write                  = pci_generic_config_write,
          .need_p2m_hwdom_mapping = pci_ecam_need_p2m_hwdom_mapping,
          .init_bus_range         = pci_generic_init_bus_range,
+        .fixup_bar              = rcar4_pcie_fixup_bar,
      }
  };
 
xen/arch/arm/platforms/Kconfig +8 -0
index 888d0b85d5..db096952c8 100644
@@ -64,6 +64,27 @@ config NO_PLAT
 
  endchoice
 
+menu "RCar Region ID Support"
+    visible if RCAR4
+
+config RCAR_REGION_ID_SUPPORT
+	bool "Renesas Region ID support for R-Car Gen4 platforms" if EXPERT
+	depends on RCAR4
+	help
+	  Enable experimental Region ID support for R-Car Gen4 platforms

xen/arch/arm/platforms/Kconfig config NO_PLAT · line 84

Quoted 10 lines
xen/arch/arm/platforms/Kconfig +10 -0
+
+config RCAR_RGID
+	int "Region ID encoded in physical address"
+	depends on RCAR_REGION_ID_SUPPORT
+	default 0
+
+config RCAR_PA_BITS
+	int "Physical address space size"
+	depends on RCAR_REGION_ID_SUPPORT
+	default 36

Options RCAR_RGID and RCAR_PA_BITS lack range constraints. A misconfigured options might cause undefined shift behavior at compile time. However, I see a valid request in a separate email to consider using a firmware table to pass these values, so just ignore this comment.

*****

I was wondering whether the RGID could be extracted at runtime from an
address that already carries it. Do addresses in the host device tree (e.g., device MMIO ranges, host bridge windows, etc) already contain RGID bits? If so, then in rcar4_pcie_fixup_bar() you could obtain the RGID from the bridge instance that is already passed as a parameter.

Something like below:

#define MADDR_GET_RGID(addr) ((addr) >> CONFIG_RCAR_PA_BITS)

paddr_t rgid = MADDR_GET_RGID(bridge->some_hw_addr_as_it_is_specified_in_dt); *addr = (rgid << CONFIG_RCAR_PA_BITS) | (*addr & MADDR_PA_MASK);

This would eliminate CONFIG_RCAR_RGID entirely by deducing it from existing platform data rather than requiring it to be passed explicitly via the device tree.

As for CONFIG_RCAR_PA_BITS, I think it could remain as-is since it is
constant across an SoC generation, unless I am mistaken. As far as I am aware, on Gen4 the RGID occupies 4 bits (0–15) starting at bit 36, and on Gen5 it occupies 5 bits (0–31) starting at bit 43. Since Xen can identify the hardware it is running on (thanks to the compatible string), the appropriate #define-s could be applied.

Would the above approach work, or is it better to pass all these values explicitly?

Please note, I am not 100% sure whether this is the right and reliable approach, but it should be considered.

Quoted 6 lines
xen/arch/arm/platforms/Kconfig +3 -0
+
+endmenu
+
  config ALL64_PLAT
  	bool
  	default (ALL_PLAT && ARM_64)
#3 lore raw

Re: [RFC PATCH 1/2] pci: Allow platforms to modify BAR adresses

In reply to Mykyta Poturai
Quoted 103 lines · Mykyta Poturai

This patch is a preparatory work for adding Region ID support on Renesas R-Car series boards. Add new host bridge op "fixup_bar" that allows platforms to modify BAR addresses before they are mapped.

Because x86 don't have support for PCI Host Bridge drivers, add another level of indirection in form of platform_pci_fixup_bar() function, that will call host bridge op on ARM and do nothing on x86.

---
 xen/arch/arm/include/asm/pci.h  |  3 +++
 xen/arch/arm/include/asm/vpci.h |  9 +++++++++
 xen/arch/arm/vpci.c             | 12 ++++++++++++
 xen/arch/x86/include/asm/vpci.h |  6 ++++++
 xen/drivers/vpci/header.c       |  2 ++
 5 files changed, 32 insertions(+)
xen/arch/arm/include/asm/pci.h +3 -0
index 7c3211823f..398a4eb746 100644
@@ -80,6 +80,9 @@ struct pci_ops {
     void (*init_bus_range)(struct dt_device_node *dev,
                            struct pci_host_bridge *bridge,
                            struct pci_config_window *cfg);
+    void (*fixup_bar)(struct pci_host_bridge *bridge,
+                      unsigned int bar_num,
+                      paddr_t *addr);
 };
 
 /*
xen/arch/arm/include/asm/vpci.h +9 -0
index 0cc6f5a105..f5c817a51c 100644
@@ -16,6 +16,10 @@ struct vpci_arch_msix_entry {
 
 int domain_vpci_init(struct domain *d);
 unsigned int domain_vpci_get_num_mmio_handlers(struct domain *d);
+
+void platform_pci_fixup_bar(const struct pci_dev *pdev, unsigned int bar_num,
+                            paddr_t *addr);
+
 #else
 static inline int domain_vpci_init(struct domain *d)
 {
@@ -26,6 +30,11 @@ static inline unsigned int domain_vpci_get_num_mmio_handlers(struct domain *d)
 {
     return 0;
 }
+
+static inline void platform_pci_fixup_bar(const struct pci_dev *pdev,
+                                          unsigned int bar_num,
+                                          paddr_t *addr)
+{}
 #endif /* CONFIG_HAS_VPCI */
 
 #endif /* ARM_VPCI_H */
xen/arch/arm/vpci.c +12 -0
index d41aa383a8..ec6efec22e 100644
@@ -189,6 +189,18 @@ unsigned int domain_vpci_get_num_mmio_handlers(struct domain *d)
     return 1;
 }
 
+void platform_pci_fixup_bar(const struct pci_dev *pdev,
+                                          unsigned int bar_num,
+                                          paddr_t *addr)
+{
+    struct pci_host_bridge *bridge = pci_find_host_bridge(pdev->sbdf.seg, pdev->sbdf.bus);
+
+    if ( bridge->ops->fixup_bar )
+    {
+        bridge->ops->fixup_bar(bridge, bar_num, addr);
+    }
+}
+
 /*
  * Local variables:
  * mode: C
xen/arch/x86/include/asm/vpci.h +6 -0
index c501ff1709..a05b70abbf 100644
@@ -16,6 +16,12 @@ struct vpci_arch_msix_entry {
     int pirq;
 };
 
+/* X86 does not require PCI BAR modifications */
+static inline void platform_pci_fixup_bar(const struct pci_dev *pdev,
+                                          unsigned int bar_num,
+                                          paddr_t *addr)
+{}
+
 #endif /* X86_VPCI_H */
 
 /*
xen/drivers/vpci/header.c +0 -0
index a760d8c32f..d89e43354c 100644

xen/drivers/vpci/header.c int vpci_init_header(struct pci_dev *pdev) · line 885

Quoted 4 lines
xen/drivers/vpci/header.c +1 -0
@@ -882,6 +882,8 @@ int vpci_init_header(struct pci_dev *pdev)
         bars[i].size = size;
         bars[i].prefetchable = val & PCI_BASE_ADDRESS_MEM_PREFETCH;
 
+        platform_pci_fixup_bar(pdev, i, &bars[i].addr);

You do this for vpci_init_header(), but don't you also need to do it for bar_write(), in case dom0 decides to relocate the BAR?

Also - I would assume that both the firmware and dom0 are aware of this limitation, and hence wonder why you need this fixup in Xen?

Thanks, Roger.