Add region id support for PCI
cover Add region id support for PCI
On Renesas R-Car platforms, the Region ID (RGID) feature encodes an identifier into the upper bits of the physical address for every CPU memory access. This address manipulations can mostly be done via system and domain congirutaion, but for PCI bar it is not possible as some BARs are 32bit only.
This series introduces a platform hook that lets the PCI host bridge driver rewrite BAR addresses at vPCI init time, and provides the R-Car Gen4 implementation that adds the configured RGID into each address.
This is sent as RFC to get early feedback on the approach:
- Is a pci_ops callback the right place for this kind of address fixup?
- Is platform Kconfig a good place for RGID related options or is it better to put them in some other place.
Mykyta Poturai (2):
pci: Allow platforms to modify BAR adresses
plat/rcar: Add region id support for PCI
xen/arch/arm/include/asm/pci.h | 3 +++ xen/arch/arm/include/asm/vpci.h | 9 +++++++++ xen/arch/arm/pci/pci-host-rcar4.c | 15 +++++++++++++++ xen/arch/arm/platforms/Kconfig | 21 +++++++++++++++++++++ xen/arch/arm/vpci.c | 12 ++++++++++++ xen/arch/x86/include/asm/vpci.h | 6 ++++++ xen/drivers/vpci/header.c | 2 ++ 7 files changed, 68 insertions(+)
Signature · 1 line
2.51.2
RFC PATCH 1/2 pci: Allow platforms to modify BAR adresses
This patch is a preparatory work for adding Region ID support on Renesas R-Car series boards. Add new host bridge op "fixup_bar" that allows platforms to modify BAR addresses before they are mapped.
Because x86 don't have support for PCI Host Bridge drivers, add another level of indirection in form of platform_pci_fixup_bar() function, that will call host bridge op on ARM and do nothing on x86.
--- xen/arch/arm/include/asm/pci.h | 3 +++ xen/arch/arm/include/asm/vpci.h | 9 +++++++++ xen/arch/arm/vpci.c | 12 ++++++++++++ xen/arch/x86/include/asm/vpci.h | 6 ++++++ xen/drivers/vpci/header.c | 2 ++ 5 files changed, 32 insertions(+)
xen/arch/arm/include/asm/pci.h +3 -0
@@ -80,6 +80,9 @@ struct pci_ops {void (*init_bus_range)(struct dt_device_node *dev,struct pci_host_bridge *bridge,struct pci_config_window *cfg); +void (*fixup_bar)(struct pci_host_bridge *bridge, +unsigned int bar_num, +paddr_t *addr); };/*
xen/arch/arm/include/asm/vpci.h +9 -0
@@ -16,6 +16,10 @@ struct vpci_arch_msix_entry {int domain_vpci_init (struct domain *d);unsigned int domain_vpci_get_num_mmio_handlers (struct domain *d); + +void platform_pci_fixup_bar (const struct pci_dev *pdev,unsigned int bar_num, +paddr_t *addr); +#else static inline int domain_vpci_init (struct domain *d) {
@@ -26,6 +30,11 @@ static inline unsigned int domain_vpci_get_num_mmio_handlers(struct domain *d) {return 0 ; } + +static inline void platform_pci_fixup_bar (const struct pci_dev *pdev, +unsigned int bar_num, +paddr_t *addr) +{}#endif /* CONFIG_HAS_VPCI */ #endif /* ARM_VPCI_H */
xen/arch/arm/vpci.c +12 -0
@@ -189,6 +189,18 @@ unsigned int domain_vpci_get_num_mmio_handlers(struct domain *d)return 1 ; } +void platform_pci_fixup_bar (const struct pci_dev *pdev, +unsigned int bar_num, +paddr_t *addr) +{ +struct pci_host_bridge *bridge =pci_find_host_bridge (pdev->sbdf.seg, pdev->sbdf.bus); + +if ( bridge->ops->fixup_bar ) + { + bridge->ops->fixup_bar (bridge, bar_num, addr); + } +} +/* * Local variables: * mode: C
xen/arch/x86/include/asm/vpci.h +6 -0
@@ -16,6 +16,12 @@ struct vpci_arch_msix_entry { int pirq; }; +/* X86 does not require PCI BAR modifications */ +static inline void platform_pci_fixup_bar (const struct pci_dev *pdev, +unsigned int bar_num, +paddr_t *addr) +{} +#endif /* X86_VPCI_H */ /*
xen/drivers/vpci/header.c +2 -0
@@ -882,6 +882,8 @@ int vpci_init_header(struct pci_dev *pdev) bars[i].size = size; bars[i].prefetchable = val &PCI_BASE_ADDRESS_MEM_PREFETCH ; +platform_pci_fixup_bar (pdev, i, &bars[i].addr); + rc = vpci_add_register(pdev->vpci, is_hwdom ? vpci_hw_read32 : guest_mem_bar_read, is_hwdom ? bar_write : guest_mem_bar_write,
Signature · 1 line
2.51.2
RFC PATCH 2/2 plat/rcar: Add region id support for PCI
In reply to Mykyta PoturaiWith Region ID enabled, all CPU memory accesses need to have rgid bits set in the physical address. This creates a problem for PCI BAR accesses, as it would require all BARs to be 64bit. Implement fixup_bar callback to add rgid bits to the address before mapping it to the guests.
Add Kconfig options to enable region id support and set the rgid value and physical address space size.
--- xen/arch/arm/pci/pci-host-rcar4.c | 15 +++++++++++++++ xen/arch/arm/platforms/Kconfig | 21 +++++++++++++++++++++ 2 files changed, 36 insertions(+)
xen/arch/arm/pci/pci-host-rcar4.c +15 -0
@@ -43,6 +43,12 @@ struct rcar4_pcie_priv {DECLARE_BITMAP (osid_regs,NUM_OSID_REGS ); }; +#define ULL (X ) _AC(X, ULL) +#define MADDR_RGID (a) (ULL(a) << CONFIG_RCAR_PA_BITS) +#define MADDR_PA_MASK ((1ULL << CONFIG_RCAR_PA_BITS) - 1) + +#define MADDR_ENCODE_RGID (a) (MADDR_RGID(CONFIG_RCAR_RGID) | (a)) +/* * PCI host bridges often have different ways to access the root and child * bus config spaces:
@@ -61,6 +67,14 @@ static int __init rcar4_child_cfg_reg_index(struct dt_device_node *np)return dt_property_match_string (np,"reg-names" ,"config" ); } +static void rcar4_pcie_fixup_bar (struct pci_host_bridge *bridge, +unsigned int bar_num, +paddr_t *addr) +{ +if (IS_ENABLED (CONFIG_RCAR_REGION_ID_SUPPORT ) ) + *addr =MADDR_ENCODE_RGID (*addr); +} +/* ECAM ops */ static const struct pci_ecam_ops rcar4_pcie_ops = { .bus_shift =20 ,
@@ -71,6 +85,7 @@ static const struct pci_ecam_ops rcar4_pcie_ops = { .write = pci_generic_config_write, .need_p2m_hwdom_mapping = pci_ecam_need_p2m_hwdom_mapping, .init_bus_range = pci_generic_init_bus_range, + .fixup_bar = rcar4_pcie_fixup_bar, } };
xen/arch/arm/platforms/Kconfig +21 -0
@@ -64,6 +64,27 @@ config NO_PLAT endchoice +menu "RCar Region ID Support" + visible if RCAR4 + +config RCAR_REGION_ID_SUPPORT + bool "Renesas Region ID support for R-Car Gen4 platforms" if EXPERT + depends on RCAR4 + help + Enable experimental Region ID support for R-Car Gen4 platforms + +config RCAR_RGID + int "Region ID encoded in physical address" + depends on RCAR_REGION_ID_SUPPORT + default 0 + +config RCAR_PA_BITS + int "Physical address space size" + depends on RCAR_REGION_ID_SUPPORT + default 36 + +endmenu + config ALL64_PLAT bool default (ALL_PLAT && ARM_64)
Signature · 1 line
2.51.2
Re: [RFC PATCH 2/2] plat/rcar: Add region id support for PCI
Quoted 3 lines · Mykyta Poturai
With Region ID enabled, all CPU memory accesses need to have rgid bits set in the physical address. This creates a problem for PCI BAR accesses, as it would require all BARs to be 64bit.
I don't understand this. CPU accesses of addresses read / derived from BARs can be massaged in any way the OS likes. That doesn't require the BARs to be 64-bit. Are you trying to arrange for RGID to be transparent to guests?
Jan
Re: [RFC PATCH 1/2] pci: Allow platforms to modify BAR adresses
In reply to Mykyta PoturaiQuoted 22 lines · Mykyta Poturai
This patch is a preparatory work for adding Region ID support on Renesas R-Car series boards. Add new host bridge op "
fixup_bar" that allows platforms to modify BAR addresses before they are mapped.Because x86 don't have support for PCI Host Bridge drivers, add another level of indirection in form of
platform_pci_fixup_bar()function, that will call host bridge op on ARM and do nothing on x86.Signed-off-by Mykyta Poturai <[email protected]>--- xen/arch/arm/include/asm/pci.h | 3 +++ xen/arch/arm/include/asm/vpci.h | 9 +++++++++ xen/arch/arm/vpci.c | 12 ++++++++++++ xen/arch/x86/include/asm/vpci.h | 6 ++++++ xen/drivers/vpci/header.c | 2 ++ 5 files changed, 32 insertions(+)xen/arch/arm/include/asm/pci.h +0
-0
xen/arch/arm/include/asm/pci.h struct pci_ops { · line 85
Quoted 6 lines
xen/arch/arm/include/asm/pci.h +3
-0@@ -80,6 +80,9 @@ struct pci_ops {void (*init_bus_range)(struct dt_device_node *dev,struct pci_host_bridge *bridge,struct pci_config_window *cfg); +void (*fixup_bar)(struct pci_host_bridge *bridge, +unsigned int bar_num, +paddr_t *addr);
I'm not an Arm maintainer, but if such a hook - used only when a certain CONFIG_* is active, as per patch 2 - was introduced in common or x86 code, I'd ask for the hook to also be conditional. Doing so avoids accidental use without ...
Quoted 6 lines
xen/arch/arm/vpci.c +0
-0@@ -189,6 +189,18 @@ unsigned int domain_vpci_get_num_mmio_handlers(struct domain *d)return 1 ; }
xen/arch/arm/vpci.c unsigned int domain_vpci_get_num_mmio_handlers(struct domain *d) · line 201
Quoted 10 lines
xen/arch/arm/vpci.c +10
-0+void platform_pci_fixup_bar (const struct pci_dev *pdev, +unsigned int bar_num, +paddr_t *addr) +{ +struct pci_host_bridge *bridge =pci_find_host_bridge (pdev->sbdf.seg, pdev->sbdf.bus); + +if ( bridge->ops->fixup_bar ) + { + bridge->ops->fixup_bar (bridge, bar_num, addr); + }
... such a conditional around it.
Jan
Re: [RFC PATCH 2/2] plat/rcar: Add region id support for PCI
In reply to Mykyta PoturaiHi Mykyta,
Quoted 16 lines · Mykyta Poturai
xen/arch/arm/platforms/Kconfig +8
-0@@ -64,6 +64,27 @@ config NO_PLAT endchoice +menu "RCar Region ID Support" + visible if RCAR4 + +config RCAR_REGION_ID_SUPPORT + bool "Renesas Region ID support for R-Car Gen4 platforms" if EXPERT + depends on RCAR4 + help + Enable experimental Region ID support for R-Car Gen4 platforms
xen/arch/arm/platforms/Kconfig config NO_PLAT · line 84
Quoted 10 lines
xen/arch/arm/platforms/Kconfig +10
-0+ +config RCAR_RGID + int "Region ID encoded in physical address" + depends on RCAR_REGION_ID_SUPPORT + default 0 + +config RCAR_PA_BITS + int "Physical address space size" + depends on RCAR_REGION_ID_SUPPORT + default 36
The 3 configs above implies that the value will change per-board. Is that correct? If so, this should be described in the firmware table so a single Xen binary can boot on all the platforms.
Cheers,
Signature · 1 line
Julien Grall
Re: [RFC PATCH 2/2] plat/rcar: Add region id support for PCI
In reply to Jan BeulichQuoted 11 lines · Jan Beulich
Quoted 3 lines · Mykyta Poturai
With Region ID enabled, all CPU memory accesses need to have rgid bits set in the physical address. This creates a problem for PCI BAR accesses, as it would require all BARs to be 64bit.
I don't understand this. CPU accesses of addresses read / derived from BARs can be massaged in any way the OS likes. That doesn't require the BARs to be 64-bit. Are you trying to arrange for RGID to be transparent to guests?
Jan
Yes, making it transparent to guests is the goal.
Signature · 1 line
Mykyta
Re: [RFC PATCH 2/2] plat/rcar: Add region id support for PCI
In reply to Mykyta PoturaiOn 4/21/26 10:57, Mykyta Poturai wrote:
Hello Mykyta
Quoted 5 lines
With Region ID enabled, all CPU memory accesses need to have rgid bits set in the physical address. This creates a problem for PCI BAR accesses, as it would require all BARs to be 64bit. Implement
fixup_barcallback to add rgid bits to the address before mapping it to the guests.
For context. When Region ID support is enabled, the RGID value is encoded in the upper bits of physical addresses for both RAM and device MMIO in host device tree. From Xen's perspective, these are just regular physical addresses, and ideally Xen should not need to be RGID-aware at all. But I understand what this series is trying to solve.
It is worth mentioning that when Region ID support is enabled, there will be no 32-bit addresses in the host device tree anymore, since encoding the RGID effectively moves every address beyond the 4GB boundary (unless the RGID is 0). This might break any code in Xen that relies on 32-bit address space for some reason, e.g. HOST_ITS_WORKAROUND_32BIT_ADDR which requires 32-bit addresses for in-memory data structures (refer gicv3_its_enable_quirk_gen4()).
Quoted 15 lines
Add
Kconfigoptions to enable region id support and set the rgid value and physical address space size.Signed-off-by Mykyta Poturai <[email protected]>--- xen/arch/arm/pci/pci-host-rcar4.c | 15 +++++++++++++++ xen/arch/arm/platforms/Kconfig | 21 +++++++++++++++++++++ 2 files changed, 36 insertions(+)xen/arch/arm/pci/pci-host-rcar4.c +0
-0
xen/arch/arm/pci/pci-host-rcar4.c struct rcar4_pcie_priv { · line 48
Quoted 6 lines
xen/arch/arm/pci/pci-host-rcar4.c +3
-0@@ -43,6 +43,12 @@ struct rcar4_pcie_priv {DECLARE_BITMAP (osid_regs,NUM_OSID_REGS ); }; +#define ULL (X ) _AC(X, ULL) +#define MADDR_RGID (a) (ULL(a) << CONFIG_RCAR_PA_BITS) +#define MADDR_PA_MASK ((1ULL << CONFIG_RCAR_PA_BITS) - 1)
NIT: MADDR_PA_MASK is defined but never used ...
Quoted 45 lines
xen/arch/arm/pci/pci-host-rcar4.c +12
-0+ +#define MADDR_ENCODE_RGID (a) (MADDR_RGID(CONFIG_RCAR_RGID) | (a)) +/* * PCI host bridges often have different ways to access the root and child * bus config spaces: @@ -61,6 +67,14 @@ static int __init rcar4_child_cfg_reg_index(struct dt_device_node *np)return dt_property_match_string (np,"reg-names" ,"config" ); } +static void rcar4_pcie_fixup_bar (struct pci_host_bridge *bridge, +unsigned int bar_num, +paddr_t *addr) +{ +if (IS_ENABLED (CONFIG_RCAR_REGION_ID_SUPPORT ) ) + *addr =MADDR_ENCODE_RGID (*addr); +} +/* ECAM ops */ static const struct pci_ecam_ops rcar4_pcie_ops = { .bus_shift =20 ,@@ -71,6 +85,7 @@ static const struct pci_ecam_ops rcar4_pcie_ops = { .write = pci_generic_config_write, .need_p2m_hwdom_mapping = pci_ecam_need_p2m_hwdom_mapping, .init_bus_range = pci_generic_init_bus_range, + .fixup_bar = rcar4_pcie_fixup_bar, } };xen/arch/arm/platforms/Kconfig +8
-0@@ -64,6 +64,27 @@ config NO_PLAT endchoice +menu "RCar Region ID Support" + visible if RCAR4 + +config RCAR_REGION_ID_SUPPORT + bool "Renesas Region ID support for R-Car Gen4 platforms" if EXPERT + depends on RCAR4 + help + Enable experimental Region ID support for R-Car Gen4 platforms
xen/arch/arm/platforms/Kconfig config NO_PLAT · line 84
Quoted 10 lines
xen/arch/arm/platforms/Kconfig +10
-0+ +config RCAR_RGID + int "Region ID encoded in physical address" + depends on RCAR_REGION_ID_SUPPORT + default 0 + +config RCAR_PA_BITS + int "Physical address space size" + depends on RCAR_REGION_ID_SUPPORT + default 36
Options RCAR_RGID and RCAR_PA_BITS lack range constraints. A misconfigured options might cause undefined shift behavior at compile time. However, I see a valid request in a separate email to consider using a firmware table to pass these values, so just ignore this comment.
*****
I was wondering whether the RGID could be extracted at runtime from an
address that already carries it. Do addresses in the host device tree (e.g., device MMIO ranges, host bridge windows, etc) already contain RGID bits? If so, then in rcar4_pcie_fixup_bar() you could obtain the RGID from the bridge instance that is already passed as a parameter.
Something like below:
#define MADDR_GET_RGID(addr) ((addr) >> CONFIG_RCAR_PA_BITS)
paddr_t rgid = MADDR_GET_RGID(bridge->some_hw_addr_as_it_is_specified_in_dt); *addr = (rgid << CONFIG_RCAR_PA_BITS) | (*addr & MADDR_PA_MASK);
This would eliminate CONFIG_RCAR_RGID entirely by deducing it from existing platform data rather than requiring it to be passed explicitly via the device tree.
As for CONFIG_RCAR_PA_BITS, I think it could remain as-is since it is
constant across an SoC generation, unless I am mistaken. As far as I am aware, on Gen4 the RGID occupies 4 bits (0–15) starting at bit 36, and on Gen5 it occupies 5 bits (0–31) starting at bit 43. Since Xen can identify the hardware it is running on (thanks to the compatible string), the appropriate #define-s could be applied.
Would the above approach work, or is it better to pass all these values explicitly?
Please note, I am not 100% sure whether this is the right and reliable approach, but it should be considered.
Quoted 6 lines
xen/arch/arm/platforms/Kconfig +3
-0+ +endmenu + config ALL64_PLAT bool default (ALL_PLAT && ARM_64)
Re: [RFC PATCH 1/2] pci: Allow platforms to modify BAR adresses
In reply to Mykyta PoturaiQuoted 103 lines · Mykyta Poturai
This patch is a preparatory work for adding Region ID support on Renesas R-Car series boards. Add new host bridge op "
fixup_bar" that allows platforms to modify BAR addresses before they are mapped.Because x86 don't have support for PCI Host Bridge drivers, add another level of indirection in form of
platform_pci_fixup_bar()function, that will call host bridge op on ARM and do nothing on x86.Signed-off-by Mykyta Poturai <[email protected]>--- xen/arch/arm/include/asm/pci.h | 3 +++ xen/arch/arm/include/asm/vpci.h | 9 +++++++++ xen/arch/arm/vpci.c | 12 ++++++++++++ xen/arch/x86/include/asm/vpci.h | 6 ++++++ xen/drivers/vpci/header.c | 2 ++ 5 files changed, 32 insertions(+)xen/arch/arm/include/asm/pci.h +3
-0@@ -80,6 +80,9 @@ struct pci_ops {void (*init_bus_range)(struct dt_device_node *dev,struct pci_host_bridge *bridge,struct pci_config_window *cfg); +void (*fixup_bar)(struct pci_host_bridge *bridge, +unsigned int bar_num, +paddr_t *addr); };/* xen/arch/arm/include/asm/vpci.h +9
-0@@ -16,6 +16,10 @@ struct vpci_arch_msix_entry {int domain_vpci_init (struct domain *d);unsigned int domain_vpci_get_num_mmio_handlers (struct domain *d); + +void platform_pci_fixup_bar (const struct pci_dev *pdev,unsigned int bar_num, +paddr_t *addr); +#else static inline int domain_vpci_init (struct domain *d) {@@ -26,6 +30,11 @@ static inline unsigned int domain_vpci_get_num_mmio_handlers(struct domain *d) {return 0 ; } + +static inline void platform_pci_fixup_bar (const struct pci_dev *pdev, +unsigned int bar_num, +paddr_t *addr) +{}#endif /* CONFIG_HAS_VPCI */ #endif /* ARM_VPCI_H */ xen/arch/arm/vpci.c +12
-0@@ -189,6 +189,18 @@ unsigned int domain_vpci_get_num_mmio_handlers(struct domain *d)return 1 ; } +void platform_pci_fixup_bar (const struct pci_dev *pdev, +unsigned int bar_num, +paddr_t *addr) +{ +struct pci_host_bridge *bridge =pci_find_host_bridge (pdev->sbdf.seg, pdev->sbdf.bus); + +if ( bridge->ops->fixup_bar ) + { + bridge->ops->fixup_bar (bridge, bar_num, addr); + } +} +/* * Local variables: * mode: C xen/arch/x86/include/asm/vpci.h +6
-0@@ -16,6 +16,12 @@ struct vpci_arch_msix_entry { int pirq; }; +/* X86 does not require PCI BAR modifications */ +static inline void platform_pci_fixup_bar (const struct pci_dev *pdev, +unsigned int bar_num, +paddr_t *addr) +{} +#endif /* X86_VPCI_H */ /* xen/drivers/vpci/header.c +0
-0
xen/drivers/vpci/header.c int vpci_init_header(struct pci_dev *pdev) · line 885
Quoted 4 lines
xen/drivers/vpci/header.c +1
-0@@ -882,6 +882,8 @@ int vpci_init_header(struct pci_dev *pdev) bars[i].size = size; bars[i].prefetchable = val &PCI_BASE_ADDRESS_MEM_PREFETCH ; +platform_pci_fixup_bar (pdev, i, &bars[i].addr);
You do this for vpci_init_header(), but don't you also need to do it for bar_write(), in case dom0 decides to relocate the BAR?
Also - I would assume that both the firmware and dom0 are aware of this limitation, and hence wonder why you need this fixup in Xen?
Thanks, Roger.