arm: mach-socfpga: PCIe Root IO TLP support for Cyclone V
Subsystems (4)
Hello ARM PCIe and especially Intel Altera SOCFPGA maintainers,
the Intel Altera Cyclone V PCIe Root Complex drivers afaik currently don’t support sending IO TLPs.
The Root Complex IP Core, seemingly unlike many other ARM Root Complexes, does not offer a memory mapping for the IO address space, but instead relies on indirect addressing via address and data registers. To my knowledge this IO space access has not been implemented in Linux, yet, and is currently only used for PCIe Configuration TLPs on this hardware, which use the same mechanism.
To support an AX99100 endpoint (which requires IO BARs for some of its features) connected to a Cyclone V PCIe root, we added support for IO TLPs to Kernels 4.14, 5.4, and 5.15. For your reference the 4.14 patches are attached. They do _not_ apply to current mainline Linux and are meant to give you an idea of our current approach. As you can see, in order to implement our own inb() and outb() family of functions, we had to remove multi-platform support for ARCH_SOCFPGA.
I would like to get some feedback on:
a) Are you interested in adding IO TLP support for this FPGA/CPU to mainline? We would implement patches for the current mainline in that case and post them for discussion.
b) Do you see an option to implement the IO space access functions without dropping multi-platform support? Is that a prerequisite for going mainline?
Thanks for your comments,
David
--- arch/arm/Kconfig | 28 ++ arch/arm/configs/socfpga_defconfig | 2 - arch/arm/mach-socfpga/Kconfig | 13 - arch/arm/mach-socfpga/Makefile | 1 + arch/arm/mach-socfpga/Makefile.boot | 1 + arch/arm/mach-socfpga/include/mach/io.h | 269 ++++++++++++++++++ arch/arm/mach-socfpga/include/mach/pcie.h | 23 ++ .../mach-socfpga/include/mach/uncompress.h | 8 + arch/arm/mach-socfpga/pcie.c | 17 ++ drivers/pci/host/pcie-altera.c | 230 ++++++++++++++- 10 files changed, 574 insertions(+), 18 deletions(-) create mode 100644 arch/arm/mach-socfpga/Makefile.boot create mode 100644 arch/arm/mach-socfpga/include/mach/io.h create mode 100644 arch/arm/mach-socfpga/include/mach/pcie.h create mode 100644 arch/arm/mach-socfpga/include/mach/uncompress.h create mode 100644 arch/arm/mach-socfpga/pcie.c
arch/arm/Kconfig +28 -0
@@ -644,6 +644,34 @@ config ARCH_OMAP1 help Support for older TI OMAP1 (omap7xx, omap15xx or omap16xx) +config ARCH_SOCFPGA + bool "Altera SOCFPGA family" + depends on MMU + select ARM_HAS_SG_CHAIN + select ARM_PATCH_PHYS_VIRT + select AUTO_ZRELADDR + select TIMER_OF + select COMMON_CLK + select GENERIC_CLOCKEVENTS + select MIGHT_HAVE_PCI + select MULTI_IRQ_HANDLER + select PCI_DOMAINS if PCI + select SPARSE_IRQ + select USE_OF + select MIGHT_HAVE_CACHE_L2X0 + select CPU_V7 + select HAVE_SMP + select ARCH_SUPPORTS_BIG_ENDIAN + select ARM_AMBA + select ARM_GIC + select CACHE_L2X0 + select DW_APB_TIMER_OF + select GPIO_PL061 if GPIOLIB + select HAVE_ARM_SCU + select HAVE_ARM_TWD if SMP + select MFD_SYSCON + select NEED_MACH_IO_H if PCI + endchoice menu "Multiple platform selection"
arch/arm/configs/socfpga_defconfig +0 -2
@@ -24,7 +24,6 @@ CONFIG_PCIE_ALTERA=y CONFIG_PCIE_ALTERA_MSI=y CONFIG_SMP=y CONFIG_NR_CPUS=2 -CONFIG_AEABI=y CONFIG_HIGHMEM=y CONFIG_ZBOOT_ROM_TEXT=0x0 CONFIG_ZBOOT_ROM_BSS=0x0
@@ -60,7 +59,6 @@ CONFIG_MTD_SPI_NOR=y # CONFIG_MTD_SPI_NOR_USE_4K_SECTORS is not set CONFIG_SPI_CADENCE_QUADSPI=y CONFIG_OF_OVERLAY=y -CONFIG_OF_CONFIGFS=y CONFIG_BLK_DEV_RAM=y CONFIG_BLK_DEV_RAM_COUNT=2 CONFIG_BLK_DEV_RAM_SIZE=8192
arch/arm/mach-socfpga/Kconfig +0 -13
@@ -1,16 +1,3 @@ -menuconfig ARCH_SOCFPGA - bool "Altera SOCFPGA family" - depends on ARCH_MULTI_V7 - select ARCH_SUPPORTS_BIG_ENDIAN - select ARM_AMBA - select ARM_GIC - select CACHE_L2X0 - select DW_APB_TIMER_OF - select GPIO_PL061 if GPIOLIB - select HAVE_ARM_SCU - select HAVE_ARM_TWD if SMP - select MFD_SYSCON - if ARCH_SOCFPGA config SOCFPGA_SUSPEND bool "Suspend to RAM on SOCFPGA"
arch/arm/mach-socfpga/Makefile +1 -0
@@ -4,6 +4,7 @@ # obj-y := socfpga.o +obj-$(CONFIG_PCI) += pcie.o obj-$(CONFIG_SMP) += headsmp.o platsmp.o obj-$(CONFIG_SOCFPGA_SUSPEND) += pm.o self-refresh.o obj-$(CONFIG_EDAC_ALTERA_L2C) += l2_cache.o
arch/arm/mach-socfpga/Makefile.boot new file +1 -0
@@ -0,0 +1 @@ +# placeholder
arch/arm/mach-socfpga/include/mach/io.h new file +269 -0
269 lines not on this page · raw message
arch/arm/mach-socfpga/include/mach/pcie.h new file +23 -0
@@ -0,0 +1,23 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef __MACH_SOCFPGA_PCIE_H +#define __MACH_SOCFPGA_PCIE_H + +#include <linux/types.h> + + +struct pci_io_ops { +void (*outb)(u8 value,u32 addr,void *priv); +void (*outw)(u16 value,u32 addr,void *priv); +void (*outl)(u32 value,u32 addr,void *priv); +u8 (*inb)(u32 addr,void *priv); +u16 (*inw)(u32 addr,void *priv); +u32 (*inl)(u32 addr,void *priv); +}; + +extern struct pci_io_ops *mach_socfpga_pci_io_ops; +extern void *mach_socfpga_pci_io_priv; + +extern void mach_socfpga_set_pci_io_ops (struct pci_io_ops *ops,void *priv); + +#endif /* __MACH_SOCFPGA_PCIE_H */
arch/arm/mach-socfpga/include/mach/uncompress.h new file +8 -0
@@ -0,0 +1,8 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifdef CONFIG_DEBUG_UNCOMPRESS +extern void putc (int c); +#else +static inline void putc (int c) {} +#endif +static inline void flush (void ) {} +static inline void arch_decomp_setup (void ) {}
arch/arm/mach-socfpga/pcie.c new file +17 -0
@@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#include <linux/module.h> +#include <mach/pcie.h> + + +struct pci_io_ops *mach_socfpga_pci_io_ops =NULL ; +EXPORT_SYMBOL (mach_socfpga_pci_io_ops); +void *mach_socfpga_pci_io_priv =NULL ; +EXPORT_SYMBOL (mach_socfpga_pci_io_priv); + +void mach_socfpga_set_pci_io_ops (struct pci_io_ops *ops,void *priv) +{ + mach_socfpga_pci_io_ops = ops; + mach_socfpga_pci_io_priv = priv; +} +EXPORT_SYMBOL (mach_socfpga_set_pci_io_ops);
drivers/pci/host/pcie-altera.c +227 -3
346 lines not on this page · raw message
Signature · 7 lines
2.34.1 _______________________________________________ linux-arm-kernel mailing list [email protected] http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
[+cc Zhichang, Gabriele, John (INDIRECT_PIO); Joyce (altera maintainer)]
Quoted 22 lines · David Epping
Hello ARM PCIe and especially Intel Altera SOCFPGA maintainers,
the Intel Altera Cyclone V PCIe Root Complex drivers afaik currently don’t support sending IO TLPs. The Root Complex IP Core, seemingly unlike many other ARM Root Complexes, does not offer a memory mapping for the IO address space, but instead relies on indirect addressing via address and data registers. To my knowledge this IO space access has not been implemented in Linux, yet, and is currently only used for PCIe Configuration TLPs on this hardware, which use the same mechanism.
To support an AX99100 endpoint (which requires IO BARs for some of its features) connected to a Cyclone V PCIe root, we added support for IO TLPs to Kernels 4.14, 5.4, and 5.15. For your reference the 4.14 patches are attached. They do _not_ apply to current mainline Linux and are meant to give you an idea of our current approach. As you can see, in order to implement our own
inb()andoutb()family of functions, we had to remove multi-platform support forARCH_SOCFPGA.I would like to get some feedback on:
Quoted 7 lines
a) Are you interested in adding IO TLP support for this FPGA/CPU to mainline? We would implement patches for the current mainline in that case and post them for discussion.
b) Do you see an option to implement the IO space access functions without dropping multi-platform support? Is that a prerequisite for going mainline?
I don't understand INDIRECT_PIO very well, but does it help with this at all?
https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/lib/Kconfig?id=v6.6#n92
https://git.kernel.org/linus/031e3601869c
Beginning of thread, including full patch (for v4.14, not current kernel) at: https://lore.kernel.org/all/[email protected]/
Bjorn
RE: mach-socfpga: PCIe Root IO TLP support for Cyclone V
In reply to David EppingFrom: David Epping
Quoted 7 lines
Sent: 31 October 2023 10:58
Hello ARM PCIe and especially Intel Altera SOCFPGA maintainers,
the Intel Altera Cyclone V PCIe Root Complex drivers afaik currently don’t support sending IO TLPs.
The Root Complex IP Core, seemingly unlike many other ARM Root Complexes,
It isn't an ARM root complex ...
I didn't think any of the Cyclone V had embedded arm cpu. I know some of the more recent Altera FPGA do, by the Cyclone V is pretty old now - although we are still using them in new cards. (Only as PCIe endpoints though.)
Quoted 2 lines
does not offer a memory mapping for the IO address space, but instead relies on indirect addressing via address and data registers.
If you are building the FPGA image then all the logic to convert the memory mapped slave cycles (into the fpga logic) is supplied as verilog source.
So you should be able to 'fix' it do generate IO TLP instead of data TLP for certain addresses.
(A few years back we had to fix it to correctly process multiple data TLP in response to a single read TLP - not a problem now.)
...
Quoted 1 line
To support an AX99100 endpoint (which requires IO BARs for some of its features)
I thought that all recent endpoints were required [1] to work with just memory BARs - even going back to the later PCI versions. So I'm surprised a PCIe endpoint need an IO BAR.
David
[1] well strongly implied that it was a really good idea :-)
-
Registered Address Lakeside, Bramley Road, Mount Farm, Milton Keynes, MK1 1PT, UK Registration No: 1397386 (Wales)
RE: mach-socfpga: PCIe Root IO TLP support for Cyclone V
From: David Laight
Quoted 20 lines
Sent: 05 November 2023 11:20
From: David Epping
Quoted 7 lines
Sent: 31 October 2023 10:58
Hello ARM PCIe and especially Intel Altera SOCFPGA maintainers,
the Intel Altera Cyclone V PCIe Root Complex drivers afaik currently don’t support sending IO TLPs.
The Root Complex IP Core, seemingly unlike many other ARM Root Complexes,It isn't an ARM root complex ...
I didn't think any of the Cyclone V had embedded arm cpu. I know some of the more recent Altera FPGA do, by the Cyclone V is pretty old now - although we are still using them in new cards. (Only as PCIe endpoints though.)Quoted 2 lines
does not offer a memory mapping for the IO address space, but instead relies on indirect addressing via address and data registers.
Quoted 7 lines
If you are building the FPGA image then all the logic to convert the memory mapped slave cycles (into the fpga logic) is supplied as verilog source.
So you should be able to 'fix' it do generate IO TLP instead of data TLP for certain addresses.
(A few years back we had to fix it to correctly process multiple data TLP in response to a single read TLP - not a problem now.)
Another idea would to be to write an Avalon slave that converts a single read/write into the required sequence of transfers into the cyclone V PCIe block to generate the IO TLP from a cpu memory access. That isn't hard to write.
It would also let you implement posted writes and asynchronous reads. Although the drivers won't expect async reads the PCIe is slow enough that they really do make sense.
In my measurements a Cyclone V endpoint (the root will be much the same) takes about 128 clocks (of the 125MHz PCIe clock) to process an incoming read TLP.
This stalls a 3GHz host for about 3000 clocks.
IIRC the time for an outgoing read is much the same, the local cpu will be slower (ours are 62.5MHz Nios) but it is still significant.
Most of the PCIe transfers we do are from a locally written multi-channel DMA block that generates 128byte TLP.
David
-
Registered Address Lakeside, Bramley Road, Mount Farm, Milton Keynes, MK1 1PT, UK Registration No: 1397386 (Wales)
Re: mach-socfpga: PCIe Root IO TLP support for Cyclone V
In reply to David LaightHi David,
Quoted 5 lines · David Laight
It isn't an ARM root complex ...
I didn't think any of the Cyclone V had embedded arm cpu. I know some of the more recent Altera FPGA do, by the Cyclone V is pretty old now - although we are still using them in new cards. (Only as PCIe endpoints though.)
You are right, the root complex itself is (likely) not an ARM design.
I should have said "... unlike many other Root Complexes used in similar ARM CPU based SoCs ...". The Cyclone V SoC sub-family of FPGAs features a dual core Cortex A9.
The "Cyclone V SoC Development Kit" is a development kit that exposes a readily usable PCIe root slot.
Quoted 3 lines
If you are building the FPGA image then all the logic to convert the memory mapped slave cycles (into the fpga logic) is supplied as verilog source.
The CPU subsystem and the PCIe IP are hard IP in silicon and can not be altered.
They are connected via the FPGA logic, though, and I agree one approach could be to intercept their communication with custom HDL. However, Linux uses the exact same PCIe hard IP registers required for IO TLPs to send Config TLPs.
Every TLP requires multiple accesses to multiple registers, so locking between FPGA logic and Linux transactions would be required. I'm not saying that this is impossible, but I don't think it can be robust without a Linux software change.
A software only solution has the benefit of being available to all users of such an FPGA, without access to that special logic.
Quoted 3 lines
I thought that all recent endpoints were required [1] to work with just memory BARs - even going back to the later PCI versions. So I'm surprised a PCIe endpoint need an IO BAR.
The AX99100 implements a so called "Legacy Endpoint" and is thus allowed to rely on support for IO space. I guess this choice was made to stay driver-compatible to the PCI version, although I don't know the ancestry if this product.
Best regards, David
RE: mach-socfpga: PCIe Root IO TLP support for Cyclone V
From: David Epping
Quoted 3 lines
Sent: 06 November 2023 10:15
On Sun, Nov 05, 2023 at 11:20:03AM +0000, David Laight wrote:
...
Quoted 5 lines
Quoted 3 lines
If you are building the FPGA image then all the logic to convert the memory mapped slave cycles (into the fpga logic) is supplied as verilog source.
The CPU subsystem and the PCIe IP are hard IP in silicon and can not be altered.
If you look carefully you'll find that the 'hard IP' block stops with a streaming interface that carries the data TLP.
All the logic to convert the TLP into Avalon memory cycles is verilog source. So, at least in principle, it is modifiable.
With a bit of effort it is possible to trace the TLP into fpga memory. (We had to modify the verilog to expose the TX TLP.)
Even allowing for development time it was probably cheaper than buying a PCIe monitor!
Quoted 9 lines
They are connected via the FPGA logic, though, and I agree one approach could be to intercept their communication with custom HDL.
However, Linux uses the exact same PCIe hard IP registers required for IO TLPs to send Config TLPs. Every TLP requires multiple accesses to multiple registers, so locking between FPGA logic and Linux transactions would be required.
I'm not saying that this is impossible, but I don't think it can be robust without a Linux software change.
A software only solution has the benefit of being available to all users of such an FPGA, without access to that special logic.
I saw/checked you'd added a lock, didn't see it was the same one used for config space accesses.
Quoted 8 lines
Quoted 3 lines
I thought that all recent endpoints were required [1] to work with just memory BARs - even going back to the later PCI versions. So I'm surprised a PCIe endpoint need an IO BAR.
The AX99100 implements a so called "Legacy Endpoint" and is thus allowed to rely on support for IO space.
I guess this choice was made to stay driver-compatible to the PCI version, although I don't know the ancestry if this product.
A lot of PCI devices solved this by adding a memory BAR that mapped exactly the same registers.
David
-
Registered Address Lakeside, Bramley Road, Mount Farm, Milton Keynes, MK1 1PT, UK Registration No: 1397386 (Wales)
Quoted 5 lines · Bjorn Helgaas
I don't understand
INDIRECT_PIOvery well, but does it help with this at all?https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/lib/Kconfig?id=v6.6#n92
https://git.kernel.org/linus/031e3601869c
Hi Bjorn,
thanks a lot for these links. They look very promising and I'll check how they can be combined with the socfpga / Altera Root Complex driver. Best regards, David