hw/pci-bridge/cxl: Add CXL Switches.
v11 cover hw/pci-bridge/cxl: Add CXL Switches.
Previously sent as patches 43-45 of
[PATCH v10 00/45] CXl 2.0 emulation Support
https://lore.kernel.org/qemu-devel/[email protected]/#r
Now the initial CXL support is upstream, patch sets applying to different parts of the CXL infrastructure can be reviewed / applied in any order.
I have just sent out arm/virt support and aim to send out DOE/CDAT patches shortly. This series is independent of those others.
Changes since v10:
- Rebase.
- Typo fix in comment about xio3110.
This series adds CXL switch support in the form of upstream and downstream ports + the additions to the interleave decoder in cxl-host to enable us to interleave across the downstream switch ports. The PCI IDs used have been allocated against Huawei's Vendor ID for use for emulation of these devices only. Primary use for this support is for testing the Linux kernel stack.
Note only 1 level of switching / switch based HDM interleave decoding is supported - consistent with typical CXL 2.0 systems. Interleaving may occur and any / some / all of:
- Across multiple CXL host bridges (static setup via Fixed Memory Windows).
- Across multiple root ports in a given CXL host bridge (dynamic config via CXL host bridge register space.
- Across multiple downstream switch ports (this series).
All comments welcome,
Thanks,
Jonathan
Jonathan Cameron (3):
pci-bridge/cxl_upstream: Add a CXL switch upstream port pci-bridge/cxl_downstream: Add a CXL switch downstream port docs/cxl: Add switch documentation
docs/system/devices/cxl.rst | 88 +++++++++++- hw/cxl/cxl-host.c | 43 +++++- hw/pci-bridge/cxl_downstream.c | 249 +++++++++++++++++++++++++++++++++ hw/pci-bridge/cxl_upstream.c | 216 ++++++++++++++++++++++++++++ hw/pci-bridge/meson.build | 2 +- include/hw/cxl/cxl.h | 5 + 6 files changed, 598 insertions(+), 5 deletions(-) create mode 100644 hw/pci-bridge/cxl_downstream.c create mode 100644 hw/pci-bridge/cxl_upstream.c
Signature · 1 line
2.32.0
PATCH v11 1/3 pci-bridge/cxl_upstream: Add a CXL switch upstream port
An initial simple upstream port emulation to allow the creation of CXL switches. The Device ID has been allocated for this use.
--- hw/pci-bridge/cxl_upstream.c | 216 +++++++++++++++++++++++++++++++++++ hw/pci-bridge/meson.build | 2 +- include/hw/cxl/cxl.h | 5 + 3 files changed, 222 insertions(+), 1 deletion(-)
hw/pci-bridge/cxl_upstream.c new file +216 -0
@@ -0,0 +1,216 @@ +/* +* Emulated CXL Switch Upstream Port +* +* Copyright (c) 2022 Huawei Technologies. +* +* Based on xio3130_upstream.c +* +* SPDX-License-Identifier: GPL-2.0-or-later +*/ + +#include "qemu/osdep.h" +#include "qemu/log.h" +#include "hw/pci/msi.h" +#include "hw/pci/pcie.h" +#include "hw/pci/pcie_port.h" + +#define CXL_UPSTREAM_PORT_MSI_NR_VECTOR 1 + +#define CXL_UPSTREAM_PORT_MSI_OFFSET 0x70 +#define CXL_UPSTREAM_PORT_PCIE_CAP_OFFSET 0x90 +#define CXL_UPSTREAM_PORT_AER_OFFSET 0x100 +#define CXL_UPSTREAM_PORT_DVSEC_OFFSET \ + (CXL_UPSTREAM_PORT_AER_OFFSET + PCI_ERR_SIZEOF) + +typedef struct CXLUpstreamPort { +/*< private >*/ +PCIEPort parent_obj; + +/*< public >*/ +CXLComponentState cxl_cstate; +}CXLUpstreamPort ; + +CXLComponentState *cxl_usp_to_cstate (CXLUpstreamPort *usp) +{ +return &usp->cxl_cstate; +} + +static void cxl_usp_dvsec_write_config (PCIDevice *dev,uint32_t addr, +uint32_t val,int len) +{ +CXLUpstreamPort *usp =CXL_USP (dev); + +if (range_contains (&usp->cxl_cstate.dvsecs[EXTENSIONS_PORT_DVSEC ], addr)) { +uint8_t *reg = &dev->config[addr]; + addr -= usp->cxl_cstate.dvsecs[EXTENSIONS_PORT_DVSEC ].lob; +if (addr ==PORT_CONTROL_OFFSET ) { +if (pci_get_word (reg) &PORT_CONTROL_UNMASK_SBR ) { +/* unmask SBR */ +qemu_log_mask (LOG_UNIMP ,"SBR mask control is not supported\n" ); + } +if (pci_get_word (reg) &PORT_CONTROL_ALT_MEMID_EN ) { +/* Alt Memory & ID Space Enable */ +qemu_log_mask (LOG_UNIMP , +"Alt Memory & ID space is not supported\n" ); + } + } + } +} + +static void cxl_usp_write_config (PCIDevice *d,uint32_t address, +uint32_t val,int len) +{ +pci_bridge_write_config (d, address, val, len); +pcie_cap_flr_write_config (d, address, val, len); +pcie_aer_write_config (d, address, val, len); + +cxl_usp_dvsec_write_config (d, address, val, len); +} + +static void latch_registers (CXLUpstreamPort *usp) +{ +uint32_t *reg_state = usp->cxl_cstate.crb.cache_mem_registers; +uint32_t *write_msk = usp->cxl_cstate.crb.cache_mem_regs_write_mask; + +cxl_component_register_init_common (reg_state, write_msk, +CXL2_UPSTREAM_PORT ); +ARRAY_FIELD_DP32 (reg_state,CXL_HDM_DECODER_CAPABILITY ,TARGET_COUNT ,8 ); +} + +static void cxl_usp_reset (DeviceState *qdev) +{ +PCIDevice *d =PCI_DEVICE (qdev); +CXLUpstreamPort *usp =CXL_USP (qdev); + +pci_bridge_reset (qdev); +pcie_cap_deverr_reset (d); +latch_registers (usp); +} + +static void build_dvsecs (CXLComponentState *cxl) +{ +uint8_t *dvsec; + + dvsec = (uint8_t *)&(CXLDVSECPortExtensions ){ + .status =0x1 ,/* Port Power Management Init Complete */ + }; +cxl_component_create_dvsec (cxl,CXL2_UPSTREAM_PORT , +EXTENSIONS_PORT_DVSEC_LENGTH , +EXTENSIONS_PORT_DVSEC , +EXTENSIONS_PORT_DVSEC_REVID , dvsec); + dvsec = (uint8_t *)&(CXLDVSECPortFlexBus ){ + .cap =0x27 ,/* Cache, IO, Mem, non-MLD */ + .ctrl =0x27 ,/* Cache, IO, Mem */ + .status =0x26 ,/* same */ + .rcvd_mod_ts_data_phase1 =0xef ,/* WTF? */ + }; +cxl_component_create_dvsec (cxl,CXL2_UPSTREAM_PORT , +PCIE_FLEXBUS_PORT_DVSEC_LENGTH_2_0 , +PCIE_FLEXBUS_PORT_DVSEC , +PCIE_FLEXBUS_PORT_DVSEC_REVID_2_0 , dvsec); + + dvsec = (uint8_t *)&(CXLDVSECRegisterLocator ){ + .rsvd =0 , + .reg0_base_lo =RBI_COMPONENT_REG |CXL_COMPONENT_REG_BAR_IDX , + .reg0_base_hi =0 , + }; +cxl_component_create_dvsec (cxl,CXL2_UPSTREAM_PORT , +REG_LOC_DVSEC_LENGTH ,REG_LOC_DVSEC , +REG_LOC_DVSEC_REVID , dvsec); +} + +static void cxl_usp_realize (PCIDevice *d,Error **errp) +{ +PCIEPort *p =PCIE_PORT (d); +CXLUpstreamPort *usp =CXL_USP (d); +CXLComponentState *cxl_cstate = &usp->cxl_cstate; +ComponentRegisters *cregs = &cxl_cstate->crb; +MemoryRegion *component_bar = &cregs->component_registers; +int rc; + +pci_bridge_initfn (d,TYPE_PCIE_BUS ); +pcie_port_init_reg (d); + + rc =msi_init (d,CXL_UPSTREAM_PORT_MSI_OFFSET , +CXL_UPSTREAM_PORT_MSI_NR_VECTOR , true, true, errp); +if (rc) { +assert (rc == -ENOTSUP ); + goto err_bridge; + } + + rc =pcie_cap_init (d,CXL_UPSTREAM_PORT_PCIE_CAP_OFFSET , +PCI_EXP_TYPE_UPSTREAM , p->port, errp); +if (rc <0 ) { + goto err_msi; + } + +pcie_cap_flr_init (d); +pcie_cap_deverr_init (d); + rc =pcie_aer_init (d,PCI_ERR_VER ,CXL_UPSTREAM_PORT_AER_OFFSET , +PCI_ERR_SIZEOF , errp); +if (rc) { + goto err_cap; + } + + cxl_cstate->dvsec_offset =CXL_UPSTREAM_PORT_DVSEC_OFFSET ; + cxl_cstate->pdev = d; +build_dvsecs (cxl_cstate); +cxl_component_register_block_init (OBJECT (d), cxl_cstate,TYPE_CXL_USP ); +pci_register_bar (d,CXL_COMPONENT_REG_BAR_IDX , +PCI_BASE_ADDRESS_SPACE_MEMORY | +PCI_BASE_ADDRESS_MEM_TYPE_64 , + component_bar); + +return ; + +err_cap: +pcie_cap_exit (d); +err_msi: +msi_uninit (d); +err_bridge: +pci_bridge_exitfn (d); +} + +static void cxl_usp_exitfn (PCIDevice *d) +{ +pcie_aer_exit (d); +pcie_cap_exit (d); +msi_uninit (d); +pci_bridge_exitfn (d); +} + +static void cxl_upstream_class_init (ObjectClass *oc,void *data) +{ +DeviceClass *dc =DEVICE_CLASS (oc); +PCIDeviceClass *k =PCI_DEVICE_CLASS (oc); + + k->is_bridge = true; + k->config_write = cxl_usp_write_config; + k->realize = cxl_usp_realize; + k->exit = cxl_usp_exitfn; + k->vendor_id =0x19e5 ;/* Huawei */ + k->device_id =0xa128 ;/* Emulated CXL Switch Upstream Port */ + k->revision =0 ; +set_bit (DEVICE_CATEGORY_BRIDGE , dc->categories); + dc->desc ="CXL Switch Upstream Port" ; + dc->reset = cxl_usp_reset; +} + +static const TypeInfo cxl_usp_info = { + .name =TYPE_CXL_USP , + .parent =TYPE_PCIE_PORT , + .instance_size =sizeof (CXLUpstreamPort), + .class_init = cxl_upstream_class_init, + .interfaces = (InterfaceInfo []) { + {INTERFACE_PCIE_DEVICE }, + {INTERFACE_CXL_DEVICE }, + { } + }, +}; + +static void cxl_usp_register_type (void ) +{ +type_register_static (&cxl_usp_info); +} + +type_init (cxl_usp_register_type);
hw/pci-bridge/meson.build +1 -1
@@ -6,7 +6,7 @@ pci_ss.add(when: 'CONFIG_PCIE_PORT', if_true: files('pcie_root_port.c', 'gen_pci pci_ss.add(when: 'CONFIG_PXB', if_true: files('pci_expander_bridge.c'), if_false: files('pci_expander_bridge_stubs.c')) pci_ss.add(when: 'CONFIG_XIO3130', if_true: files('xio3130_upstream.c', 'xio3130_downstream.c')) -pci_ss.add(when: 'CONFIG_CXL', if_true: files('cxl_root_port.c')) +pci_ss.add(when: 'CONFIG_CXL', if_true: files('cxl_root_port.c', 'cxl_upstream.c')) # NewWorld PowerMac pci_ss.add(when: 'CONFIG_DEC_PCI', if_true: files('dec.c'))
include/hw/cxl/cxl.h +5 -0
@@ -53,4 +53,9 @@ struct CXLHost {#define TYPE_PXB_CXL_HOST "pxb-cxl-host" OBJECT_DECLARE_SIMPLE_TYPE(CXLHost, PXB_CXL_HOST) +#define TYPE_CXL_USP "cxl-upstream" + +typedef struct CXLUpstreamPort CXLUpstreamPort ; +DECLARE_INSTANCE_CHECKER(CXLUpstreamPort, CXL_USP, TYPE_CXL_USP) +CXLComponentState *cxl_usp_to_cstate (CXLUpstreamPort *usp);#endif
Signature · 1 line
2.32.0
PATCH v11 2/3 pci-bridge/cxl_downstream: Add a CXL switch downstream port
In reply to Jonathan CameronEmulation of a simple CXL Switch downstream port.
The Device ID has been allocated for this use.
--- hw/cxl/cxl-host.c | 43 +++++- hw/pci-bridge/cxl_downstream.c | 249 +++++++++++++++++++++++++++++++++ hw/pci-bridge/meson.build | 2 +- 3 files changed, 291 insertions(+), 3 deletions(-)
hw/cxl/cxl-host.c +41 -2
@@ -129,8 +129,9 @@ static bool cxl_hdm_find_target(uint32_t *cache_mem, hwaddr addr,static PCIDevice *cxl_cfmws_find_device (CXLFixedWindow *fw,hwaddr addr) { -CXLComponentState *hb_cstate; +CXLComponentState *hb_cstate, *usp_cstate;PCIHostState *hb; +CXLUpstreamPort *usp;int rb_index;uint32_t *cache_mem;uint8_t target;
@@ -164,8 +165,46 @@ static PCIDevice *cxl_cfmws_find_device(CXLFixedWindow *fw, hwaddr addr) } d =pci_bridge_get_sec_bus (PCI_BRIDGE (rp))->devices[0 ]; +if (!d) { +return NULL ; + } + +if (object_dynamic_cast (OBJECT (d),TYPE_CXL_TYPE3 )) { +return d; + } + +/* +* Could also be a switch. Note only one level of switching currently +* supported. +*/ +if (!object_dynamic_cast (OBJECT (d),TYPE_CXL_USP )) { +return NULL ; + } + usp =CXL_USP (d); + + usp_cstate =cxl_usp_to_cstate (usp); +if (!usp_cstate) { +return NULL ; + } + + cache_mem = usp_cstate->crb.cache_mem_registers; + + target_found =cxl_hdm_find_target (cache_mem, addr, &target); +if (!target_found) { +return NULL ; + } + + d =pcie_find_port_by_pn (&PCI_BRIDGE (d)->sec_bus, target); +if (!d) { +return NULL ; + } + + d =pci_bridge_get_sec_bus (PCI_BRIDGE (d))->devices[0 ]; +if (!d) { +return NULL ; + } -if (!d || !object_dynamic_cast (OBJECT (d),TYPE_CXL_TYPE3 )) { +if (!object_dynamic_cast (OBJECT (d),TYPE_CXL_TYPE3 )) {return NULL ; }
hw/pci-bridge/cxl_downstream.c new file +249 -0
249 lines not on this page · raw message
hw/pci-bridge/meson.build +1 -1
@@ -6,7 +6,7 @@ pci_ss.add(when: 'CONFIG_PCIE_PORT', if_true: files('pcie_root_port.c', 'gen_pci pci_ss.add(when: 'CONFIG_PXB', if_true: files('pci_expander_bridge.c'), if_false: files('pci_expander_bridge_stubs.c')) pci_ss.add(when: 'CONFIG_XIO3130', if_true: files('xio3130_upstream.c', 'xio3130_downstream.c')) -pci_ss.add(when: 'CONFIG_CXL', if_true: files('cxl_root_port.c', 'cxl_upstream.c')) +pci_ss.add(when: 'CONFIG_CXL', if_true: files('cxl_root_port.c', 'cxl_upstream.c', 'cxl_downstream.c')) # NewWorld PowerMac pci_ss.add(when: 'CONFIG_DEC_PCI', if_true: files('dec.c'))
Signature · 1 line
2.32.0
PATCH v11 3/3 docs/cxl: Add switch documentation
In reply to Jonathan CameronSwitches were already introduced, but now we support them update the documentation to provide an example in diagram and qemu command line parameter forms.
--- docs/system/devices/cxl.rst | 88 ++++++++++++++++++++++++++++++++++++- 1 file changed, 86 insertions(+), 2 deletions(-)
docs/system/devices/cxl.rst +86 -2
@@ -118,8 +118,6 @@ and associated component register access via PCI bars. CXL Switch ~~~~~~~~~~ -Not yet implemented in QEMU. - Here we consider a simple CXL switch with only a single virtual hierarchy. Whilst more complex devices exist, their visibility to a particular host is generally the same as for
@@ -137,6 +135,10 @@ BARs. The Upstream Port has the configuration interfaces for the HDM decoders which route incoming memory accesses to the appropriate downstream port. +A CXL switch is created in a similar fashion to PCI switches +by creating an upstream port (cxl-upstream) and a number of +downstream ports on the internal switch bus (cxl-downstream). + CXL Memory Devices - Type 3 ~~~~~~~~~~~~~~~~~~~~~~~~~~~ CXL type 3 devices use a PCI class code and are intended to be supported
@@ -240,6 +242,62 @@ Notes: they will take the Host Physical Addresses of accesses and map them to their own local Device Physical Address Space (DPA). +Example topology involving a switch:: + + |<------------------SYSTEM PHYSICAL ADDRESS MAP (1)----------------->| + | __________ __________________________________ __________ | + | | | | | | | | + | | CFMW 0 | | CXL Fixed Memory Window 1 | | CFMW 1 | | + | | HB0 only | | Configured to interleave memory | | HB1 only | | + | | | | memory accesses across HB0/HB1 | | | | + | |____x_____| |__________________________________| |__________| | + | | | | + | | | | + | | | + Interleave Decoder | | | + Matches this HB | | | + \_____________| |_____________/ + __________|__________ _____|_______________ + | | | | + | CXL HB 0 | | CXL HB 1 | + | HB IntLv Decoders | | HB IntLv Decoders | + | PCI/CXL Root Bus 0c | | PCI/CXL Root Bus 0d | + | | | | + |___x_________________| |_____________________| + | | | | + | + A HB 0 HDM Decoder + matches this Port + ___________|___ + | Root Port 0 | + | Appears in | + | PCI topology | + | As 0c:00.0 | + |___________x___| + | + | + \_____________________ + | + | + --------------------------------------------------- + | Switch 0 USP as PCI 0d:00.0 | + | USP has HDM decoder which direct traffic to | + | appropiate downstream port | + | Switch BUS appears as 0e | + |x__________________________________________________| + | | | | + | | | | + _____|_________ ______|______ ______|_____ ______|_______ + (4)| x | | | | | | | + | CXL Type3 0 | | CXL Type3 1 | | CXL type3 2| | CLX Type 3 3 | + | | | | | | | | + | PMEM0(Vol LSA)| | PMEM1 (...) | | PMEM2 (...)| | PMEM3 (...) | + | Decoder to go | | | | | | | + | from host PA | | PCI 10:00.0 | | PCI 11:00.0| | PCI 12:00.0 | + | to device PA | | | | | | | + | PCI as 0f:00.0| | | | | | | + |_______________| |_____________| |____________| |______________| + Example command lines --------------------- A very simple setup with just one directly attached CXL Type 3 device::
@@ -279,6 +337,32 @@ the CXL Type3 device directly attached (no switches).:: -device cxl-type3,bus=root_port16,memdev=cxl-mem4,lsa=cxl-lsa4,id=cxl-pmem3 \ -M cxl-fmw.0.targets.0=cxl.1,cxl-fmw.0.targets.1=cxl.2,cxl-fmw.0.size=4G,cxl-fmw.0.interleave-granularity=8k +An example of 4 devices below a switch suitable for 1, 2 or 4 way interleave:: + + qemu-system-aarch64 -M virt,gic-version=3,cxl=on -m 4g,maxmem=8G,slots=8 -cpu max \ + ... + -object memory-backend-file,id=cxl-mem0,share=on,mem-path=/tmp/cxltest.raw,size=256M \ + -object memory-backend-file,id=cxl-mem1,share=on,mem-path=/tmp/cxltest1.raw,size=256M \ + -object memory-backend-file,id=cxl-mem2,share=on,mem-path=/tmp/cxltest2.raw,size=256M \ + -object memory-backend-file,id=cxl-mem3,share=on,mem-path=/tmp/cxltest3.raw,size=256M \ + -object memory-backend-file,id=cxl-lsa0,share=on,mem-path=/tmp/lsa0.raw,size=256M \ + -object memory-backend-file,id=cxl-lsa1,share=on,mem-path=/tmp/lsa1.raw,size=256M \ + -object memory-backend-file,id=cxl-lsa2,share=on,mem-path=/tmp/lsa2.raw,size=256M \ + -object memory-backend-file,id=cxl-lsa3,share=on,mem-path=/tmp/lsa3.raw,size=256M \ + -device pxb-cxl,bus_nr=12,bus=pcie.0,id=cxl.1 \ + -device cxl-rp,port=0,bus=cxl.1,id=root_port0,chassis=0,slot=0 \ + -device cxl-rp,port=1,bus=cxl.1,id=root_port1,chassis=0,slot=1 \ + -device cxl-upstream,bus=root_port0,id=us0 \ + -device cxl-downstream,port=0,bus=us0,id=swport0,chassis=0,slot=4 \ + -device cxl-type3,bus=swport0,memdev=cxl-mem0,lsa=cxl-lsa0,id=cxl-pmem0,size=256M \ + -device cxl-downstream,port=1,bus=us0,id=swport1,chassis=0,slot=5 \ + -device cxl-type3,bus=swport1,memdev=cxl-mem1,lsa=cxl-lsa1,id=cxl-pmem1,size=256M \ + -device cxl-downstream,port=2,bus=us0,id=swport2,chassis=0,slot=6 \ + -device cxl-type3,bus=swport2,memdev=cxl-mem2,lsa=cxl-lsa2,id=cxl-pmem2,size=256M \ + -device cxl-downstream,port=3,bus=us0,id=swport3,chassis=0,slot=7 \ + -device cxl-type3,bus=swport3,memdev=cxl-mem3,lsa=cxl-lsa3,id=cxl-pmem3,size=256M \ + -M cxl-fmw.0.targets.0=cxl.1,cxl-fmw.0.size=4G,cxl-fmw.0.interleave-granularity=4k + Kernel Configuration Options ----------------------------
Signature · 1 line
2.32.0