Skip to contentSkip to conversation

hw/pci: Move 'x-256b-flit' property from cxl_root_port to pcie_root_port.

RFC In review Shrihari E S 2 participants 23 messages Aug 26 – Sep 10 2 messages not in the archive

Subsystems (2)
lore
Share & tools
lore

A message that is not in the archive (sent off-list or not yet received): [email protected]

#1 lore raw

RFC v2 1/10 hw/pci: Refactor flitmode from PCIESlot to PCIEPort.

Move the 'flitmode' property from PCIESlot to PCIEPort. This change allows all the derived ports (PCIe ports or CXL ports) to use this property.

This is a structural refactor only. There is no functional change in existing behavior.

Note: This property was added from Davidlohr's BI patch series[1].

[1]: https://lore.kernel.org/all/[email protected]/

---
 hw/pci-bridge/cxl_downstream.c | 6 +++---
 hw/pci-bridge/cxl_root_port.c  | 6 +++---
 hw/pci/pcie.c                  | 2 +-
 include/hw/pci/pcie_port.h     | 2 +-
 4 files changed, 8 insertions(+), 8 deletions(-)
hw/pci-bridge/cxl_downstream.c +3 -3
index d1e27994a3..7c7a1d1221 100644
@@ -118,7 +118,7 @@ static void cxl_dsp_reset(DeviceState *qdev)
 
 static void build_dvsecs(PCIDevice *d, CXLComponentState *cxl)
 {
-    PCIESlot *s = PCIE_SLOT(d);
+    PCIEPort *p = PCIE_PORT(d);
     CXLDVSECRegisterLocator *regloc_dvsec;
     uint8_t *dvsec;
     int i;
@@ -132,7 +132,7 @@ static void build_dvsecs(PCIDevice *d, CXLComponentState *cxl)
     dvsec = (uint8_t *)&(CXLDVSECPortFlexBus){
         .cap                     = 0x27, /* Cache, IO, Mem, non-MLD */
         .ctrl                    = 0x02, /* IO always enabled */
-        .status                  = s->flitmode ? 0x6 : 0x26, /* lack of 68B */
+        .status                  = p->flitmode ? 0x6 : 0x26, /* lack of 68B */
         .rcvd_mod_ts_data_phase1 = 0xef, /* WTF? */
     };
     cxl_component_create_dvsec(cxl, CXL2_DOWNSTREAM_PORT,
@@ -259,7 +259,7 @@ static const Property cxl_dsp_props[] = {
                                 speed, PCIE_LINK_SPEED_64),
     DEFINE_PROP_PCIE_LINK_WIDTH("x-width", PCIESlot,
                                 width, PCIE_LINK_WIDTH_16),
-    DEFINE_PROP_BOOL("x-256b-flit", PCIESlot, flitmode, true),
+    DEFINE_PROP_BOOL("x-256b-flit", PCIEPort, flitmode, true),
 };
 
 static void cxl_dsp_class_init(ObjectClass *oc, const void *data)
hw/pci-bridge/cxl_root_port.c +3 -3
index 0ee18ef5c3..df69c5200f 100644
@@ -112,7 +112,7 @@ static void latch_registers(CXLRootPort *crp)
 
 static void build_dvsecs(PCIDevice *d, CXLComponentState *cxl)
 {
-    PCIESlot *s = PCIE_SLOT(d);
+    PCIEPort *p = PCIE_PORT(d);
     CXLDVSECRegisterLocator *regloc_dvsec;
     uint8_t *dvsec;
     int i;
@@ -135,7 +135,7 @@ static void build_dvsecs(PCIDevice *d, CXLComponentState *cxl)
     dvsec = (uint8_t *)&(CXLDVSECPortFlexBus){
         .cap                     = 0x26, /* IO, Mem, non-MLD */
         .ctrl                    = 0x2,
-        .status                  = s->flitmode ? 0x6 : 0x26, /* lack of 68B */
+        .status                  = p->flitmode ? 0x6 : 0x26, /* lack of 68B */
         .rcvd_mod_ts_data_phase1 = 0xef,
     };
     cxl_component_create_dvsec(cxl, CXL2_ROOT_PORT,
@@ -237,7 +237,7 @@ static const Property gen_rp_props[] = {
                                 speed, PCIE_LINK_SPEED_64),
     DEFINE_PROP_PCIE_LINK_WIDTH("x-width", PCIESlot,
                                 width, PCIE_LINK_WIDTH_32),
-    DEFINE_PROP_BOOL("x-256b-flit", PCIESlot, flitmode, true),
+    DEFINE_PROP_BOOL("x-256b-flit", PCIEPort, flitmode, true),
 };
 
 static void cxl_rp_dvsec_write_config(PCIDevice *dev, uint32_t addr,
hw/pci/pcie.c +1 -1
index 1de0b1e4fd..4f6fc9db61 100644
@@ -217,7 +217,7 @@ static void pcie_cap_fill_slot_lnk(PCIDevice *dev)
         /* the PCI_EXP_LNKSTA_DLLLA will be set in the hotplug function */
     }
 
-    pcie_cap_fill_lnk(exp_cap, s->width, s->speed, s->flitmode);
+    pcie_cap_fill_lnk(exp_cap, s->width, s->speed, PCIE_PORT(s)->flitmode);
 }
 
 int pcie_cap_init(PCIDevice *dev, uint8_t offset,
include/hw/pci/pcie_port.h +1 -1
index 53cd64c5ed..1bcc734649 100644
@@ -33,6 +33,7 @@ struct PCIEPort {
     /*< private >*/
     PCIBridge   parent_obj;
     /*< public >*/
+    bool flitmode;
 
     /* pci express switch port */
     uint8_t     port;
@@ -58,7 +59,6 @@ struct PCIESlot {
 
     PCIExpLinkSpeed speed;
     PCIExpLinkWidth width;
-    bool flitmode;
 
     /* Disable ACS (really for a pcie_root_port) */
     bool        disable_acs;
Signature · 1 line
2.34.1
#2 lore raw

RFC v2 2/10 hw/pci: Move 'x-256b-flit' property from cxl_root_port to pcie_root_port.

In reply to a message not in the archive

Relocate 'x-256b-flit' property from cxl_root_port to its parent pcie_root_port so that both can utilize this property.

This is a pure refactor and does not impact any existing behavior.

---
 hw/pci-bridge/cxl_root_port.c  | 1 -
 hw/pci-bridge/pcie_root_port.c | 1 +
 2 files changed, 1 insertion(+), 1 deletion(-)
hw/pci-bridge/cxl_root_port.c +0 -1
index df69c5200f..e82685d1ab 100644
@@ -237,7 +237,6 @@ static const Property gen_rp_props[] = {
                                 speed, PCIE_LINK_SPEED_64),
     DEFINE_PROP_PCIE_LINK_WIDTH("x-width", PCIESlot,
                                 width, PCIE_LINK_WIDTH_32),
-    DEFINE_PROP_BOOL("x-256b-flit", PCIEPort, flitmode, true),
 };
 
 static void cxl_rp_dvsec_write_config(PCIDevice *dev, uint32_t addr,
hw/pci-bridge/pcie_root_port.c +1 -0
index fe3ced5685..eeee24e042 100644
@@ -152,6 +152,7 @@ static const Property rp_props[] = {
     DEFINE_PROP_BIT(COMPAT_PROP_PCP, PCIDevice, cap_present,
                     QEMU_PCIE_SLTCAP_PCP_BITNR, true),
     DEFINE_PROP_BOOL("disable-acs", PCIESlot, disable_acs, false),
+    DEFINE_PROP_BOOL("x-256b-flit", PCIEPort, flitmode, true),
 };
 
 static void rp_instance_post_init(Object *obj)
Signature · 1 line
2.34.1
#3 lore raw

Hi Shrihari,

hw/pci-bridge/pcie_root_port.c static const Property rp_props[] = { · line 153

Quoted 3 lines · Shrihari E S
hw/pci-bridge/pcie_root_port.c +1 -0
@@ -152,6 +152,7 @@ static const Property rp_props[] = {
+    DEFINE_PROP_BOOL("x-256b-flit", PCIEPort, flitmode, true),
 };

This moves the default=true along with the property, so it now applies to every derived root port (pcie-root-port, ioh3420, pnv-phb-root-port, aspeed.pcie-root-port), not just cxl-rp. On an empty port the LNKSTA2 flit status bit flips at this patch alone -- pcie_cap_fill_slot_lnk() consumes flitmode, and pcie_sync_bridge_lnk() only overwrites it once a child is plugged -- so "does not impact any existing behavior" doesn't cover the derived ports; with patch 6's pcie_cap_fill_lnk() additions the ports also come up with Flit Mode Supported (Flags bit 15) and LNKCTL bit 13 guest-writable. I checked "-device pcie-root-port" and "-device ioh3420" on q35 with the series applied: the Express Flags read 0x8142 on both; before the series the property didn't exist on these devices and bit 15 stayed 0.

That's a guest-visible config space change on non-CXL machines with no compat handling. Migration already breaks at this patch -- pc-q35-9.2 at both ends, plain "-device pcie-root-port", a build at the series base on the source and one at 2/10 on the destination:

get_pci_config_device: Bad config data: i=0x87 read: 0 device: 4 cmask: ff ... load of migration failed: ... '0000:00:03.0/pcie-root-port'

0x87 is the high byte of PCI_EXP_LNKSTA2; with the whole series on the destination the first mismatch comes earlier, at 0x57, PCI_EXP_FLAGS. Fresh q35 guests see different hardware too. Was extending the default beyond cxl-rp intended? If yes, it needs hw_compat entries for older machine types. If not, defaulting to off here and setting flitmode in a cxl-rp instance_init would preserve current behaviour for non-CXL ports: a second DEFINE_PROP in a subclass asserts on the duplicate name, and instance_init still runs before qdev applies globals, so -global and hw_compat keep working.

Many thanks,
Junjie

#4 lore raw

RFC v2 3/10 hw/pci: Add SVC capability and UIO properties to PCIe ports

In reply to a message not in the archive

From: Dongjoo Seo <[email protected]>

Implement the Streamlined Virtual Channel (SVC) extended capability for PCIe ports (Root, Upstream, and Downstream). This capability is mandatory for enabling Unordered IO (UIO) traffic as per PCIe 6.4 specification [1].

UIO functionality depends on two key prerequisites:
1. SVC
2. 256-byte Flit mode support

This patch adds the following experimental properties to PCIe ports:
- x-svc: Enable SVC in the port.
- x-256b-flit: Enable 256B flit mode in the port.

This change lays the groundwork for UIO emulation in QEMU.

[1]: PCIe 6.4 Specification, Table 2-46 (Streamlined Virtual Channel).

---
 hw/pci-bridge/pcie_root_port.c     | 1 +
 hw/pci-bridge/xio3130_downstream.c | 2 ++
 hw/pci-bridge/xio3130_upstream.c   | 7 +++++++
 include/hw/pci/pcie_port.h         | 3 +++
 4 files changed, 13 insertions(+)
hw/pci-bridge/pcie_root_port.c +1 -0
index eeee24e042..0ae7b10fbd 100644
@@ -153,6 +153,7 @@ static const Property rp_props[] = {
                     QEMU_PCIE_SLTCAP_PCP_BITNR, true),
     DEFINE_PROP_BOOL("disable-acs", PCIESlot, disable_acs, false),
     DEFINE_PROP_BOOL("x-256b-flit", PCIEPort, flitmode, true),
+    DEFINE_PROP_BOOL("x-svc", PCIEPort, svc, false),
 };
 
 static void rp_instance_post_init(Object *obj)
hw/pci-bridge/xio3130_downstream.c +2 -0
index 0c3fed3053..090a3fdfc9 100644
@@ -137,6 +137,8 @@ static void xio3130_downstream_exitfn(PCIDevice *d)
 static const Property xio3130_downstream_props[] = {
     DEFINE_PROP_BIT(COMPAT_PROP_PCP, PCIDevice, cap_present,
                     QEMU_PCIE_SLTCAP_PCP_BITNR, true),
+    DEFINE_PROP_BOOL("x-svc", PCIEPort, svc, false),
+    DEFINE_PROP_BOOL("x-256b-flit", PCIEPort, flitmode, false),
 };
 
 static const VMStateDescription vmstate_xio3130_downstream = {
hw/pci-bridge/xio3130_upstream.c +7 -0
index 40057b749b..767b7d17f2 100644
@@ -24,6 +24,7 @@
 #include "hw/pci/msi.h"
 #include "hw/pci/pcie.h"
 #include "hw/pci/pcie_port.h"
+#include "hw/core/qdev-properties.h"
 #include "migration/vmstate.h"
 #include "qemu/module.h"
 
@@ -123,6 +124,11 @@ static const VMStateDescription vmstate_xio3130_upstream = {
     }
 };
 
+static const Property xio3130_upstream_props[] = {
+    DEFINE_PROP_BOOL("x-svc", PCIEPort, svc, false),
+    DEFINE_PROP_BOOL("x-256b-flit", PCIEPort, flitmode, false),
+};
+
 static void xio3130_upstream_class_init(ObjectClass *klass, const void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
@@ -138,6 +144,7 @@ static void xio3130_upstream_class_init(ObjectClass *klass, const void *data)
     dc->desc = "TI X3130 Upstream Port of PCI Express Switch";
     device_class_set_legacy_reset(dc, xio3130_upstream_reset);
     dc->vmsd = &vmstate_xio3130_upstream;
+    device_class_set_props(dc, xio3130_upstream_props);
 }
 
 static const TypeInfo xio3130_upstream_info = {
include/hw/pci/pcie_port.h +3 -0
index 1bcc734649..0b3e0efb8c 100644
@@ -37,6 +37,9 @@ struct PCIEPort {
 
     /* pci express switch port */
     uint8_t     port;
+
+    /* Streamlined Virtual Channel (SVC) capability */
+    bool    svc;
 };
 
 void pcie_port_init_reg(PCIDevice *d);
Signature · 1 line
2.34.1
#5 lore raw

Re: [RFC V2 03/10] hw/pci: Add SVC capability and UIO properties to PCIe ports

Hi Shrihari,

hw/pci-bridge/xio3130_upstream.c static const VMStateDescription vmstate_xio3130_upstream = { · line 130

Quoted 4 lines · Shrihari E S
hw/pci-bridge/xio3130_upstream.c +4 -0
+static const Property xio3130_upstream_props[] = {
+    DEFINE_PROP_BOOL("x-svc", PCIEPort, svc, false),
+    DEFINE_PROP_BOOL("x-256b-flit", PCIEPort, flitmode, false),
+};

x-256b-flit is a no-op on x3130-upstream: nothing in xio3130_upstream.c reads PCIEPort::flitmode. pcie_cap_fill_slot_lnk() does run from pcie_cap_init(), but returns early -- an upstream port is a PCIEPort, not a PCIESlot -- and pcie_cap_fill_link_ep_usp(), the path cxl-upstream and cxl-type3 take, isn't called here. So with x-256b-flit=on the Flags bit 15 and LNKSTA2 flit bits stay 0. I'd either wire up pcie_cap_fill_link_ep_usp() the way cxl-upstream does, from its reset handler, or drop x-256b-flit from this hunk.

x-svc has no reader until 7/10 either, so x-svc=on has no effect for four commits. Same thing Jonathan asked for on v1: bring the state in with the patch that uses it. Here that would fold 3 and 4 into 7.

Many thanks,
Junjie

#6 lore raw

RFC v2 4/10 hw/cxl: Add Streamlined Virtual Channel (SVC) property to CXL ports

In reply to a message not in the archive

Add Unordered IO (UIO) dependent property SVC to CXL ports.

The following properties are added to CXL ports:
- x-svc: To enable SVC capability in the port

Note: 256B flit mode property was already added in CXL ports, so reused it.

---
 hw/pci-bridge/cxl_downstream.c | 1 +
 hw/pci-bridge/cxl_upstream.c   | 1 +
 2 files changed, 2 insertions(+)
hw/pci-bridge/cxl_downstream.c +1 -0
index 7c7a1d1221..1c8c3b4636 100644
@@ -260,6 +260,7 @@ static const Property cxl_dsp_props[] = {
     DEFINE_PROP_PCIE_LINK_WIDTH("x-width", PCIESlot,
                                 width, PCIE_LINK_WIDTH_16),
     DEFINE_PROP_BOOL("x-256b-flit", PCIEPort, flitmode, true),
+    DEFINE_PROP_BOOL("x-svc", PCIEPort, svc, false),
 };
 
 static void cxl_dsp_class_init(ObjectClass *oc, const void *data)
hw/pci-bridge/cxl_upstream.c +1 -0
index ef5480b116..999ee0b56b 100644
@@ -437,6 +437,7 @@ static const Property cxl_upstream_props[] = {
     DEFINE_PROP_PCIE_LINK_WIDTH("x-width", CXLUpstreamPort,
                                 width, PCIE_LINK_WIDTH_16),
     DEFINE_PROP_BOOL("x-256b-flit", CXLUpstreamPort, flitmode, false),
+    DEFINE_PROP_BOOL("x-svc", PCIEPort, svc, false),
 };
 
 static void cxl_upstream_class_init(ObjectClass *oc, const void *data)
Signature · 1 line
2.34.1
#7 lore raw

RFC v2 5/10 hw/cxl: Wire UIO capability into HDM decoder and DVSEC registers

In reply to a message not in the archive

Plumb the 'uio_capable' flag to CXL HDM decoder capability and control register interfaces. The UIO bit in the capability register is now set for CXL Type3 devices and ports when UIO support is enabled in them.

Per CXL 4.0 specification Section 8.2.4.20.7, the decoder control UIO bit is validated against the advertised capability during HDM decoder commit operations.

Also for DSP, enabled write mask to "UIO to HDM" bit in CXL DVSEC Port Control Extension register.

---
 hw/cxl/cxl-component-utils.c              | 23 +++++++++++++++++------
 hw/mem/cxl_type3.c                        | 10 +++++++++-
 hw/pci-bridge/cxl_downstream.c            |  2 +-
 hw/pci-bridge/cxl_root_port.c             |  3 ++-
 hw/pci-bridge/cxl_upstream.c              |  3 ++-
 hw/pci-bridge/pci_expander_bridge.c       |  3 ++-
 include/hw/cxl/cxl_component.h            |  2 +-
 include/hw/cxl/cxl_device.h               |  2 ++
 include/hw/pci-bridge/cxl_upstream_port.h |  1 +
 9 files changed, 37 insertions(+), 12 deletions(-)
hw/cxl/cxl-component-utils.c +17 -6
index 31bbedb502..0c92c41266 100644
@@ -305,7 +305,7 @@ static void ras_init_common(uint32_t *reg_state, uint32_t *write_msk)
 }
 
 static void hdm_init_common(uint32_t *reg_state, uint32_t *write_msk,
-                            enum reg_type type, bool bi)
+                            enum reg_type type, bool bi, bool uio)
 {
     int decoder_count = CXL_HDM_DECODER_COUNT;
     int hdm_inc = R_CXL_HDM_DECODER1_BASE_LO - R_CXL_HDM_DECODER0_BASE_LO;
@@ -325,9 +325,13 @@ static void hdm_init_common(uint32_t *reg_state, uint32_t *write_msk,
         ARRAY_FIELD_DP32(reg_state, CXL_HDM_DECODER_CAPABILITY, 3_6_12_WAY, 0);
         ARRAY_FIELD_DP32(reg_state, CXL_HDM_DECODER_CAPABILITY, 16_WAY, 0);
     }
-    ARRAY_FIELD_DP32(reg_state, CXL_HDM_DECODER_CAPABILITY, UIO, 0);
+    ARRAY_FIELD_DP32(reg_state, CXL_HDM_DECODER_CAPABILITY, UIO,
+                     (type == CXL2_TYPE3_DEVICE || type == CXL2_UPSTREAM_PORT
+                     || type == CXL2_ROOT_PORT) && uio);
     ARRAY_FIELD_DP32(reg_state, CXL_HDM_DECODER_CAPABILITY,
-                     UIO_DECODER_COUNT, 0);
+                     UIO_DECODER_COUNT,
+                     (type == CXL2_TYPE3_DEVICE || type == CXL2_UPSTREAM_PORT
+                     || type == CXL2_ROOT_PORT) && uio ? decoder_count : 0);
     ARRAY_FIELD_DP32(reg_state, CXL_HDM_DECODER_CAPABILITY, MEMDATA_NXM_CAP, 0);
     ARRAY_FIELD_DP32(reg_state, CXL_HDM_DECODER_CAPABILITY,
                      SUPPORTED_COHERENCY_MODEL,
@@ -342,6 +346,12 @@ static void hdm_init_common(uint32_t *reg_state, uint32_t *write_msk,
         write_msk[R_CXL_HDM_DECODER0_SIZE_LO + i * hdm_inc] = 0xf0000000;
         write_msk[R_CXL_HDM_DECODER0_SIZE_HI + i * hdm_inc] = 0xffffffff;
         write_msk[R_CXL_HDM_DECODER0_CTRL + i * hdm_inc] = 0x13ff;
+        if (uio) {
+            write_msk[R_CXL_HDM_DECODER0_CTRL + i * hdm_inc] |=
+                R_CXL_HDM_DECODER0_CTRL_UIO_MASK |
+                R_CXL_HDM_DECODER0_CTRL_UIG_MASK |
+                R_CXL_HDM_DECODER0_CTRL_UIW_MASK;
+        }
         if (type == CXL2_DEVICE ||
             type == CXL2_TYPE3_DEVICE ||
             type == CXL2_LOGICAL_DEVICE) {
@@ -391,7 +401,7 @@ static void bi_decoder_init_common(uint32_t *reg_state, uint32_t *write_msk,
 void cxl_component_register_init_common(uint32_t *reg_state,
                                         uint32_t *write_msk,
                                         enum reg_type type,
-                                        bool bi)
+                                        bool bi, bool uio)
 {
     int caps = 0;
 
@@ -431,7 +441,7 @@ void cxl_component_register_init_common(uint32_t *reg_state,
     case CXL2_LOGICAL_DEVICE:
         /* + HDM */
         init_cap_reg(HDM, 5, 1);
-        hdm_init_common(reg_state, write_msk, type, bi);
+        hdm_init_common(reg_state, write_msk, type, bi, uio);
         /* fallthrough */
     case CXL2_DOWNSTREAM_PORT:
     case CXL2_DEVICE:
@@ -523,7 +533,8 @@ void cxl_component_create_dvsec(CXLComponentState *cxl,
     case NON_CXL_FUNCTION_MAP_DVSEC:
         break; /* Not yet implemented */
     case EXTENSIONS_PORT_DVSEC:
-        wmask[offset + offsetof(CXLDVSECPortExt, control)] = 0x0F;
+        wmask[offset + offsetof(CXLDVSECPortExt, control)] =
+            cxl_dev_type == CXL2_DOWNSTREAM_PORT ? 0x1F : 0x0F;
         wmask[offset + offsetof(CXLDVSECPortExt, control) + 1] = 0x40;
         wmask[offset + offsetof(CXLDVSECPortExt, alt_bus_base)] = 0xFF;
         wmask[offset + offsetof(CXLDVSECPortExt, alt_bus_limit)] = 0xFF;
hw/mem/cxl_type3.c +9 -1
index b7ad437cbc..24bf4f8309 100644
@@ -590,6 +590,11 @@ static void hdm_decoder_commit(CXLType3Dev *ct3d, int which)
     /* TODO: Sanity checks that the decoder is possible */
     ctrl = FIELD_DP32(ctrl, CXL_HDM_DECODER0_CTRL, ERR, 0);
     ctrl = FIELD_DP32(ctrl, CXL_HDM_DECODER0_CTRL, COMMITTED, 1);
+    if (ct3d->uio_comp_capable) {
+        ct3d->uio_enabled = FIELD_EX32(ctrl, CXL_HDM_DECODER0_CTRL, UIO);
+    } else {
+        ctrl = FIELD_DP32(ctrl, CXL_HDM_DECODER0_CTRL, UIO, 0);
+    }
 
     /* Get interleave details for chmu */
     ig = FIELD_EX32(ctrl, CXL_HDM_DECODER0_CTRL, IG);
@@ -657,6 +662,7 @@ static void hdm_decoder_uncommit(CXLType3Dev *ct3d, int which)
     ctrl = FIELD_DP32(ctrl, CXL_HDM_DECODER0_CTRL, COMMITTED, 0);
 
     stl_le_p(cache_mem + R_CXL_HDM_DECODER0_CTRL + which * hdm_inc, ctrl);
+    ct3d->uio_enabled = false;
 }
 
 static int ct3d_qmp_uncor_err_to_cxl(CxlUncorErrorType qmp_err)
@@ -1512,7 +1518,8 @@ void ct3d_reset(DeviceState *dev)
     pcie_cap_fill_link_ep_usp(PCI_DEVICE(dev), ct3d->width, ct3d->speed,
                               ct3d->flitmode);
     cxl_component_register_init_common(reg_state, write_msk,
-                                       CXL2_TYPE3_DEVICE, ct3d->hdmdb);
+                                       CXL2_TYPE3_DEVICE, ct3d->hdmdb,
+                                       ct3d->uio_comp_capable);
     cxl_device_register_init_t3(ct3d, CXL_T3_MSIX_MBOX);
 
     /*
@@ -1552,6 +1559,7 @@ static const Property ct3_props[] = {
                                 width, PCIE_LINK_WIDTH_16),
     DEFINE_PROP_BOOL("x-256b-flit", CXLType3Dev, flitmode, false),
     DEFINE_PROP_BOOL("hdm-db", CXLType3Dev, hdmdb, false),
+    DEFINE_PROP_BOOL("x-uio", CXLType3Dev, uio_comp_capable, false),
     DEFINE_PROP_UINT16("chmu-port", CXLType3Dev, cxl_dstate.chmu[0].port, 0),
 };
 
hw/pci-bridge/cxl_downstream.c +1 -1
index 1c8c3b4636..8f034991d8 100644
@@ -42,7 +42,7 @@ static void latch_registers(CXLDownstreamPort *dsp)
     uint32_t *write_msk = dsp->cxl_cstate.crb.cache_mem_regs_write_mask;
 
     cxl_component_register_init_common(reg_state, write_msk,
-                                       CXL2_DOWNSTREAM_PORT, true);
+                                       CXL2_DOWNSTREAM_PORT, true, false);
 }
 
 /* TODO: Look at sharing this code across all CXL port types */
hw/pci-bridge/cxl_root_port.c +2 -1
index e82685d1ab..4be2b400f9 100644
@@ -53,6 +53,7 @@ typedef struct CXLRootPort {
     CPMUState cpmu;
     MemoryRegion cpmu_registers;
     PCIResReserve res_reserve;
+    bool uio_capable;
 } CXLRootPort;
 
 #define TYPE_CXL_ROOT_PORT "cxl-rp"
@@ -107,7 +108,7 @@ static void latch_registers(CXLRootPort *crp)
     uint32_t *write_msk = crp->cxl_cstate.crb.cache_mem_regs_write_mask;
 
     cxl_component_register_init_common(reg_state, write_msk, CXL2_ROOT_PORT,
-                                       true);
+                                       true, crp->uio_capable);
 }
 
 static void build_dvsecs(PCIDevice *d, CXLComponentState *cxl)
hw/pci-bridge/cxl_upstream.c +2 -1
index 999ee0b56b..e8f2e57819 100644
@@ -136,7 +136,8 @@ static void latch_registers(CXLUpstreamPort *usp)
     uint32_t *write_msk = usp->cxl_cstate.crb.cache_mem_regs_write_mask;
 
     cxl_component_register_init_common(reg_state, write_msk,
-                                       CXL2_UPSTREAM_PORT, usp->flitmode);
+                                       CXL2_UPSTREAM_PORT, usp->flitmode,
+                                       usp->uio_capable);
     ARRAY_FIELD_DP32(reg_state, CXL_HDM_DECODER_CAPABILITY, TARGET_COUNT, 8);
 }
 
hw/pci-bridge/pci_expander_bridge.c +2 -1
index 25dfee6a9b..18b61eca20 100644
@@ -301,7 +301,8 @@ static void pxb_cxl_dev_reset(DeviceState *dev)
     uint32_t *write_msk = cxl_cstate->crb.cache_mem_regs_write_mask;
     int dsp_count = 0;
 
-    cxl_component_register_init_common(reg_state, write_msk, CXL2_RC, false);
+    cxl_component_register_init_common(reg_state, write_msk, CXL2_RC, false,
+                                       false);
     /*
      * The CXL specification allows for host bridges with no HDM decoders
      * if they only have a single root port.
include/hw/cxl/cxl_component.h +1 -1
index d734f88d2c..662fdb0833 100644
@@ -315,7 +315,7 @@ void cxl_component_register_block_init(Object *obj,
                                        const char *type);
 void cxl_component_register_init_common(uint32_t *reg_state,
                                         uint32_t *write_msk,
-                                        enum reg_type type, bool bi);
+                                        enum reg_type type, bool bi, bool uio);
 
 void cxl_component_create_dvsec(CXLComponentState *cxl_cstate,
                                 enum reg_type cxl_dev_type, uint16_t length,
include/hw/cxl/cxl_device.h +2 -0
index 3b5dcb5aec..3d92d3f46c 100644
@@ -883,6 +883,8 @@ struct CXLType3Dev {
     PCIExpLinkSpeed speed;
     PCIExpLinkWidth width;
     bool flitmode;
+    bool uio_comp_capable;
+    bool uio_enabled;
 
     /* DOE */
     DOECap doe_cdat;
include/hw/pci-bridge/cxl_upstream_port.h +1 -0
index d4186234c8..d23541e23a 100644
@@ -22,6 +22,7 @@ typedef struct CXLUpstreamPort {
     PCIExpLinkSpeed speed;
     PCIExpLinkWidth width;
     bool flitmode;
+    bool uio_capable;
 
     DOECap doe_cdat;
     uint64_t sn;
Signature · 1 line
2.34.1
#8 lore raw

Re: [RFC V2 05/10] hw/cxl: Wire UIO capability into HDM decoder and DVSEC registers

Hi Shrihari,

hw/cxl/cxl-component-utils.c static void hdm_init_common(uint32_t *reg_state, uint32_t *write_msk, · line 334

Quoted 4 lines · Shrihari E S

ARRAY_FIELD_DP32(reg_state, CXL_HDM_DECODER_CAPABILITY,

hw/cxl/cxl-component-utils.c +3 -0
+                     UIO_DECODER_COUNT,
+                     (type == CXL2_TYPE3_DEVICE || type == CXL2_UPSTREAM_PORT
+                     || type == CXL2_ROOT_PORT) && uio ? decoder_count : 0);

The always-true expression from v1 is a real type test now, but UIO Capable Decoder Count uses the same encoding as Decoder Count -- CXL 3.2 8.2.4.20.1: "See the Decoder Count field in this register for enumeration" -- so writing decoder_count raw advertises 4h = 8 UIO decoders while Decoder Count reads 2h = 4. I read the register back on a cxl-rp: 0x00042312, bits[3:0] = 2h but bits[19:16] = 4h. cxl_decoder_count_enc() would line the two up. The same field description also marks it reserved for CXL.mem devices ("not permitted to limit the number of UIO-capable HDM decoders"), so CXL2_TYPE3_DEVICE should drop out of the count expression.

On the type list itself: 8.2.4.20.1 enumerates UIO Capable for CXL.mem devices, USPs and CXL Host Bridges, the Host Bridge case being the one that means "all the root ports within this Host Bridge are capable of routing UIO requests". This sets it per root port instead, and pci_expander_bridge.c passes uio=false for CXL2_RC unconditionally, so the host bridge never advertises it. On v1 I said a root port may well be intended, going off crp->uio_capable in the SVC wiring, but the enumeration above doesn't list one. Is the root port the intended carrier?

hw/cxl/cxl-component-utils.c static void hdm_init_common(uint32_t *reg_state, uint32_t *write_msk, · line 354

Quoted 6 lines
hw/cxl/cxl-component-utils.c +6 -0
+        if (uio) {
+            write_msk[R_CXL_HDM_DECODER0_CTRL + i * hdm_inc] |=
+                R_CXL_HDM_DECODER0_CTRL_UIO_MASK |
+                R_CXL_HDM_DECODER0_CTRL_UIG_MASK |
+                R_CXL_HDM_DECODER0_CTRL_UIW_MASK;
+        }

The if (uio) block adds UIG and UIW to the write mask for every type, type3 included. 8.2.4.20.7 makes both "RWL for a switch and a Host Bridge if the UIO Capable bit ... is set" and then "reserved for CXL.mem devices." I wrote 0x00ff0000 to decoder 0's Control register on an x-uio=on type3 and read it straight back; the same write on a type3 without x-uio reads back 0. Gating UIG/UIW on the port types, and leaving UIO alone, would match the table.

Many thanks,
Junjie

#9 lore raw

Re: [RFC V2 05/10] hw/cxl: Wire UIO capability into HDM decoder and DVSEC registers

Quoted 9 lines · Junjie Cao

On the type list itself: 8.2.4.20.1 enumerates UIO Capable for CXL.mem devices, USPs and CXL Host Bridges, the Host Bridge case being the one that means "all the root ports within this Host Bridge are capable of routing UIO requests". This sets it per root port instead, and pci_expander_bridge.c passes uio=false for CXL2_RC unconditionally, so the host bridge never advertises it. On v1 I said a root port may well be intended, going off crp->uio_capable in the SVC wiring, but the enumeration above doesn't list one. Is the root port the intended carrier?

Hi Junjie,

Thank you pointing this out. I agree that enabling UIO in Host Bridge (HB) should enable UIO in all the root ports, but the challenge is that the HB might not know which root port attached to it holds UIO PCIe capability. That's why the implementation went to root port. Will extend this capability field to HB (x-uio) in the next version of this patch. CIIW, Root port is intended to hold the UIO dependent capablites (SVC, 256b flit), so that eventhough the HB enables UIO HDM field, the attached root ports can be checked for the PCIe capabilities before enabling UIO in their respective HDM decoder.

Thanks and regards,
Shrihari

#10 lore raw

RFC v2 6/10 hw/pci: Add PCIe Streamlined Virtual Channel (SVC) capability.

In reply to a message not in the archive

From: Dongjoo Seo <[email protected]>

Implement the PCIe Streamlined Virtual Channel (SVC) Extended Capability by adding support of capability, control and status registers per PCIe 6.4 section 7.9.29. This capability is one of the main requisites for UIO support in both PCIe and CXL ports.

Key changes include:
- New pcie_svc.c file for SVC capability management.
- Updated pcie_cap_fill_lnk() to handle flitmode signaling.
- Implement Lifecycle hooks (reset and config_write) to manage SVC state.

---
 hw/pci/meson.build         |   2 +-
 hw/pci/pci.c               |   4 +
 hw/pci/pcie.c              |  53 +++++++++-
 hw/pci/pcie_svc.c          | 192 +++++++++++++++++++++++++++++++++++++
 include/hw/pci/pcie.h      |   7 ++
 include/hw/pci/pcie_regs.h |  55 +++++++++++
 include/hw/pci/pcie_svc.h  |  26 +++++
 7 files changed, 333 insertions(+), 6 deletions(-)
 create mode 100644 hw/pci/pcie_svc.c
 create mode 100644 include/hw/pci/pcie_svc.h
hw/pci/meson.build +1 -1
index b9c34b2acf..d55882a0c9 100644
@@ -14,7 +14,7 @@ pci_ss.add(files(
 # The functions in these modules can be used by devices too.  Since we
 # allow plugging PCIe devices into PCI buses, include them even if
 # CONFIG_PCI_EXPRESS=n.
-pci_ss.add(files('pcie.c', 'pcie_aer.c'))
+pci_ss.add(files('pcie.c', 'pcie_aer.c', 'pcie_svc.c'))
 pci_ss.add(files('pcie_doe.c'))
 system_ss.add(when: 'CONFIG_PCI_EXPRESS', if_true: files('pcie_port.c', 'pcie_host.c'))
 system_ss.add_all(when: 'CONFIG_PCI', if_true: pci_ss)
hw/pci/pci.c +4 -0
index 5996229c81..0469250f42 100644
@@ -30,6 +30,7 @@
 #include "hw/pci/pci_bridge.h"
 #include "hw/pci/pci_bus.h"
 #include "hw/pci/pci_host.h"
+#include "hw/pci/pcie_svc.h"
 #include "hw/core/qdev-properties.h"
 #include "hw/core/qdev-properties-system.h"
 #include "migration/cpr.h"
@@ -568,6 +569,7 @@ static void pci_do_device_reset(PCIDevice *dev)
     msi_reset(dev);
     msix_reset(dev);
     pcie_sriov_pf_reset(dev);
+    pcie_svc_cap_reset(dev);
 }
 
 /*
@@ -1814,6 +1816,8 @@ void pci_default_write_config(PCIDevice *d, uint32_t addr, uint32_t val_in, int
     msi_write_config(d, addr, val_in, l);
     msix_write_config(d, addr, val_in, l);
     pcie_sriov_config_write(d, addr, val_in, l);
+    pcie_cap_flit_write_config(d, addr, val_in, l);
+    pcie_svc_cap_write_config(d, addr, val_in, l);
 }
 
 /***********************************************************/
hw/pci/pcie.c +48 -5

80 lines not on this page · raw message

hw/pci/pcie_svc.c new file +192 -0

192 lines not on this page · raw message

include/hw/pci/pcie.h +7 -0
index 79808126dc..2feeb022f0 100644
@@ -25,6 +25,7 @@
 #include "hw/pci/pcie_regs.h"
 #include "hw/pci/pcie_aer.h"
 #include "hw/pci/pcie_sriov.h"
+#include "hw/pci/pcie_svc.h"
 #include "hw/core/hotplug.h"
 
 typedef struct PCIEPort PCIEPort;
@@ -82,6 +83,10 @@ struct PCIExpressDevice {
     uint16_t sriov_cap;
     PCIESriovPF sriov_pf;
     PCIESriovVF sriov_vf;
+
+    /* Streamlined Virtual Channel (SVC) introduced from PCIe 6.1 */
+    uint32_t svc_cap;
+    PCIESvcCap svc;
 };
 
 #define COMPAT_PROP_PCP "power_controller_present"
@@ -143,6 +148,8 @@ void pcie_dev_ser_num_init(PCIDevice *dev, uint16_t offset, uint64_t ser_num);
 void pcie_ats_init(PCIDevice *dev, uint16_t offset, bool aligned);
 void pcie_cap_fill_link_ep_usp(PCIDevice *dev, PCIExpLinkWidth width,
                                PCIExpLinkSpeed speed, bool flitmode);
+void pcie_cap_flit_write_config(PCIDevice *dev,
+                                uint32_t addr, uint32_t val, int len);
 
 void pcie_cap_slot_pre_plug_cb(HotplugHandler *hotplug_dev, DeviceState *dev,
                                Error **errp);
include/hw/pci/pcie_regs.h +55 -0

65 lines not on this page · raw message

include/hw/pci/pcie_svc.h new file +26 -0
new file mode 100644
index 0000000000..23957752c9
@@ -0,0 +1,26 @@
+/*
+ * SPDX-License-Identifier: GPL-2.0-or-later
+ *
+ * PCIe Streamlined Virtual Channel (SVC) Extended Capability
+ *
+ * Copyright (c) 2026 Samsung Electronics Co., Ltd.
+ */
+
+#ifndef HW_PCIE_SVC_H
+#define HW_PCIE_SVC_H
+
+#include "hw/pci/pci.h"
+
+typedef struct PCIESvcCap {
+    uint8_t svc_evcc;
+    bool uio_mand_svc;
+    bool uio_opt_svc;
+    bool tc_vc_map[8];
+} PCIESvcCap;
+
+int  pcie_svc_cap_init(PCIDevice *dev, uint32_t offset, Error **errp);
+void pcie_svc_cap_reset(PCIDevice *dev);
+void pcie_svc_cap_write_config(PCIDevice *dev,
+                               uint32_t addr, uint32_t val, int len);
+
+#endif /* HW_PCIE_SVC_H */
Signature · 1 line
2.34.1
#11 lore raw

Re: [RFC V2 06/10] hw/pci: Add PCIe Streamlined Virtual Channel (SVC) capability.

Hi Shrihari,

hw/pci/pcie_svc.c line 177

Quoted 8 lines · Shrihari E S
hw/pci/pcie_svc.c new file +8 -0
+            /*
+             * SVC3 is only meant for UIO TLPs and The non-UIO protocol 0000
+             * should be for VC0 only. Also checking the TC-VC mapping.
+             */
+            if ((i == 3) || (res_ctrl & PCI_SVC_VC0_PROTOCOL) ||
+                 !(res_ctrl & BIT(i))) {
+                valid = false;
+            }

PCI_SVC_VC0_PROTOCOL is (0x0 << 8), so "res_ctrl & PCI_SVC_VC0_PROTOCOL" is always 0 and the protocol-0000 rejection this comment describes can never fire. I tested it: wrote 0x80000010 (VC Enable | TC/VC map bit 4, protocol select = 0000) to the VC4 Resource Control register of a cxl-rp; the readback keeps VC Enable set (and, going by the code, pcie_svc_update_map() then marks uio_opt_svc), where per the comment the write should have been rejected. Protocol == 0000 needs the whole field tested, e.g. !(res_ctrl & PCI_SVC_VC_PROTOCOL_SELECTED).

The UIO branch has the same pattern:

Quoted 1 line

+ if (res_ctrl & SVC_UIO_PROTOCOL_SELECTED) {

Matches any protocol value with bit 9 set -- including the vendor-defined 1111b -- not just 0010b. Comparing the extracted 4-bit field against the expected value in both branches would close that.

Same run: Port Cap1 reads back EVCC=7, but the init loop only populates sets 0, 3 and 4 -- the rest read VC ID 000b, the same as VC0, which 7.9.29.6 wants unique. Was EVCC=2 with sets 1/2 carrying 3/4 the shape you were after? (Quoting 6.2 throughout -- newest spec I have.)

Quoted 2 lines
(quoted diff) +2 -0
+void pcie_svc_cap_reset(PCIDevice *dev)
+{

[...]

hw/pci/pcie_svc.c line 119

Quoted 3 lines
hw/pci/pcie_svc.c new file +3 -0
+    pci_set_long(dev->config + offset + PCI_SVC_CTL_OFFSET, 0);
+    pci_set_long(dev->config + offset + PCI_SVC_STA_OFFSET, 0);
+}

The init function sets USE_VC_MFVC and this clears it again, before a cold-plugged guest ever runs -- SVC status reads 0 on all four port types at first boot. 0 is the conformant value anyway (7.9.29.5, and the comment above the set says as much), so I'd drop the set at init rather than restore the bit here.

The asymmetry does bite for the rest of the state: a guest-set VC Enable in RES_CTRL(3)/(4) survives a system reset while CTL/STA are cleared, and the dev->exp.svc shadow flags set by pcie_svc_update_map() aren't cleared either.

On the register defines: v1 ended with routing these through pci_regs.h plus a note on when the kernel header picks them up; they went into QEMU's pcie_regs.h instead. Still the plan for a later spin, or intentional?

Many thanks,
Junjie

#12 lore raw

Re: [RFC V2 06/10] hw/pci: Add PCIe Streamlined Virtual Channel (SVC) capability.

Quoted 7 lines · Junjie Cao

On the register defines: v1 ended with routing these through pci_regs.h plus a note on when the kernel header picks them up; they went into QEMU's pcie_regs.h instead. Still the plan for a later spin, or intentional?

Many thanks,
Junjie

Hi Junjie,

The register definitions are intentionally placed in QEMU's include/hw/pci/pci_regs.h for now. Once The corresponding macros are merged into Linux Kernel's include/uapi/linux/pci_regs.h, will sync them through include/standard headers/linux/pci_regs.h in a follow-up patch.

Thanks and regards,
Shrihari

#13 lore raw

RFC v2 7/10 hw/pci: hw/cxl: Wire SVC initialization into port realize functions.

In reply to a message not in the archive

Integrate Streamlined Virtual Channel (SVC) capability initialization into the realize() functions of both PCIe and CXL ports. This change ensures that the 'uio_capable' flag is correctly populated in CXL ports during the initialization sequence.

'uio_capable' in CXL ports is responsible for enabling UIO capability in HDM decoder registers.

---
 hw/pci-bridge/cxl_downstream.c     |  7 ++++++-
 hw/pci-bridge/cxl_root_port.c      | 11 ++++++++++-
 hw/pci-bridge/cxl_upstream.c       | 10 +++++++++-
 hw/pci-bridge/gen_pcie_root_port.c |  3 +++
 hw/pci-bridge/pcie_root_port.c     |  4 ++++
 hw/pci-bridge/xio3130_downstream.c |  3 +++
 hw/pci-bridge/xio3130_upstream.c   |  3 +++
 include/hw/pci/pcie_port.h         |  1 +
 8 files changed, 39 insertions(+), 3 deletions(-)
hw/pci-bridge/cxl_downstream.c +6 -1
index 8f034991d8..de6d2f7306 100644
@@ -33,8 +33,10 @@ typedef struct CXLDownstreamPort {
 #define CXL_DOWNSTREAM_PORT_MSI_NR_VECTOR 2
 #define CXL_DOWNSTREAM_PORT_EXP_OFFSET 0x90
 #define CXL_DOWNSTREAM_PORT_AER_OFFSET 0x100
-#define CXL_DOWNSTREAM_PORT_DVSEC_OFFSET        \
+#define CXL_DOWNSTREAM_PORT_SVC_OFFSET          \
     (CXL_DOWNSTREAM_PORT_AER_OFFSET + PCI_ERR_SIZEOF)
+#define CXL_DOWNSTREAM_PORT_DVSEC_OFFSET        \
+    (CXL_DOWNSTREAM_PORT_SVC_OFFSET + PCI_SVC_SIZEOF)
 
 static void latch_registers(CXLDownstreamPort *dsp)
 {
@@ -229,6 +231,9 @@ static void cxl_dsp_realize(PCIDevice *d, Error **errp)
                      PCI_BASE_ADDRESS_SPACE_MEMORY |
                          PCI_BASE_ADDRESS_MEM_TYPE_64,
                      &dsp->bar);
+    if (p->svc) {
+        pcie_svc_cap_init(d, CXL_DOWNSTREAM_PORT_SVC_OFFSET, errp);
+    }
 
     return;
 
hw/pci-bridge/cxl_root_port.c +10 -1
index 4be2b400f9..19dd75b79f 100644
@@ -40,8 +40,10 @@
 #define GEN_PCIE_ROOT_PORT_AER_OFFSET 0x100
 #define GEN_PCIE_ROOT_PORT_ACS_OFFSET \
     (GEN_PCIE_ROOT_PORT_AER_OFFSET + PCI_ERR_SIZEOF)
-#define CXL_ROOT_PORT_DVSEC_OFFSET \
+#define GEN_PCIE_ROOT_PORT_SVC_OFFSET \
     (GEN_PCIE_ROOT_PORT_ACS_OFFSET + PCI_ACS_SIZEOF)
+#define CXL_ROOT_PORT_DVSEC_OFFSET \
+    (GEN_PCIE_ROOT_PORT_SVC_OFFSET + PCI_SVC_SIZEOF)
 
 typedef struct CXLRootPort {
     /*< private >*/
@@ -164,6 +166,7 @@ static void cxl_rp_realize(DeviceState *dev, Error **errp)
 {
     PCIDevice *pci_dev     = PCI_DEVICE(dev);
     PCIERootPortClass *rpc = PCIE_ROOT_PORT_GET_CLASS(dev);
+    PCIEPort *p            = PCIE_PORT(dev);
     CXLRootPort *crp       = CXL_ROOT_PORT(dev);
     CXLComponentState *cxl_cstate = &crp->cxl_cstate;
     ComponentRegisters *cregs = &cxl_cstate->crb;
@@ -212,6 +215,12 @@ static void cxl_rp_realize(DeviceState *dev, Error **errp)
                      PCI_BASE_ADDRESS_SPACE_MEMORY |
                      PCI_BASE_ADDRESS_MEM_TYPE_64,
                      &crp->bar);
+    if (p->svc) {
+        rc = pcie_svc_cap_init(pci_dev, GEN_PCIE_ROOT_PORT_SVC_OFFSET, errp);
+        if (p->flitmode && rc >= 0) {
+            crp->uio_capable = true;
+        }
+    }
 }
 
 static void cxl_rp_reset_hold(Object *obj, ResetType type)
hw/pci-bridge/cxl_upstream.c +9 -1
index e8f2e57819..3ed6a89bf3 100644
@@ -27,8 +27,10 @@
 #define CXL_UPSTREAM_PORT_MSI_OFFSET 0x70
 #define CXL_UPSTREAM_PORT_PCIE_CAP_OFFSET 0x90
 #define CXL_UPSTREAM_PORT_AER_OFFSET 0x100
+#define CXL_UPSTREAM_PORT_SVC_OFFSET \
+     (CXL_UPSTREAM_PORT_AER_OFFSET + PCI_ERR_SIZEOF)
 #define CXL_UPSTREAM_PORT_SN_OFFSET \
-    (CXL_UPSTREAM_PORT_AER_OFFSET + PCI_ERR_SIZEOF)
+    (CXL_UPSTREAM_PORT_SVC_OFFSET + PCI_SVC_SIZEOF)
 #define CXL_UPSTREAM_PORT_DVSEC_OFFSET \
     (CXL_UPSTREAM_PORT_SN_OFFSET + PCI_EXT_CAP_DSN_SIZEOF)
 
@@ -411,6 +413,12 @@ static void cxl_usp_realize(PCIDevice *d, Error **errp)
         goto err_cap;
     }
 
+    if (p->svc) {
+        rc = pcie_svc_cap_init(d, CXL_UPSTREAM_PORT_SVC_OFFSET, errp);
+        if (p->flitmode && rc >= 0) {
+            usp->uio_capable = true;
+        }
+    }
     return;
 
 err_cap:
hw/pci-bridge/gen_pcie_root_port.c +3 -0
index 2f7257d166..d4df4a7c16 100644
@@ -26,6 +26,8 @@ OBJECT_DECLARE_SIMPLE_TYPE(GenPCIERootPort, GEN_PCIE_ROOT_PORT)
 #define GEN_PCIE_ROOT_PORT_AER_OFFSET           0x100
 #define GEN_PCIE_ROOT_PORT_ACS_OFFSET \
         (GEN_PCIE_ROOT_PORT_AER_OFFSET + PCI_ERR_SIZEOF)
+#define GEN_PCIE_ROOT_PORT_SVC_OFFSET \
+        (GEN_PCIE_ROOT_PORT_ACS_OFFSET + PCI_ACS_SIZEOF)
 
 #define GEN_PCIE_ROOT_PORT_MSIX_NR_VECTOR       1
 #define GEN_PCIE_ROOT_DEFAULT_IO_RANGE          4096
@@ -166,6 +168,7 @@ static void gen_rp_dev_class_init(ObjectClass *klass, const void *data)
     rpc->interrupts_uninit = gen_rp_interrupts_uninit;
     rpc->aer_offset = GEN_PCIE_ROOT_PORT_AER_OFFSET;
     rpc->acs_offset = GEN_PCIE_ROOT_PORT_ACS_OFFSET;
+    rpc->svc_offset = GEN_PCIE_ROOT_PORT_SVC_OFFSET;
 }
 
 static const TypeInfo gen_rp_dev_info = {
hw/pci-bridge/pcie_root_port.c +4 -0
index 0ae7b10fbd..9c325aa5f4 100644
@@ -120,6 +120,10 @@ static void rp_realize(PCIDevice *d, Error **errp)
     if (rpc->acs_offset && !s->disable_acs) {
         pcie_acs_init(d, rpc->acs_offset);
     }
+    if (!pci_is_cxl(d) && p->svc) {
+        pcie_svc_cap_init(d, rpc->svc_offset, errp);
+    }
+
     return;
 
 err:
hw/pci-bridge/xio3130_downstream.c +3 -0
index 090a3fdfc9..a2546d5060 100644
@@ -110,6 +110,9 @@ static void xio3130_downstream_realize(PCIDevice *d, Error **errp)
     if (rc < 0) {
         goto err;
     }
+    if (p->svc) {
+        pcie_svc_cap_init(d, XIO3130_AER_OFFSET + PCI_ERR_SIZEOF, errp);
+    }
 
     return;
 
hw/pci-bridge/xio3130_upstream.c +3 -0
index 767b7d17f2..fd15e8c943 100644
@@ -92,6 +92,9 @@ static void xio3130_upstream_realize(PCIDevice *d, Error **errp)
     if (rc < 0) {
         goto err;
     }
+    if (p->svc) {
+        pcie_svc_cap_init(d, XIO3130_AER_OFFSET + PCI_ERR_SIZEOF, errp);
+    }
 
     return;
 
include/hw/pci/pcie_port.h +1 -0
index 0b3e0efb8c..8de95d71ef 100644
@@ -97,6 +97,7 @@ struct PCIERootPortClass {
     int aer_offset;
     int ssvid_offset;
     int acs_offset;    /* If nonzero, optional ACS capability offset */
+    int svc_offset;    /* optional SVC capability */
     int ssid;
 };
 
Signature · 1 line
2.34.1
#14 lore raw

Re: [RFC V2 07/10] hw/pci: hw/cxl: Wire SVC initialization into port realize functions.

Hi Shrihari,

hw/pci-bridge/cxl_upstream.c static void cxl_usp_realize(PCIDevice *d, Error **errp) · line 421

Quoted 6 lines · Shrihari E S
hw/pci-bridge/cxl_upstream.c +6 -0
+    if (p->svc) {
+        rc = pcie_svc_cap_init(d, CXL_UPSTREAM_PORT_SVC_OFFSET, errp);
+        if (p->flitmode && rc >= 0) {
+            usp->uio_capable = true;
+        }
+    }

This gate never opens on cxl-upstream: p->flitmode is the PCIEPort field, but the USP's x-256b-flit property still targets its own CXLUpstreamPort::flitmode rather than the field patch 1 moved to PCIEPort, and nothing sets the PCIEPort one there. Everything else on the USP (pcie_cap_fill_link_ep_usp(), the DVSEC status, latch_registers()) reads usp->flitmode, so the port reports flit active while uio_capable stays false. I read the HDM Decoder Capability register back on the patch-10 example topology with x-svc=on,x-256b-flit=on everywhere: the USP reads 0x00000382 -- UIO clear, UIO decoder count 0 -- while cxl-rp reads 0x00042312 with UIO set. A switch topology, the case the cover letter leads with, can't advertise UIO through the USP. What worked for me was dropping the CXLUpstreamPort copy so both readers take the PCIEPort field: with that the USP reads 0x00042382 and nothing else in the topology moves. I'd put it in 1/10, which left CXLUpstreamPort::flitmode behind while its message says the refactor "allows all the derived ports ... to use this property".

rpc->svc_offset is only set by gen_pcie_root_port; ioh3420, pnv-phb-root-port and aspeed.pcie-root-port leave it 0, so

qemu-system-x86_64 -M q35 -display none -device ioh3420,x-svc=on

aborts:

../hw/pci/pcie.c:1140: pcie_add_capability: Assertion `offset >= PCI_CONFIG_SPACE_SIZE' failed.

I'd either give them svc_offsets or fail realize with a proper error when svc_offset is 0 and x-svc is set.

Neither of those shows up in make check. A qtest instantiating ioh3420,x-svc=on would have caught the abort; the USP gate only shows up in the component BAR, where I read it in the HDM Decoder Capability register, so a chain walk wouldn't catch that one. Worth some coverage for the two new capabilities.

The v1 SVC/AER collision is fixed -- I walked the extended chains over ECAM: gen pcie-root-port has SVC at 0x150 after AER/ACS, cxl-rp's four DVSECs moved out to 0x1c4 with SVC taking 0x150, and usp/dsp have SVC at 0x148 with SN/DVSEC offsets shifted to match.

The DVSEC moves are unconditional, though. CXL_ROOT_PORT_DVSEC_OFFSET and the usp/dsp equivalents are compile-time chains through PCI_SVC_SIZEOF, so x-svc=off changes nothing: on a topology with no x-svc anywhere I read cxl-rp DVSECs at 0x1c4 (0x150 on the base branch), usp DSN at 0x1bc (0x148), dsp DVSEC at 0x1bc (0x148). That's a config space change on existing CXL machines with no property to gate it. Migration won't notice -- none of the CXL devices carries a VMStateDescription, so their config space never reaches the stream -- but a guest on an unchanged machine type sees the DVSECs move across QEMU versions, and there's no knob to hang a hw_compat entry on.

Many thanks,
Junjie

#15 lore raw

RFC v2 8/10 hw/pci: Add PCIe Device3 capability support

In reply to a message not in the archive

This commit adds Device3 support to the PCIe devices, with which an endpoint can be exposed as a UIO requester or completer, based on its configuration.

---
 hw/mem/cxl_type3.c         |   1 +
 hw/pci/meson.build         |   2 +-
 hw/pci/pci.c               |   2 +
 hw/pci/pcie.c              |  30 +++++++
 hw/pci/pcie_dev3.c         | 165 +++++++++++++++++++++++++++++++++++++
 include/hw/pci/pcie.h      |   6 ++
 include/hw/pci/pcie_dev3.h |  29 +++++++
 include/hw/pci/pcie_regs.h |  30 +++++++
 8 files changed, 264 insertions(+), 1 deletion(-)
 create mode 100644 hw/pci/pcie_dev3.c
 create mode 100644 include/hw/pci/pcie_dev3.h
hw/mem/cxl_type3.c +1 -0
index 24bf4f8309..0adf2e8b3e 100644
@@ -17,6 +17,7 @@
 #include "hw/mem/memory-device.h"
 #include "hw/mem/pc-dimm.h"
 #include "hw/pci/pci.h"
+#include "hw/pci/pcie_dev3.h"
 #include "hw/core/qdev-properties.h"
 #include "hw/core/qdev-properties-system.h"
 #include "qapi/error.h"
hw/pci/meson.build +1 -1
index d55882a0c9..0a7f3b162a 100644
@@ -14,7 +14,7 @@ pci_ss.add(files(
 # The functions in these modules can be used by devices too.  Since we
 # allow plugging PCIe devices into PCI buses, include them even if
 # CONFIG_PCI_EXPRESS=n.
-pci_ss.add(files('pcie.c', 'pcie_aer.c', 'pcie_svc.c'))
+pci_ss.add(files('pcie.c', 'pcie_aer.c', 'pcie_svc.c', 'pcie_dev3.c'))
 pci_ss.add(files('pcie_doe.c'))
 system_ss.add(when: 'CONFIG_PCI_EXPRESS', if_true: files('pcie_port.c', 'pcie_host.c'))
 system_ss.add_all(when: 'CONFIG_PCI', if_true: pci_ss)
hw/pci/pci.c +2 -0
index 0469250f42..600f41fc95 100644
@@ -570,6 +570,7 @@ static void pci_do_device_reset(PCIDevice *dev)
     msix_reset(dev);
     pcie_sriov_pf_reset(dev);
     pcie_svc_cap_reset(dev);
+    pcie_dev3_cap_reset(dev);
 }
 
 /*
@@ -1818,6 +1819,7 @@ void pci_default_write_config(PCIDevice *d, uint32_t addr, uint32_t val_in, int
     pcie_sriov_config_write(d, addr, val_in, l);
     pcie_cap_flit_write_config(d, addr, val_in, l);
     pcie_svc_cap_write_config(d, addr, val_in, l);
+    pcie_dev3_cap_write_config(d, addr, val_in, l);
 }
 
 /***********************************************************/
hw/pci/pcie.c +30 -0
index cc1cc500ab..6b8cc0f948 100644
@@ -190,6 +190,32 @@ void pcie_cap_fill_link_ep_usp(PCIDevice *dev, PCIExpLinkWidth width,
     pcie_cap_fill_lnk(dev, exp_cap, width, speed, flitmode);
 }
 
+bool pcie_link_flit_active(PCIDevice *dev)
+{
+    uint8_t *exp_cap;
+    uint16_t lnksta2;
+    uint16_t lnkctl;
+    uint16_t flags;
+
+    if (!pci_is_express(dev) || !dev->exp.exp_cap) {
+        return false;
+    }
+
+    exp_cap = dev->config + dev->exp.exp_cap;
+    flags = pci_get_word(exp_cap + PCI_EXP_FLAGS);
+    if (!(flags & PCI_EXP_FLAGS_FLIT)) {
+        return false;
+    }
+    lnkctl = pci_get_word(exp_cap + PCI_EXP_LNKCTL);
+    lnksta2 = pci_get_word(exp_cap + PCI_EXP_LNKSTA2);
+
+    if (lnkctl & PCI_EXP_LNKCTL_FLIT_DIS) {
+        return false;
+    }
+
+    return lnksta2 & PCI_EXP_LNKSTA2_FLIT;
+}
+
 void pcie_cap_flit_write_config(PCIDevice *dev, uint32_t addr, uint32_t val,
                                 int len)
 {
@@ -223,6 +249,10 @@ void pcie_cap_flit_write_config(PCIDevice *dev, uint32_t addr, uint32_t val,
     }
 
     pci_set_word(exp_cap + PCI_EXP_LNKSTA2, lnksta2);
+    if (dev->exp.dev3_cap) {
+        pcie_dev3_cap_flit_update(dev);
+    }
+
  }
 
 static void pcie_cap_fill_slot_lnk(PCIDevice *dev)
hw/pci/pcie_dev3.c new file +165 -0

165 lines not on this page · raw message

include/hw/pci/pcie.h +6 -0
index 2feeb022f0..e997f3a456 100644
@@ -26,6 +26,7 @@
 #include "hw/pci/pcie_aer.h"
 #include "hw/pci/pcie_sriov.h"
 #include "hw/pci/pcie_svc.h"
+#include "hw/pci/pcie_dev3.h"
 #include "hw/core/hotplug.h"
 
 typedef struct PCIEPort PCIEPort;
@@ -87,6 +88,10 @@ struct PCIExpressDevice {
     /* Streamlined Virtual Channel (SVC) introduced from PCIe 6.1 */
     uint32_t svc_cap;
     PCIESvcCap svc;
+
+    /* PCIe Device 3 Capability */
+    uint32_t dev3_cap;
+    struct PCIEDev3Cap dev3;
 };
 
 #define COMPAT_PROP_PCP "power_controller_present"
@@ -150,6 +155,7 @@ void pcie_cap_fill_link_ep_usp(PCIDevice *dev, PCIExpLinkWidth width,
                                PCIExpLinkSpeed speed, bool flitmode);
 void pcie_cap_flit_write_config(PCIDevice *dev,
                                 uint32_t addr, uint32_t val, int len);
+bool pcie_link_flit_active(PCIDevice *dev);
 
 void pcie_cap_slot_pre_plug_cb(HotplugHandler *hotplug_dev, DeviceState *dev,
                                Error **errp);
include/hw/pci/pcie_dev3.h new file +29 -0
new file mode 100644
index 0000000000..0a49835ab4
@@ -0,0 +1,29 @@
+/*
+ * SPDX-License-Identifier: GPL-2.0-or-later
+ *
+ * PCIe Device3 Extended Capability
+ *
+ * Copyright (c) 2026 Samsung Electronics Co., Ltd.
+ */
+
+#ifndef HW_PCIE_DEV3_H
+#define HW_PCIE_DEV3_H
+
+#include "hw/pci/pci.h"
+
+typedef struct PCIEDev3Cap {
+    bool uio_req;
+    bool uio_comp;
+} PCIEDev3Cap;
+
+int pcie_dev3_cap_init(PCIDevice *dev, uint32_t offset,
+                       bool uio_req, bool uio_comp, Error **errp);
+void pcie_dev3_cap_reset(PCIDevice *dev);
+void pcie_dev3_cap_write_config(PCIDevice *dev,
+                                uint32_t addr, uint32_t val, int len);
+void pcie_dev3_cap_flit_update(PCIDevice *dev);
+bool pcie_dev3_uio_enabled(PCIDevice *dev);
+bool pcie_dev3_tag_req_enabled(PCIDevice *dev);
+void pcie_dev3_apply_gating(PCIDevice *dev);
+
+#endif /* HW_PCIE_DEV3_H */
include/hw/pci/pcie_regs.h +30 -0
index 566754646c..326195a199 100644
@@ -237,4 +237,34 @@ typedef enum PCIExpLinkWidth {
 #define PCI_SVC_RES_STATUS(n)           (PCI_SVC_RES_STATUS_BASE + ((n) * 0x0c))
 #define PCI_SVC_VC_NEGO                 BIT(1)
 
+/* PCIe Device 3 capability structure. PCIe 6.4 Specification Section 7.7.9 */
+#define PCI_EXT_CAP_ID_DEV3 0x2f
+
+#define PCI_DEV3_CAP_HEAD              0x00
+#define PCI_DEV3_CAP_OFFSET            0x04
+#define PCI_DEV3_CTL_OFFSET            0x08
+#define PCI_DEV3_STA_OFFSET            0x0c
+#define PCI_DEV3_SIZEOF                0x10
+
+/* Section 7.7.9.1 Device 3 Extended Capability Header. */
+#define PCI_DEV3_CAP_ID                (0x2f << 0)
+#define PCI_DEV3_CAP_VER               (0x1 << 16)
+#define PCI_DEV3_NEXT_CAP_OFF          (0x0 << 20)
+
+/* Section 7.7.9.3 Device 3 capability register */
+#define PCI_DMWR_REQ_SUPPORTED         BIT(0)
+#define PCI_DEV3_CAP_TAG_COMP          BIT(1)
+#define PCI_DEV3_CAP_TAG_REQ           BIT(2)
+#define PCI_L0P_SUPPORTED              BIT(3)
+#define PCI_DEV3_CAP_UIO_COMP          BIT(10)
+#define PCI_DEV3_CAP_UIO_REQ           BIT(11)
+
+/* Section 7.7.9.3 Device Control 3 Register */
+#define PCI_DEV3_CTL_TAG_REQ_EN        BIT(2)
+#define PCI_DEV3_CTL_UIO_REQ_EN        BIT(7)
+#define PCI_DEV3_CTL_UIO_256B_DIS      BIT(8)
+
+/* Section 7.7.9.4 Device Status 3 Register */
+#define PCI_DEV3_STA_SEGMENT_CAP       BIT(3)
+
 #endif /* QEMU_PCIE_REGS_H */
Signature · 1 line
2.34.1
#16 lore raw

Re: [RFC V2 08/10] hw/pci: Add PCIe Device3 capability support

Hi Shrihari,

hw/pci/pcie_dev3.c line 60

Quoted 9 lines · Shrihari E S
hw/pci/pcie_dev3.c new file +9 -0
+        /*
+         * A UIO requester must be able to issue DMA transactions.
+         * Enable PCI_COMMAND_MASTER in the PCI command register so the
+         * device's bus master bit is set when the requester capability
+         * is advertised.
+         */
+        pci_set_word(dev->config + PCI_COMMAND,
+                     pci_get_word(dev->config + PCI_COMMAND) |
+                     PCI_COMMAND_MASTER);

As posted this is dead code -- the series' only caller, in 09/10, passes uio_req=false (the type3 is wired as completer only). If a requester shows up later, the write still doesn't do what the comment
wants: pci_do_device_reset() clears the writable COMMAND bits, so on cold boot the machine reset discards it before the guest runs. On a hotplug path there is no bus reset, and pci_qdev_realize() ends in pci_set_power() -> pci_set_enabled() -> pci_set_master(), so there the write does take effect and the device surfaces with bus mastering already on. Bus master enable stays under firmware/OS control; I'd drop the write.

The Dev3 defines already exist in Linux, and they reached QEMU in
January: 49f6b93d0752 ("linux-headers: Update to Linux v6.19-rc1") put PCI_EXT_CAP_ID_DEV3 0x2F into include/standard-headers/linux/pci_regs.h. The cxl-2026-01-09-draft base predates that sync, which is why this builds here; on master the 0x2f added to pcie_regs.h redefines it, and pcie.h pulls both headers into the same TU, so that's a redefinition warning under the default -Werror. The register offsets don't collide -- PCI_DEV3_CAP_OFFSET here against the header's PCI_DEV3_CAP -- so PCI_EXT_CAP_ID_DEV3 is the one to drop. SVC is the other case: 0x35 genuinely isn't in the kernel header yet.

Many thanks,
Junjie

#17 lore raw

RFC v2 9/10 hw/cxl: Wire SVC and Dev3 capability to CXL Type 3 device

In reply to a message not in the archive

Adds SVC and Dev3 capability to CXL type 3 device to make it work as a UIO completer.

---
 hw/mem/cxl_type3.c          | 15 +++++++++++++++
 include/hw/cxl/cxl_device.h |  1 +
 2 files changed, 16 insertions(+)
hw/mem/cxl_type3.c +15 -0
index 0adf2e8b3e..74ddcf6e58 100644
@@ -1195,6 +1195,20 @@ void ct3_realize(PCIDevice *pci_dev, Error **errp)
     pcie_doe_init(pci_dev, &ct3d->doe_comp, 0x1b0, doe_comp_prot, true,
                   CXL_T3_MSIX_PCIE_DOE_COMPLIANCE);
 
+    if (ct3d->svc) {
+        /* SVC capability of the type3 device */
+        rc = pcie_svc_cap_init(pci_dev, 0x200 + PCI_ERR_SIZEOF, errp);
+        if ((rc >= 0) && ct3d->flitmode && ct3d->uio_comp_capable) {
+
+            /*
+             * The Dev3 Capability is initialized  after AER in the PCIe config
+             * space. Also initializing CXL type3 device as a UIO completer.
+             */
+            pcie_dev3_cap_init(pci_dev, 0x200 + PCI_ERR_SIZEOF + PCI_SVC_SIZEOF,
+                               false, true, errp);
+        }
+    }
+
     /* Set default value for patrol scrub attributes */
     ct3d->patrol_scrub_attrs.scrub_cycle_cap =
                            CXL_MEMDEV_PS_SCRUB_CYCLE_CHANGE_CAP_DEFAULT |
@@ -1560,6 +1574,7 @@ static const Property ct3_props[] = {
                                 width, PCIE_LINK_WIDTH_16),
     DEFINE_PROP_BOOL("x-256b-flit", CXLType3Dev, flitmode, false),
     DEFINE_PROP_BOOL("hdm-db", CXLType3Dev, hdmdb, false),
+    DEFINE_PROP_BOOL("x-svc", CXLType3Dev, svc, false),
     DEFINE_PROP_BOOL("x-uio", CXLType3Dev, uio_comp_capable, false),
     DEFINE_PROP_UINT16("chmu-port", CXLType3Dev, cxl_dstate.chmu[0].port, 0),
 };
include/hw/cxl/cxl_device.h +1 -0
index 3d92d3f46c..5505aefe37 100644
@@ -885,6 +885,7 @@ struct CXLType3Dev {
     bool flitmode;
     bool uio_comp_capable;
     bool uio_enabled;
+    bool svc;
 
     /* DOE */
     DOECap doe_cdat;
Signature · 1 line
2.34.1
#18 lore raw

Re: [RFC V2 09/10] hw/cxl: Wire SVC and Dev3 capability to CXL Type 3 device

Hi Shrihari,

hw/mem/cxl_type3.c void ct3_realize(PCIDevice *pci_dev, Error **errp) · line 1201

Quoted 4 lines · Shrihari E S
hw/mem/cxl_type3.c +4 -0
+    if (ct3d->svc) {
+        /* SVC capability of the type3 device */
+        rc = pcie_svc_cap_init(pci_dev, 0x200 + PCI_ERR_SIZEOF, errp);
+        if ((rc >= 0) && ct3d->flitmode && ct3d->uio_comp_capable) {

The cover letter says x-uio depends on x-svc and x-256b-flit, but this only gates the config space capabilities. ct3d_reset() passes ct3d->uio_comp_capable straight to cxl_component_register_init_common(), so those dependencies don't reach the component registers. I checked:
a type3 with x-uio=on and neither x-svc nor x-256b-flit boots silently and its HDM Decoder Capability reads 0x00043b12 -- UIO set, UIO decoder count filled -- with no SVC or Dev3 capability in config space, and hdm_decoder_commit() will then latch ct3d->uio_enabled from the guest write. Failing realize ("x-uio requires x-svc and x-256b-flit") would keep the device consistent; alternatively deriving one uio gate at realize and using it for both the capabilities and the register init would do the same.

It's asymmetric the other way as well: there's no x-uio on the ports, and x-256b-flit defaults to on for a cxl-rp, so x-svc=on alone is enough -- a cxl-rp with just x-svc=on reads 0x00042312 -- while the type3 has to spell it out. That assumes root ports keep the bit at all; see my note on 5/10.

Related: ct3d->uio_enabled, the dev->exp.svc flags and the two pcie_dev3_*_enabled() helpers are written but never read anywhere in the series. Same as on 3/10, I'd land them with the code that consumes them -- here that's the data-plane series.

With the full docs topology (patch 10's example) the layout itself looks good: on the type3 device SVC lands at 0x248 behind AER and Dev3 at 0x2bc, chain intact, and Dev3 advertises UIO/14-bit-tag completer with Segment Captured tracking flit mode.

Many thanks,
Junjie

#19 lore raw

RFC v2 10/10 cxl: Add documentation for CXL UIO support.

In reply to a message not in the archive

Add a command-line example demonstrating a topology with three CXL Type-3 devices connected via a switch. This example enables Unordered I/O (UIO) on all ports and endpoints to illustrate the correct usage of UIO-related properties.

---
 docs/system/devices/cxl.rst | 47 +++++++++++++++++++++++++++++++++++++
 1 file changed, 47 insertions(+)
docs/system/devices/cxl.rst +47 -0
index 32b1b5d773..378fb5b2c1 100644
@@ -407,6 +407,53 @@ use HDM-DB for coherence, which requires operating in Flit mode::
   -device cxl-type3,bus=swport3,volatile-memdev=cxl-mem3,id=cxl-mem3,sn=0x4 \
   -M cxl-fmw.0.targets.0=cxl.1,cxl-fmw.0.size=4G,cxl-fmw.0.interleave-granularity=4k
 
+Unordered I/O (UIO) Support
+---------------------------
+CXL 3.2 and PCIe 6.1 introduce Unordered I/O (UIO), which allows memory
+transactions to bypass strict ordering requirements, potentially improving
+performance for certain workloads. UIO support requires three coordinated
+features:
+
+1. **256-byte Flit Mode**: Enabled via ``x-256b-flit=on`` on all ports
+   and devices in the UIO path
+2. **Streamlined Virtual Channel (SVC)**: Enabled via ``x-svc`` on all ports
+   and CXL type3 device. SVC is a PCIe 6.1 extended capability
+   (PCIe 6.4 section 7.9.29) that provides dedicated virtual channels for
+   UIO traffic.
+3. **Device 3 Extended Capability**: Enabled via ``x-uio`` on CXL type3
+   device. Device3 is a PCIe 6.4 extended capability (PCIe 6.4 section 7.7.9)
+   that advertises a device's UIO requester/completer support.
+
+An example of 3 type3 devices with volatile memory below a switch. All the devices
+and ports support Unordered IO (UIO) feature which requires Streamlined Virtual Channel
+(SVC) capability, Device3 capability and requires operating in Flit mode::
+
+  qemu-system-x86_64 -M q35,cxl=on -m 4G,maxmem=8G,slots=8 -smp 4 \
+  ...
+  -object memory-backend-ram,id=target-mem1,size=512M \
+  -object memory-backend-ram,id=target-mem2,size=512M \
+  -object memory-backend-ram,id=target-mem3,size=512M \
+  -device pxb-cxl,bus_nr=12,bus=pcie.0,id=cxl.1 \
+  -device cxl-rp,port=0,bus=cxl.1,id=rp13,chassis=0,slot=0,\
+         x-svc=on,x-256b-flit=on,x-speed=64,x-width=16 \
+  -device cxl-rp,port=1,bus=cxl.1,id=root_port1,chassis=0,slot=1,\
+         x-svc=on,x-256b-flit=on,x-speed=64,x-width=16 \
+  -device cxl-upstream,port=2,sn=1234,bus=rp13,id=us0,\
+         x-svc=on,x-256b-flit=on,x-speed=64,x-width=16 \
+  -device cxl-downstream,port=0,bus=us0,id=swport0,chassis=0,slot=4,\
+         x-svc=on,x-256b-flit=on,x-speed=64,x-width=16 \
+  -device cxl-downstream,port=1,bus=us0,id=swport1,chassis=0,slot=5,\
+         x-svc=on,x-256b-flit=on,x-speed=64,x-width=16 \
+  -device cxl-downstream,port=3,bus=us0,id=swport2,chassis=0,slot=6,\
+         x-svc=on,x-256b-flit=on,x-speed=64,x-width=16 \
+  -device cxl-type3,id=mem1,bus=swport0,volatile-memdev=target-mem1,\
+         x-uio=on,x-svc=on,x-256b-flit=on,sn=1233 \
+  -device cxl-type3,id=mem2,bus=swport1,volatile-memdev=target-mem2,\
+         x-uio=on,x-svc=on,x-256b-flit=on,sn=1234 \
+  -device cxl-type3,id=mem3,bus=swport2,volatile-memdev=target-mem3,\
+         x-uio=on,x-svc=on,x-256b-flit=on,sn=1235 \
+  -M cxl-fmw.0.targets.0=cxl.1,cxl-fmw.0.size=4G,cxl-fmw.0.interleave-granularity=4k
+
 A simple arm/virt example featuring a single direct connected CXL Type 3
 Volatile Memory device::
 
Signature · 1 line
2.34.1
#20 lore raw

Re: [RFC V2 10/10] cxl: Add documentation for CXL UIO support.

Hi Shrihari,

Quoted 1 line · Shrihari E S

+3. **Device 3 Extended Capability**: Enabled via ``x-uio`` on CXL type3

x-uio on its own doesn't get you Dev3: ct3_realize() adds it only when x-svc and x-256b-flit are set as well. It does set the HDM UIO bit with neither of them, which is the asymmetry I raised on 9/10.

docs/system/devices/cxl.rst use HDM-DB for coherence, which requires operating in Flit mode:: · line 428

Quoted 2 lines
docs/system/devices/cxl.rst +2 -0
+An example of 3 type3 devices with volatile memory below a switch. All the devices
+and ports support Unordered IO (UIO) feature which requires Streamlined Virtual Channel

The topology below this doesn't advertise UIO through the switch. I ran it -- joined onto single lines, the indented continuations don't survive a paste -- and the upstream port reads 0x00000382 in its HDM Decoder Capability register, UIO clear, against 0x00042312 on cxl-rp. That's the gate in 7/10; worth re-running the example once it's settled.

Many thanks,
Junjie

A message that is not in the archive (sent off-list or not yet received): CGME20260826051958epcas5p3db6cf2ef9115168c5d8dcec7bdb9b8c3@epcas5p3.samsung.com

#22 lore raw

v2 cover hw/pci: hw/cxl: Add UIO support in CXL and PCIe stack.

This is the RFC V2 version of "UIO feature enablement in Qemu", By addressing the comments received from the maintainers in the last series [1], this series adds datapath enablement in both PCIe and CXL stack.

Features added in RFC V2,

- Device 3 capability in PCIe stack and hooked it to the CXL type 3 device.
- DVSEC registers related to UIO in CXL stack.
- Modified UIO as a generic capability, while last series binds it to UIO.
- extended SVC capability to CXL type 3 device.

New Properties added,

- x-256b-flit: Apart from the small refactoring added in RFC V1 [1], remains unchanged.

- x-svc : Added currently in all the CXL/PCIe ports and in CXL type 3 device to enable SVC capability in them.

- x-uio : Added in CXL type 3 device so that it can act as a UIO completer. This property enables Device3 capability and is dependent on "x-svc" and "x-256b-flit".

This patch series introduces enumeration, configuration and data plane support for the PCIe Unordered I/O (UIO) feature across standard PCIe and CXL QEMU stack. UIO, is introduced from PCIe 6.1, a transaction layer enhancement that relaxes strict transaction ordering rules and shifts the ordering control from the fabric to the requester device.

This implementation establishes the essential architectural foundation required for the OS to discover, negotiate, and configure UIO capabilities.

UIO in CXL follows CXL.io protocol, where a CXL device can send read/write request to peer .mem device's HDM decoder at fabric level.

      __________________________________________
     |                                          |
     |              CXL Host Bridge             |
     |              (Root Port/RC)              |
     |__________________________________________|
                         |
      ___________________|______________________
     |                                          |
     |               CXL Switch                 |
     |__________________________________________|
        |                  |                 |
     ___|____           ___|_____         ___|____
    |        |         |         |       |        |
    | Dev A  |         | Dev B   |       | Dev C  |
    | (Req)  |         |(Target) |       |(Target)|
    |________|         |_________|       |________|

For example, consider the topology shown above, in that if the requester device (Dev A) wants to read data from the target device (Dev C), then Dev A can prepare UIO command with the targeted HPA of Dev C and can send directly to the Dev C's HDM decoder at switch level. Here the host gets bypassed in the data path. For this to happen, both the devices and the intermediate fabrics should support UIO and its dependencies.

The implementation relies on three key dependencies:
- 256B Flit Mode
- Streamlined Virtual Channel (SVC) Extended Capability (from PCIe 6.1)
- PCIe Device 3 capability

Patch Layout:

Patch 01-02: Refactor 'flitmode' property from PCIESlot to PCIEPort so
that all derived ports (PCIe and CXL) can use it. This is a pure structural refactor with no functional change.

Patch 03-04: Add new properties to PCIe and CXL ports:
- x-256b-flit: Enable 256B flit mode
- x-svc: Enable Streamlined Virtual Channel capability

Patch 05: Wire UIO capability into CXL HDM decoder capability and
control registers per CXL 3.2 8.2.4.20.7. The UIO bit in the capability register is set for CXL Type 3 devices and ports when UIO support is enabled.

Patch 06: Implement the PCIe SVC Extended Capability (PCIe 6.4
7.9.29). This is the largest patch, adding:
- hw/pci/pcie_svc.c: SVC capability management
- include/hw/pci/pcie_svc.h: SVC data structures
- Updated pcie_cap_fill_lnk() for flitmode signaling
- Lifecycle hooks (reset and config_write) for SVC state

Patch 07: Wire SVC initialization into the realize() functions of
PCIe/CXL Root, Upstream, Downstream ports, and The 'uio_capable' flag is set in CXL Ports when both SVC and flitmode are enabled.

Patch 08: Implement the PCIe Device 3 Extended Capability (PCIe 6.4
7.7.9). This adds:
- hw/pci/pcie_dev3.c: Dev3 capability management
- include/hw/pci/pcie_dev3.h: Dev3 data structures
- pcie_link_flit_active() helper for flit mode queries
- Gating logic that ties Dev3 control bits to flit mode

Patch 09: Wire SVC and Dev3 capabilities into the CXL Type 3 device
realize() function, enabling it to act as a UIO completer.

Patch 10: Adds CXL UIO documentation

This series is written on top of:

Base commit: 3c99db7705c1a885c795b7c14f751c7a3ba29ed8
Repository: https://gitlab.com/jic23/qemu
Branch: cxl-2026-01-09-draft

Testing:
This series has been validated through the following:

1. Enumeration testing with Linux kernel
- Verified SVC capability discovery via lspci -vvv
- Verified Dev3 capability advertisement in config space
- Verified HDM decoder UIO bits are set correctly
- Verified capabilities' write_config() path.

2. Dataplane testing:
- Added UIO requester capability to an existing PCIe device and verified DMA data flow using Linux Kernel test driver.

3. Backward compatibility:
- Verified existing CXL Type 3 configurations (without x-svc/x-uio) continue to work unchanged
- Confirmed no regression in standard PCIe enumeration

(The Linux kernel patches will be upstreamed later)

Future work:
1. Emulation of CXL type 2 device; To add a CXL test device with UIO requester capability and verify the CXL-CXL P2P DMA datapath (mimicking CXL type2 to CXL type3 P2P DMA via UIO)

2. Emulation of PCIe UIO test device; To add a PCIe UIO requester test device and verify the PCIe-CXL P2P DMA path (mimicking PCIe GPU to CXL P2P DMA). the CXL type2 to CXL type 3 UIO communication.

Thanks!

References
[1] https://lore.kernel.org/qemu-devel/[email protected]/

Shrihari E S and Dongjoo Seo (9):
hw/pci: Refactor flitmode from PCIESlot to PCIEPort.
hw/pci: Move 'x-256b-flit' property from cxl_root_port to pcie_root_port. hw/pci: Add SVC capability and UIO properties to PCIe ports hw/cxl: Add Streamlined Virtual Channel (SVC) property to CXL ports hw/cxl: Wire UIO capability into HDM decoder and DVSEC registers hw/pci: Add PCIe Streamlined Virtual Channel (SVC) capability. hw/pci: hw/cxl: Wire SVC initialization into port realize functions. hw/pci: Adding Device3 capabilities support
hw/cxl: Wire SVC and Dev3 capability to CXL Type 3 device
cxl: Add documentation for CXL UIO support.

 hw/cxl/cxl-component-utils.c              |  23 ++-
 hw/mem/cxl_type3.c                        |  26 ++-
 hw/pci-bridge/cxl_downstream.c            |  16 +-
 hw/pci-bridge/cxl_root_port.c             |  19 ++-
 hw/pci-bridge/cxl_upstream.c              |  14 +-
 hw/pci-bridge/gen_pcie_root_port.c        |   3 +
 hw/pci-bridge/pci_expander_bridge.c       |   3 +-
 hw/pci-bridge/pcie_root_port.c            |   6 +
 hw/pci-bridge/xio3130_downstream.c        |   5 +
 hw/pci-bridge/xio3130_upstream.c          |  10 ++
 hw/pci/meson.build                        |   2 +-
 hw/pci/pci.c                              |   6 +
 hw/pci/pcie.c                             |  83 +++++++++-
 hw/pci/pcie_dev3.c                        | 163 ++++++++++++++++++
 hw/pci/pcie_svc.c                         | 192 ++++++++++++++++++++++
 include/hw/cxl/cxl_component.h            |   2 +-
 include/hw/cxl/cxl_device.h               |   3 +
 include/hw/pci-bridge/cxl_upstream_port.h |   1 +
 include/hw/pci/pcie.h                     |  13 ++
 include/hw/pci/pcie_dev3.h                |  26 +++
 include/hw/pci/pcie_port.h                |   6 +-
 include/hw/pci/pcie_regs.h                |  85 ++++++++++
 include/hw/pci/pcie_svc.h                 |  26 +++
 23 files changed, 705 insertions(+), 28 deletions(-)
 create mode 100644 hw/pci/pcie_dev3.c
 create mode 100644 hw/pci/pcie_svc.c
 create mode 100644 include/hw/pci/pcie_dev3.h
 create mode 100644 include/hw/pci/pcie_svc.h
Signature · 1 line
2.34.1
#23 lore raw

Re: [RFC V2 0/9] hw/pci: hw/cxl: Add UIO support in CXL and PCIe stack.

Hi Shrihari,

Quoted 5 lines · Shrihari E S

This series is written on top of:

Base commit: 3c99db7705c1a885c795b7c14f751c7a3ba29ed8
Repository: https://gitlab.com/jic23/qemu
Branch: cxl-2026-01-09-draft

Applies clean there and builds; x86_64 cxl-test still passes. I booted the patch-10 example topology and walked the extended capability chains and the HDM decoder registers: the v1 SVC/AER collision is fixed. Two things are broken, both on 7/10: the switch USP never advertises HDM UIO -- its uio gate reads a flitmode field that nothing sets -- so the example topology can't advertise UIO end to end, and "-device ioh3420,x-svc=on" aborts QEMU. Per-patch comments on 2, 3 and 5 through 10.

Testing point 3 looks at whether existing configurations still work; the compat question is whether they still look the same to a guest. 2/10 moves x-256b-flit onto pcie-root-port with the default left on, so every pcie-root-port on q35 comes up in flit mode with no compat entry -- an inbound migration already fails at that patch, I tried it -- and 7/10 shifts the CXL DVSEC offsets whether or not x-svc is set.

Worth adding the PCI maintainers on v3 -- eight of the ten patches touch files under their entry, and both new capabilities land in hw/pci.

I'll re-run the topology once the USP gate is settled.

Many thanks,
Junjie

#24 lore raw

Re: [RFC V2 0/9] hw/pci: hw/cxl: Add UIO support in CXL and PCIe stack.

Quoted 25 lines · Junjie Cao

Hi Shrihari,

Quoted 5 lines · Shrihari E S

This series is written on top of:

Base commit: 3c99db7705c1a885c795b7c14f751c7a3ba29ed8
Repository: https://gitlab.com/jic23/qemu
Branch: cxl-2026-01-09-draft

Applies clean there and builds; x86_64 cxl-test still passes. I booted the patch-10 example topology and walked the extended capability chains and the HDM decoder registers: the v1 SVC/AER collision is fixed. Two things are broken, both on 7/10: the switch USP never advertises HDM UIO -- its uio gate reads a flitmode field that nothing sets -- so the example topology can't advertise UIO end to end, and "-device ioh3420,x-svc=on" aborts QEMU. Per-patch comments on 2, 3 and 5 through 10.

Testing point 3 looks at whether existing configurations still work; the compat question is whether they still look the same to a guest. 2/10 moves x-256b-flit onto pcie-root-port with the default left on, so every pcie-root-port on q35 comes up in flit mode with no compat entry -- an inbound migration already fails at that patch, I tried it -- and 7/10 shifts the CXL DVSEC offsets whether or not x-svc is set.

Quoted 8 lines

Worth adding the PCI maintainers on v3 -- eight of the ten patches touch files under their entry, and both new capabilities land in hw/pci.

I'll re-run the topology once the USP gate is settled.

Many thanks,
Junjie

Hi Junjie,

Thank you for the detailed review and testing. Your feedback helped catch several issues that were missing in this patch series (particularly the derived ports abort scenario). Will address your comments in the next version of this patch series.

Thanks and regards,
Shrihari