PCIe DOE for PCIe and CXL 2.0
cover PCIe DOE for PCIe and CXL 2.0
PCIe Data Object Exchange (DOE) protocol for PCIe and CXL is available
https://gitlab.com/avery-qemu/cxl2.0-v3-doe/
based on Ben Widawsky's CXL QEMU cxl2.0-v3 gitlab branch
https://lore.kernel.org/qemu-devel/[email protected]
which is located at
https://gitlab.com/bwidawsk/qemu
The changes from Ben’s latest cxl-2.0v3 are:
MAINTAINERS | 7 + hw/cxl/cxl-component-utils.c | 151 +++++++++++ hw/mem/cxl_type3.c | 121 +++++++++ hw/pci/meson.build | 1 + hw/pci/pcie.c | 4 +- hw/pci/pcie_doe.c | 360 +++++++++++++++++++++++++ include/hw/cxl/cxl_component.h | 120 +++++++++ include/hw/cxl/cxl_pci.h | 428 ++++++++++++++++++++++++++++++ include/hw/pci/pcie.h | 5 + include/hw/pci/pcie_doe.h | 130 +++++++++ include/hw/pci/pcie_regs.h | 4 + include/standard-headers/linux/pci_regs.h | 3 +- 12 files changed, 1332 insertions(+), 2 deletions(-)
The DOE protocol defines a mailbox method that allows either UEFI or OS methods to read the device and do further setup of ACPI tables, etc.
There are 2 PCIe DOE protocols (PCI-SIG ECN Data Object Exchange (DOE) March 2020)
- Discovery
- Component Measurement (CMA)
And 2 CXL specific ones:
- Compliance Mode (Compute Express Link Specification September 2, 2020
Revision: 2.0, Version 1.0)
- CDAT (Coherent Device Attribute Table (CDAT) Specification
October 2020 Revision 1.02)For CXL, the CDAT table defines the memory device so that UEFI or OS can read it out of device using DOE and then can configure the system’s ACPI SRAT/HMAT tables for system memory, and DEVSEC, Component, and Device registers in CXL
device.
Current version provides fixed CDAT table defined in the CXL Type3 device model. Updates are planned shortly to allow for user to provide CDAT tables through -device option property to vary from run to run. The format will be ASCII with structure/field-values pairs that are read by the device during initialization as shown here:
-device cxl-type3,bus=rp0,memdev=cxl-mem1,id=cxl-pmem0,size=256M,cdat_file=<filename>
For testing, a cxl_app.c user program is enhanced to test all supported DOE protocols which are comprised of sequences of CFG RD/WR to various DOE cap registers. The Linux kernel updates and CXL Type3 Device driver provide sufficient ioctl() support to exercise the DOE protocol. See
https://lore.kernel.org/linux-cxl/[email protected]
======== cxl_app.c
#include <stdio.h>
#include <unistd.h>
#include <sys/ioctl.h>
#include <string.h>
#include <fcntl.h>
#include "cxl_mem_wrapper.h"
const char* help= "\
-h help message\n\ -query IOCTL CXL_MEM_QUERY_COMMANDS\n\ -cfg_rd [0xoffset] IOCTL CXL_MEM_CONFIG_WR Read Hex\n\ -cfg_wr [0xoffset] [0xaddr] IOCTL CXL_MEM_CONFIG_WR Write Hex\n\
-doe_discovery [0xindex=0-3] IOCTL CXL_MEM_CONFIG_WR Write Hex\n\
-doe_cxl [0xprotocol=0 or 2] [0xreq_code=0,1 for protocol=0]\n\
-doe_cma [0xnum = 0] IOCTL CXL_MEM_CONFIG_WR Write Hex\n\
example:\n\
./cxl_app.exe -cfg_rd 0x00\n\
./cxl_app.exe -cfg_wr 0x10 0x00ff0004\n\
./cxl_app.exe -doe_discovery 0\n\
./cxl_app.exe -doe_cxl 2\n\
./cxl_app.exe -doe_cxl 0 1\n\
./cxl_app.exe -doe_cma 0\n\
";
#define READ 0 #define WRITE 1
int FD;
typedef struct cxl_pdev_config cxl_pdev_config;
int cxl_query() {
typedef struct cxl_mem_query_commands cxl_mem_query_commands;
typedef struct cxl_command_info cxl_command_info;
int n_cmds= 0;
// QUERY with n_commands == 0 to get command size
ioctl(FD, CXL_MEM_QUERY_COMMANDS, &n_cmds);
printf("Querying\n");
cxl_mem_query_commands* cmds= malloc(sizeof(cxl_mem_query_commands)
+ n_cmds * sizeof(cxl_command_info));
cmds->n_commands= n_cmds;
// QUERY with command size & pre-alloc memory
ioctl(FD, CXL_MEM_QUERY_COMMANDS, cmds);
for (int i= 0; i < (int)cmds->n_commands; i++) {
printf(" id %d", cmds->commands[i].id);
printf(" flags %d", cmds->commands[i].flags);
printf(" size_in %d", cmds->commands[i].size_in);
printf(" size_out %d\n", cmds->commands[i].size_out);
}
return 0;
};int cxl_config(char* offset_s, char* data_s) {
int offset, data, is_write;
cxl_pdev_config* config_payload= malloc(sizeof(cxl_pdev_config));
if (data_s == NULL)
is_write= 0;
else {
is_write= 1;
data= strtol(data_s, NULL, 16);
}
offset= strtol(offset_s, NULL, 16);
config_payload->offset= offset;
config_payload->data= data;
config_payload->is_write= is_write;
ioctl(FD, CXL_MEM_CONFIG_WR, config_payload);
printf("CONFIG_WR %s [%0x] ", (is_write)? "write" : "read",
config_payload->offset);
for (int i= 0; i < 32; i += 8) printf(" %02x", (config_payload->data >> i) & 0xff);
printf("\n");
return 0;
};void doe_config(cxl_pdev_config* config_payload, uint32_t offset, uint32_t data, uint32_t is_write) {
config_payload->offset= offset;
config_payload->data= data;
config_payload->is_write= is_write;
ioctl(FD, CXL_MEM_CONFIG_WR, config_payload);
printf("CONFIG_%s [%0x] ", (is_write)? "WR": "RD",
config_payload->offset);
for (int i= 0; i < 32; i += 8) printf(" %02x", (config_payload->data >> i) & 0xff);
printf("\n");
};int cxl_doe_cxl(char* entry_s, char* data_s) {
cxl_pdev_config* config_payload= malloc(sizeof(cxl_pdev_config));
uint32_t data_obj[3];
int length;
uint32_t do_type;
uint32_t req_type;
do_type = strtol(entry_s, NULL, 16);
req_type = strtol(data_s, NULL, 16);
printf("DOE TYPE=%0x\n", do_type);
printf("REQ TYPE=%0x\n", req_type);
if (do_type == 0)
data_obj[0] = 0x00001e98;
if (do_type == 2)
data_obj[0] = 0x00021e98;
data_obj[1] = 0x3;
data_obj[2] = req_type;
printf("data_obj[0]=%x\n", data_obj[0]);
printf("data_obj[1]=%x\n", data_obj[1]);
printf("data_obj[2]=%x\n", data_obj[2]);
printf("DOE\n");
doe_config(config_payload, 0x170, data_obj[0], WRITE);
doe_config(config_payload, 0x170, data_obj[1], WRITE);
doe_config(config_payload, 0x170, data_obj[2], WRITE);
/* Set GO */
doe_config(config_payload, 0x168, 0x80000000, WRITE);
/* check status READY is set 16c */
doe_config(config_payload, 0x16c, 0, READ);
/* read cdat response */
doe_config(config_payload, 0x174, 0, READ);
/* write cdat response success */
doe_config(config_payload, 0x174, 0x00000001, WRITE);
/* read cdat response */
doe_config(config_payload, 0x174, 0, READ);
length = config_payload->data & 0x0000ffff;
printf("DOE RSP LENGTH = %0d\n", length);
/* write cdat response success */
doe_config(config_payload, 0x174, 0x00000001, WRITE);
for (int j= 0; j < length-2; j=j+1) {
/* read cdat response */
doe_config(config_payload, 0x174, 0, READ);
/* write cdat response success */
doe_config(config_payload, 0x174, 0x00000001, WRITE);
}
return 0;};
int cxl_doe_cma(char* entry_s, char* data_s) {
cxl_pdev_config* config_payload= malloc(sizeof(cxl_pdev_config));
uint32_t data_obj[3];
data_obj[0] = 0x00010001;
data_obj[1] = 0x3;
data_obj[2] = strtol(entry_s, NULL, 16);
printf("DOE\n");
doe_config(config_payload, 0x170, data_obj[0], WRITE);
doe_config(config_payload, 0x170, data_obj[1], WRITE);
doe_config(config_payload, 0x170, data_obj[2], WRITE);
/* Set GO */
doe_config(config_payload, 0x168, 0x80000000, WRITE);
/* check status READY is set 16c */
doe_config(config_payload, 0x16c, 0, READ);
/* read Discovery response */
doe_config(config_payload, 0x174, 0, READ);
doe_config(config_payload, 0x168, 0x1, WRITE);
/* check status READY is set 16c */
doe_config(config_payload, 0x16c, 0, READ);
};int cxl_doe_discovery(char* entry_s, char* data_s) {
cxl_pdev_config* config_payload= malloc(sizeof(cxl_pdev_config));
uint32_t data_obj[3];
data_obj[0] = 0x00000001;
data_obj[1] = 0x3;
data_obj[2] = strtol(entry_s, NULL, 16);
printf("DOE\n");
doe_config(config_payload, 0x170, data_obj[0], WRITE);
doe_config(config_payload, 0x170, data_obj[1], WRITE);
doe_config(config_payload, 0x170, data_obj[2], WRITE);
/* Set GO */
doe_config(config_payload, 0x168, 0x80000000, WRITE);
/* check status READY is set 16c */
doe_config(config_payload, 0x16c, 0, READ);
/* read Discovery response */
doe_config(config_payload, 0x174, 0, READ);
/* write Discovery response success */
doe_config(config_payload, 0x174, 0x00000001, WRITE);
doe_config(config_payload, 0x174, 0, READ);
/* write Discovery response success */
doe_config(config_payload, 0x174, 0x00000002, WRITE);
doe_config(config_payload, 0x174, 0, READ);
/* write Discovery response success */
doe_config(config_payload, 0x174, 0x00000003, WRITE);
return 0;
};int parse_input(int argc, char** argv) {
int idx;
if (argc < 2) return -1;
for (idx= 0; idx < argc; idx++) {
if (strcmp(argv[idx], "-h") == 0)
return -1;
if (strcmp(argv[idx], "-query") == 0)
return cxl_query();
if (strcmp(argv[idx], "-cfg_rd") == 0)
return cxl_config(argv[idx + 1], NULL);
if (strcmp(argv[idx], "-cfg_wr") == 0)
return cxl_config(argv[idx + 1], argv[idx + 2]);
if (strcmp(argv[idx], "-doe_discovery") == 0)
return cxl_doe_discovery(argv[idx + 1], NULL);
if (strcmp(argv[idx], "-doe_cxl") == 0)
return cxl_doe_cxl(argv[idx + 1], argv[idx + 2]);
if (strcmp(argv[idx], "-doe_cma") == 0)
return cxl_doe_cma(argv[idx + 1], NULL);
}
return 0;
};int main(int argc, char** argv) {
int ret;
char* dev_path= "/dev/cxl/mem0";
if ((FD= open(dev_path, O_RDWR)) < 0) {
printf("Open error loc: %s\n", dev_path);
printf("Try sudo %s\n", argv[0]);
exit(0);
}
if ((ret= parse_input(argc, argv)) < 0) {
printf("Please specify input ");
for (int i= 0; i < argc; i++) printf(" %s", argv[i]);;
printf("\n%s\n", help);
}
printf("DOE\n");
close(FD);
exit(0);
}========
The QEMU script to run is
sudo $QEMU_HOME/qemu/build/x86_64-softmmu/qemu-system-x86_64 \
-nic user,hostfwd=tcp::2222-:22,hostfwd=tcp::1234-:1234 \
-machine q35,accel=kvm,nvdimm,cxl \
-m 8192M,slots=4,maxmem=40964M \
-smp 8,sockets=2,cores=2,threads=2 \
-enable-kvm \
-boot order=d \
-k 'en-us' \
-vga virtio \
-drive if=pflash,format=raw,readonly,file=$TIANOCORE_HOME/edk2/Build/OvmfX64/DEBUG_GCC5/FV/OVMF_CODE.fd \
-drive if=pflash,format=raw,file=$TIANOCORE_HOME/edk2/Build/OvmfX64/DEBUG_GCC5/FV/OVMF_VARS.fd \
-drive file=$QCOW_HOME/ubuntu_20.qcow,format=qcow2 \
-object memory-backend-file,id=cxl-mem1,share,mem-path=cxl-type3,size=512M \
-object memory-backend-file,id=cxl-mem2,share,mem-path=cxl-mem2,size=512M" \
-device pxb-cxl,id=cxl.0,bus=pcie.0,bus_nr=52,uid=0,len-window-base=1,window-base[0]=0x4c0000000,memdev[0]=cxl-mem1 \
-device cxl-rp,id=rp0,bus=cxl.0,addr=0.0,chassis=0,slot=0 \
-device cxl-type3,bus=rp0,memdev=cxl-mem1,id=cxl-pmem0,size=256M,lsa=cxl-mem2 \
-object memory-backend-ram,id=mem0,size=2048M \
-numa node,nodeid=0,memdev=mem0, \
-numa cpu,node-id=0,socket-id=0 \
-object memory-backend-ram,id=mem1,size=2048M \
-numa node,nodeid=1,memdev=mem1, \
-numa cpu,node-id=1,socket-id=1 \
-object memory-backend-ram,id=mem2,size=2048M \
-numa node,nodeid=2,memdev=mem2, \
-object memory-backend-ram,id=mem3,size=2048M \
-numa node,nodeid=3,memdev=mem3, \
-numa node,nodeid=4, \
-object memory-backend-file,id=nvmem0,share,mem-path=nvdimm-0,size=16384M,align=1G \
-device nvdimm,memdev=nvmem0,id=nv0,label-size=2M,node=4 \
-numa node,nodeid=5, \
-object memory-backend-file,id=nvmem1,share,mem-path=nvdimm-1,size=16384M,align=1G \
-device nvdimm,memdev=nvmem1,id=nv1,label-size=2M,node=5 \
-numa dist,src=0,dst=0,val=10 \
-numa dist,src=0,dst=1,val=21 \
-numa dist,src=0,dst=2,val=12 \
-numa dist,src=0,dst=3,val=21 \
-numa dist,src=0,dst=4,val=17 \
-numa dist,src=0,dst=5,val=28 \
-numa dist,src=1,dst=1,val=10 \
-numa dist,src=1,dst=2,val=21 \
-numa dist,src=1,dst=3,val=12 \
-numa dist,src=1,dst=4,val=28 \
-numa dist,src=1,dst=5,val=17 \
-numa dist,src=2,dst=2,val=10 \
-numa dist,src=2,dst=3,val=21 \
-numa dist,src=2,dst=4,val=28 \
-numa dist,src=2,dst=5,val=28 \ -numa dist,src=3,dst=3,val=10 \
-numa dist,src=3,dst=4,val=28 \
-numa dist,src=3,dst=5,val=28 \
-numa dist,src=4,dst=4,val=10 \
-numa dist,src=4,dst=5,val=28 \
-numa dist,src=5,dst=5,val=10
diff --git a/MAINTAINERS b/MAINTAINERS
index 981dc92e25..4fb865e0b3 100644
MAINTAINERS +7 -0
@@ -1655,6 +1655,13 @@ F: docs/pci* F: docs/specs/*pci* F: default-configs/pci.mak +PCIE DOE +M: Huai-Cheng Kuo <[email protected]> +M: Chris Browy <[email protected]> +S: Supported +F: include/hw/pci/pcie_doe.h +F: hw/pci/pcie_doe.c + ACPI/SMBIOS M: Michael S. Tsirkin <[email protected]> M: Igor Mammedov <[email protected]>
hw/cxl/cxl-component-utils.c +151 -0
154 lines not on this page · raw message
hw/mem/cxl_type3.c +121 -0
139 lines not on this page · raw message
hw/pci/meson.build +1 -0
@@ -12,6 +12,7 @@ pci_ss.add(files( # allow plugging PCIe devices into PCI buses, include them even if # CONFIG_PCI_EXPRESS=n. pci_ss.add(files('pcie.c', 'pcie_aer.c')) +pci_ss.add(files('pcie.c', 'pcie_doe.c')) softmmu_ss.add(when: 'CONFIG_PCI_EXPRESS', if_true: files('pcie_port.c', 'pcie_host.c')) softmmu_ss.add_all(when: 'CONFIG_PCI', if_true: pci_ss)
hw/pci/pcie.c +2 -1
@@ -19,6 +19,8 @@*/ #include "qemu/osdep.h" +#include "qemu/log.h" +#include "qemu/error-report.h" #include "qapi/error.h" #include "hw/mem/memory-device.h" #include "hw/pci/pci_bridge.h"
@@ -735,7 +737,7 @@ void pcie_cap_slot_write_config(PCIDevice *dev,hotplug_event_notify (dev); -/* + /* * 6.7.3.2 Command Completed Events * * Software issues a command to a hot-plug capable Downstream Port by
hw/pci/pcie_doe.c new file +360 -0
360 lines not on this page · raw message
include/hw/cxl/cxl_component.h +120 -0
136 lines not on this page · raw message
include/hw/cxl/cxl_pci.h +428 -0
440 lines not on this page · raw message
include/hw/pci/pcie.h +5 -0
@@ -25,6 +25,7 @@#include "hw/pci/pcie_regs.h" #include "hw/pci/pcie_aer.h" #include "hw/hotplug.h" +#include "hw/pci/pcie_doe.h" typedef enum { /* for attention and power indicator */
@@ -81,6 +82,10 @@ struct PCIExpressDevice {/* ACS */ uint16_t acs_cap; + +/* DOE */ +uint16_t doe_cap; +DOEState doe_state; };#define COMPAT_PROP_PCP "power_controller_present"
include/hw/pci/pcie_doe.h new file +130 -0
130 lines not on this page · raw message
include/hw/pci/pcie_regs.h +4 -0
@@ -179,4 +179,8 @@ typedef enum PCIExpLinkWidth {#define PCI_ACS_VER 0x1#define PCI_ACS_SIZEOF 8 +/* DOE Capability Register Fields */ +#define PCI_DOE_VER 0x1 +#define PCI_DOE_SIZEOF 24 +#endif /* QEMU_PCIE_REGS_H */
include/standard-headers/linux/pci_regs.h +2 -1
@@ -730,7 +730,8 @@#define PCI_EXT_CAP_ID_DVSEC 0x23/* Designated Vendor-Specific */ #define PCI_EXT_CAP_ID_DLF 0x25/* Data Link Feature */ #define PCI_EXT_CAP_ID_PL_16GT 0x26/* Physical Layer 16.0 GT/s */ -#define PCI_EXT_CAP_ID_MAX PCI_EXT_CAP_ID_PL_16GT +#define PCI_EXT_CAP_ID_DOE 0x2E/* Data Object Exchange */ +#define PCI_EXT_CAP_ID_MAX PCI_EXT_CAP_ID_DOE#define PCI_EXT_CAP_DSN_SIZEOF 12#define PCI_EXT_CAP_MCAST_ENDPOINT_SIZEOF 40
On Tue, 2 Feb 2021 15:43:28 -0500
Chris Browy <[email protected]> wrote:
Hi Chris,
Whilst I appreciate that this is very much an RFC and so not in the form you would eventually aim to present it in, please look for a v2 to break this into a series of functionally separate patches. Probably.
1. Introduce DOE support with no users - probably including the discovery protocol
2. CMA support
3. CDAT support for CXL
4. Compliance part.
It's also well worth jumping through the hoops needed to get a git send-email workflow up and running as you seem to have had some trouble with getting the thread to send in one go etc.
Clearly we now have two possible implementations for this functionality. Personally I don't care which one we take forwards - if nothing else the exercise has highlighted some disagreements in spec interpretation that need clearing up. I've mailed one big one to the SSWG list today.
I found a few things I definitely got wrong as well whilst reading this :) Always advantages in having multiple implementations given we don't have hardware yet.
Jonathan
Quoted 68 lines
MAINTAINERS +7
-0@@ -1655,6 +1655,13 @@ F: docs/pci* F: docs/specs/*pci* F: default-configs/pci.mak +PCIE DOE +M: Huai-Cheng Kuo <[email protected]> +M: Chris Browy <[email protected]> +S: Supported +F: include/hw/pci/pcie_doe.h +F: hw/pci/pcie_doe.c + ACPI/SMBIOS M: Michael S. Tsirkin <[email protected]> M: Igor Mammedov <[email protected]>hw/cxl/cxl-component-utils.c +42
-0@@ -195,3 +195,154 @@ void cxl_component_create_dvsec(CXLComponentState *cxl, uint16_t length,range_init_nofail (&cxl->dvsecs[type], cxl->dvsec_offset, length); cxl->dvsec_offset += length; } + +uint32_t cxl_doe_compliance_init (CXLComponentState *cxl_cstate) +{ +PCIDevice *pci_dev = cxl_cstate->pdev; +uint32_t req; +uint32_t byte_cnt =0 ; + +DOE_DBG (">> %s\n" , __func__); + + req = ((struct cxl_compliance_mode_cap *)pcie_doe_get_req (pci_dev)) + ->req_code; +switch (req) { +case CXL_COMP_MODE_CAP : + byte_cnt =sizeof (struct cxl_compliance_mode_cap_rsp ); + cxl_cstate->doe_resp.cap_rsp.header.vendor_id =CXL_VENDOR_ID ; + cxl_cstate->doe_resp.cap_rsp.header.doe_type =CXL_DOE_COMPLIANCE ; + cxl_cstate->doe_resp.cap_rsp.header.reserved =0x0 ; + cxl_cstate->doe_resp.cap_rsp.header.length = + dwsizeof(struct cxl_compliance_mode_cap_rsp); + cxl_cstate->doe_resp.cap_rsp.rsp_code =0x0 ; + cxl_cstate->doe_resp.cap_rsp.version =0x1 ; + cxl_cstate->doe_resp.cap_rsp.length =0x1c ; + cxl_cstate->doe_resp.cap_rsp.status =0x0 ; + cxl_cstate->doe_resp.cap_rsp.available_cap_bitmask =0x3 ; + cxl_cstate->doe_resp.cap_rsp.enabled_cap_bitmask =0x3 ; +break ; +case CXL_COMP_MODE_STATUS : + byte_cnt =sizeof (struct cxl_compliance_mode_status_rsp ); + cxl_cstate->doe_resp.status_rsp.header.vendor_id =CXL_VENDOR_ID ; + cxl_cstate->doe_resp.status_rsp.header.doe_type =CXL_DOE_COMPLIANCE ; + cxl_cstate->doe_resp.status_rsp.header.reserved =0x0 ; + cxl_cstate->doe_resp.status_rsp.header.length = + dwsizeof(struct cxl_compliance_mode_status_rsp); + cxl_cstate->doe_resp.status_rsp.rsp_code =0x1 ; + cxl_cstate->doe_resp.status_rsp.version =0x1 ; + cxl_cstate->doe_resp.status_rsp.length =0x14 ; + cxl_cstate->doe_resp.status_rsp.cap_bitfield =0x3 ; + cxl_cstate->doe_resp.status_rsp.cache_size =0 ; + cxl_cstate->doe_resp.status_rsp.cache_size_units =0 ; +break ; +default : +break ;
hw/cxl/cxl-component-utils.c void cxl_component_create_dvsec(CXLComponentState *cxl, uint16_t length, · line 249
Quoted 10 lines
hw/cxl/cxl-component-utils.c +10
-0+ } + +DOE_DBG (" REQ=%x, RSP BYTE_CNT=%d\n" , req, byte_cnt); +DOE_DBG ("<< %s\n" , __func__); +return byte_cnt; +} + +void cxl_doe_cdat_init (CXLComponentState *cxl_cstate) +{ +
As noted elsewhere. There will be more than one of some of these...
Quoted 124 lines
hw/cxl/cxl-component-utils.c +99
-0+DOE_DBG (">> %s\n" , __func__); + + cxl_cstate->doe_resp.cdat_rsp.header.vendor_id =CXL_VENDOR_ID ; + cxl_cstate->doe_resp.cdat_rsp.header.doe_type =CXL_DOE_TABLE_ACCESS ; + cxl_cstate->doe_resp.cdat_rsp.header.reserved =0x0 ; + cxl_cstate->doe_resp.cdat_rsp.header.length =0 ; + cxl_cstate->doe_resp.cdat_rsp.rsp_code =0x0 ; + cxl_cstate->doe_resp.cdat_rsp.table_type =0x0 ; + cxl_cstate->doe_resp.cdat_rsp.next_entry_handle =0xffff ; + +/* copy the DSMAS entry */ + cxl_cstate->dsmas.type =CDAT_TYPE_DSMAS ; + cxl_cstate->dsmas.reserved =0x0 ; + cxl_cstate->dsmas.length =0x0 ; + cxl_cstate->dsmas.DSMADhandle =0x0 ; + cxl_cstate->dsmas.flags =0x0 ; + cxl_cstate->dsmas.reserved2 =0x0 ; + cxl_cstate->dsmas.DPA_base =0x0 ; + cxl_cstate->dsmas.DPA_length =0x40000 ; + +/* copy the DSLBIS entry */ + cxl_cstate->dslbis.type =CDAT_TYPE_DSLBIS ; + cxl_cstate->dslbis.reserved =0 ; + cxl_cstate->dslbis.length =16 ; + cxl_cstate->dslbis.handle =0 ; + cxl_cstate->dslbis.flags =0 ; + cxl_cstate->dslbis.data_type =0 ; + cxl_cstate->dslbis.reserved2 =0 ; + cxl_cstate->dslbis.entry_base_unit =0 ; + cxl_cstate->dslbis.entry[0 ] =0 ; + cxl_cstate->dslbis.entry[1 ] =0 ; + cxl_cstate->dslbis.entry[2 ] =0 ; + cxl_cstate->dslbis.reserved3 =0 ; + +/* copy the DSMSCIS entry */ + cxl_cstate->dsmscis.type =CDAT_TYPE_DSMSCIS ; + cxl_cstate->dsmscis.reserved =0 ; + cxl_cstate->dsmscis.length =20 ; + cxl_cstate->dsmscis.DSMASH_handle =0 ; + cxl_cstate->dsmscis.reserved2[0 ] =0 ; + cxl_cstate->dsmscis.reserved2[1 ] =0 ; + cxl_cstate->dsmscis.reserved2[2 ] =0 ; + cxl_cstate->dsmscis.memory_side_cache_size =0 ; + cxl_cstate->dsmscis.cache_attributes =0 ; + +/* copy the DSIS entry */ + cxl_cstate->dsis.type =CDAT_TYPE_DSIS ; + cxl_cstate->dsis.reserved =0 ; + cxl_cstate->dsis.length =8 ; + cxl_cstate->dsis.flags =0 ; + cxl_cstate->dsis.handle =0 ; + cxl_cstate->dsis.reserved2 =0 ; + cxl_cstate->dsis.DPA_offset =0 ; + cxl_cstate->dsis.DPA_length =0 ; + +/* copy the DSEMTS entry */ + cxl_cstate->dsemts.type =CDAT_TYPE_DSEMTS ; + cxl_cstate->dsemts.reserved =0 ; + cxl_cstate->dsemts.length =24 ; + cxl_cstate->dsemts.DSMAS_handle =0 ; + cxl_cstate->dsemts.EFI_memory_type_attr =0 ; + cxl_cstate->dsemts.reserved2 =0 ; + +/* copy the SSLBE entry */ + cxl_cstate->sslbe.port_x_id =0 ; + cxl_cstate->sslbe.port_y_id =0 ; + cxl_cstate->sslbe.latency_bandwidth =0 ; + cxl_cstate->sslbe.reserved =0 ; + +/* copy the SSLBIS entry */ + cxl_cstate->sslbis.type =CDAT_TYPE_SSLBIS ; + cxl_cstate->sslbis.reserved =0 ; + cxl_cstate->sslbis.length =0 ; + cxl_cstate->sslbis.data_type =0 ; + cxl_cstate->sslbis.reserved2[0 ] =0 ; + cxl_cstate->sslbis.reserved2[1 ] =0 ; + cxl_cstate->sslbis.reserved2[2 ] =0 ; + cxl_cstate->sslbis.cdat_sslbe_array[0 ] = cxl_cstate->sslbe; + +DOE_DBG ("<< %s\n" , __func__); +} + +/* +* Helper to creates a DOE header for a CXL entity. The caller is responsible +* for tracking the valid offset. +* +* This function will build the DOE header on behalf of the caller and then +* copy in the remaining bits. +*/ +void cxl_component_create_doe (CXLComponentState *cxl,uint16_t offset) +{ +PCIDevice *pdev = cxl->pdev; + +pcie_cap_doe_init (pdev, offset); +pcie_doe_register_protocol (pdev,CXL_VENDOR_ID ,CXL_DOE_COMPLIANCE , + cxl_doe_compliance_rsp); +pcie_doe_register_protocol (pdev,CXL_VENDOR_ID ,CXL_DOE_TABLE_ACCESS , + cxl_doe_cdat_rsp); +}hw/mem/cxl_type3.c +17
-0@@ -14,6 +14,123 @@#include "sysemu/hostmem.h" #include "hw/cxl/cxl.h" +bool cxl_doe_compliance_rsp (PCIDevice *pci_dev) +{ +CXLComponentState *cxl_cstate = &CT3 (pci_dev)->cxl_cstate; +uint32_t byte_cnt; +uint32_t dw_cnt; + +DOE_DBG (">> %s\n" , __func__); + + byte_cnt =cxl_doe_compliance_init (cxl_cstate); + dw_cnt = byte_cnt /4 ; +memcpy (pci_dev->exp.doe_state.read_mbox, + cxl_cstate->doe_resp.data_byte, byte_cnt); + + pci_dev->exp.doe_state.read_mbox_len += dw_cnt; +DOE_DBG (" read_mbox_len=%x\n" , + pci_dev->exp.doe_state.read_mbox_len); +
hw/mem/cxl_type3.c line 43
Quoted 10 lines
hw/mem/cxl_type3.c +10
-0+DOE_DBG (" RD MBOX[%d]=%x\n" , dw_cnt -1 , + *(pci_dev->exp.doe_state.read_mbox + dw_cnt -1 )); + +DOE_DBG ("<< %s\n" , __func__); + +return DOE_SUCCESS ; +} + +bool cxl_doe_cdat_rsp (PCIDevice *pci_dev) +{
So this is the bit I've queried with SSWG. Not clear to me whether we are supposed to be using the entryHandle value in request for anything. I was assuming it was to allow us to get one entry at a time (I'd flagged it as something to check though as tricky to tell!)
hw/mem/cxl_type3.c line 50
Quoted 7 lines
hw/mem/cxl_type3.c +7
-0+CXLComponentState *cxl_cstate = &CT3 (pci_dev)->cxl_cstate; +uint32_t cnt; +uint32_t dw_cnt; + +DOE_DBG (">> %s\n" , __func__); + + cnt =sizeof (struct cxl_cdat_rsp );
Perhaps better to set cnt after writing (for consistency with below). Also, use cnt = sizeof(cxl_cstate->doe_resp.cdat_rsp);
Quoted 59 lines
hw/mem/cxl_type3.c +59
-0+cxl_doe_cdat_init (cxl_cstate); +memcpy (pci_dev->exp.doe_state.read_mbox, + cxl_cstate->doe_resp.data_byte, cnt); + +/* copy the DSMAS entry */ +memcpy (pci_dev->exp.doe_state.read_mbox + cnt /4 , + &cxl_cstate->dsmas,sizeof (struct cdat_dsmas )); + cnt +=sizeof (struct cdat_dsmas ); + +/* copy the DSLBIS entry */ +memcpy (pci_dev->exp.doe_state.read_mbox + cnt /4 , + &cxl_cstate->dslbis,sizeof (struct cdat_dslbis )); + cnt +=sizeof (struct cdat_dslbis ); + +/* copy the DSMSCIS entry */ +memcpy (pci_dev->exp.doe_state.read_mbox + cnt /4 , + &cxl_cstate->dsmscis,sizeof (struct cdat_dsmscis )); + cnt +=sizeof (struct cdat_dsmscis ); + +/* copy the DSIS entry */ +memcpy (pci_dev->exp.doe_state.read_mbox + cnt /4 , + &cxl_cstate->dsis,sizeof (struct cdat_dsis )); + cnt +=sizeof (struct cdat_dsis ); + +/* copy the DSEMTS entry */ +memcpy (pci_dev->exp.doe_state.read_mbox + cnt /4 , + &cxl_cstate->dsemts,sizeof (struct cdat_dsemts )); + cnt +=sizeof (struct cdat_dsemts ); + +/* copy the SSLBE entry */ + +/* copy the SSLBIS entry */ + +memcpy (pci_dev->exp.doe_state.read_mbox + cnt /4 , + &cxl_cstate->sslbis,4 ); + cnt +=4 ; + dw_cnt = cnt /4 ; + pci_dev->exp.doe_state.read_mbox_len += dw_cnt; +DOE_DBG (" read_mbox_len=%x\n" , + pci_dev->exp.doe_state.read_mbox_len); +memcpy (pci_dev->exp.doe_state.read_mbox +1 , &dw_cnt,4 ); +DOE_DBG (" RD MBOX[0]=%x" , *(pci_dev->exp.doe_state.read_mbox)); +DOE_DBG (" RD MBOX[%d]=%x\n" , + dw_cnt -1 , *(pci_dev->exp.doe_state.read_mbox + dw_cnt -1 )); +for (int i =0 ; i < dw_cnt; i++) +DOE_DBG ("\033[31m RD MBOX[%d]=%x\033[m\n" , i, + pci_dev->exp.doe_state.read_mbox[i]); + +DOE_DBG ("<< %s\n" , __func__); + +return DOE_SUCCESS ; +} + +static uint32_t ct3d_config_read (PCIDevice *pci_dev, +uint32_t addr,int size) +{ +DOE_DBG (">> %s addr: %" PRIx32" size: %x\n" , __func__, addr, size); + +DOE_DBG ("<< %s\n" , __func__);
hw/mem/cxl_type3.c line 119
Quoted 10 lines
hw/mem/cxl_type3.c +10
-0+return pcie_doe_read_config (pci_dev, addr, size); ; +} + +static void ct3d_config_write (PCIDevice *pci_dev, +uint32_t addr,uint32_t val,int size) +{ +DOE_DBG (">> %s\n" , __func__); +DOE_DBG (" addr=%x, val=%x, size=%x\n" , addr, val, size); + +pcie_doe_write_config (pci_dev, addr, val, size);
Useful to know if the write has 'hit' for doe as no point in trying other types once we introduce them. Should also do the default_config_write if not to make sure we can write the other config space elements.
hw/mem/cxl_type3.c line 129
Quoted 10 lines
hw/mem/cxl_type3.c +10
-0+ +DOE_DBG ("<< %s\n" , __func__); +} + +static void build_doe (CXLType3Dev *ct3d) +{ +CXLComponentState *cxl_cstate = &ct3d->cxl_cstate; + +DOE_DBG (">> %s\n" , __func__); +cxl_component_create_doe (cxl_cstate,0x160 );
This wrapper doesn't seem to add anything. I'd just call cx_component_create_doe(&ct3d->cxl_cstate, 0x160); Also need that offset to be automatically calculated to not flash with anything else.
Quoted 30 lines
hw/mem/cxl_type3.c +8
-0+ +DOE_DBG ("<< %s\n" , __func__); +} +static void build_dvsecs (CXLType3Dev *ct3d) {CXLComponentState *cxl_cstate = &ct3d->cxl_cstate;@@ -239,6 +356,7 @@ static void ct3_realize(PCIDevice *pci_dev, Error **errp)PCI_BASE_ADDRESS_SPACE_MEMORY |PCI_BASE_ADDRESS_MEM_TYPE_64 , &ct3d->cxl_dstate.device_registers); +build_doe (ct3d); }static uint64_t cxl_md_get_addr (const MemoryDeviceState *md)@@ -357,6 +475,9 @@ static void ct3_class_init(ObjectClass *oc, void *data)DeviceClass *dc =DEVICE_CLASS (oc);PCIDeviceClass *pc =PCI_DEVICE_CLASS (oc);MemoryDeviceClass *mdc =MEMORY_DEVICE_CLASS (oc); + + pc->config_write = ct3d_config_write; + pc->config_read = ct3d_config_read;CXLType3Class *cvc =CXL_TYPE3_DEV_CLASS (oc); pc->realize = ct3_realize;hw/pci/meson.build +0
-0
hw/pci/meson.build pci_ss.add(files( · line 15
Quoted 4 lines
hw/pci/meson.build +1
-0@@ -12,6 +12,7 @@ pci_ss.add(files( # allow plugging PCIe devices into PCI buses, include them even if # CONFIG_PCI_EXPRESS=n. pci_ss.add(files('pcie.c', 'pcie_aer.c')) +pci_ss.add(files('pcie.c', 'pcie_doe.c'))
why add pcie.c twice?
Quoted 17 lines
hw/pci/meson.build +0
-0softmmu_ss.add(when: 'CONFIG_PCI_EXPRESS', if_true: files('pcie_port.c', 'pcie_host.c')) softmmu_ss.add_all(when: 'CONFIG_PCI', if_true: pci_ss)hw/pci/pcie.c +2
-0@@ -19,6 +19,8 @@*/ #include "qemu/osdep.h" +#include "qemu/log.h" +#include "qemu/error-report.h" #include "qapi/error.h" #include "hw/mem/memory-device.h" #include "hw/pci/pci_bridge.h"
hw/pci/pcie.c void pcie_cap_slot_write_config(PCIDevice *dev, · line 740
Quoted 5 lines
hw/pci/pcie.c +1
-1@@ -735,7 +737,7 @@ void pcie_cap_slot_write_config(PCIDevice *dev,hotplug_event_notify (dev); -/* +/*
Do a quick read through and make sure to clean out accidental changes like this as they add noise during review.
Quoted 24 lines
hw/pci/pcie.c +0
-0* 6.7.3.2 Command Completed Events * * Software issues a command to a hot-plug capable Downstream Port by hw/pci/pcie_doe.c new file +15
-0@@ -0,0 +1,360 @@ +#include "qemu/osdep.h" +#include "qemu/log.h" +#include "qemu/error-report.h" +#include "qapi/error.h" +#include "qemu/range.h" +#include "hw/pci/pci.h" +#include "hw/pci/pcie.h" +#include "hw/pci/pcie_doe.h" + +/* +* DOE Default Protocols (Discovery, CMA) +*/ +/* Discovery Request Object */ +struct doe_discovery { +DOEHeader header;
hw/pci/pcie_doe.c line 25
Quoted 10 lines
hw/pci/pcie_doe.c new file +10
-0+uint8_t index; +uint8_t reserved[3 ]; +} __attribute__((__packed__)); + +/* Discovery Response Object */ +struct doe_discovery_rsp { +DOEHeader header; +uint16_t vendor_id; +uint8_t doe_type; +uint8_t next_index;
We probably need to think about qemu host endianness. This whole spec is defined in terms of DW so need to be careful to ensure the bit order is what we expect.
Quoted 69 lines
hw/pci/pcie_doe.c new file +69
-0+} __attribute__((__packed__)); + +/* Callback for Discovery */ +static bool pcie_doe_discovery_rsp (PCIDevice *dev) +{ +struct doe_discovery *req =pcie_doe_get_req (dev); +uint8_t index = req->index; +DOEProtocol *prot =NULL ; + +/* Length mismatch, discard */ + if (req->header.length < dwsizeof(struct doe_discovery)) { +return DOE_DISCARD ; + } + +if (index < dev->exp.doe_state.protocol_num) { + prot = &dev->exp.doe_state.protocols[index]; + } + +struct doe_discovery_rsp response = { + .header = { + .vendor_id =PCI_DOE_PCI_SIG_VID , + .doe_type =PCI_SIG_DOE_DISCOVERY , + .reserved =0x0 , + .length = dwsizeof(struct doe_discovery_rsp), + }, + .vendor_id = (prot) ? prot->vendor_id :0xFFFF , + .doe_type = (prot) ? prot->doe_type :0xFF , + .next_index = (index +1 ) < dev->exp.doe_state.protocol_num ? + (index +1 ) :0 , + }; + +pcie_doe_set_rsp (dev, &response); + +return DOE_SUCCESS ; +} + +/* Callback for CMA */ +static bool pcie_doe_cma_rsp (PCIDevice *dev) +{ +uint32_t local_val; + + local_val = +pci_get_long (dev->config + dev->exp.doe_cap +PCI_EXP_DOE_STATUS ); + local_val =FIELD_DP32 (local_val,PCI_DOE_CAP_STATUS ,DOE_ERROR ,0x1 ); +pci_set_long (dev->config + dev->exp.doe_cap +PCI_EXP_DOE_STATUS , + local_val); +memset (dev->exp.doe_state.read_mbox,0 , +PCI_DOE_MAX_DW_SIZE *sizeof (uint32_t )); + + dev->exp.doe_state.write_mbox_len =0 ; + +return DOE_DISCARD ; +} + +/* +* DOE Utilities +*/ +static void pcie_doe_reset_mbox (PCIDevice *dev) +{ +DOEState *st = &dev->exp.doe_state; +uint16_t offset = dev->exp.doe_cap; + + st->read_mbox_idx =0 ; + + st->read_mbox_len =0 ; + st->write_mbox_len =0 ; + +memset (st->read_mbox,0 ,PCI_DOE_MAX_DW_SIZE *sizeof (uint32_t )); +memset (st->write_mbox,0 ,PCI_DOE_MAX_DW_SIZE *sizeof (uint32_t ));
hw/pci/pcie_doe.c line 104
Quoted 10 lines
hw/pci/pcie_doe.c new file +10
-0+ +pci_set_word (dev->config + offset +PCI_EXP_DOE_WR_DATA_MBOX ,0 ); +pci_set_word (dev->config + offset +PCI_EXP_DOE_RD_DATA_MBOX ,0 ); +} + +void pcie_cap_doe_init (PCIDevice *dev,uint16_t offset) +{ +pcie_add_capability (dev,PCI_EXT_CAP_ID_DOE ,PCI_DOE_VER , offset, +PCI_DOE_SIZEOF ); + dev->exp.doe_cap = offset;
As mentioned earlier there can be more than one DOE. "It is permitted to implement DOE more than once in a single Function or RCRB."
Quoted 13 lines
hw/pci/pcie_doe.c new file +13
-0+ +/* Set wmask on all registers */ +pci_set_long (dev->wmask + offset +PCI_EXP_DOE_CTRL ,0xffffffff ); +pci_set_long (dev->wmask + offset +PCI_EXP_DOE_WR_DATA_MBOX ,0xffffffff ); +pci_set_long (dev->wmask + offset +PCI_EXP_DOE_RD_DATA_MBOX ,0xffffffff ); + + dev->exp.doe_state.write_mbox = +calloc (PCI_DOE_MAX_DW_SIZE ,sizeof (uint32_t )); +if (dev->exp.doe_state.write_mbox ==NULL ) { +DOE_DBG ("%s could not allocate DOE write mbox memory\n" , __func__); + } + + dev->exp.doe_state.read_mbox =
hw/pci/pcie_doe.c line 127
Quoted 10 lines
hw/pci/pcie_doe.c new file +10
-0+calloc (PCI_DOE_MAX_DW_SIZE ,sizeof (uint32_t )); +if (dev->exp.doe_state.read_mbox ==NULL ) { +DOE_DBG ("%s could not allocate DOE read mbox memory\n" , __func__); + } + + dev->exp.doe_state.protocol_num =0 ; +pcie_doe_register_protocol (dev,PCI_DOE_PCI_SIG_VID , +PCI_SIG_DOE_DISCOVERY , pcie_doe_discovery_rsp); +pcie_doe_register_protocol (dev,PCI_DOE_PCI_SIG_VID , +PCI_SIG_DOE_CMA , pcie_doe_cma_rsp);
No particular reason to assume that having a DOE means we support CMA. Should register this separately.
Quoted 52 lines
hw/pci/pcie_doe.c new file +52
-0+ +pcie_doe_reset_mbox (dev); +} + +void pcie_doe_register_protocol (PCIDevice *dev,uint16_t vendor_id, +uint8_t doe_type,bool (*set_rsp)(PCIDevice *)) +{ +DOEState *st; + + st = &dev->exp.doe_state; + +/* Protocol num should not exceed 256 */ +assert (st->protocol_num <PCI_DOE_PROTOCOL_MAX ); + + st->protocols[st->protocol_num].vendor_id = vendor_id; + st->protocols[st->protocol_num].doe_type = doe_type; + st->protocols[st->protocol_num].set_rsp = set_rsp; + + st->protocol_num++; +} + +void pcie_cap_doe_reset (PCIDevice *dev) +{ +uint16_t offset; + + offset = dev->exp.doe_cap; +if (offset) { +pci_set_word (dev->config + offset +PCI_EXP_DOE_CTRL ,0 ); +pcie_doe_reset_mbox (dev); + } +} + +uint32_t pcie_doe_build_protocol (DOEProtocol *p) +{ +return DATA_OBJ_BUILD_HEADER1 (p->vendor_id, p->doe_type); +} + +/* Return the pointer of DOE request in write mailbox buffer */ +void *pcie_doe_get_req (PCIDevice *dev) +{ +return dev->exp.doe_state.write_mbox; +} + +/* Copy the response to read mailbox buffer */ +void pcie_doe_set_rsp (PCIDevice *dev,void *rsp) +{ +DOEState *st = &dev->exp.doe_state; +uint32_t len = ((DOEHeader *)rsp)->length; + +memcpy (st->read_mbox + st->read_mbox_len, rsp, len *sizeof (uint32_t )); + st->read_mbox_len += len; +}
hw/pci/pcie_doe.c line 189
Quoted 10 lines
hw/pci/pcie_doe.c new file +10
-0+ +static void pcie_doe_write_mbox (DOEState *st,uint32_t val) +{ +memcpy (st->write_mbox + st->write_mbox_len, &val,sizeof (uint32_t )); + +if (st->write_mbox_len ==1 ) { +DOE_DBG (" Capture DOE request DO length = %d\n" , val &0x0003ffff ); + } + + st->write_mbox_len +=1 ;
++;
Quoted 34 lines
hw/pci/pcie_doe.c new file +34
-0+} + +static bool pcie_doe_check_ready (PCIDevice *p) +{ +uint32_t val; + + val =pci_get_long (p->config + p->exp.doe_cap +PCI_EXP_DOE_STATUS ); +DOE_DBG (" STATUS_REG=%x\n" , val); + + val =FIELD_EX32 (val,PCI_DOE_CAP_STATUS ,DATA_OBJ_RDY ); +DOE_DBG (" DATA OBJECT READY=%x\n" , val); + +return val; +} + +static void pcie_doe_set_ready (PCIDevice *p,bool rdy) +{ +uint32_t val; + + val =pci_get_long (p->config + p->exp.doe_cap +PCI_EXP_DOE_STATUS ); + val =FIELD_DP32 (val,PCI_DOE_CAP_STATUS ,DATA_OBJ_RDY , rdy); +pci_set_long (p->config + p->exp.doe_cap +PCI_EXP_DOE_STATUS , val); +} + +static void pcie_doe_set_error (PCIDevice *p,bool err) +{ +uint32_t val; + + val =pci_get_long (p->config + p->exp.doe_cap +PCI_EXP_DOE_STATUS ); + val =FIELD_DP32 (val,PCI_DOE_CAP_STATUS ,DOE_ERROR , err); +pci_set_long (p->config + p->exp.doe_cap +PCI_EXP_DOE_STATUS , val); +} + +uint32_t pcie_doe_read_config(PCIDevice *pci_dev,
hw/pci/pcie_doe.c line 233
Quoted 10 lines
hw/pci/pcie_doe.c new file +10
-0+ uint32_t addr, int size) +{ + uint32_t ret_val; +uint16_t doe_offset = pci_dev->exp.doe_cap; +uint32_t doe_reg = addr - doe_offset; +DOEState *st = &pci_dev->exp.doe_state; + +/* Decode address and process DOE protocol if overlaps DOE cap range */ +if (!range_covers_byte (doe_offset,PCI_DOE_SIZEOF , addr)) + ret_val =pci_default_read_config (pci_dev, addr, size);
As below. This means we can only have 1 DOE and nothing else that needs to do something on config space reads.
So pass whether you got a hit back to the caller.
hw/pci/pcie_doe.c line 239
Quoted 6 lines
hw/pci/pcie_doe.c new file +6
-0+else { +switch (doe_reg) { +case PCI_EXP_DOE_CTRL : +/* must return ABORT=0 and GO=0 */ + ret_val =pci_get_long (pci_dev->config + addr); + ret_val &=PCI_EXP_DOE_CTRL_RMASK ;
I found it easier to never write to the underlying memory, but
instead maintain this state separately. I think it ended up
neater than this reading the value then changing it before return. There aren't many bits of states to maintain.
Quoted 6 lines
hw/pci/pcie_doe.c new file +6
-0+DOE_DBG (" CONTROL REG=%x\n" , ret_val); +break ; +case PCI_EXP_DOE_RD_DATA_MBOX : +/* check that DOE_READY is set */ +if (!pcie_doe_check_ready (pci_dev)) { +/* return 0 if not ready */
hw/pci/pcie_doe.c line 255
Quoted 10 lines
hw/pci/pcie_doe.c new file +10
-0+ ret_val =0 ; +DOE_DBG (" RD MBOX RETURN=%x\n" , ret_val); + }else { +/* deposit next DO DW into read mbox */ +DOE_DBG (" RD MBOX, DATA OBJECT READY," +" CURRENT DO DW OFFSET=%x\n" , + st->read_mbox_idx); + + ret_val = st->read_mbox[st->read_mbox_idx]; +pci_set_long (pci_dev->config + addr, ret_val);
Given we always read this register directly from this function, what is advantage in writing the underlying storage?
Quoted 25 lines
hw/pci/pcie_doe.c new file +25
-0+ +DOE_DBG (" RD MBOX DW=%x\n" , ret_val); +DOE_DBG (" RD MBOX DW OFFSET=%d, RD MBOX LENGTH=%d\n" , + st->read_mbox_idx, st->read_mbox_len); + } +break ; +case PCI_EXP_DOE_WR_DATA_MBOX : + ret_val =0 ; +DOE_DBG (" WR MBOX, local_val =%x\n" , ret_val); +break ; +default : + ret_val =pci_default_read_config (pci_dev, addr, size); +DOE_DBG (" ADDR=%x, VAL =%x\n" , addr, ret_val); +break ; + } +DOE_DBG (" RETURN=%x\n" , ret_val); + } + +return ret_val; +} + +void pcie_doe_write_config (PCIDevice *pci_dev, +uint32_t addr,uint32_t val,int size) +{ +uint16_t doe_offset = pci_dev->exp.doe_cap;
hw/pci/pcie_doe.c line 290
Quoted 10 lines
hw/pci/pcie_doe.c new file +10
-0+uint32_t doe_reg = addr - doe_offset; +DOEState *st = &pci_dev->exp.doe_state; +int p; +bool discard; + +DOE_DBG (" addr=%x, val=%x, size=%x\n" , addr, val, size); + +/* if accessing DOE cap read or write mailbox */ +if (!range_covers_byte (doe_offset,PCI_DOE_SIZEOF , addr)) +pci_default_write_config (pci_dev, addr, val, size);
Don't call this in here. One a few more bits of the device are emulated, it is likely other elements will need custom config space handlers. Use a return value to say if we 'hit' on doe.
Quoted 30 lines
hw/pci/pcie_doe.c new file +30
-0+else { +switch (doe_reg) { +case PCI_EXP_DOE_CTRL : +DOE_DBG (" CONTROL=%x\n" , val); +/* control reg */ +if (FIELD_EX32 (val,PCI_DOE_CAP_CONTROL ,DOE_ABORT )) { +/* If ABORT, clear status reg DOE Error and DOE Ready */ +DOE_DBG (" Setting ABORT\n" ); +pcie_doe_set_ready (pci_dev,0 ); +pcie_doe_set_error (pci_dev,0 ); +pcie_doe_reset_mbox (pci_dev); + }else if (FIELD_EX32 (val,PCI_DOE_CAP_CONTROL ,DOE_GO )) { +DOE_DBG (" CONTROL GO=%x\n" , val); +/* +* Check protocol the incoming request in write_mbox and +* memcpy the corresponding response to read_mbox. +* +* "discard" should be set up if the response callback +* function could not deal with request (e.g. length +* mismatch) or the protocol of request was not found. +*/ + p =0 ; + discard =DOE_DISCARD ; +while (p < st->protocol_num) { +if (st->write_mbox[0 ] == +pcie_doe_build_protocol (&st->protocols[p])) { +/* Found */ +DOE_DBG (" DOE PROTOCOL=%x\n" , st->write_mbox[0 ]); +/* +* Spec:
hw/pci/pcie_doe.c line 330
Quoted 10 lines
hw/pci/pcie_doe.c new file +10
-0+* If the number of DW transferred does not match the +* indicated Length for a data object, then the +* data object must be silently discarded. +*/ +if (st->write_mbox_len == + ((DOEHeader *)pcie_doe_get_req (pci_dev))->length) + discard = st->protocols[p].set_rsp (pci_dev); +break ; + } + p++;
A for loop (with early exit) would be simpler here than while.
Quoted 4 lines
hw/pci/pcie_doe.c new file +4
-0+ } + +/* set DOE Ready */ +if (!discard) {
hw/pci/pcie_doe.c line 344
Quoted 10 lines
hw/pci/pcie_doe.c new file +10
-0+pcie_doe_set_ready (pci_dev,1 ); + }else { +pcie_doe_reset_mbox (pci_dev); + } + } +break ; +case PCI_EXP_DOE_RD_DATA_MBOX : +/* read mbox */ +DOE_DBG (" INCR RD MBOX DO DW OFFSET=%d\n" , st->read_mbox_idx); + st->read_mbox_idx +=1 ;
Ah. I got this bit wrong - had missed that you need to write this to advance the counter. Make sense, but I was clearly dozing when I read that bit of the spec.
Quoted 85 lines
hw/pci/pcie_doe.c new file +16
-0+/* Last DW */ +if (st->read_mbox_idx >= st->read_mbox_len) { +pcie_doe_reset_mbox (pci_dev); +pcie_doe_set_ready (pci_dev,0 ); + } +break ; +case PCI_EXP_DOE_WR_DATA_MBOX : +/* write mbox */ +DOE_DBG (" WR MBOX=%x, DW OFFSET = %d\n" , val, st->write_mbox_len); +pcie_doe_write_mbox (st, val); +break ; +default : +break ; + } + } +}include/hw/cxl/cxl_component.h +54
-0@@ -132,6 +132,23 @@ HDM_DECODER_INIT(0);_Static_assert ((CXL_SNOOP_REGISTERS_OFFSET +CXL_SNOOP_REGISTERS_SIZE ) <0x1000 ,"No space for registers" ); +/* 14.16.4 - Compliance Mode */ +#define CXL_COMP_MODE_CAP 0x0 +#define CXL_COMP_MODE_STATUS 0x1 +#define CXL_COMP_MODE_HALT 0x2 +#define CXL_COMP_MODE_MULT_WR_STREAM 0x3 +#define CXL_COMP_MODE_PRO_CON 0x4 +#define CXL_COMP_MODE_BOGUS 0x5 +#define CXL_COMP_MODE_INJ_POISON 0x6 +#define CXL_COMP_MODE_INJ_CRC 0x7 +#define CXL_COMP_MODE_INJ_FC 0x8 +#define CXL_COMP_MODE_TOGGLE_CACHE 0x9 +#define CXL_COMP_MODE_INJ_MAC 0xa +#define CXL_COMP_MODE_INS_UNEXP_MAC 0xb +#define CXL_COMP_MODE_INJ_VIRAL 0xc +#define CXL_COMP_MODE_INJ_ALMP 0xd +#define CXL_COMP_MODE_IGN_ALMP 0xe + typedef struct component_registers { /* * Main memory region to be registered with QEMU core.@@ -173,6 +190,103 @@ typedef struct cxl_component {struct PCIDevice *pdev; }; }; + +/* +* SW write write mailbox and GO on last DW +* device sets READY of offset DW in DO types to copy +* to read mailbox register on subsequent cfg_read +* of read mailbox register and then on cfg_write to +* denote success read increments offset to next DW +*/ +struct cdat_table_header cdat_header; + +union doe_request_u { +/* Compliance DOE Data Objects Type=0*/ +struct cxl_compliance_mode_cap + mode_cap; +struct cxl_compliance_mode_status + mode_status; +struct cxl_compliance_mode_halt + mode_halt; +struct cxl_compliance_mode_multiple_write_streaming + multiple_write_streaming; +struct cxl_compliance_mode_producer_consumer + producer_consumer; +struct cxl_compliance_mode_inject_bogus_writes + inject_bogus_writes; +struct cxl_compliance_mode_inject_poison + inject_poison; +struct cxl_compliance_mode_inject_crc + inject_crc; +struct cxl_compliance_mode_inject_flow_control + inject_flow_control; +struct cxl_compliance_mode_toggle_cache_flush + toggle_cache_flush; +struct cxl_compliance_mode_inject_mac_delay + inject_mac_delay; +struct cxl_compliance_mode_insert_unexp_mac + insert_unexp_mac; +struct cxl_compliance_mode_inject_viral
include/hw/cxl/cxl_component.h typedef struct cxl_component { · line 239
Quoted 10 lines
include/hw/cxl/cxl_component.h +10
-0+ inject_viral; +struct cxl_compliance_mode_inject_almp + inject_almp; +struct cxl_compliance_mode_ignore_almp + ignore_almp; +struct cxl_compliance_mode_ignore_bit_error + ignore_bit_error; +/* CDAT DOE Data Objects Type=2*/ +struct cxl_cdat cdat; +char data_byte[128 ];
Where does this size come from?
Quoted 29 lines
include/hw/cxl/cxl_component.h +29
-0+ } doe_request; + +union doe_resp_u { +/* Compliance DOE Data Objects Type=0*/ +struct cxl_compliance_mode_cap_rsp + cap_rsp; +struct cxl_compliance_mode_status_rsp + status_rsp; +struct cxl_compliance_mode_halt_rsp + halt_rsp; +struct cxl_compliance_mode_multiple_write_streaming_rsp + multiple_write_streaming_rsp; +struct cxl_compliance_mode_producer_consumer_rsp + producer_consumer_rsp; +struct cxl_compliance_mode_inject_bogus_writes_rsp + inject_bogus_writes_rsp; +struct cxl_compliance_mode_inject_poison_rsp + inject_poison_rsp; +struct cxl_compliance_mode_inject_crc_rsp + inject_crc_rsp; +struct cxl_compliance_mode_inject_flow_control_rsp + inject_flow_control_rsp; +struct cxl_compliance_mode_toggle_cache_flush_rsp + toggle_cache_flush_rsp; +struct cxl_compliance_mode_inject_mac_delay_rsp + inject_mac_delay_rsp; +struct cxl_compliance_mode_insert_unexp_mac_rsp + insert_unexp_mac_rsp; +struct cxl_compliance_mode_inject_viral
include/hw/cxl/cxl_component.h typedef struct cxl_component { · line 278
Quoted 10 lines
include/hw/cxl/cxl_component.h +10
-0+ inject_viral_rsp; +struct cxl_compliance_mode_inject_almp_rsp + inject_almp_rsp; +struct cxl_compliance_mode_ignore_almp_rsp + ignore_almp_rsp; +struct cxl_compliance_mode_ignore_bit_error_rsp + ignore_bit_error_rsp; +/* CDAT DOE Data Objects Type=2*/ +struct cxl_cdat_rsp cdat_rsp; +char data_byte[520 *4 ];
Likewise, where does this come from?
include/hw/cxl/cxl_component.h typedef struct cxl_component { · line 286
Quoted 8 lines
include/hw/cxl/cxl_component.h +8
-0+uint32_t data_dword[520 ]; + } doe_resp; + +/* Table entry types */ +struct cdat_dsmas dsmas; +struct cdat_dslbis dslbis; +struct cdat_dsmscis dsmscis; +struct cdat_dsis dsis;
There will be multiples of some of these.
Quoted 40 lines
include/hw/cxl/cxl_component.h +9
-0+struct cdat_dsemts dsemts; +struct cdat_sslbe sslbe; +struct cdat_sslbis sslbis; } CXLComponentState; void cxl_component_register_block_init(Object *obj,@@ -184,4 +298,10 @@ void cxl_component_register_init_common(uint32_t *reg_state,void cxl_component_create_dvsec (CXLComponentState *cxl_cstate,uint16_t length,uint16_t type,uint8_t rev,uint8_t *body); +void cxl_component_create_doe (CXLComponentState *cxl_cstate,uint16_t offset); + +uint32_t cxl_doe_compliance_init (CXLComponentState *cxl_cstate); +bool cxl_doe_compliance_rsp (PCIDevice *pci_dev); +void cxl_doe_cdat_init (CXLComponentState *cxl_cstate); +bool cxl_doe_cdat_rsp (PCIDevice *pci_dev);#endif include/hw/cxl/cxl_pci.h +9
-0@@ -34,6 +34,15 @@#define REG_LOC_DVSEC_LENGTH 0x24#define REG_LOC_DVSEC_REVID 0 +enum { +CXL_DOE_COMPLIANCE =0 , +CXL_DOE_TABLE_ACCESS =2 , +CXL_DOE_MAX_TYPE +}; + +#define CXL_DOE_PROTOCOL_COMPLIANCE ((CXL_DOE_COMPLIANCE << 16) | CXL_VENDOR_ID) +#define CXL_DOE_PROTOCOL_CDAT ((CXL_DOE_TABLE_ACCESS << 16) | CXL_VENDOR_ID) +enum {PCIE_CXL_DEVICE_DVSEC =0 ,NON_CXL_FUNCTION_MAP_DVSEC =2 ,@@ -54,6 +63,425 @@ struct dvsec_header {_Static_assert (sizeof (struct dvsec_header ) ==10 ,"dvsec header size incorrect" );
include/hw/cxl/cxl_pci.h struct dvsec_header { · line 74
Quoted 10 lines
include/hw/cxl/cxl_pci.h +9
-0+/* CXL 2.0 - 8.1.11 */ +/* +* CDAT - Coherence Device Attributes Table +* Version 1 +*/ + +/* +* CXL 2.0 devices may implement certain DOE Cap +*/
This comment doesn't tell us anything and is in an odd location.
include/hw/cxl/cxl_pci.h struct dvsec_header { · line 87
Quoted 13 lines
include/hw/cxl/cxl_pci.h +13
-0+ +/* +* DOE CDAT Table Protocol +*/ + +/* Data object header */ +struct cdat_table_header { +uint32_t length;/* Length of table in bytes, including this header */ +uint8_t revision;/* ACPI Specification minor version number */ +uint8_t checksum;/* To make sum of entire table == 0 */ +char reserved[6 ]; +char sequence[4 ];/* ASCII table signature */ +} __attribute__((__packed__ ));
Subject to that question on the SSWG reflector around what is actually intended for this protocol and hence under the assumption that we should be generating the full table, then can we use the standard aml building code that used for other ACPI tables? They will build into a GArray and handle things like building the header etc for us. For example see what is done in hw/acpi/hmat.c
Main advantage is that it would be in a familiar form for QEMU developers.
Quoted 28 lines
include/hw/cxl/cxl_pci.h +28
-0+ +/* Values for subtable type in CDAT structures */ + +enum cdat_type { +CDAT_TYPE_DSMAS =0 , +CDAT_TYPE_DSLBIS =1 , +CDAT_TYPE_DSMSCIS =2 , +CDAT_TYPE_DSIS =3 , +CDAT_TYPE_DSEMTS =4 , +CDAT_TYPE_SSLBIS =5 +}; + +/* +* CDAT Structure Subtables +*/ + +struct cxl_cdat { +DOEHeader header; +uint8_t req_code; +uint8_t table_type; +uint16_t entry_handle; +} __attribute__((__packed__ )); + +struct cxl_cdat_rsp { +DOEHeader header; +uint8_t rsp_code; +uint8_t table_type; +uint16_t next_entry_handle;
include/hw/cxl/cxl_pci.h struct dvsec_header { · line 125
Quoted 10 lines
include/hw/cxl/cxl_pci.h +10
-0+uint32_t *entry;/* CDAT Table Entry content */ +} __attribute__((__packed__)); + +struct cdat_dsmas { +uint8_t type; +uint8_t reserved; +uint16_t length; +uint8_t DSMADhandle; +uint8_t flags; +uint16_t reserved2;
Be consistent - sometimes you have char for reserved fields, sometimes an element of the expected size.
Quoted 12 lines
include/hw/cxl/cxl_pci.h +12
-0+uint64_t DPA_base; +uint64_t DPA_length; +} __attribute__((__packed__)); + +struct cdat_dslbis { +uint8_t type; +uint8_t reserved; +uint16_t length; +uint8_t handle; +uint8_t flags; +uint8_t data_type; +uint8_t reserved2;
include/hw/cxl/cxl_pci.h struct dvsec_header { · line 147
Quoted 10 lines
include/hw/cxl/cxl_pci.h +10
-0+uint64_t entry_base_unit; +uint16_t entry[3 ]; +uint16_t reserved3; +} __attribute__((__packed__)); + +struct cdat_dsmscis { +uint8_t type; +uint8_t reserved; +uint16_t length; +uint8_t DSMASH_handle;
DSMAS_handle
Quoted 4 lines
include/hw/cxl/cxl_pci.h +4
-0+char reserved2[3 ]; +uint64_t memory_side_cache_size; +uint32_t cache_attributes; +} __attribute__((__packed__));
include/hw/cxl/cxl_pci.h struct dvsec_header { · line 161
Quoted 10 lines
include/hw/cxl/cxl_pci.h +10
-0+ +struct cdat_dsis { +uint8_t type; +uint8_t reserved; +uint16_t length; +uint8_t flags; +uint8_t handle; +uint16_t reserved2; +uint64_t DPA_offset; +uint64_t DPA_length;
Version I'm looking at for CDAT doesn't have these last two. Rev 1.02 October 2020
include/hw/cxl/cxl_pci.h struct dvsec_header { · line 170
Quoted 9 lines
include/hw/cxl/cxl_pci.h +9
-0+} __attribute__((__packed__)); + +struct cdat_dsemts { +uint8_t type; +uint8_t reserved; +uint16_t length; +uint8_t DSMAS_handle; +uint8_t EFI_memory_type_attr; +uint16_t reserved2;
whereas this one does have DPA_offset and DPA_length.
Quoted 5 lines
include/hw/cxl/cxl_pci.h +5
-0+} __attribute__((__packed__)); + +struct cdat_sslbe { +uint16_t port_x_id; +uint16_t port_y_id;
include/hw/cxl/cxl_pci.h struct dvsec_header { · line 185
Quoted 10 lines
include/hw/cxl/cxl_pci.h +10
-0+uint16_t latency_bandwidth; +uint16_t reserved; +} __attribute__((__packed__)); + +struct cdat_sslbis { +uint8_t type; +uint8_t reserved; +uint16_t length; +uint8_t data_type; +char reserved2[3 ];
Entry base unit?
include/hw/cxl/cxl_pci.h struct dvsec_header { · line 186
Quoted 1 line
include/hw/cxl/cxl_pci.h +1
-0+struct cdat_sslbe cdat_sslbe_array[256 ];
Can we avoid this fixed large length using a variable sized element and appropriate allocator to fit what is actually going in it?
include/hw/cxl/cxl_pci.h struct dvsec_header { · line 195
Quoted 9 lines
include/hw/cxl/cxl_pci.h +9
-0+} __attribute__((__packed__)); + +/* +* DOE Compliance Mode Protocol +* Version 1 +*/ + +/* +* CXL 2.0 devices may implement certain DOE Cap
As before doesn't mean much so drop this comment.
I'd be tempted to break this and the cdat part into different files.
include/hw/cxl/cxl_pci.h struct dvsec_header { · line 198
Quoted 3 lines
include/hw/cxl/cxl_pci.h +3
-0+*/ + +struct cxl_compliance_mode_cap {
This confusingly is called Compliance Mode Availability within a section called Compliance Mode Capability (that has nothing else in it).
Btw it's helpful to add a section or table reference to these. Not too hard to find, but good to be extra nice to a reviewer
Quoted 5 lines
include/hw/cxl/cxl_pci.h +5
-0+DOEHeader header; +uint8_t req_code; +uint8_t version; +uint16_t reserved; +} __attribute__((__packed__));
include/hw/cxl/cxl_pci.h struct dvsec_header { · line 213
Quoted 10 lines
include/hw/cxl/cxl_pci.h +10
-0+ +struct cxl_compliance_mode_cap_rsp { +DOEHeader header; +uint8_t rsp_code; +uint8_t version; +uint8_t length; +uint8_t status; +uint64_t available_cap_bitmask; +uint64_t enabled_cap_bitmask; +} __attribute__((__packed__ ));
Side note - there is an oddity in the spec where address of last field in here has a leading 0 for no apparently reason. One for the tidy up list..
Quoted 153 lines
include/hw/cxl/cxl_pci.h +153
-0+ +struct cxl_compliance_mode_status { +DOEHeader header; +uint8_t req_code; +uint8_t version; +uint16_t reserved; +} __attribute__((__packed__ )); + +struct cxl_compliance_mode_status_rsp { +DOEHeader header; +uint8_t rsp_code; +uint8_t version; +uint8_t length; +uint32_t cap_bitfield; +uint16_t cache_size; +uint8_t cache_size_units; +} __attribute__((__packed__ )); + +struct cxl_compliance_mode_halt { +DOEHeader header; +uint8_t req_code; +uint8_t version; +uint16_t reserved; +} __attribute__((__packed__ )); + +struct cxl_compliance_mode_halt_rsp { +DOEHeader header; +uint8_t rsp_code; +uint8_t version; +uint8_t length; +uint8_t status; +} __attribute__((__packed__ )); + +struct cxl_compliance_mode_multiple_write_streaming { +DOEHeader header; +uint8_t req_code; +uint8_t version; +uint16_t reserved; +uint8_t protocol; +uint8_t virtual_addr; +uint8_t self_checking; +uint8_t verify_read_semantics; +uint8_t num_inc; +uint8_t num_sets; +uint8_t num_loops; +uint8_t reserved2; +uint64_t start_addr; +uint64_t write_addr; +uint64_t writeback_addr; +uint64_t byte_mask; +uint32_t addr_incr; +uint32_t set_offset; +uint32_t pattern_p; +uint32_t inc_pattern_b; +} __attribute__((__packed__ )); + +struct cxl_compliance_mode_multiple_write_streaming_rsp { +DOEHeader header; +uint8_t rsp_code; +uint8_t version; +uint8_t length; +uint8_t status; +} __attribute__((__packed__ )); + +struct cxl_compliance_mode_producer_consumer { +DOEHeader header; +uint8_t req_code; +uint8_t version; +uint16_t reserved; +uint8_t protocol; +uint8_t num_inc; +uint8_t num_sets; +uint8_t num_loops; +uint8_t write_semantics; +char reserved2[3 ]; +uint64_t start_addr; +uint64_t byte_mask; +uint32_t addr_incr; +uint32_t set_offset; +uint32_t pattern; +} __attribute__((__packed__ )); + +struct cxl_compliance_mode_producer_consumer_rsp { +DOEHeader header; +uint8_t rsp_code; +uint8_t version; +uint8_t length; +uint8_t status; +} __attribute__((__packed__ )); + +struct cxl_compliance_mode_inject_bogus_writes { +DOEHeader header; +uint8_t req_code; +uint8_t version; +uint16_t reserved; +uint8_t count; +uint8_t reserved2; +uint32_t pattern; +} __attribute__((__packed__ )); + +struct cxl_compliance_mode_inject_bogus_writes_rsp { +DOEHeader header; +uint8_t rsp_code; +uint8_t version; +uint8_t length; +uint8_t status; +} __attribute__((__packed__ )); + +struct cxl_compliance_mode_inject_poison { +DOEHeader header; +uint8_t req_code; +uint8_t version; +uint16_t reserved; +uint8_t protocol; +} __attribute__((__packed__ )); + +struct cxl_compliance_mode_inject_poison_rsp { +DOEHeader header; +uint8_t rsp_code; +uint8_t version; +uint8_t length; +uint8_t status; +} __attribute__((__packed__ )); + +struct cxl_compliance_mode_inject_crc { +DOEHeader header; +uint8_t req_code; +uint8_t version; +uint16_t reserved; +uint8_t num_bits_flip; +uint8_t num_flits_inj; +} __attribute__((__packed__ )); + +struct cxl_compliance_mode_inject_crc_rsp { +DOEHeader header; +uint8_t rsp_code; +uint8_t version; +uint8_t length; +uint8_t status; +} __attribute__((__packed__ )); + +struct cxl_compliance_mode_inject_flow_control { +DOEHeader header; +uint8_t req_code; +uint8_t version; +uint16_t reserved; +uint8_t inj_flow_control; +} __attribute__((__packed__ )); + +struct cxl_compliance_mode_inject_flow_control_rsp { +DOEHeader header; +uint8_t rsp_code; +uint8_t version;
include/hw/cxl/cxl_pci.h struct dvsec_header { · line 376
Quoted 10 lines
include/hw/cxl/cxl_pci.h +10
-0+uint8_t length; +uint8_t status; +} __attribute__((__packed__)); + +struct cxl_compliance_mode_toggle_cache_flush { +DOEHeader header; +uint8_t req_code; +uint8_t version; +uint16_t reserved; +uint8_t inj_flow_control;
cache_flush_enable perhaps?
Quoted 7 lines
include/hw/cxl/cxl_pci.h +7
-0+} __attribute__((__packed__)); + +struct cxl_compliance_mode_toggle_cache_flush_rsp { +DOEHeader header; +uint8_t rsp_code; +uint8_t version; +uint8_t length;
include/hw/cxl/cxl_pci.h struct dvsec_header { · line 393
Quoted 10 lines
include/hw/cxl/cxl_pci.h +10
-0+uint8_t status; +} __attribute__((__packed__)); + +struct cxl_compliance_mode_inject_mac_delay { +DOEHeader header; +uint8_t req_code; +uint8_t version; +uint16_t reserved; +uint8_t enable; +uint8_t mode;
Perhaps good to have defines for the possible values.
Quoted 16 lines
include/hw/cxl/cxl_pci.h +16
-0+uint8_t delay; +} __attribute__((__packed__)); + +struct cxl_compliance_mode_inject_mac_delay_rsp { +DOEHeader header; +uint8_t rsp_code; +uint8_t version; +uint8_t length; +uint8_t status; +} __attribute__((__packed__ )); + +struct cxl_compliance_mode_insert_unexp_mac { +DOEHeader header; +uint8_t req_code; +uint8_t version; +uint16_t reserved;
include/hw/cxl/cxl_pci.h struct dvsec_header { · line 419
Quoted 10 lines
include/hw/cxl/cxl_pci.h +10
-0+uint8_t opcode; +uint8_t mode; +} __attribute__((__packed__)); + +struct cxl_compliance_mode_insert_unexp_mac_rsp { +DOEHeader header; +uint8_t rsp_code; +uint8_t version; +uint8_t length; +uint8_t status;
Curiously this one doesn't seem to have a status.
Might be subject to errata - I've not checked.
Quoted 14 lines
include/hw/cxl/cxl_pci.h +14
-0+} __attribute__((__packed__)); + +struct cxl_compliance_mode_inject_viral { +DOEHeader header; +uint8_t req_code; +uint8_t version; +uint16_t reserved; +uint8_t protocol; +} __attribute__((__packed__ )); + +struct cxl_compliance_mode_inject_viral_rsp { +DOEHeader header; +uint8_t rsp_code; +uint8_t version;
include/hw/cxl/cxl_pci.h struct dvsec_header { · line 443
Quoted 10 lines
include/hw/cxl/cxl_pci.h +10
-0+uint8_t length; +uint8_t status; +} __attribute__((__packed__)); + +struct cxl_compliance_mode_inject_almp { +DOEHeader header; +uint8_t req_code; +uint8_t version; +uint16_t reserved; +uint8_t opcode;
Why opcode? Doesn't have a name in the spec but this doesn't feel like a particularly obvious one. enable perhaps?
Quoted 22 lines
include/hw/cxl/cxl_pci.h +22
-0+char reserved2[3 ]; +} __attribute__((__packed__)); + +struct cxl_compliance_mode_inject_almp_rsp { +DOEHeader header; +uint8_t rsp_code; +uint8_t version; +uint8_t reserved[6 ]; +} __attribute__((__packed__ )); + +struct cxl_compliance_mode_ignore_almp { +DOEHeader header; +uint8_t req_code; +uint8_t version; +uint16_t reserved; +uint8_t opcode; +uint8_t reserved2[3 ]; +} __attribute__((__packed__ )); + +struct cxl_compliance_mode_ignore_almp_rsp { +DOEHeader header; +uint8_t rsp_code;
include/hw/cxl/cxl_pci.h struct dvsec_header { · line 475
Quoted 10 lines
include/hw/cxl/cxl_pci.h +10
-0+uint8_t version; +uint8_t reserved[6 ]; +} __attribute__((__packed__)); + +struct cxl_compliance_mode_ignore_bit_error { +DOEHeader header; +uint8_t req_code; +uint8_t version; +uint16_t reserved; +uint8_t opcode;
Another one where opcode doesn't feel like the right control.
Quoted 25 lines
include/hw/cxl/cxl_pci.h +9
-0+} __attribute__((__packed__)); + +struct cxl_compliance_mode_ignore_bit_error_rsp { +DOEHeader header; +uint8_t rsp_code; +uint8_t version; +uint8_t reserved[6 ]; +} __attribute__((__packed__ )); +/* * CXL 2.0 devices must implement certain DVSEC IDs, and can [optionally] * implement others. include/hw/pci/pcie.h +1
-0@@ -25,6 +25,7 @@#include "hw/pci/pcie_regs.h" #include "hw/pci/pcie_aer.h" #include "hw/hotplug.h" +#include "hw/pci/pcie_doe.h" typedef enum { /* for attention and power indicator */
include/hw/pci/pcie.h struct PCIExpressDevice { · line 88
Quoted 7 lines
include/hw/pci/pcie.h +4
-0@@ -81,6 +82,10 @@ struct PCIExpressDevice {/* ACS */ uint16_t acs_cap; + +/* DOE */ +uint16_t doe_cap; +DOEState doe_state;
Given a PCI device may may have 0..N Doe mailboxes, I don't think this belongs in the generic PCIExpressDevice structure.
It should be in a device type specific structure.
include/hw/pci/pcie_doe.h
Quoted 9 lines
include/hw/pci/pcie.h +0
-0};#define COMPAT_PROP_PCP "power_controller_present"include/hw/pci/pcie_doe.h new file +0
-0@@ -0,0 +1,130 @@
Don't forget to add a copyright / license header.
include/hw/pci/pcie_doe.h line 8
Quoted 8 lines
include/hw/pci/pcie_doe.h new file +8
-0+#ifndef PCIE_DOE_H +#define PCIE_DOE_H + +#include "qemu/range.h" +#include "qemu/typedefs.h" +#include "hw/register.h" + +/* PCI DOE register defines 7.9.xx */
Reference the ECN here as this will seem oddly vague once the PCI 6.0 spec is out and we actually know the numbers.
include/hw/pci/pcie_doe.h line 12
Quoted 4 lines
include/hw/pci/pcie_doe.h new file +4
-0+/* DOE Capabilities Register */ +#define PCI_EXP_DOE_CAP 0x04 +#define PCI_EXP_DOE_CAP_INTR_SUPP 0x00000001 +#define PCI_EXP_DOE_CAP_INTR (x) ((x) >> 11)
Make it clear this is extracting the interrupt rather than putting it in the register. It is also wrong as it's doing an 11 bit shift rather than a 1 bit shift of an 11 bit value.
Having defined fields below, use them throughout and drop the alternative definitions.
include/hw/pci/pcie_doe.h line 20
Quoted 9 lines
include/hw/pci/pcie_doe.h new file +8
-0+REG32(PCI_DOE_CAP_REG, 0) + FIELD(PCI_DOE_CAP_REG, INTR_SUPP, 0, 1) +FIELD (PCI_DOE_CAP_REG ,DOE_INTR_MSG_NUM ,1 ,11 ) +/* DOE Control Register */ +#define PCI_EXP_DOE_CTRL 0x08 +#define PCI_EXP_DOE_CTRL_ABORT 0x00000001 +#define PCI_EXP_DOE_CTRL_INTR_EN 0x00000002 +#define PCI_EXP_DOE_CTRL_GO 0x80000000
Again, drop these defines and just keep the REG32 / FIELD as they are defining the same things.
Quoted 1 line
+REG32(PCI_DOE_CAP_CONTROL, 0)
Consistent naming. If PCI_DOE_CAP_REG, above then PCI_DOE_CONTROL_REG here (possible PCIE given it's only in the PCI Express spec).
include/hw/pci/pcie_doe.h line 26
Quoted 5 lines
include/hw/pci/pcie_doe.h new file +5
-0+ FIELD(PCI_DOE_CAP_CONTROL, DOE_ABORT, 0, 1) + FIELD(PCI_DOE_CAP_CONTROL, DOE_INTR_EN, 1, 1) + FIELD(PCI_DOE_CAP_CONTROL, DOE_GO, 31, 1) +#define PCI_EXP_DOE_CTRL_RMASK \ + (~(PCI_EXP_DOE_CTRL_ABORT | PCI_EXP_DOE_CTRL_GO))
Build this from the field defines, not the extra version of the same thing.
include/hw/pci/pcie_doe.h line 37
Quoted 11 lines
include/hw/pci/pcie_doe.h new file +11
-0+/* DOE Status Register */ +#define PCI_EXP_DOE_STATUS 0x0c +#define PCI_EXP_DOE_STATUS_BUSY 0x00000001 +#define PCI_EXP_DOE_STATUS_INTR 0x00000002 +#define PCI_EXP_DOE_STATUS_ERR 0x00000004 +#define PCI_EXP_DOE_STATUS_DO_RDY 0x80000000 +REG32(PCI_DOE_CAP_STATUS, 0) + FIELD(PCI_DOE_CAP_STATUS, DOE_BUSY, 0, 1) + FIELD(PCI_DOE_CAP_STATUS, DOE_INTR_STATUS, 1, 1) + FIELD(PCI_DOE_CAP_STATUS, DOE_ERROR, 2, 1) +FIELD (PCI_DOE_CAP_STATUS ,DATA_OBJ_RDY ,31 ,1 )
I'd use a few more blank lines to help readability.
include/hw/pci/pcie_doe.h line 41
Quoted 4 lines
include/hw/pci/pcie_doe.h new file +4
-0+/* DOE Write Data Mailbox Register */ +#define PCI_EXP_DOE_WR_DATA_MBOX 0x10 +/* DOE Read Data Mailbox Register */ +#define PCI_EXP_DOE_RD_DATA_MBOX 0x14
Use REG32 for these as well to maintain consistency.
include/hw/pci/pcie_doe.h line 44
Quoted 3 lines
include/hw/pci/pcie_doe.h new file +3
-0+ +/* Table 7-x2 */ +#define PCI_DOE_PCI_SIG_VID 0x0001
This isn't DOE specific, it's the PCI SIG Vendor ID in general. Put it in a generic header rather than down here in DOE.
include/hw/pci/pcie_doe.h line 48
Quoted 4 lines
include/hw/pci/pcie_doe.h new file +4
-0+#define PCI_SIG_DOE_DISCOVERY 0x00 +#define PCI_SIG_DOE_CMA 0x01 + +#define DATA_OBJ_BUILD_HEADER1 (v, p) ((p << 16) | v)
Probably want masking for those fields or some verification that they fit.
include/hw/pci/pcie_doe.h line 52
Quoted 4 lines
include/hw/pci/pcie_doe.h new file +4
-0+#define PCI_DOE_PROTOCOL_DISCOVERY \ + DATA_OBJ_BUILD_HEADER1(PCI_DOE_PCI_SIG_VID, PCI_SIG_DOE_DISCOVERY) +#define PCI_DOE_PROTOCOL_CMA \ + DATA_OBJ_BUILD_HEADER1(PCI_DOE_PCI_SIG_VID, PCI_SIG_DOE_CMA)
These two defines don't seem to be used. Given that I'd be tempted to put the DATA_OBJ_BUILD_HEADER1 code directly in the one place it's used.
Whilst it isn't strictly a 'register' it seems like it may be useful to just pretend it is to get the convenient macros.
Quoted 4 lines
include/hw/pci/pcie_doe.h new file +4
-0+ +/* Table 7-x3 */ +#define DOE_DISCOVERY_IDX_MASK 0x000000ff +
include/hw/pci/pcie_doe.h line 66
Quoted 10 lines
include/hw/pci/pcie_doe.h new file +10
-0+/* Figure 6-x1 */ +#define DATA_OBJECT_HEADER1_OFFSET 0 +#define DATA_OBJECT_HEADER2_OFFSET 1 +#define DATA_OBJECT_CONTENT_OFFSET 2 + +#define PCI_DOE_MAX_DW_SIZE (1 << 18) +#define PCI_DOE_PROTOCOL_MAX 256 + +#define DOE_SUCCESS 0 +#define DOE_DISCARD 1
These two defines effectively just redefining a bool for whether a given function succeeded. I'd just use true and false directly, or map to more standard error codes
Quoted 4 lines
include/hw/pci/pcie_doe.h new file +4
-0+/* +* DOE Protocol - Data Object +*/ +typedef struct DOEHeader DOEHeader ;
include/hw/pci/pcie_doe.h line 80
Quoted 10 lines
include/hw/pci/pcie_doe.h new file +10
-0+typedef struct DOEProtocol DOEProtocol ; +typedef struct DOEState DOEState ; + +struct DOEHeader { +uint16_t vendor_id; +uint8_t doe_type; +uint8_t reserved; +struct { +uint32_t length:18 ; +uint32_t reserved2:14 ;
Bitfields are notoriously badly defined in the C spec. (layout is compiler specific)
I'll note that the only place I can find this done in Qemu from a quick grep is in the intel iommu drivers. So it might be fine, but given the huge number of places where these would be useful and aren't used, I'd avoid them.
(subject to a QEMU specialist saying they are fine :)
include/hw/pci/pcie_doe.h
Quoted 2 lines
include/hw/pci/pcie_doe.h new file +2
-0+ }; +} __attribute__((__packed__));
Use QEMU_PACKED Seems there are inconsistencies in how different compilers do packing so can't use this gcc attribute directly.
include/hw/pci/pcie_doe.h line 89
Quoted 7 lines
include/hw/pci/pcie_doe.h new file +7
-0+ +/* Protocol infos and rsp function callback */ +struct DOEProtocol { +uint16_t vendor_id; +uint8_t doe_type; + +bool (*set_rsp)(PCIDevice *);
As noted above, I think DOE should not in the PCIDevice structure. To do that you'll need to provide a few more parameters to the callback as the DOEState is not in a known location.
Quoted 4 lines
include/hw/pci/pcie_doe.h new file +4
-0+}; + +struct DOEState { +/* Mailbox buffer for device */
include/hw/pci/pcie_doe.h line 103
Quoted 10 lines
include/hw/pci/pcie_doe.h new file +10
-0+uint32_t *write_mbox; +uint32_t *read_mbox; + +/* Mailbox position indicator */ +uint32_t read_mbox_idx; +uint32_t read_mbox_len; +uint32_t write_mbox_len; + +/* Protocols and its callback response */ +DOEProtocol protocols[PCI_DOE_PROTOCOL_MAX ];
Seems a list would more appropriate for this given max is huge and likely actual elements is small.
Quoted 13 lines
include/hw/pci/pcie_doe.h new file +13
-0+uint32_t protocol_num; +}; + +void pcie_cap_doe_init (PCIDevice *dev,uint16_t offset); +void pcie_cap_doe_reset (PCIDevice *dev); +uint32_t pcie_doe_build_protocol (DOEProtocol *p); +uint32_t pcie_doe_read_config (PCIDevice *pci_dev,uint32_t addr,int size); +void pcie_doe_write_config (PCIDevice *pci_dev,uint32_t addr, +uint32_t val,int size); +void pcie_doe_register_protocol (PCIDevice *dev,uint16_t vendor_id, +uint8_t doe_type, +bool (*set_rsp)(PCIDevice *)); +
include/hw/pci/pcie_doe.h line 126
Quoted 10 lines
include/hw/pci/pcie_doe.h new file +10
-0+/* Utility functions for set_rsp in DOEProtocol */ +void *pcie_doe_get_req (PCIDevice *dev); +void pcie_doe_set_rsp (PCIDevice *dev,void *rsp); + +#define DOE_DEBUG +#ifdef DOE_DEBUG +#define DOE_DBG (...) printf(__VA_ARGS__) +#else +#define DOE_DBG (...) {} +#endif
Either rip this out, or replace with standard QEMU logging. Fine to have it in for development of course, but it's mostly noise once you have things working.
include/hw/pci/pcie_doe.h line 128
Quoted 2 lines
include/hw/pci/pcie_doe.h new file +2
-0+ +#define dwsizeof (s) ((sizeof(s) + 4 - 1) / 4)
Please put this maths inline where you need it or propose a standard definition, not one buried in this DOE code.
Quoted 30 lines
include/hw/pci/pcie_doe.h new file +2
-0+ +#endif /* PCIE_DOE_H */ include/hw/pci/pcie_regs.h +4
-0@@ -179,4 +179,8 @@ typedef enum PCIExpLinkWidth {#define PCI_ACS_VER 0x1#define PCI_ACS_SIZEOF 8 +/* DOE Capability Register Fields */ +#define PCI_DOE_VER 0x1 +#define PCI_DOE_SIZEOF 24 +#endif /* QEMU_PCIE_REGS_H */ include/standard-headers/linux/pci_regs.h +2
-1@@ -730,7 +730,8 @@#define PCI_EXT_CAP_ID_DVSEC 0x23/* Designated Vendor-Specific */ #define PCI_EXT_CAP_ID_DLF 0x25/* Data Link Feature */ #define PCI_EXT_CAP_ID_PL_16GT 0x26/* Physical Layer 16.0 GT/s */ -#define PCI_EXT_CAP_ID_MAX PCI_EXT_CAP_ID_PL_16GT +#define PCI_EXT_CAP_ID_DOE 0x2E/* Data Object Exchange */ +#define PCI_EXT_CAP_ID_MAX PCI_EXT_CAP_ID_DOE#define PCI_EXT_CAP_DSN_SIZEOF 12#define PCI_EXT_CAP_MCAST_ENDPOINT_SIZEOF 40
cover PCIe DOE for PCIe and CXL 2.0
In reply to Chris BrowyPCIe Data Object Exchange (DOE) protocol for PCIe and CXL is available
Quoted 382 lines
https://gitlab.com/avery-qemu/cxl2.0-v3-doe/
based on Ben Widawsky's CXL QEMU cxl2.0-v3 gitlab branch
https://lore.kernel.org/qemu-devel/[email protected]
which is located at
https://gitlab.com/bwidawsk/qemu
The changes from Ben’s latest cxl-2.0v3 are:
MAINTAINERS | 7 + hw/cxl/cxl-component-utils.c | 151 +++++++++++ hw/mem/cxl_type3.c | 121 +++++++++ hw/pci/meson.build | 1 + hw/pci/pcie.c | 4 +- hw/pci/pcie_doe.c | 360 +++++++++++++++++++++++++ include/hw/cxl/cxl_component.h | 120 +++++++++ include/hw/cxl/cxl_pci.h | 428 ++++++++++++++++++++++++++++++ include/hw/pci/pcie.h | 5 + include/hw/pci/pcie_doe.h | 130 +++++++++ include/hw/pci/pcie_regs.h | 4 + include/standard-headers/linux/pci_regs.h | 3 +- 12 files changed, 1332 insertions(+), 2 deletions(-)The DOE protocol defines a mailbox method that allows either UEFI or OS methods to read the device and do further setup of ACPI tables, etc.
There are 2 PCIe DOE protocols (PCI-SIG ECN Data Object Exchange (DOE) March 2020)
- Discovery
- Component Measurement (CMA)And 2 CXL specific ones:
- Compliance Mode (Compute Express Link Specification September 2, 2020Revision: 2.0, Version 1.0) - CDAT (Coherent Device Attribute Table (CDAT) Specification October 2020 Revision 1.02)For CXL, the CDAT table defines the memory device so that UEFI or OS can read it out of device using DOE and then can configure the system’s ACPI SRAT/HMAT tables for system memory, and DEVSEC, Component, and Device registers in CXL
device.Current version provides fixed CDAT table defined in the CXL Type3 device model. Updates are planned shortly to allow for user to provide CDAT tables through -device option property to vary from run to run. The format will be ASCII with structure/field-values pairs that are read by the device during initialization as shown here:
-device cxl-type3,bus=rp0,memdev=cxl-mem1,id=cxl-pmem0,size=256M,cdat_file=<filename>
For testing, a
cxl_app.cuser program is enhanced to test all supported DOE protocols which are comprised of sequences of CFG RD/WR to various DOE cap registers. The Linux kernel updates and CXL Type3 Device driver provide sufficientioctl()support to exercise the DOE protocol. Seehttps://lore.kernel.org/linux-cxl/[email protected]
========
cxl_app.c
#include <stdio.h>
#include <unistd.h>
#include <sys/ioctl.h>
#include <string.h>
#include <fcntl.h>#include "
cxl_mem_wrapper.h"const char* help= "\
(quoted diff) +0
-7-h help message\n\ -query IOCTL CXL_MEM_QUERY_COMMANDS\n\ -cfg_rd [0xoffset] IOCTL CXL_MEM_CONFIG_WR Read Hex\n\ -cfg_wr [0xoffset] [0xaddr] IOCTL CXL_MEM_CONFIG_WR Write Hex\n\ -doe_discovery [0xindex=0-3] IOCTL CXL_MEM_CONFIG_WR Write Hex\n\ -doe_cxl [0xprotocol=0 or 2] [0xreq_code=0,1 for protocol=0]\n\ -doe_cma [0xnum = 0] IOCTL CXL_MEM_CONFIG_WR Write Hex\n\example:\n\
./cxl_app.exe -cfg_rd 0x00\n\
./cxl_app.exe -cfg_wr0x100x00ff0004\n\
./cxl_app.exe -doe_discovery 0\n\
./cxl_app.exe -doe_cxl 2\n\
./cxl_app.exe -doe_cxl 0 1\n\
./cxl_app.exe -doe_cma 0\n\
";
#define READ 0 #define WRITE 1int FD;
typedef structcxl_pdev_configcxl_pdev_config;int cxl_query() { typedef struct cxl_mem_query_commands cxl_mem_query_commands; typedef struct cxl_command_info cxl_command_info; int n_cmds= 0; // QUERY with n_commands == 0 to get command size ioctl(FD, CXL_MEM_QUERY_COMMANDS, &n_cmds); printf("Querying\n"); cxl_mem_query_commands* cmds= malloc(sizeof(cxl_mem_query_commands) + n_cmds * sizeof(cxl_command_info)); cmds->n_commands= n_cmds; // QUERY with command size & pre-alloc memory ioctl(FD, CXL_MEM_QUERY_COMMANDS, cmds); for (int i= 0; i < (int)cmds->n_commands; i++) { printf(" id %d", cmds->commands[i].id); printf(" flags %d", cmds->commands[i].flags); printf(" size_in %d", cmds->commands[i].size_in); printf(" size_out %d\n", cmds->commands[i].size_out); }return 0; };int cxl_config(char* offset_s, char* data_s) { int offset, data, is_write; cxl_pdev_config* config_payload= malloc(sizeof(cxl_pdev_config)); if (data_s == NULL) is_write= 0; else { is_write= 1; data= strtol(data_s, NULL, 16); } offset= strtol(offset_s, NULL, 16); config_payload->offset= offset; config_payload->data= data; config_payload->is_write= is_write; ioctl(FD, CXL_MEM_CONFIG_WR, config_payload); printf("CONFIG_WR %s [%0x] ", (is_write)? "write" : "read", config_payload->offset); for (int i= 0; i < 32; i += 8) printf(" %02x", (config_payload->data >> i) & 0xff); printf("\n");return 0; };void doe_config(cxl_pdev_config* config_payload, uint32_t offset, uint32_t data, uint32_t is_write) { config_payload->offset= offset; config_payload->data= data; config_payload->is_write= is_write; ioctl(FD, CXL_MEM_CONFIG_WR, config_payload); printf("CONFIG_%s [%0x] ", (is_write)? "WR": "RD", config_payload->offset); for (int i= 0; i < 32; i += 8) printf(" %02x", (config_payload->data >> i) & 0xff); printf("\n"); };int cxl_doe_cxl(char* entry_s, char* data_s) { cxl_pdev_config* config_payload= malloc(sizeof(cxl_pdev_config)); uint32_t data_obj[3]; int length; uint32_t do_type; uint32_t req_type; do_type = strtol(entry_s, NULL, 16); req_type = strtol(data_s, NULL, 16); printf("DOE TYPE=%0x\n", do_type); printf("REQ TYPE=%0x\n", req_type); if (do_type == 0) data_obj[0] = 0x00001e98; if (do_type == 2) data_obj[0] = 0x00021e98; data_obj[1] = 0x3; data_obj[2] = req_type; printf("data_obj[0]=%x\n", data_obj[0]); printf("data_obj[1]=%x\n", data_obj[1]); printf("data_obj[2]=%x\n", data_obj[2]);printf("DOE\n");doe_config(config_payload, 0x170, data_obj[0], WRITE); doe_config(config_payload, 0x170, data_obj[1], WRITE); doe_config(config_payload, 0x170, data_obj[2], WRITE); /* Set GO */ doe_config(config_payload, 0x168, 0x80000000, WRITE); /* check status READY is set 16c */ doe_config(config_payload, 0x16c, 0, READ); /* read cdat response */ doe_config(config_payload, 0x174, 0, READ); /* write cdat response success */ doe_config(config_payload, 0x174, 0x00000001, WRITE); /* read cdat response */ doe_config(config_payload, 0x174, 0, READ); length = config_payload->data & 0x0000ffff; printf("DOE RSP LENGTH = %0d\n", length); /* write cdat response success */ doe_config(config_payload, 0x174, 0x00000001, WRITE); for (int j= 0; j < length-2; j=j+1) {/* read cdat response */
doe_config(config_payload, 0x174, 0, READ);/* write cdat response success */ doe_config(config_payload, 0x174, 0x00000001, WRITE); } return 0;};
int cxl_doe_cma(char* entry_s, char* data_s) { cxl_pdev_config* config_payload= malloc(sizeof(cxl_pdev_config)); uint32_t data_obj[3]; data_obj[0] = 0x00010001; data_obj[1] = 0x3; data_obj[2] = strtol(entry_s, NULL, 16); printf("DOE\n"); doe_config(config_payload, 0x170, data_obj[0], WRITE); doe_config(config_payload, 0x170, data_obj[1], WRITE); doe_config(config_payload, 0x170, data_obj[2], WRITE); /* Set GO */ doe_config(config_payload, 0x168, 0x80000000, WRITE); /* check status READY is set 16c */ doe_config(config_payload, 0x16c, 0, READ); /* read Discovery response */ doe_config(config_payload, 0x174, 0, READ);doe_config(config_payload, 0x168, 0x1, WRITE);/* check status READY is set 16c */ doe_config(config_payload, 0x16c, 0, READ); };int cxl_doe_discovery(char* entry_s, char* data_s) { cxl_pdev_config* config_payload= malloc(sizeof(cxl_pdev_config)); uint32_t data_obj[3]; data_obj[0] = 0x00000001; data_obj[1] = 0x3; data_obj[2] = strtol(entry_s, NULL, 16); printf("DOE\n"); doe_config(config_payload, 0x170, data_obj[0], WRITE); doe_config(config_payload, 0x170, data_obj[1], WRITE); doe_config(config_payload, 0x170, data_obj[2], WRITE); /* Set GO */ doe_config(config_payload, 0x168, 0x80000000, WRITE); /* check status READY is set 16c */ doe_config(config_payload, 0x16c, 0, READ); /* read Discovery response */ doe_config(config_payload, 0x174, 0, READ); /* write Discovery response success */ doe_config(config_payload, 0x174, 0x00000001, WRITE);doe_config(config_payload, 0x174, 0, READ);/* write Discovery response success */ doe_config(config_payload, 0x174, 0x00000002, WRITE);doe_config(config_payload, 0x174, 0, READ);/* write Discovery response success */ doe_config(config_payload, 0x174, 0x00000003, WRITE);return 0; };int parse_input(int argc, char** argv) { int idx; if (argc < 2) return -1; for (idx= 0; idx < argc; idx++) { if (strcmp(argv[idx], "-h") == 0) return -1; if (strcmp(argv[idx], "-query") == 0) return cxl_query(); if (strcmp(argv[idx], "-cfg_rd") == 0) return cxl_config(argv[idx + 1], NULL); if (strcmp(argv[idx], "-cfg_wr") == 0) return cxl_config(argv[idx + 1], argv[idx + 2]); if (strcmp(argv[idx], "-doe_discovery") == 0) return cxl_doe_discovery(argv[idx + 1], NULL); if (strcmp(argv[idx], "-doe_cxl") == 0) return cxl_doe_cxl(argv[idx + 1], argv[idx + 2]); if (strcmp(argv[idx], "-doe_cma") == 0) return cxl_doe_cma(argv[idx + 1], NULL); } return 0; };int main(int argc, char** argv) { int ret; char* dev_path= "/dev/cxl/mem0"; if ((FD= open(dev_path, O_RDWR)) < 0) { printf("Open error loc: %s\n", dev_path); printf("Try sudo %s\n", argv[0]); exit(0); } if ((ret= parse_input(argc, argv)) < 0) { printf("Please specify input "); for (int i= 0; i < argc; i++) printf(" %s", argv[i]);; printf("\n%s\n", help); } printf("DOE\n"); close(FD); exit(0); }========
The QEMU script to run is
sudo $QEMU_HOME/qemu/build/x86_64-softmmu/qemu-system-x86_64 \
(quoted diff) +0
-9-nic user,hostfwd=tcp::2222-:22,hostfwd=tcp::1234-:1234 \ -machine q35,accel=kvm,nvdimm,cxl \ -m 8192M,slots=4,maxmem=40964M \ -smp 8,sockets=2,cores=2,threads=2 \ -enable-kvm \ -boot order=d \ -k 'en-us' \ -vga virtio \ -driveif=pflash,format=raw,readonly,file=$TIANOCORE_HOME/edk2/Build/OvmfX64/DEBUG_GCC5/FV/OVMF_CODE.fd \
-drive if=pflash,format=raw,file=$TIANOCORE_HOME/edk2/Build/OvmfX64/DEBUG_GCC5/FV/OVMF_VARS.fd \(quoted diff) +0
-4-drive file=$QCOW_HOME/ubuntu_20.qcow,format=qcow2 \ -object memory-backend-file,id=cxl-mem1,share,mem-path=cxl-type3,size=512M \ -object memory-backend-file,id=cxl-mem2,share,mem-path=cxl-mem2,size=512M" \ -devicepxb-cxl,id=cxl.0,bus=pcie.0,bus_nr=52,uid=0,len-window-base=1,window-base[0]=0x4c0000000,memdev[0]=cxl-mem1 \
(quoted diff) +0
-2-device cxl-rp,id=rp0,bus=cxl.0,addr=0.0,chassis=0,slot=0 \ -device cxl-type3,bus=rp0,memdev=cxl-mem1,id=cxl-pmem0,size=256M,lsa=cxl-mem2\
(quoted diff) +0
-12-object memory-backend-ram,id=mem0,size=2048M \ -numa node,nodeid=0,memdev=mem0, \ -numa cpu,node-id=0,socket-id=0 \ -object memory-backend-ram,id=mem1,size=2048M \ -numa node,nodeid=1,memdev=mem1, \ -numa cpu,node-id=1,socket-id=1 \ -object memory-backend-ram,id=mem2,size=2048M \ -numa node,nodeid=2,memdev=mem2, \ -object memory-backend-ram,id=mem3,size=2048M \ -numa node,nodeid=3,memdev=mem3, \ -numa node,nodeid=4, \ -objectmemory-backend-file,id=nvmem0,share,mem-path=nvdimm-0,size=16384M,align=1G \
(quoted diff) +0
-3-device nvdimm,memdev=nvmem0,id=nv0,label-size=2M,node=4 \ -numa node,nodeid=5, \ -objectmemory-backend-file,id=nvmem1,share,mem-path=nvdimm-1,size=16384M,align=1G \
(quoted diff) +0
-21-device nvdimm,memdev=nvmem1,id=nv1,label-size=2M,node=5 \ -numa dist,src=0,dst=0,val=10 \ -numa dist,src=0,dst=1,val=21 \ -numa dist,src=0,dst=2,val=12 \ -numa dist,src=0,dst=3,val=21 \ -numa dist,src=0,dst=4,val=17 \ -numa dist,src=0,dst=5,val=28 \ -numa dist,src=1,dst=1,val=10 \ -numa dist,src=1,dst=2,val=21 \ -numa dist,src=1,dst=3,val=12 \ -numa dist,src=1,dst=4,val=28 \ -numa dist,src=1,dst=5,val=17 \ -numa dist,src=2,dst=2,val=10 \ -numa dist,src=2,dst=3,val=21 \ -numa dist,src=2,dst=4,val=28 \ -numa dist,src=2,dst=5,val=28 \ -numa dist,src=3,dst=3,val=10 \ -numa dist,src=3,dst=4,val=28 \ -numa dist,src=3,dst=5,val=28 \ -numa dist,src=4,dst=4,val=10 \ -numa dist,src=4,dst=5,val=28 \ -numa dist,src=5,dst=5,val=10