CXL: Standalone switch CCI driver
v4 cover CXL: Standalone switch CCI driver
RFC v4 changes:
- Add missing core/switch-cci.c
- Add missing header io-64-nonatomic-lo-hi.h to core/mbox.c (both Reported-by kernel test robot <[email protected]>)
RFC v3 changes:
- Rebase
- Break out the CXL mailbox infrastructure to allow reuse.
- Cleaner separation of the switch cci driver from the type 3 pci driver.
- Introduction of rough tunneling support.
Note this is a rough and ready RFC rather than a formal proposal. I'm sharing it at this stage to provide a test path for the QEMU rework that enables emulation of the various different types of CCI (PCI Mailbox / MCTP over I2C / Tunneled versions of each). There are open questions from earlier versions around how we manage the security of these. To a certain extent we wouldn't expect to often see a switch-cci connected to a CXL host (except when emulating), but we should probably not assume that.
I would like feedback on the basic form of the refactoring. Doing the rebase this time was painful so I'd rather not have that pain too often in the future.
CXL rev 3.0 introduced the option for a PCI function, intended to sit on an upstream port of a CXL switch. This function provides a mailbox interface similar to that seen on CXL type 3 devices. However, the command set is mostly different and intended for Fabric management. Note however that as we add support for multi headed devices (MHDs) a subset of commands will be available on selected MHD type 3 mailboxes. (tunnelling DCD commands for example)
See: CXL rev 3.0
7.2.9 Switch Mailbox CCI
8.1.13 Switch Malibox CCI Configuration Space Layout 8.2.8.6 Switch Mailbox CCI capability
It is probably relatively unusual that a typical host of CXL devices will have access to the one of these devices, in many cases they will be on a port connected to a BMC or similar. There are a few use cases where the host might be in charge of the configuration.
These are very convenient for testing in conjunction with the QEMU emulation though so far CXL switch and type 3 emulation is in QEMU is not complex enough to make these particular interesting.
This initial support provides only a few commands but I'm sending it out as an RFC to get some input on how we should refactor the CXL core code to support these devices that use some of the provide functionality.
Example command line based on CXL QEMU emulation branch at
https://gitlab.com/jic23/qemu cxl-2023-07-17
qemu-system-aarch64 -M virt,nvdimm=on,gic-version=3,cxl=on -m 4g,maxmem=8G,slots=8 -cpu max -smp 4 \
...
-object memory-backend-file,id=cxl-mem1,share=on,mem-path=/tmp/t3_cxl1.raw,size=256M,align=256M \
-object memory-backend-file,id=cxl-lsa1,share=on,mem-path=/tmp/t3_lsa1.raw,size=1M,align=1M \
-object memory-backend-file,id=cxl-mem2,share=on,mem-path=/tmp/t3_cxl2.raw,size=256M,align=256M \
-object memory-backend-file,id=cxl-lsa2,share=on,mem-path=/tmp/t3_lsa2.raw,size=1M,align=1M \
-device pxb-cxl,bus_nr=12,bus=pcie.0,id=cxl.1,hdm_for_passthrough=true \
-device cxl-rp,port=0,bus=cxl.1,id=cxl_rp_port0,chassis=0,slot=2 \
-device cxl-upstream,bus=cxl_rp_port0,id=us0,addr=0.0,multifunction=on, \
-device cxl-switch-mailbox-cci,bus=cxl_rp_port0,addr=0.1,target=us0 \
-device cxl-downstream,port=0,bus=us0,id=swport0,chassis=0,slot=4 \
-device cxl-downstream,port=1,bus=us0,id=swport1,chassis=0,slot=5 \
-device cxl-type3,bus=swport0,memdev=cxl-mem1,id=cxl-pmem1,lsa=cxl-lsa1,sn=3 \
-device cxl-type3,bus=swport1,memdev=cxl-mem2,id=cxl-pmem2,lsa=cxl-lsa2,sn=4 \
-machine cxl-fmw.0.targets.0=cxl.1,cxl-fmw.0.size=4G,cxl-fmw.0.interleave-granularity=1k
Really hacky test program:
//Testswitch.c
#include <linux/types.h>
#include <stdint.h>
#include <sys/ioctl.h>
#include <stdio.h>
#include <fcntl.h>
#include <stdlib.h>
#define __user
#include "cxl_mem.h" /* include/uapi/linux/cxl_mem.h */
/* Move to appropriate header later */
struct cxl_cmd_infostat_identify_rsp {
uint16_t pcie_vid;
uint16_t pcie_did;
uint16_t pcie_subsys_vid;
uint16_t pcie_subsys_id;
uint64_t sn;
uint8_t max_message_size;
uint8_t component_type;
};
struct cxl_cmd_infostat_get_bg_cmd_sts_rsp {
uint8_t status;
uint8_t rsvd;
uint16_t opcode;
uint16_t returncode;
uint16_t vendor_ext_status;
};
struct cxl_cmd_identify_switch_device_rsp {
uint8_t ingress_port_id;
uint8_t rsvd;
uint8_t num_physical_ports;
uint8_t num_vcs;
uint8_t active_port_bm[0x20];
uint8_t vcs_bm[0x20];
uint16_t total_num_vPPBs;
uint16_t num_bound_vPPBs;
uint8_t num_hdm_decoders;
} __attribute__((packed));
struct cci_message {
uint8_t message_cat;
uint8_t tag;
uint8_t resv;
uint8_t command;
uint8_t command_set;
uint8_t pl_length[3];
uint16_t ret_code;
uint16_t vendor_ext;
uint8_t payload[];
};
struct cxl_cmd_tunnel_command_req {
uint8_t id;
uint8_t target_type;
#define TUNNEL_TARGET_TYPE_PORT_OR_LD 0
#define TUNNEL_TARGET_TYPE_LD_POOL_CCI 1
uint16_t command_size;
struct cci_message message;
};
struct cxl_cmd_tunnel_command_resp {
uint16_t length;
uint16_t resv;
struct cci_message message;
};
int main()
{
struct cxl_send_command cmd = {};
struct cxl_cmd_infostat_identify_rsp is_identify; struct cxl_cmd_identify_switch_device_rsp switch_identify; struct cxl_cmd_infostat_get_bg_cmd_sts_rsp bg_cmd_status; struct cxl_cmd_tunnel_command_req *tunnel_req; struct cxl_cmd_tunnel_command_resp *tunnel_resp; int fd;
int rc, i;
int tun_cmd_size, tun_resp_size;
printf("Starting\n");
fd = open("/dev/cxl/swcci0", O_RDWR);
if (fd < 0) {
printf("could not open file\n");
return 0;
}
cmd.id = CXL_MEM_COMMAND_ID_RAW;
cmd.id = CXL_MEM_COMMAND_ID_INFO_STAT_IDENTIFY;
cmd.out.size = sizeof(is_identify);
cmd.out.payload = (__u64)&is_identify;
rc = ioctl(fd, CXL_MEM_SEND_COMMAND, &cmd);
if (rc) {
printf("rc %d\n", rc);
if (rc < 0)
return rc;
}
printf("Identify on switch:\n");
printf("VID:0x%04x DID:0x%04x\n", is_identify.pcie_vid, is_identify.pcie_did);
printf("Subsys: VID:0x%04x DID:0x%04x\n", is_identify.pcie_subsys_vid, is_identify.pcie_subsys_id);
cmd.id = CXL_MEM_COMMAND_ID_GET_BG_CMD_STATUS;
cmd.out.size = sizeof(bg_cmd_status);
cmd.out.payload = (__u64)&bg_cmd_status;
rc = ioctl(fd, CXL_MEM_SEND_COMMAND, &cmd);
if (rc) {
printf("rc %d\n", rc);
if (rc < 0)
return rc;
}
cmd.id = CXL_MEM_COMMAND_ID_IDENTIFY_SWITCH_DEVICE;
cmd.out.size = sizeof(switch_identify);
cmd.out.payload = (__u64)&switch_identify;
rc = ioctl(fd, CXL_MEM_SEND_COMMAND, &cmd);
if (rc) {
printf("rc %d\n", rc);
if (rc < 0)
return rc;
}printf("Switch indent ingress=%#x #ports=%d\n",
switch_identify.ingress_port_id,
switch_identify.num_physical_ports);
for (i = 0; i < sizeof(switch_identify.active_port_bm); i++) {
int j;
for (j = 0; j < 8; j++) {
if (switch_identify.active_port_bm[i] & 1 << j) {
printf("Port %x active\n", i * 8 + j);
}
}}
tun_cmd_size = sizeof(*tunnel_req); tunnel_req = malloc(tun_cmd_size); tun_resp_size = sizeof(*tunnel_resp) + sizeof(struct cxl_cmd_infostat_identify_rsp); tunnel_resp = malloc(tun_resp_size); tunnel_req->target_type = TUNNEL_TARGET_TYPE_PORT_OR_LD; tunnel_req->command_size = sizeof(tunnel_req->message); tunnel_req->message.command = 0x01; tunnel_req->message.command_set = 0x0; // tunnels within tunnels cmd.id = CXL_MEM_COMMAND_ID_TUNNEL_MANAGEMENT_COMMAND; cmd.in.payload = tunnel_req; cmd.in.size = tun_cmd_size; cmd.out.size = tun_resp_size; cmd.out.payload = tunnel_resp;
rc = ioctl(fd, CXL_MEM_SEND_COMMAND, &cmd);
if (rc) {
printf("rc from tunnel cmd is %d\n", rc);
if (rc < 0)
return rc;
}
printf("message response size %d\n", cmd.out.size);
{
struct cxl_cmd_infostat_identify_rsp *pl = (void *)tunnel_resp->message.payload;
printf("Vendor id : %04x\n", pl->pcie_vid);
printf("Device id : %04x\n", pl->pcie_did);
printf("Subsys vid : %04x\n", pl->pcie_subsys_vid);
printf("Subsys id : %04x\n", pl->pcie_subsys_id);
switch (pl->component_type) {
case 0x00:
printf("Switch!\n");
break;
case 0x03:
printf("Type3!\n");
break;
} }
return 0;
}
Jonathan Cameron (4):
cxl: mbox: Preparatory move of functions to core/mbox.c
cxl: mbox: Factor out the mbox specific data for reuse in switch cci
PCI: Add PCI_CLASS_SERIAL_CXL_SWITCH_CCI class ID to pci_ids.h cxl/pci: Add support for stand alone CXL Switch mailbox CCI
drivers/cxl/Kconfig | 14 ++ drivers/cxl/Makefile | 2 + drivers/cxl/core/Makefile | 1 + drivers/cxl/core/core.h | 4 +- drivers/cxl/core/mbox.c | 446 +++++++++++++++++++++++++++------- drivers/cxl/core/memdev.c | 32 +-- drivers/cxl/core/port.c | 4 + drivers/cxl/core/regs.c | 33 ++- drivers/cxl/core/switch-cci.c | 144 +++++++++++ drivers/cxl/cxl.h | 4 +- drivers/cxl/cxlmbox.h | 49 ++++ drivers/cxl/cxlmem.h | 41 ++-- drivers/cxl/pci.c | 430 +++++++++----------------------- drivers/cxl/pmem.c | 6 +- drivers/cxl/security.c | 13 +- drivers/cxl/switch-cci.c | 181 ++++++++++++++ include/linux/pci_ids.h | 1 + include/uapi/linux/cxl_mem.h | 4 + 18 files changed, 965 insertions(+), 444 deletions(-) create mode 100644 drivers/cxl/core/switch-cci.c create mode 100644 drivers/cxl/cxlmbox.h create mode 100644 drivers/cxl/switch-cci.c
Signature · 1 line
2.39.2
RFC PATCH v4 1/4 cxl: mbox: Preparatory move of functions to core/mbox.c
A later patch will modify this code to separate out the mbox functionality from the memdev. This patch is intended to make that a little more readable.
--- drivers/cxl/core/mbox.c | 261 ++++++++++++++++++++++++++++++++++++++- drivers/cxl/cxlmbox.h | 21 ++++ drivers/cxl/pci.c | 265 +--------------------------------------- 3 files changed, 281 insertions(+), 266 deletions(-)
drivers/cxl/core/mbox.c +259 -2
@@ -1,10 +1,12 @@// SPDX-License-Identifier: GPL-2.0-only /* Copyright(c) 2020 Intel Corporation. All rights reserved. */ +#include <linux/io-64-nonatomic-lo-hi.h> #include <linux/security.h> #include <linux/debugfs.h> #include <linux/ktime.h> #include <linux/mutex.h> #include <asm/unaligned.h> +#include <cxlmbox.h> #include <cxlpci.h> #include <cxlmem.h> #include <cxl.h>
@@ -12,6 +14,14 @@#include "core.h" #include "trace.h" + +/* CXL 2.0 - 8.2.8.4 */ +#define CXL_MAILBOX_TIMEOUT_MS (2 * HZ) + +#define cxl_doorbell_busy (cxlds) \ + (readl((cxlds)->regs.mbox + CXLDEV_MBOX_CTRL_OFFSET) & \ + CXLDEV_MBOX_CTRL_DOORBELL) +static bool cxl_raw_allow_all;/**
@@ -180,6 +190,253 @@ static const char *cxl_mem_opcode_to_name(u16 opcode)return cxl_command_names[c->info.id].name; } +int cxl_pci_mbox_wait_for_doorbell (struct cxl_dev_state *cxlds) +{ +const unsigned long start = jiffies; +unsigned long end = start; + +while (cxl_doorbell_busy (cxlds)) { + end = jiffies; + +if (time_after (end, start +CXL_MAILBOX_TIMEOUT_MS )) { +/* Check again in case preempted before timeout test */ +if (!cxl_doorbell_busy (cxlds)) +break ; +return -ETIMEDOUT ; + } +cpu_relax (); + } + +dev_dbg (cxlds->dev,"Doorbell wait took %dms" , +jiffies_to_msecs (end) -jiffies_to_msecs (start)); +return 0 ; +} +EXPORT_SYMBOL_NS_GPL (cxl_pci_mbox_wait_for_doorbell,CXL ); + +bool cxl_mbox_background_complete (struct cxl_dev_state *cxlds) +{ +u64 reg; + + reg =readq (cxlds->regs.mbox +CXLDEV_MBOX_BG_CMD_STATUS_OFFSET ); +return FIELD_GET (CXLDEV_MBOX_BG_CMD_COMMAND_PCT_MASK , reg) ==100 ; +} +EXPORT_SYMBOL_NS_GPL (cxl_mbox_background_complete,CXL ); + +/** +* __cxl_pci_mbox_send_cmd() - Execute a mailbox command +* @mds: The memory device driver data +* @mbox_cmd: Command to send to the memory device. +* +* Context: Any context. Expects mbox_mutex to be held. +* Return: -ETIMEDOUT if timeout occurred waiting for completion. 0 on success. +* Caller should check the return code in @mbox_cmd to make sure it +* succeeded. +* +* This is a generic form of the CXL mailbox send command thus only using the +* registers defined by the mailbox capability ID - CXL 2.0 8.2.8.4. Memory +* devices, and perhaps other types of CXL devices may have further information +* available upon error conditions. Driver facilities wishing to send mailbox +* commands should use the wrapper command. +* +* The CXL spec allows for up to two mailboxes. The intention is for the primary +* mailbox to be OS controlled and the secondary mailbox to be used by system +* firmware. This allows the OS and firmware to communicate with the device and +* not need to coordinate with each other. The driver only uses the primary +* mailbox. +*/ +static int __cxl_pci_mbox_send_cmd (struct cxl_memdev_state *mds, +struct cxl_mbox_cmd *mbox_cmd) +{ +struct cxl_dev_state *cxlds = &mds->cxlds; +void __iomem *payload = cxlds->regs.mbox +CXLDEV_MBOX_PAYLOAD_OFFSET ; +struct device *dev = cxlds->dev; +u64 cmd_reg, status_reg; +size_t out_len; +int rc; + +lockdep_assert_held (&mds->mbox_mutex); + +/* +* Here are the steps from 8.2.8.4 of the CXL 2.0 spec. +* 1. Caller reads MB Control Register to verify doorbell is clear +* 2. Caller writes Command Register +* 3. Caller writes Command Payload Registers if input payload is non-empty +* 4. Caller writes MB Control Register to set doorbell +* 5. Caller either polls for doorbell to be clear or waits for interrupt if configured +* 6. Caller reads MB Status Register to fetch Return code +* 7. If command successful, Caller reads Command Register to get Payload Length +* 8. If output payload is non-empty, host reads Command Payload Registers +* +* Hardware is free to do whatever it wants before the doorbell is rung, +* and isn't allowed to change anything after it clears the doorbell. As +* such, steps 2 and 3 can happen in any order, and steps 6, 7, 8 can +* also happen in any order (though some orders might not make sense). +*/ + +/* #1 */ +if (cxl_doorbell_busy (cxlds)) { +u64 md_status = +readq (cxlds->regs.memdev +CXLMDEV_STATUS_OFFSET ); + +cxl_cmd_err (cxlds->dev, mbox_cmd, md_status, +"mailbox queue busy" ); +return -EBUSY ; + } + +/* +* With sanitize polling, hardware might be done and the poller still +* not be in sync. Ensure no new command comes in until so. Keep the +* hardware semantics and only allow device health status. +*/ +if (mds->security.poll_tmo_secs >0 ) { +if (mbox_cmd->opcode !=CXL_MBOX_OP_GET_HEALTH_INFO ) +return -EBUSY ; + } + + cmd_reg =FIELD_PREP (CXLDEV_MBOX_CMD_COMMAND_OPCODE_MASK , + mbox_cmd->opcode); +if (mbox_cmd->size_in) { +if (WARN_ON (!mbox_cmd->payload_in)) +return -EINVAL ; + + cmd_reg |=FIELD_PREP (CXLDEV_MBOX_CMD_PAYLOAD_LENGTH_MASK , + mbox_cmd->size_in); +memcpy_toio (payload, mbox_cmd->payload_in, mbox_cmd->size_in); + } + +/* #2, #3 */ +writeq (cmd_reg, cxlds->regs.mbox +CXLDEV_MBOX_CMD_OFFSET ); + +/* #4 */ +dev_dbg (dev,"Sending command: 0x%04x\n" , mbox_cmd->opcode); +writel (CXLDEV_MBOX_CTRL_DOORBELL , + cxlds->regs.mbox +CXLDEV_MBOX_CTRL_OFFSET ); + +/* #5 */ + rc =cxl_pci_mbox_wait_for_doorbell (cxlds); +if (rc == -ETIMEDOUT ) { +u64 md_status =readq (cxlds->regs.memdev +CXLMDEV_STATUS_OFFSET ); + +cxl_cmd_err (cxlds->dev, mbox_cmd, md_status,"mailbox timeout" ); +return rc; + } + +/* #6 */ + status_reg =readq (cxlds->regs.mbox +CXLDEV_MBOX_STATUS_OFFSET ); + mbox_cmd->return_code = +FIELD_GET (CXLDEV_MBOX_STATUS_RET_CODE_MASK , status_reg); + +/* +* Handle the background command in a synchronous manner. +* +* All other mailbox commands will serialize/queue on the mbox_mutex, +* which we currently hold. Furthermore this also guarantees that +* cxl_mbox_background_complete() checks are safe amongst each other, +* in that no new bg operation can occur in between. +* +* Background operations are timesliced in accordance with the nature +* of the command. In the event of timeout, the mailbox state is +* indeterminate until the next successful command submission and the +* driver can get back in sync with the hardware state. +*/ +if (mbox_cmd->return_code ==CXL_MBOX_CMD_RC_BACKGROUND ) { +u64 bg_status_reg; +int i, timeout; + +/* +* Sanitization is a special case which monopolizes the device +* and cannot be timesliced. Handle asynchronously instead, +* and allow userspace to poll(2) for completion. +*/ +if (mbox_cmd->opcode ==CXL_MBOX_OP_SANITIZE ) { +if (mds->security.poll) { +/* hold the device throughout */ +get_device (cxlds->dev); + +/* give first timeout a second */ + timeout =1 ; + mds->security.poll_tmo_secs = timeout; +queue_delayed_work (system_wq, + &mds->security.poll_dwork, + timeout *HZ ); + } + +dev_dbg (dev,"Sanitization operation started\n" ); + goto success; + } + +dev_dbg (dev,"Mailbox background operation (0x%04x) started\n" , + mbox_cmd->opcode); + + timeout = mbox_cmd->poll_interval_ms; +for (i =0 ; i < mbox_cmd->poll_count; i++) { +if (rcuwait_wait_event_timeout (&mds->mbox_wait, +cxl_mbox_background_complete (cxlds), +TASK_UNINTERRUPTIBLE , +msecs_to_jiffies (timeout)) >0 ) +break ; + } + +if (!cxl_mbox_background_complete (cxlds)) { +dev_err (dev,"timeout waiting for background (%d ms)\n" , + timeout * mbox_cmd->poll_count); +return -ETIMEDOUT ; + } + + bg_status_reg =readq (cxlds->regs.mbox + +CXLDEV_MBOX_BG_CMD_STATUS_OFFSET ); + mbox_cmd->return_code = +FIELD_GET (CXLDEV_MBOX_BG_CMD_COMMAND_RC_MASK , + bg_status_reg); +dev_dbg (dev, +"Mailbox background operation (0x%04x) completed\n" , + mbox_cmd->opcode); + } + +if (mbox_cmd->return_code !=CXL_MBOX_CMD_RC_SUCCESS ) { +dev_dbg (dev,"Mailbox operation had an error: %s\n" , +cxl_mbox_cmd_rc2str (mbox_cmd)); +return 0 ;/* completed but caller must check return_code */ + } + +success: +/* #7 */ + cmd_reg =readq (cxlds->regs.mbox +CXLDEV_MBOX_CMD_OFFSET ); + out_len =FIELD_GET (CXLDEV_MBOX_CMD_PAYLOAD_LENGTH_MASK , cmd_reg); + +/* #8 */ +if (out_len && mbox_cmd->payload_out) { +/* +* Sanitize the copy. If hardware misbehaves, out_len per the +* spec can actually be greater than the max allowed size (21 +* bits available but spec defined 1M max). The caller also may +* have requested less data than the hardware supplied even +* within spec. +*/ +size_t n; + + n =min3 (mbox_cmd->size_out, mds->payload_size, out_len); +memcpy_fromio (mbox_cmd->payload_out, payload, n); + mbox_cmd->size_out = n; + }else { + mbox_cmd->size_out =0 ; + } + +return 0 ; +} + +static int cxl_pci_mbox_send (struct cxl_memdev_state *mds, +struct cxl_mbox_cmd *cmd) +{ +int rc; + +mutex_lock_io (&mds->mbox_mutex); + rc =__cxl_pci_mbox_send_cmd (mds, cmd); +mutex_unlock (&mds->mbox_mutex); + +return rc; +} +/** * cxl_internal_send_cmd() - Kernel internal interface to send a mailbox command * @mds: The driver data for the operation
@@ -210,7 +467,7 @@ int cxl_internal_send_cmd(struct cxl_memdev_state *mds, out_size = mbox_cmd->size_out; min_out = mbox_cmd->min_out; - rc = mds->mbox_send (mds, mbox_cmd); + rc =cxl_pci_mbox_send (mds, mbox_cmd);/* * EIO is reserved for a payload size mismatch and mbox_send() * may not return this error.
@@ -549,7 +806,7 @@ static int handle_mailbox_cmd_from_user(struct cxl_memdev_state *mds,cxl_mem_opcode_to_name (mbox_cmd->opcode), mbox_cmd->opcode, mbox_cmd->size_in); - rc = mds->mbox_send (mds, mbox_cmd); + rc =cxl_pci_mbox_send (mds, mbox_cmd);if (rc) goto out;
drivers/cxl/cxlmbox.h new file +21 -0
@@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +#ifndef __CXLMBOX_H__ +#define __CXLMBOX_H__ + +struct cxl_dev_state ; +int cxl_pci_mbox_wait_for_doorbell (struct cxl_dev_state *cxlds); +bool cxl_mbox_background_complete (struct cxl_dev_state *cxlds); + +#define cxl_err (dev, status, msg) \ + dev_err_ratelimited(dev, msg ", device state %s%s\n", \ + status & CXLMDEV_DEV_FATAL ? " fatal" : "", \ + status & CXLMDEV_FW_HALT ? " firmware-halt" : "") + +#define cxl_cmd_err (dev, cmd, status, msg) \ + dev_err_ratelimited(dev, msg " (opcode: %#x), device state %s%s\n", \ + (cmd)->opcode, \ + status & CXLMDEV_DEV_FATAL ? " fatal" : "", \ + status & CXLMDEV_FW_HALT ? " firmware-halt" : "") + +#endif /* __CXLMBOX_H__ */
drivers/cxl/pci.c +1 -264
301 lines not on this page · raw message
Signature · 1 line
2.39.2
RFC PATCH v4 2/4 cxl: mbox: Factor out the mbox specific data for reuse in switch cci
In reply to Jonathan CameronThe mbox implementation should be reusuable on devices that are not CXL type 3 memory devices. The implementation has a number of direct calls that assume it is such a device. Move the data to a separate structure under struct cxl_memdev_state and add callbacks to deal with the non generic corners.
--- drivers/cxl/core/core.h | 3 +- drivers/cxl/core/mbox.c | 303 +++++++++++++++++++------------------- drivers/cxl/core/memdev.c | 32 ++-- drivers/cxl/core/regs.c | 33 ++++- drivers/cxl/cxl.h | 4 +- drivers/cxl/cxlmbox.h | 34 ++++- drivers/cxl/cxlmem.h | 28 ++-- drivers/cxl/pci.c | 199 +++++++++++++++++-------- drivers/cxl/pmem.c | 6 +- drivers/cxl/security.c | 13 +- 10 files changed, 394 insertions(+), 261 deletions(-)
drivers/cxl/core/core.h +2 -1
@@ -51,9 +51,10 @@ static inline void cxl_region_exit(void)struct cxl_send_command ;struct cxl_mem_query_commands ; +struct cxl_mbox ;int cxl_query_cmd (struct cxl_memdev *cxlmd,struct cxl_mem_query_commands __user *q); -int cxl_send_cmd (struct cxl_memdev *cxlmd,struct cxl_send_command __user *s); +int cxl_send_cmd (struct cxl_mbox *mbox,struct cxl_send_command __user *s);void __iomem *devm_cxl_iomap_block (struct device *dev,resource_size_t addr,resource_size_t length);
drivers/cxl/core/mbox.c +153 -150
809 lines not on this page · raw message
drivers/cxl/core/memdev.c +18 -14
@@ -58,7 +58,7 @@ static ssize_t payload_max_show(struct device *dev,if (!mds)return sysfs_emit (buf,"\n" ); -return sysfs_emit (buf,"%zu\n" , mds->payload_size); +return sysfs_emit (buf,"%zu\n" , mds->mbox.payload_size); }static DEVICE_ATTR_RO (payload_max);
@@ -125,7 +125,8 @@ static ssize_t security_state_show(struct device *dev,struct cxl_memdev *cxlmd =to_cxl_memdev (dev);struct cxl_dev_state *cxlds = cxlmd->cxlds;struct cxl_memdev_state *mds =to_cxl_memdev_state (cxlds); -u64 reg =readq (cxlds->regs.mbox +CXLDEV_MBOX_BG_CMD_STATUS_OFFSET ); +//accessor? +u64 reg =readq (mds->mbox.mbox +CXLDEV_MBOX_BG_CMD_STATUS_OFFSET );u32 pct =FIELD_GET (CXLDEV_MBOX_BG_CMD_COMMAND_PCT_MASK , reg);u16 cmd =FIELD_GET (CXLDEV_MBOX_BG_CMD_COMMAND_OPCODE_MASK , reg);unsigned long state = mds->security.state;
@@ -349,7 +350,7 @@ int cxl_inject_poison(struct cxl_memdev *cxlmd, u64 dpa) .size_in = sizeof(inject), .payload_in = &inject, }; - rc =cxl_internal_send_cmd (mds, &mbox_cmd); + rc =cxl_internal_send_cmd (&mds->mbox, &mbox_cmd);if (rc) goto out;
@@ -406,7 +407,7 @@ int cxl_clear_poison(struct cxl_memdev *cxlmd, u64 dpa) .payload_in = &clear, }; - rc =cxl_internal_send_cmd (mds, &mbox_cmd); + rc =cxl_internal_send_cmd (&mds->mbox, &mbox_cmd);if (rc) goto out;
@@ -516,7 +517,7 @@ void set_exclusive_cxl_commands(struct cxl_memdev_state *mds, unsigned long *cmds) {down_write (&cxl_memdev_rwsem); -bitmap_or (mds->exclusive_cmds, mds->exclusive_cmds, cmds, +bitmap_or (mds->mbox.exclusive_cmds, mds->mbox.exclusive_cmds, cmds,CXL_MEM_COMMAND_ID_MAX );up_write (&cxl_memdev_rwsem); }
@@ -531,7 +532,7 @@ void clear_exclusive_cxl_commands(struct cxl_memdev_state *mds, unsigned long *cmds) {down_write (&cxl_memdev_rwsem); -bitmap_andnot (mds->exclusive_cmds, mds->exclusive_cmds, cmds, +bitmap_andnot (mds->mbox.exclusive_cmds, mds->mbox.exclusive_cmds, cmds,CXL_MEM_COMMAND_ID_MAX );up_write (&cxl_memdev_rwsem); }
@@ -617,11 +618,14 @@ static struct cxl_memdev *cxl_memdev_alloc(struct cxl_dev_state *cxlds,static long __cxl_memdev_ioctl (struct cxl_memdev *cxlmd,unsigned int cmd,unsigned long arg) { +struct cxl_dev_state *cxlds = cxlmd->cxlds; + struct cxl_memdev_state *mds = container_of(cxlds, struct cxl_memdev_state, cxlds); +switch (cmd) {case CXL_MEM_QUERY_COMMANDS :return cxl_query_cmd (cxlmd, (void __user *)arg);case CXL_MEM_SEND_COMMAND : -return cxl_send_cmd (cxlmd, (void __user *)arg); +return cxl_send_cmd (&mds->mbox, (void __user *)arg);default :return -ENOTTY ; }
@@ -686,7 +690,7 @@ static int cxl_mem_get_fw_info(struct cxl_memdev_state *mds) .payload_out = &info, }; - rc =cxl_internal_send_cmd (mds, &mbox_cmd); + rc =cxl_internal_send_cmd (&mds->mbox, &mbox_cmd);if (rc <0 )return rc;
@@ -726,7 +730,7 @@ static int cxl_mem_activate_fw(struct cxl_memdev_state *mds, int slot) activate.action =CXL_FW_ACTIVATE_OFFLINE ; activate.slot = slot; -return cxl_internal_send_cmd (mds, &mbox_cmd); +return cxl_internal_send_cmd (&mds->mbox, &mbox_cmd); }/**
@@ -760,7 +764,7 @@ static int cxl_mem_abort_fw_xfer(struct cxl_memdev_state *mds) transfer->action =CXL_FW_TRANSFER_ACTION_ABORT ; - rc =cxl_internal_send_cmd (mds, &mbox_cmd); + rc =cxl_internal_send_cmd (&mds->mbox, &mbox_cmd);kfree (transfer);return rc; }
@@ -796,7 +800,7 @@ static enum fw_upload_err cxl_fw_prepare(struct fw_upload *fwl, const u8 *data,return FW_UPLOAD_ERR_INVALID_SIZE ; mds->fw.oneshot =struct_size (transfer, data, size) < - mds->payload_size; + mds->mbox.payload_size;if (cxl_mem_get_fw_info (mds))return FW_UPLOAD_ERR_HW_ERROR ;
@@ -839,7 +843,7 @@ static enum fw_upload_err cxl_fw_write(struct fw_upload *fwl, const u8 *data,* sizeof(*transfer) is 128. These constraints imply that @cur_size * will always be 128b aligned. */ - cur_size =min_t (size_t, size, mds->payload_size -sizeof (*transfer)); + cur_size =min_t (size_t, size, mds->mbox.payload_size -sizeof (*transfer)); remaining = size - cur_size; size_in =struct_size (transfer, data, cur_size);
@@ -883,7 +887,7 @@ static enum fw_upload_err cxl_fw_write(struct fw_upload *fwl, const u8 *data, .poll_count = 30, }; - rc =cxl_internal_send_cmd (mds, &mbox_cmd); + rc =cxl_internal_send_cmd (&mds->mbox, &mbox_cmd);if (rc <0 ) { rc =FW_UPLOAD_ERR_RW_ERROR ; goto out_free;
@@ -954,7 +958,7 @@ int cxl_memdev_setup_fw_upload(struct cxl_memdev_state *mds)struct fw_upload *fwl;int rc; -if (!test_bit (CXL_MEM_COMMAND_ID_GET_FW_INFO , mds->enabled_cmds)) +if (!test_bit (CXL_MEM_COMMAND_ID_GET_FW_INFO , mds->mbox.enabled_cmds))return 0 ; fwl = firmware_upload_register(THIS_MODULE, dev, dev_name(dev),
drivers/cxl/core/regs.c +32 -1
@@ -244,7 +244,6 @@ int cxl_map_device_regs(const struct cxl_register_map *map,void __iomem **addr; } mapinfo[] = { { &map->device_map.status, ®s->status, }, - { &map->device_map.mbox, ®s->mbox, }, { &map->device_map.memdev, ®s->memdev, }, }; int i;
@@ -268,6 +267,38 @@ int cxl_map_device_regs(const struct cxl_register_map *map, }EXPORT_SYMBOL_NS_GPL (cxl_map_device_regs,CXL ); +int cxl_map_mbox_regs (const struct cxl_register_map *map, +void __iomem **mbox_regs) +{ +struct device *dev = map->dev; +resource_size_t phys_addr = map->resource; +struct mapinfo { +const struct cxl_reg_map *rmap; +void __iomem **addr; + } mapinfo[] = { + { &map->device_map.mbox, mbox_regs, }, + }; +int i; + +for (i =0 ; i <ARRAY_SIZE (mapinfo); i++) { +struct mapinfo *mi = &mapinfo[i]; +resource_size_t length; +resource_size_t addr; + +if (!mi->rmap || !mi->rmap->valid) +continue ; + + addr = phys_addr + mi->rmap->offset; + length = mi->rmap->size; + *(mi->addr) =devm_cxl_iomap_block (dev, addr, length); +if (!*(mi->addr)) +return -ENOMEM ; + } + +return 0 ; +} +EXPORT_SYMBOL_NS_GPL (cxl_map_mbox_regs,CXL ); +static bool cxl_decode_regblock (struct pci_dev *pdev,u32 reg_lo,u32 reg_hi,struct cxl_register_map *map) {
drivers/cxl/cxl.h +3 -1
@@ -215,7 +215,7 @@ struct cxl_regs {* @memdev: CXL 2.0 8.2.8.5 Memory Device Registers */ struct_group_tagged (cxl_device_regs, device_regs, - void __iomem *status, *mbox, *memdev; + void __iomem *status, *memdev; ); struct_group_tagged(cxl_pmu_regs, pmu_regs,
@@ -278,6 +278,8 @@ int cxl_map_component_regs(const struct cxl_register_map *map, unsigned long map_mask);int cxl_map_device_regs (const struct cxl_register_map *map,struct cxl_device_regs *regs); +int cxl_map_mbox_regs (const struct cxl_register_map *map, +void __iomem **mbox_reg);int cxl_map_pmu_regs (struct pci_dev *pdev,struct cxl_pmu_regs *regs,struct cxl_register_map *map);
drivers/cxl/cxlmbox.h +31 -3
@@ -3,9 +3,36 @@#ifndef __CXLMBOX_H__#define __CXLMBOX_H__ -struct cxl_dev_state ; -int cxl_pci_mbox_wait_for_doorbell (struct cxl_dev_state *cxlds); -bool cxl_mbox_background_complete (struct cxl_dev_state *cxlds); +#include <linux/irqreturn.h> +#include <linux/export.h> +#include <linux/io.h> + +#include <uapi/linux/cxl_mem.h> + +struct device ; +struct cxl_mbox_cmd ; +struct cxl_mbox { +struct device *dev;/* Used for debug prints */ +size_t payload_size; +struct mutex mbox_mutex;/* Protects device mailbox and firmware */ + DECLARE_BITMAP(enabled_cmds, CXL_MEM_COMMAND_ID_MAX); + DECLARE_BITMAP(exclusive_cmds, CXL_MEM_COMMAND_ID_MAX); +struct rcuwait mbox_wait; +int (*mbox_send)(struct cxl_mbox *mbox, +struct cxl_mbox_cmd *cmd); +bool (*special_irq)(struct cxl_mbox *mbox,u16 opcode); +void (*special_init_poll)(struct cxl_mbox *mbox); +bool (*special_bg)(struct cxl_mbox *mbox,u16 opcode); +u64 (*get_status)(struct cxl_mbox *mbox); +bool (*can_run)(struct cxl_mbox *mbox,u16 opcode); +void (*extra_cmds)(struct cxl_mbox *mbox,u16 opcode); +/* Also needs access to registers */ +void __iomem *status, *mbox; +}; + +irqreturn_t cxl_mbox_irq (int irq,struct cxl_mbox *mbox); +int cxl_pci_mbox_wait_for_doorbell (struct cxl_mbox *mbox); +bool cxl_mbox_background_complete (struct cxl_mbox *mbox);#define cxl_err (dev, status, msg) \ dev_err_ratelimited(dev, msg ", device state %s%s\n", \
@@ -19,3 +46,4 @@ bool cxl_mbox_background_complete(struct cxl_dev_state *cxlds); status & CXLMDEV_FW_HALT ? " firmware-halt" : "")#endif /* __CXLMBOX_H__ */ +
drivers/cxl/cxlmem.h +11 -17
@@ -6,6 +6,7 @@#include <linux/cdev.h> #include <linux/uuid.h> #include <linux/rcuwait.h> +#include "cxlmbox.h" #include "cxl.h" /* CXL 2.0 8.2.8.5.1.1 Memory Device Status Register */
@@ -416,14 +417,10 @@ struct cxl_dev_state {* the functionality related to that like Identify Memory Device and Get * Partition Info * @cxlds: Core driver state common across Type-2 and Type-3 devices -* @payload_size: Size of space for payload -* (CXL 2.0 8.2.8.4.3 Mailbox Capabilities Register) +* @mbox: Mailbox instance. * @lsa_size: Size of Label Storage Area * (CXL 2.0 8.2.9.5.1.1 Identify Memory Device) -* @mbox_mutex: Mutex to synchronize mailbox access. * @firmware_version: Firmware version for the memory device. -* @enabled_cmds: Hardware commands found enabled in CEL. -* @exclusive_cmds: Commands that are kernel-internal only * @total_bytes: sum of all possible capacities * @volatile_only_bytes: hard volatile capacity * @persistent_only_bytes: hard persistent capacity
@@ -435,19 +432,16 @@ struct cxl_dev_state {* @event: event log driver state * @poison: poison driver state info * @fw: firmware upload / activation state -* @mbox_send: @dev specific transport for transmitting mailbox commands * * See CXL 3.0 8.2.9.8.2 Capacity Configuration and Label Storage for * details on capacity parameters. */ struct cxl_memdev_state {struct cxl_dev_state cxlds; -size_t payload_size; +struct cxl_mbox mbox; +size_t lsa_size; -struct mutex mbox_mutex;/* Protects device mailbox and firmware */ char firmware_version[0x10 ]; - DECLARE_BITMAP(enabled_cmds, CXL_MEM_COMMAND_ID_MAX); - DECLARE_BITMAP(exclusive_cmds, CXL_MEM_COMMAND_ID_MAX);u64 total_bytes;u64 volatile_only_bytes;u64 persistent_only_bytes;
@@ -460,10 +454,6 @@ struct cxl_memdev_state {struct cxl_poison_state poison;struct cxl_security_state security;struct cxl_fw_state fw; - -struct rcuwait mbox_wait; -int (*mbox_send)(struct cxl_memdev_state *mds, -struct cxl_mbox_cmd *cmd); }; static inline struct cxl_memdev_state *
@@ -835,11 +825,15 @@ enum { CXL_PMEM_SEC_PASS_USER, }; -int cxl_internal_send_cmd (struct cxl_memdev_state *mds, -struct cxl_mbox_cmd *cmd); +int cxl_internal_send_cmd (struct cxl_mbox *mbox, +struct cxl_mbox_cmd *cmd); +bool cxl_is_poison_command (u16 opcode); +void cxl_set_poison_cmd_enabled (struct cxl_poison_state *poison, +u16 opcode); +int cxl_dev_state_identify (struct cxl_memdev_state *mds);int cxl_await_media_ready (struct cxl_dev_state *cxlds); -int cxl_enumerate_cmds (struct cxl_memdev_state *mds); +int cxl_enumerate_cmds (struct cxl_mbox *mbox);int cxl_mem_create_range_info (struct cxl_memdev_state *mds);struct cxl_memdev_state *cxl_memdev_state_create (struct device *dev); void set_exclusive_cxl_commands(struct cxl_memdev_state *mds,
drivers/cxl/pci.c +134 -65
377 lines not on this page · raw message
drivers/cxl/pmem.c +3 -3
@@ -110,7 +110,7 @@ static int cxl_pmem_get_config_size(struct cxl_memdev_state *mds, *cmd = (struct nd_cmd_get_config_size ){ .config_size = mds->lsa_size, .max_xfer = - mds->payload_size -sizeof (struct cxl_mbox_set_lsa ), + mds->mbox.payload_size -sizeof (struct cxl_mbox_set_lsa ), };return 0 ;
@@ -141,7 +141,7 @@ static int cxl_pmem_get_config_data(struct cxl_memdev_state *mds, .payload_out = cmd->out_buf, }; - rc =cxl_internal_send_cmd (mds, &mbox_cmd); + rc =cxl_internal_send_cmd (&mds->mbox, &mbox_cmd); cmd->status =0 ;return rc;
@@ -177,7 +177,7 @@ static int cxl_pmem_set_config_data(struct cxl_memdev_state *mds, .size_in = struct_size(set_lsa, data, cmd->in_length), }; - rc =cxl_internal_send_cmd (mds, &mbox_cmd); + rc =cxl_internal_send_cmd (&mds->mbox, &mbox_cmd);/* * Set "firmware" status (4-packed bytes at the end of the input
drivers/cxl/security.c +7 -6
@@ -6,6 +6,7 @@#include <linux/async.h> #include <linux/slab.h> #include <linux/memregion.h> +#include "cxlmbox.h" #include "cxlmem.h" #include "cxl.h"
@@ -29,7 +30,7 @@ static unsigned long cxl_pmem_get_security_flags(struct nvdimm *nvdimm, .payload_out = &out, }; - rc =cxl_internal_send_cmd (mds, &mbox_cmd); + rc =cxl_internal_send_cmd (&mds->mbox, &mbox_cmd);if (rc <0 )return 0 ;
@@ -87,7 +88,7 @@ static int cxl_pmem_security_change_key(struct nvdimm *nvdimm, .payload_in = &set_pass, }; -return cxl_internal_send_cmd (mds, &mbox_cmd); +return cxl_internal_send_cmd (&mds->mbox, &mbox_cmd); } static int __cxl_pmem_security_disable(struct nvdimm *nvdimm,
@@ -112,7 +113,7 @@ static int __cxl_pmem_security_disable(struct nvdimm *nvdimm, .payload_in = &dis_pass, }; -return cxl_internal_send_cmd (mds, &mbox_cmd); +return cxl_internal_send_cmd (&mds->mbox, &mbox_cmd); } static int cxl_pmem_security_disable(struct nvdimm *nvdimm,
@@ -136,7 +137,7 @@ static int cxl_pmem_security_freeze(struct nvdimm *nvdimm) .opcode = CXL_MBOX_OP_FREEZE_SECURITY, }; -return cxl_internal_send_cmd (mds, &mbox_cmd); +return cxl_internal_send_cmd (&mds->mbox, &mbox_cmd); } static int cxl_pmem_security_unlock(struct nvdimm *nvdimm,
@@ -156,7 +157,7 @@ static int cxl_pmem_security_unlock(struct nvdimm *nvdimm, .payload_in = pass, }; - rc =cxl_internal_send_cmd (mds, &mbox_cmd); + rc =cxl_internal_send_cmd (&mds->mbox, &mbox_cmd);if (rc <0 )return rc;
@@ -185,7 +186,7 @@ static int cxl_pmem_security_passphrase_erase(struct nvdimm *nvdimm, .payload_in = &erase, }; - rc =cxl_internal_send_cmd (mds, &mbox_cmd); + rc =cxl_internal_send_cmd (&mds->mbox, &mbox_cmd);if (rc <0 )return rc;
Signature · 1 line
2.39.2
Davidlohr Bueso
maintainerRe: [RFC PATCH v4 2/4] cxl: mbox: Factor out the mbox specific data for reuse in switch cci
Quoted 2 lines · Jonathan Cameron
#ifndef
__CXLMBOX_H__
#define__CXLMBOX_H__
Unrelated but looks like cxlmem.h needs s/__CXL_MEM_H__/__CXLMEM_H__
drivers/cxl/cxlmbox.h line 13
Quoted 12 lines
drivers/cxl/cxlmbox.h +8
-3-struct cxl_dev_state ; -int cxl_pci_mbox_wait_for_doorbell (struct cxl_dev_state *cxlds); -bool cxl_mbox_background_complete (struct cxl_dev_state *cxlds); +#include <linux/irqreturn.h> +#include <linux/export.h> +#include <linux/io.h> + +#include <uapi/linux/cxl_mem.h> + +struct device ; +struct cxl_mbox_cmd ;
Would it make sense to instead move the whole cxl_mbox_cmd out of cxlmem.h into here? Same for the cmd rc table stuff. Then cxlmem can include cxlmbox.
Quoted 5 lines
drivers/cxl/cxlmbox.h +5
-0+struct cxl_mbox { +struct device *dev;/* Used for debug prints */ +size_t payload_size; +struct mutex mbox_mutex;/* Protects device mailbox and firmware */ + DECLARE_BITMAP(enabled_cmds, CXL_MEM_COMMAND_ID_MAX);
drivers/cxl/cxlmbox.h line 28
Quoted 10 lines
drivers/cxl/cxlmbox.h +10
-0+DECLARE_BITMAP (exclusive_cmds,CXL_MEM_COMMAND_ID_MAX ); +struct rcuwait mbox_wait; +int (*mbox_send)(struct cxl_mbox *mbox, +struct cxl_mbox_cmd *cmd); +bool (*special_irq)(struct cxl_mbox *mbox,u16 opcode); +void (*special_init_poll)(struct cxl_mbox *mbox); +bool (*special_bg)(struct cxl_mbox *mbox,u16 opcode); +u64 (*get_status)(struct cxl_mbox *mbox); +bool (*can_run)(struct cxl_mbox *mbox,u16 opcode); +void (*extra_cmds)(struct cxl_mbox *mbox,u16 opcode);
Ok, so most of these corner cases are wrt Sanitize. Do you have anything in mind what would require such any additional users in the future (such as completely taking over the device), beyond pci mailbox? Otherwise this feels too ad-hoc with only the naming being generic. Perhaps instead have some sort of mbox->type and handle accordingly directly in the core mbox calls? It would be nice to have these callbacks somewhat documented.
Also the 'can_run' name is a bit disconnected from the sanitize special case, maybe be rename to something like 'special_canrun'?
drivers/cxl/cxlmbox.h line 32
Quoted 4 lines
drivers/cxl/cxlmbox.h +4
-0+/* Also needs access to registers */ +void __iomem *status, *mbox; +}; +
Thanks,
Davidlohr
Re: [RFC PATCH v4 2/4] cxl: mbox: Factor out the mbox specific data for reuse in switch cci
Quoted 8 lines · On Fri, 21 Jul 2023 09:48:16 -0700 Davidlohr Bueso <[email protected]> wrote:
Quoted 2 lines · Jonathan Cameron
#ifndef
__CXLMBOX_H__
#define__CXLMBOX_H__Unrelated but looks like
cxlmem.hneeds s/__CXL_MEM_H__/__CXLMEM_H__Quoted 1 line
Quoted 15 lines
drivers/cxl/cxlmbox.h line 13
Quoted 11 lines
drivers/cxl/cxlmbox.h +8
-3-struct cxl_dev_state ; -int cxl_pci_mbox_wait_for_doorbell (struct cxl_dev_state *cxlds); -bool cxl_mbox_background_complete (struct cxl_dev_state *cxlds); +#include <linux/irqreturn.h> +#include <linux/export.h> +#include <linux/io.h> + +#include <uapi/linux/cxl_mem.h> + +struct device ; +struct cxl_mbox_cmd ;Would it make sense to instead move the whole
cxl_mbox_cmdout ofcxlmem.hinto here? Same for the cmd rc table stuff. Then cxlmem can include cxlmbox.
That makes sense but I'll do it in an additional patch as the chances of that sort of move being rebase pain is very high. Rebasing this is nasty already (just did so on top of the currently cxl/fixes and it wasn't as bad, but still not trivial)
Maybe we can squish it in with this patch for a final merge.
Having had a go at this, it gets a little fiddly to work out what to move - for example the event logs are general and can turn up on the switch cci (I think anyway) but hopefully not a DRAM event record.
For now I've gone for moving less rather that more, particularly as not that many messages are yet supported on the switch-cci.
Ideally the switch-cci.c file wouldn't include cxlmem.h at all and would not use a struct cxl_dev_state. That makes the handling of register mapping a bit more ugly as the status may or may not have been mapped before the mbox mappings. Let's see how bad it is for RFC v5.
Quoted 15 lines
Quoted 15 lines
drivers/cxl/cxlmbox.h +14
-0+struct cxl_mbox { +struct device *dev;/* Used for debug prints */ +size_t payload_size; +struct mutex mbox_mutex;/* Protects device mailbox and firmware */ + DECLARE_BITMAP(enabled_cmds, CXL_MEM_COMMAND_ID_MAX); + DECLARE_BITMAP(exclusive_cmds, CXL_MEM_COMMAND_ID_MAX); +struct rcuwait mbox_wait; +int (*mbox_send)(struct cxl_mbox *mbox, +struct cxl_mbox_cmd *cmd); +bool (*special_irq)(struct cxl_mbox *mbox,u16 opcode); +void (*special_init_poll)(struct cxl_mbox *mbox); +bool (*special_bg)(struct cxl_mbox *mbox,u16 opcode); +u64 (*get_status)(struct cxl_mbox *mbox); +bool (*can_run)(struct cxl_mbox *mbox,u16 opcode);
Quoted 8 lines
drivers/cxl/cxlmbox.h line 28
Quoted 1 line
drivers/cxl/cxlmbox.h +1
-0+void (*extra_cmds)(struct cxl_mbox *mbox,u16 opcode);Ok, so most of these corner cases are wrt Sanitize. Do you have anything in mind what would require such any additional users in the future (such as completely taking over the device), beyond pci mailbox? Otherwise this feels too ad-hoc with only the naming being generic. Perhaps instead have some sort of
mbox->typeand handle accordingly directly in the core mbox calls?
I don't like the idea of dragging the stuff related to sanitize and poison into the general mbox code and I do like the ease this gives of doing something special for the weird corners without making too much spaghetti. So whilst I agree it's a fair bit of complexity for a few small corner cases, I think it's striking roughly the right balance between device specific and generic.
Quoted 2 lines
It would be
nice to have these callbacks somewhat documented
And spoil the fun? Sure I'll add some stuff.
Quoted 11 lines
Also the '
can_run' name is a bit disconnected from the sanitize special case, maybe be rename to something like 'special_canrun'?drivers/cxl/cxlmbox.h line 32
Quoted 4 lines
drivers/cxl/cxlmbox.h +4
-0+/* Also needs access to registers */ +void __iomem *status, *mbox; +}; +Thanks,
Davidlohr
Re: [RFC PATCH v4 2/4] cxl: mbox: Factor out the mbox specific data for reuse in switch cci
Quoted 81 lines · On Thu, 3 Aug 2023 17:47:51 +0100 Jonathan Cameron…
Quoted 8 lines · On Fri, 21 Jul 2023 09:48:16 -0700 Davidlohr Bueso <[email protected]> wrote:
Quoted 2 lines · Jonathan Cameron
#ifndef
__CXLMBOX_H__
#define__CXLMBOX_H__Unrelated but looks like
cxlmem.hneeds s/__CXL_MEM_H__/__CXLMEM_H__Quoted 1 line
Quoted 15 lines
drivers/cxl/cxlmbox.h line 13
Quoted 11 lines
drivers/cxl/cxlmbox.h +8
-3-struct cxl_dev_state ; -int cxl_pci_mbox_wait_for_doorbell (struct cxl_dev_state *cxlds); -bool cxl_mbox_background_complete (struct cxl_dev_state *cxlds); +#include <linux/irqreturn.h> +#include <linux/export.h> +#include <linux/io.h> + +#include <uapi/linux/cxl_mem.h> + +struct device ; +struct cxl_mbox_cmd ;Would it make sense to instead move the whole
cxl_mbox_cmdout ofcxlmem.hinto here? Same for the cmd rc table stuff. Then cxlmem can include cxlmbox.That makes sense but I'll do it in an additional patch as the chances of that sort of move being rebase pain is very high. Rebasing this is nasty already (just did so on top of the currently cxl/fixes and it wasn't as bad, but still not trivial)
Maybe we can squish it in with this patch for a final merge.
Having had a go at this, it gets a little fiddly to work out what to move - for example the event logs are general and can turn up on the switch cci (I think anyway) but hopefully not a DRAM event record.
For now I've gone for moving less rather that more, particularly as not that many messages are yet supported on the switch-cci.
Ideally the
switch-cci.cfile wouldn't includecxlmem.hat all and would not use a structcxl_dev_state. That makes the handling of register mapping a bit more ugly as the status may or may not have been mapped before the mbox mappings. Let's see how bad it is for RFC v5.Quoted 15 lines
Quoted 15 lines
drivers/cxl/cxlmbox.h +14
-0+struct cxl_mbox { +struct device *dev;/* Used for debug prints */ +size_t payload_size; +struct mutex mbox_mutex;/* Protects device mailbox and firmware */ + DECLARE_BITMAP(enabled_cmds, CXL_MEM_COMMAND_ID_MAX); + DECLARE_BITMAP(exclusive_cmds, CXL_MEM_COMMAND_ID_MAX); +struct rcuwait mbox_wait; +int (*mbox_send)(struct cxl_mbox *mbox, +struct cxl_mbox_cmd *cmd); +bool (*special_irq)(struct cxl_mbox *mbox,u16 opcode); +void (*special_init_poll)(struct cxl_mbox *mbox); +bool (*special_bg)(struct cxl_mbox *mbox,u16 opcode); +u64 (*get_status)(struct cxl_mbox *mbox); +bool (*can_run)(struct cxl_mbox *mbox,u16 opcode);Quoted 8 lines
drivers/cxl/cxlmbox.h line 28
Quoted 1 line
drivers/cxl/cxlmbox.h +1
-0+void (*extra_cmds)(struct cxl_mbox *mbox,u16 opcode);Ok, so most of these corner cases are wrt Sanitize. Do you have anything in mind what would require such any additional users in the future (such as completely taking over the device), beyond pci mailbox? Otherwise this feels too ad-hoc with only the naming being generic. Perhaps instead have some sort of
mbox->typeand handle accordingly directly in the core mbox calls?I don't like the idea of dragging the stuff related to sanitize and poison into the general mbox code and I do like the ease this gives of doing something special for the weird corners without making too much spaghetti. So whilst I agree it's a fair bit of complexity for a few small corner cases, I think it's striking roughly the right balance between device specific and generic.
Quoted 4 lines
Quoted 2 lines
It would be
nice to have these callbacks somewhat documentedAnd spoil the fun? Sure I'll add some stuff.
Was definitely a useful exercise. One was never called and the other one was only used in one local path in pci.c so no need to have a callback.
Thanks :)
Jonathan
Quoted 13 lines
Quoted 11 lines
Also the '
can_run' name is a bit disconnected from the sanitize special case, maybe be rename to something like 'special_canrun'?drivers/cxl/cxlmbox.h line 32
Quoted 4 lines
drivers/cxl/cxlmbox.h +4
-0+/* Also needs access to registers */ +void __iomem *status, *mbox; +}; +Thanks,
Davidlohr
Re: [RFC PATCH v4 2/4] cxl: mbox: Factor out the mbox specific data for reuse in switch cci
In reply to Davidlohr BuesoQuoted 48 lines · On Fri, 21 Jul 2023 09:48:16 -0700 Davidlohr Bueso <[email protected]> wrote:
Quoted 2 lines · Jonathan Cameron
#ifndef
__CXLMBOX_H__
#define__CXLMBOX_H__Unrelated but looks like
cxlmem.hneeds s/__CXL_MEM_H__/__CXLMEM_H__drivers/cxl/cxlmbox.h line 13
Quoted 12 lines
drivers/cxl/cxlmbox.h +8
-3-struct cxl_dev_state ; -int cxl_pci_mbox_wait_for_doorbell (struct cxl_dev_state *cxlds); -bool cxl_mbox_background_complete (struct cxl_dev_state *cxlds); +#include <linux/irqreturn.h> +#include <linux/export.h> +#include <linux/io.h> + +#include <uapi/linux/cxl_mem.h> + +struct device ; +struct cxl_mbox_cmd ;Would it make sense to instead move the whole
cxl_mbox_cmdout ofcxlmem.hinto here? Same for the cmd rc table stuff. Then cxlmem can include cxlmbox.Quoted 5 lines
drivers/cxl/cxlmbox.h +5
-0+struct cxl_mbox { +struct device *dev;/* Used for debug prints */ +size_t payload_size; +struct mutex mbox_mutex;/* Protects device mailbox and firmware */ + DECLARE_BITMAP(enabled_cmds, CXL_MEM_COMMAND_ID_MAX);drivers/cxl/cxlmbox.h line 28
Quoted 10 lines
drivers/cxl/cxlmbox.h +10
-0+DECLARE_BITMAP (exclusive_cmds,CXL_MEM_COMMAND_ID_MAX ); +struct rcuwait mbox_wait; +int (*mbox_send)(struct cxl_mbox *mbox, +struct cxl_mbox_cmd *cmd); +bool (*special_irq)(struct cxl_mbox *mbox,u16 opcode); +void (*special_init_poll)(struct cxl_mbox *mbox); +bool (*special_bg)(struct cxl_mbox *mbox,u16 opcode); +u64 (*get_status)(struct cxl_mbox *mbox); +bool (*can_run)(struct cxl_mbox *mbox,u16 opcode); +void (*extra_cmds)(struct cxl_mbox *mbox,u16 opcode);Ok, so most of these corner cases are wrt Sanitize. Do you have anything in mind what would require such any additional users in the future (such as completely taking over the device), beyond pci mailbox? Otherwise this feels too ad-hoc with only the naming being generic. Perhaps instead have some sort of
mbox->typeand handle accordingly directly in the core mbox calls? It would be nice to have these callbacks somewhat documented.
Quoted 2 lines
Also the '
can_run' name is a bit disconnected from the sanitize special case, maybe be rename to something like 'special_canrun'?
I thought a bit more on this and the special_can_run is too specific as it's not the special command that is restricted, but rather could be any command as a result of a special command being in flight. I can't think of a better name for that :(
Quoted 8 lines
drivers/cxl/cxlmbox.h line 32
Quoted 5 lines
drivers/cxl/cxlmbox.h +4
-0+/* Also needs access to registers */ +void __iomem *status, *mbox; +}; +Thanks,
Davidlohr
RFC PATCH v4 3/4 PCI: Add PCI_CLASS_SERIAL_CXL_SWITCH_CCI class ID to pci_ids.h
In reply to Jonathan CameronThis ID is used for CXL Switch Mailbox CCIs. Incorporate the Programming Interface in the definition in common with similar entries.
See CXL rev3.0 81.1.3.1 PCI Header Class Code Register
--- include/linux/pci_ids.h | 1 + 1 file changed, 1 insertion(+)
include/linux/pci_ids.h +1 -0
@@ -128,6 +128,7 @@#define PCI_CLASS_SERIAL_IPMI_SMIC 0x0c0700#define PCI_CLASS_SERIAL_IPMI_KCS 0x0c0701#define PCI_CLASS_SERIAL_IPMI_BT 0x0c0702 +#define PCI_CLASS_SERIAL_CXL_SWITCH_CCI 0x0c0b00#define PCI_BASE_CLASS_WIRELESS 0x0d#define PCI_CLASS_WIRELESS_RF_CONTROLLER 0x0d10
Signature · 1 line
2.39.2
RFC PATCH v4 4/4 cxl/pci: Add support for stand alone CXL Switch mailbox CCI
In reply to Jonathan CameronCXL 3.0 defines a mailbox PCI function independent of any other CXL components. The intent is that instances of this mailbox will be found as additional PCI functions of upstream CXL switch ports.
--- drivers/cxl/Kconfig | 14 +++ drivers/cxl/Makefile | 2 + drivers/cxl/core/Makefile | 1 + drivers/cxl/core/core.h | 1 + drivers/cxl/core/mbox.c | 6 ++ drivers/cxl/core/port.c | 4 + drivers/cxl/core/switch-cci.c | 144 +++++++++++++++++++++++++++ drivers/cxl/cxlmem.h | 15 +++ drivers/cxl/switch-cci.c | 181 ++++++++++++++++++++++++++++++++++ include/uapi/linux/cxl_mem.h | 4 + 10 files changed, 372 insertions(+)
drivers/cxl/Kconfig +14 -0
@@ -152,5 +152,19 @@ config CXL_PMU Say 'y/m' to enable a driver that will attach to performance monitoring units and provide standard perf based interfaces. + If unsure say 'm'. + +config CXL_SWITCH + tristate "CXL switch mailbox access" + help + The CXL r3.0 specification defines a "CXL switch CCI" sub-class in the + PCI "Serial" base class of devices. Device's identified by + this class code provide a mailbox interface to allow control of CXL + switch configuration over inband PCI. + + Say 'y/m' to enable a driver that will attach to CXL Switch CCI + devices enumerated by the CXL switch CCI class code for configuration + and management primarily via the mailbox interface. + If unsure say 'm'. endif
drivers/cxl/Makefile +2 -0
@@ -5,9 +5,11 @@ obj-$(CONFIG_CXL_MEM) += cxl_mem.o obj-$(CONFIG_CXL_ACPI) += cxl_acpi.o obj-$(CONFIG_CXL_PMEM) += cxl_pmem.o obj-$(CONFIG_CXL_PORT) += cxl_port.o +obj-$(CONFIG_CXL_SWITCH) += cxl_switch_cci.o cxl_mem-y := mem.o cxl_pci-y := pci.o cxl_acpi-y := acpi.o cxl_pmem-y := pmem.o security.o cxl_port-y := port.o +cxl_switch_cci-y := switch-cci.o
drivers/cxl/core/Makefile +1 -0
@@ -13,5 +13,6 @@ cxl_core-y += mbox.o cxl_core-y += pci.o cxl_core-y += hdm.o cxl_core-y += pmu.o +cxl_core-y += switch-cci.o cxl_core-$(CONFIG_TRACING) += trace.o cxl_core-$(CONFIG_CXL_REGION) += region.o
drivers/cxl/core/core.h +1 -0
@@ -80,6 +80,7 @@ extern struct rw_semaphore cxl_dpa_rwsem;int cxl_memdev_init (void );void cxl_memdev_exit (void );void cxl_mbox_init (void ); +int cxl_switch_cci_init (void );enum cxl_poison_trace_type {CXL_POISON_TRACE_LIST ,
drivers/cxl/core/mbox.c +6 -0
@@ -9,6 +9,7 @@#include <cxlmbox.h> #include <cxlpci.h> #include <cxlmem.h> +#include <cxlpci.h> #include <cxl.h> #include "core.h"
@@ -56,6 +57,8 @@ static bool cxl_raw_allow_all;* 0, and the user passed in 1, it is an error. */ static struct cxl_mem_command cxl_mem_commands[CXL_MEM_COMMAND_ID_MAX ] = { +CXL_CMD (INFO_STAT_IDENTIFY ,0 ,0x12 ,0 ), +CXL_CMD (GET_BG_CMD_STATUS ,0 ,8 ,0 ),CXL_CMD (IDENTIFY ,0 ,0x43 ,CXL_CMD_FLAG_FORCE_ENABLE ), #ifdef CONFIG_CXL_MEM_RAW_COMMANDS CXL_CMD(RAW, CXL_VARIABLE_PAYLOAD, CXL_VARIABLE_PAYLOAD, 0),
@@ -73,6 +76,9 @@ static struct cxl_mem_command cxl_mem_commands[CXL_MEM_COMMAND_ID_MAX] = {CXL_CMD (GET_SHUTDOWN_STATE ,0 ,0x1 ,0 ),CXL_CMD (SET_SHUTDOWN_STATE ,0x1 ,0 ,0 ),CXL_CMD (GET_SCAN_MEDIA_CAPS ,0x10 ,0x4 ,0 ), +CXL_CMD (IDENTIFY_SWITCH_DEVICE ,0 ,0x49 ,0 ), +CXL_CMD (TUNNEL_MANAGEMENT_COMMAND ,CXL_VARIABLE_PAYLOAD , + CXL_VARIABLE_PAYLOAD, 0), };/*
drivers/cxl/core/port.c +4 -0
@@ -2020,6 +2020,10 @@ static __init int cxl_core_init(void)if (rc)return rc; + rc =cxl_switch_cci_init (); +if (rc) +return rc; + cxl_bus_wq =alloc_ordered_workqueue ("cxl_port" ,0 );if (!cxl_bus_wq) { rc = -ENOMEM ;
drivers/cxl/core/switch-cci.c new file +144 -0
144 lines not on this page · raw message
drivers/cxl/cxlmem.h +15 -0
@@ -456,6 +456,17 @@ struct cxl_memdev_state {struct cxl_fw_state fw; }; +struct cxl_swdev { +struct device dev; +struct cdev cdev; +struct cxl_mbox mbox; +struct cxl_dev_state cxlds; +int id; +}; + +struct cxl_swdev *cxl_swdev_alloc (struct device *parent); +void cxl_swdev_shutdown (struct cxl_swdev *cxlswd); +static inline struct cxl_memdev_state *to_cxl_memdev_state (struct cxl_dev_state *cxlds) {
@@ -466,6 +477,8 @@ to_cxl_memdev_state(struct cxl_dev_state *cxlds)enum cxl_opcode {CXL_MBOX_OP_INVALID =0x0000 , +CXL_MBOX_OP_INFO_STAT_IDENTIFY =0x0001 , +CXL_MBOX_OP_GET_BG_CMD_STATUS =0x0002 ,CXL_MBOX_OP_RAW =CXL_MBOX_OP_INVALID ,CXL_MBOX_OP_GET_EVENT_RECORD =0x0100 ,CXL_MBOX_OP_CLEAR_EVENT_RECORD =0x0101 ,
@@ -501,6 +514,8 @@ enum cxl_opcode {CXL_MBOX_OP_UNLOCK =0x4503 ,CXL_MBOX_OP_FREEZE_SECURITY =0x4504 ,CXL_MBOX_OP_PASSPHRASE_SECURE_ERASE =0x4505 , +CXL_MBOX_OP_IDENTIFY_SWITCH_DEVICE =0x5100 , +CXL_MBOX_OP_TUNNEL_MANAGEMENT_COMMAND =0x5300 , CXL_MBOX_OP_MAX = 0x10000 };
drivers/cxl/switch-cci.c new file +181 -0
181 lines not on this page · raw message
include/uapi/linux/cxl_mem.h +4 -0
@@ -46,6 +46,10 @@___C (GET_SCAN_MEDIA_CAPS ,"Get Scan Media Capabilities" ), \___DEPRECATED (SCAN_MEDIA ,"Scan Media" ), \___DEPRECATED (GET_SCAN_MEDIA ,"Get Scan Media Results" ), \ +___C (INFO_STAT_IDENTIFY ,"Get Information" ), \ +___C (GET_BG_CMD_STATUS ,"Background Command Status" ), \ +___C (IDENTIFY_SWITCH_DEVICE ,"Identify Switch Device" ), \ +___C (TUNNEL_MANAGEMENT_COMMAND ,"Tunnel Management Command" ), \ ___C(MAX, "invalid / last command") #define ___C(a, b) CXL_MEM_COMMAND_ID_##a
Signature · 1 line
2.39.2
Davidlohr Bueso
maintainerRe: [RFC PATCH v4 4/4] cxl/pci: Add support for stand alone CXL Switch mailbox CCI
Quoted 509 lines · Jonathan Cameron
CXL 3.0 defines a mailbox PCI function independent of any other CXL components. The intent is that instances of this mailbox will be found as additional PCI functions of upstream CXL switch ports.
Signed-off-by Jonathan Cameron <[email protected]>---
drivers/cxl/Kconfig | 14 +++ drivers/cxl/Makefile | 2 + drivers/cxl/core/Makefile | 1 + drivers/cxl/core/core.h | 1 + drivers/cxl/core/mbox.c | 6 ++ drivers/cxl/core/port.c | 4 + drivers/cxl/core/switch-cci.c | 144 +++++++++++++++++++++++++++ drivers/cxl/cxlmem.h | 15 +++ drivers/cxl/switch-cci.c | 181 ++++++++++++++++++++++++++++++++++ include/uapi/linux/cxl_mem.h | 4 + 10 files changed, 372 insertions(+)drivers/cxl/Kconfig +14
-0@@ -152,5 +152,19 @@ config CXL_PMU Say 'y/m' to enable a driver that will attach to performance monitoring units and provide standard perf based interfaces. + If unsure say 'm'. + +config CXL_SWITCH + tristate "CXL switch mailbox access" + help + The CXL r3.0 specification defines a "CXL switch CCI" sub-class in the + PCI "Serial" base class of devices. Device's identified by + this class code provide a mailbox interface to allow control of CXL + switch configuration over inband PCI. + + Say 'y/m' to enable a driver that will attach to CXL Switch CCI + devices enumerated by the CXL switch CCI class code for configuration + and management primarily via the mailbox interface. + If unsure say 'm'.endif
drivers/cxl/Makefile +0
-0@@ -5,9 +5,11 @@ obj-$(CONFIG_CXL_MEM) += cxl_mem.oobj-$(CONFIG_CXL_ACPI) += cxl_acpi.o
obj-$(CONFIG_CXL_PMEM) += cxl_pmem.o
obj-$(CONFIG_CXL_PORT) += cxl_port.o
+obj-$(CONFIG_CXL_SWITCH) += cxl_switch_cci.ocxl_mem-y := mem.o
cxl_pci-y := pci.o
cxl_acpi-y := acpi.o
cxl_pmem-y := pmem.o security.o
cxl_port-y := port.odrivers/cxl/Makefile +1
-0+cxl_switch_cci-y := switch-cci.odrivers/cxl/core/Makefile +0
-0@@ -13,5 +13,6 @@ cxl_core-y += mbox.ocxl_core-y += pci.o
cxl_core-y += hdm.o
cxl_core-y += pmu.o
+cxl_core-y += switch-cci.o
cxl_core-$(CONFIG_TRACING) += trace.o
cxl_core-$(CONFIG_CXL_REGION) += region.odrivers/cxl/core/core.h +1
-0@@ -80,6 +80,7 @@ extern struct rw_semaphore cxl_dpa_rwsem;int cxl_memdev_init (void );void cxl_memdev_exit (void );void cxl_mbox_init (void ); +int cxl_switch_cci_init (void );enum cxl_poison_trace_type {CXL_POISON_TRACE_LIST ,drivers/cxl/core/mbox.c +6
-0@@ -9,6 +9,7 @@#include <cxlmbox.h> #include <cxlpci.h> #include <cxlmem.h> +#include <cxlpci.h> #include <cxl.h> #include "core.h" @@ -56,6 +57,8 @@ static bool cxl_raw_allow_all;* 0, and the user passed in 1, it is an error. */ static struct cxl_mem_command cxl_mem_commands[CXL_MEM_COMMAND_ID_MAX ] = { +CXL_CMD (INFO_STAT_IDENTIFY ,0 ,0x12 ,0 ), +CXL_CMD (GET_BG_CMD_STATUS ,0 ,8 ,0 ),CXL_CMD (IDENTIFY ,0 ,0x43 ,CXL_CMD_FLAG_FORCE_ENABLE ), #ifdef CONFIG_CXL_MEM_RAW_COMMANDS CXL_CMD(RAW, CXL_VARIABLE_PAYLOAD, CXL_VARIABLE_PAYLOAD, 0),@@ -73,6 +76,9 @@ static struct cxl_mem_command cxl_mem_commands[CXL_MEM_COMMAND_ID_MAX] = {CXL_CMD (GET_SHUTDOWN_STATE ,0 ,0x1 ,0 ),CXL_CMD (SET_SHUTDOWN_STATE ,0x1 ,0 ,0 ),CXL_CMD (GET_SCAN_MEDIA_CAPS ,0x10 ,0x4 ,0 ), +CXL_CMD (IDENTIFY_SWITCH_DEVICE ,0 ,0x49 ,0 ), +CXL_CMD (TUNNEL_MANAGEMENT_COMMAND ,CXL_VARIABLE_PAYLOAD , + CXL_VARIABLE_PAYLOAD, 0), };/* drivers/cxl/core/port.c +4
-0@@ -2020,6 +2020,10 @@ static __init int cxl_core_init(void)if (rc)return rc; + rc =cxl_switch_cci_init (); +if (rc) +return rc; + cxl_bus_wq =alloc_ordered_workqueue ("cxl_port" ,0 );if (!cxl_bus_wq) { rc = -ENOMEM ;drivers/cxl/core/switch-cci.c new file +144
-0@@ -0,0 +1,144 @@ +// SPDX-License-Identifier: GPL-2.0-only +#include "cxlmem.h" /* For now to get the cxl_device_state */ +#include "cxlpci.h" +#include "core.h" + +static int cxl_sw_major; +static DEFINE_IDA (cxl_swdev_ida); +static DECLARE_RWSEM (cxl_swdev_rwsem); + +static inline struct cxl_swdev *to_cxl_swdev (struct device *dev) +{ + return container_of(dev, struct cxl_swdev, dev); +} + +static char *cxl_swdev_devnode (const struct device *dev,umode_t *mode,kuid_t *uid, +kgid_t *gid) +{ +return kasprintf (GFP_KERNEL ,"cxl/%s" ,dev_name (dev)); +} + +static long __cxl_swdev_ioctl (struct cxl_swdev *cxlswd,unsigned int cmd, +unsigned long arg) +{ +switch (cmd) { +case CXL_MEM_SEND_COMMAND : +return cxl_send_cmd (&cxlswd->mbox, (void __user *)arg); +default : +return -ENOTTY ; + } +} + +static long cxl_swdev_ioctl (struct file *file,unsigned int cmd, +unsigned long arg) +{ +struct cxl_swdev *cxlswd = file->private_data; +int rc = -ENXIO ; + +down_read (&cxl_swdev_rwsem); + rc =__cxl_swdev_ioctl (cxlswd, cmd, arg); +up_read (&cxl_swdev_rwsem); + +return rc; +} + +static int cxl_swdev_open (struct inode *inode,struct file *file) +{ +struct cxl_memdev *cxlswd = +container_of (inode->i_cdev,typeof (*cxlswd), cdev); + +get_device (&cxlswd->dev); + file->private_data = cxlswd; + +return 0 ; +} + +static int cxl_swdev_release_file (struct inode *inode,struct file *file) +{ +struct cxl_swdev *cxlswd = +container_of (inode->i_cdev,typeof (*cxlswd), cdev); + +put_device (&cxlswd->dev); + +return 0 ; +} + +static const struct file_operations cxl_swdev_fops = { + .owner =THIS_MODULE , + .unlocked_ioctl = cxl_swdev_ioctl, + .open = cxl_swdev_open, + .release = cxl_swdev_release_file, + .compat_ioctl = compat_ptr_ioctl, + .llseek = noop_llseek, +}; + +void cxl_swdev_shutdown (struct cxl_swdev *cxlswd) +{ +down_write (&cxl_swdev_rwsem); +up_write (&cxl_swdev_rwsem); +} +EXPORT_SYMBOL_NS_GPL (cxl_swdev_shutdown,CXL ); + +static void cxl_swdev_release (struct device *dev) +{ +struct cxl_swdev *cxlswd =to_cxl_swdev (dev); + +ida_free (&cxl_swdev_ida, cxlswd->id); +kfree (cxlswd); +} + +static const struct device_type cxl_swdev_type = { + .name ="cxl_swdev" , + .release = cxl_swdev_release, + .devnode = cxl_swdev_devnode, +}; + +struct cxl_swdev *cxl_swdev_alloc (struct device *parent) +{ +struct cxl_swdev *cxlswd; +struct device *dev; +struct cdev *cdev; +int rc; + + cxlswd =kzalloc (sizeof (*cxlswd),GFP_KERNEL ); +if (!cxlswd) +return ERR_PTR (-ENOMEM ); + + rc =ida_alloc_range (&cxl_swdev_ida,0 ,10 ,GFP_KERNEL ); +if (rc <0 ) { +kfree (cxlswd); +return ERR_PTR (rc); + } + + cxlswd->id = rc; + dev = &cxlswd->dev; +device_initialize (dev); + dev->bus = &cxl_bus_type; + dev->parent = parent; + dev->devt =MKDEV (cxl_sw_major, cxlswd->id); + dev->type = &cxl_swdev_type; +device_set_pm_not_required (dev); + cdev = &cxlswd->cdev; +cdev_init (cdev, &cxl_swdev_fops); + rc =dev_set_name (dev,"swcci%d" , cxlswd->id); +if (rc) { +put_device (dev); +return ERR_PTR (rc); + } + +return cxlswd; +} +EXPORT_SYMBOL_NS_GPL (cxl_swdev_alloc,CXL ); + +__init int cxl_switch_cci_init (void ) +{ +dev_t devt; +int rc; + + rc =alloc_chrdev_region (&devt,0 ,10 ,"cxlsw" ); +if (rc) +return rc; + cxl_sw_major =MAJOR (devt); + +return 0 ; +}drivers/cxl/cxlmem.h +11
-0@@ -456,6 +456,17 @@ struct cxl_memdev_state {struct cxl_fw_state fw; }; +struct cxl_swdev { +struct device dev; +struct cdev cdev; +struct cxl_mbox mbox; +struct cxl_dev_state cxlds; +int id; +}; + +struct cxl_swdev *cxl_swdev_alloc (struct device *parent); +void cxl_swdev_shutdown (struct cxl_swdev *cxlswd); +static inline struct
cxl_memdev_state*to_cxl_memdev_state(struct cxl_dev_state *cxlds)
{drivers/cxl/cxlmem.h +4
-0@@ -466,6 +477,8 @@ to_cxl_memdev_state(struct cxl_dev_state *cxlds)enum cxl_opcode {CXL_MBOX_OP_INVALID =0x0000 , +CXL_MBOX_OP_INFO_STAT_IDENTIFY =0x0001 , +CXL_MBOX_OP_GET_BG_CMD_STATUS =0x0002 ,CXL_MBOX_OP_RAW =CXL_MBOX_OP_INVALID ,CXL_MBOX_OP_GET_EVENT_RECORD =0x0100 ,CXL_MBOX_OP_CLEAR_EVENT_RECORD =0x0101 ,@@ -501,6 +514,8 @@ enum cxl_opcode {CXL_MBOX_OP_UNLOCK =0x4503 ,CXL_MBOX_OP_FREEZE_SECURITY =0x4504 ,CXL_MBOX_OP_PASSPHRASE_SECURE_ERASE =0x4505 , +CXL_MBOX_OP_IDENTIFY_SWITCH_DEVICE =0x5100 , +CXL_MBOX_OP_TUNNEL_MANAGEMENT_COMMAND =0x5300 , CXL_MBOX_OP_MAX = 0x10000 };drivers/cxl/switch-cci.c new file +181
-0@@ -0,0 +1,181 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* +* Copyright(c) Huawei Technologies +* Based on cxl/pci.c Copyright(c) 2020 Intel Corporation. All rights reserved. +*/ + +#include <linux/io-64-nonatomic-lo-hi.h> +#include <linux/moduleparam.h> +#include <linux/module.h> +#include <linux/delay.h> +#include <linux/sizes.h> +#include <linux/mutex.h> +#include <linux/list.h> +#include <linux/pci.h> +#include <linux/io.h> +#include "cxlmem.h" +#include "cxlpci.h" +#include "cxl.h" + +static irqreturn_t cxl_swmbcci_mbox_irq (int irq,void *d) +{ +return cxl_mbox_irq (irq, d); +} + +static int cxl_swmbcci_setup_mailbox (struct cxl_mbox *mbox) +{ +const int cap =readl (mbox->mbox +CXLDEV_MBOX_CAPS_OFFSET ); + +/* +* A command may be in flight from a previous driver instance, +* think kexec, do one doorbell wait so that +* __cxl_pci_mbox_send_cmd() can assume that it is the only +* source for future doorbell busy events. +*/ +if (cxl_pci_mbox_wait_for_doorbell (mbox) !=0 ) { +u64 md_status =0 ; + +if (mbox->get_status) + md_status = mbox->get_status (mbox); +cxl_err (mbox->dev, md_status,"timeout awaiting mailbox idle" ); + +return -ETIMEDOUT ; + } + + mbox->payload_size = +1 <<FIELD_GET (CXLDEV_MBOX_CAP_PAYLOAD_SIZE_MASK , cap); + +/* +* CXL 2.0 8.2.8.4.3 Mailbox Capabilities Register +* +* If the size is too small, mandatory commands will not work and so +* there's no point in going forward. If the size is too large, there's +* no harm is soft limiting it. +*/ + mbox->payload_size =min_t (size_t, mbox->payload_size,SZ_1M ); +if (mbox->payload_size <256 ) { +dev_err (mbox->dev,"Mailbox is too small (%zub)" , + mbox->payload_size); +return -ENXIO ; + } + +dev_dbg (mbox->dev,"Mailbox payload sized %zu" , mbox->payload_size); + +rcuwait_init (&mbox->mbox_wait); + +if (cap &CXLDEV_MBOX_CAP_BG_CMD_IRQ ) { +u32 ctrl; +int irq, msgnum, rc; +struct pci_dev *pdev =to_pci_dev (mbox->dev); + + msgnum =FIELD_GET (CXLDEV_MBOX_CAP_IRQ_MSGNUM_MASK , cap); + irq =pci_irq_vector (pdev, msgnum); +if (irq <0 ) + goto mbox_poll; + + rc =devm_request_threaded_irq (mbox->dev, irq, cxl_swmbcci_mbox_irq, +NULL ,IRQF_SHARED |IRQF_ONESHOT , +NULL , mbox); +if (rc) + goto mbox_poll; + +/* enable background command mbox irq support */ + ctrl =readl (mbox->mbox +CXLDEV_MBOX_CTRL_OFFSET ); + ctrl |=CXLDEV_MBOX_CTRL_BG_CMD_IRQ ; +writel (ctrl, mbox->mbox +CXLDEV_MBOX_CTRL_OFFSET ); + +return 0 ; + } + +mbox_poll: +if (mbox->special_init_poll) + mbox->special_init_poll (mbox); + +dev_dbg (mbox->dev,"Mailbox interrupts are unsupported" ); +return 0 ; +} + + +static int cxl_swmbcci_probe (struct pci_dev *pdev,const struct pci_device_id *id) +{ +struct cxl_register_map map; +struct cxl_swdev *cxlswd; +int rc; + + rc =pcim_enable_device (pdev); +if (rc) +return rc; + + cxlswd =cxl_swdev_alloc (&pdev->dev); +if (IS_ERR (cxlswd)) +return PTR_ERR (cxlswd); + +mutex_init (&cxlswd->mbox.mbox_mutex); + rc =cxl_find_regblock (pdev,CXL_REGLOC_RBI_MEMDEV , &map); +if (rc) +return rc; + rc =cxl_setup_regs (&map); +if (rc) +return rc; + + rc =cxl_map_device_regs (&map, &cxlswd->cxlds.regs.device_regs); +if (rc) +return rc; + + rc =cxl_map_mbox_regs (&map, &cxlswd->mbox.mbox); +if (rc) +return rc; + + cxlswd->mbox.status = cxlswd->cxlds.regs.status; + cxlswd->mbox.dev = &pdev->dev; + + rc =cxl_swmbcci_setup_mailbox (&cxlswd->mbox); +if (rc) +return rc; + + +pci_set_drvdata (pdev, cxlswd); + + rc =cxl_enumerate_cmds (&cxlswd->mbox); +if (rc) + goto error_put_device; + + rc =cdev_device_add (&cxlswd->cdev, &cxlswd->dev); +if (rc) + goto error_put_device; + +return 0 ; + +error_put_device: +cxl_swdev_shutdown (cxlswd); +put_device (&cxlswd->dev); +return rc; +} + +static void cxl_swbmcci_remove (struct pci_dev *pdev) +{ +struct cxl_swdev *cxlswd =pci_get_drvdata (pdev); +struct device *dev = &cxlswd->dev; + +cxl_swdev_shutdown (cxlswd); +cdev_device_del (&cxlswd->cdev, dev); +put_device (&cxlswd->dev); +} + +static const struct pci_device_id cxl_swmbcci_pci_tbl[] = { + {PCI_DEVICE_CLASS (PCI_CLASS_SERIAL_CXL_SWITCH_CCI , ~0 ) }, + {} +}; +MODULE_DEVICE_TABLE (pci, cxl_swmbcci_pci_tbl); + +static struct pci_driver cxl_swmbcci_driver = { + .name =KBUILD_MODNAME , + .id_table = cxl_swmbcci_pci_tbl, + .probe = cxl_swmbcci_probe, + .remove = cxl_swbmcci_remove, +}; + +module_pci_driver (cxl_swmbcci_driver); +MODULE_DESCRIPTION ("CXL Switch CCI mailbox access driver" ); +MODULE_LICENSE ("GPL" ); +MODULE_IMPORT_NS (CXL );include/uapi/linux/cxl_mem.h +0
-0
include/uapi/linux/cxl_mem.h line 50
Quoted 5 lines
include/uapi/linux/cxl_mem.h +2
-0@@ -46,6 +46,10 @@___C (GET_SCAN_MEDIA_CAPS ,"Get Scan Media Capabilities" ), \___DEPRECATED (SCAN_MEDIA ,"Scan Media" ), \___DEPRECATED (GET_SCAN_MEDIA ,"Get Scan Media Results" ), \ +___C (INFO_STAT_IDENTIFY ,"Get Information" ), \ + ___C(GET_BG_CMD_STATUS, "Background Command Status"), \
There has been concerns in the past about exposing this and the fact that it's conceptually broken for userspace.
https://lore.kernel.org/all/[email protected]/
Thanks,
Davidlohr
Quoted 8 lines
include/uapi/linux/cxl_mem.h +2
-1+___C (IDENTIFY_SWITCH_DEVICE ,"Identify Switch Device" ), \ +___C (TUNNEL_MANAGEMENT_COMMAND ,"Tunnel Management Command" ), \ ___C(MAX, "invalid / last command")#define ___C (a, b) CXL_MEM_COMMAND_ID_##a --2.39.2
Davidlohr Bueso
maintainerRe: [RFC PATCH v4 4/4] cxl/pci: Add support for stand alone CXL Switch mailbox CCI
In reply to Jonathan CameronQuoted 31 lines · Jonathan Cameron
drivers/cxl/switch-cci.c new file +31
-0+static int cxl_swmbcci_probe (struct pci_dev *pdev,const struct pci_device_id *id) +{ +struct cxl_register_map map; +struct cxl_swdev *cxlswd; +int rc; + + rc =pcim_enable_device (pdev); +if (rc) +return rc; + + cxlswd =cxl_swdev_alloc (&pdev->dev); +if (IS_ERR (cxlswd)) +return PTR_ERR (cxlswd); + +mutex_init (&cxlswd->mbox.mbox_mutex); + rc =cxl_find_regblock (pdev,CXL_REGLOC_RBI_MEMDEV , &map); +if (rc) +return rc; + rc =cxl_setup_regs (&map); +if (rc) +return rc; + + rc =cxl_map_device_regs (&map, &cxlswd->cxlds.regs.device_regs); +if (rc) +return rc; + + rc =cxl_map_mbox_regs (&map, &cxlswd->mbox.mbox); +if (rc) +return rc; + + cxlswd->mbox.status = cxlswd->cxlds.regs.status;
drivers/cxl/switch-cci.c line 139
Quoted 10 lines
drivers/cxl/switch-cci.c new file +10
-0+ cxlswd->mbox.dev = &pdev->dev; + + rc =cxl_swmbcci_setup_mailbox (&cxlswd->mbox); +if (rc) +return rc; + + +pci_set_drvdata (pdev, cxlswd); + + rc =cxl_enumerate_cmds (&cxlswd->mbox);
Making the whole mbox more generically available makes me wonder if we want to start using Get Supported Logs Sub-List instead of the current gsl? The spec is pretty explicit about this:
""
It is strongly recommended that the Get Supported Logs Sub-List (see Section 8.2.9.5.6) is supported by Components and used by software instead of Get Supported Logs so that requestors may control the output payload size, as needed. Type 3 Devices that implement support for the Get Supported Logs opcode on an MCTP-based CCI shall also support the Get Supported Logs Sub-List opcode. ""
Thanks,
Davidlohr
Quoted 14 lines
drivers/cxl/switch-cci.c new file +14
-0+if (rc) + goto error_put_device; + + rc =cdev_device_add (&cxlswd->cdev, &cxlswd->dev); +if (rc) + goto error_put_device; + +return 0 ; + +error_put_device: +cxl_swdev_shutdown (cxlswd); +put_device (&cxlswd->dev); +return rc; +}
Re: [RFC PATCH v4 4/4] cxl/pci: Add support for stand alone CXL Switch mailbox CCI
Quoted 48 lines · On Wed, 26 Jul 2023 13:00:31 -0700 Davidlohr Bueso <[email protected]> wrote:
Quoted 31 lines · Jonathan Cameron
drivers/cxl/switch-cci.c new file +31
-0+static int cxl_swmbcci_probe (struct pci_dev *pdev,const struct pci_device_id *id) +{ +struct cxl_register_map map; +struct cxl_swdev *cxlswd; +int rc; + + rc =pcim_enable_device (pdev); +if (rc) +return rc; + + cxlswd =cxl_swdev_alloc (&pdev->dev); +if (IS_ERR (cxlswd)) +return PTR_ERR (cxlswd); + +mutex_init (&cxlswd->mbox.mbox_mutex); + rc =cxl_find_regblock (pdev,CXL_REGLOC_RBI_MEMDEV , &map); +if (rc) +return rc; + rc =cxl_setup_regs (&map); +if (rc) +return rc; + + rc =cxl_map_device_regs (&map, &cxlswd->cxlds.regs.device_regs); +if (rc) +return rc; + + rc =cxl_map_mbox_regs (&map, &cxlswd->mbox.mbox); +if (rc) +return rc; + + cxlswd->mbox.status = cxlswd->cxlds.regs.status;drivers/cxl/switch-cci.c line 139
Quoted 10 lines
drivers/cxl/switch-cci.c new file +10
-0+ cxlswd->mbox.dev = &pdev->dev; + + rc =cxl_swmbcci_setup_mailbox (&cxlswd->mbox); +if (rc) +return rc; + + +pci_set_drvdata (pdev, cxlswd); + + rc =cxl_enumerate_cmds (&cxlswd->mbox);Making the whole mbox more generically available makes me wonder if we want to start using Get Supported Logs Sub-List instead of the current gsl? The spec is pretty explicit about this:
Quoted 7 lines
""
It is strongly recommended that the Get Supported Logs Sub-List (see Section 8.2.9.5.6) is supported by Components and used by software instead of Get Supported Logs so that requestors may control the output payload size, as needed. Type 3 Devices that implement support for the Get Supported Logs opcode on an MCTP-based CCI shall also support the Get Supported Logs Sub-List opcode. ""
Agreed - it would be good to make that move on any devices that support the command. Patches welcome :)
Jonathan
Quoted 18 lines
Thanks,
DavidlohrQuoted 14 lines
drivers/cxl/switch-cci.c new file +14
-0+if (rc) + goto error_put_device; + + rc =cdev_device_add (&cxlswd->cdev, &cxlswd->dev); +if (rc) + goto error_put_device; + +return 0 ; + +error_put_device: +cxl_swdev_shutdown (cxlswd); +put_device (&cxlswd->dev); +return rc; +}