ARM CCA Device Assignment support
Subsystems (13)
Aneesh Kumar K.V (Arm)
maintainercover ARM CCA Device Assignment support
This patch series implements support for Device Assignment in the ARM CCA architecture. The code changes are based on Alp12 specification published here [1].
The code builds on the TSM framework patches posted at [2]. We add extension to that framework so that TSM is now used in both the host and the guest.
A DA workflow can be summarized as below:
Host:
step 1.
echo ${DEVICE} > /sys/bus/pci/devices/${DEVICE}/driver/unbind echo vfio-pci > /sys/bus/pci/devices/${DEVICE}/driver_override echo ${DEVICE} > /sys/bus/pci/drivers_probe
step 2.
echo 1 > /sys/bus/pci/devices/$DEVICE/tsm/connect
Now in the guest we follow the below steps
step 1:
echo ${DEVICE} > /sys/bus/pci/devices/${DEVICE}/driver/unbind
step 2: Move the device to TDISP LOCK state
echo 1 > /sys/bus/pci/devices/${DEVICE}/tsm/lock
step 3: Moves the device to TDISP RUN state echo 1 > /sys/bus/pci/devices/${DEVICE}/tsm/accept
step 4: Load the driver again.
echo ${DEVICE} > /sys/bus/pci/drivers_probe
I'm currently working against TSM v3, as TSM v4 lacks the necessary callbacks—bind, unbind, and guest_req—required for guest interactions.
The implementation also makes use of RHI interfaces that fall outside the current RHI specification [5]. Once the spec is finalized, the code will be aligned accordingly.
For now, I’ve retained validate_mmio and vdev_req exit handling within KVM. This will transition to a guest_req-based mechanism once the specification is updated.
At that point, all device assignment (DA)-specific VM exits will exit directly to the VMM, and will use the guest_req ioctl to handle exit reasons. As part of this change, the handlers realm_exit_vdev_req_handler, realm_exit_vdev_comm_handler, and realm_exit_dev_mem_map_handler will be removed.
Full patchset for the kernel and kvmtool can be found at [3] and [4]
[1] https://developer.arm.com/-/cdn-downloads/permalink/Architectures/Armv9/DEN0137_1.1-alp12.zip
[2] https://lore.kernel.org/all/[email protected]
[3] https://git.gitlab.arm.com/linux-arm/linux-cca.git cca/tdisp-upstream-post-v1
[4] https://git.gitlab.arm.com/linux-arm/kvmtool-cca.git cca/tdisp-upstream-post-v1
[5] https://developer.arm.com/documentation/den0148/latest/
Aneesh Kumar K.V (Arm) (35):
tsm: Add tsm_bind/unbind helpers
tsm: Move tsm core outside the host directory
tsm: Move dsm_dev from pci_tdi to pci_tsm
tsm: Support DMA Allocation from private memory
tsm: Don't overload connect
iommufd: Add and option to request for bar mapping with
IORESOURCE_EXCLUSIVE
iommufd/viommu: Add support to associate viommu with kvm instance iommufd/tsm: Add tsm_op iommufd ioctls
iommufd/vdevice: Add TSM Guest request uAPI iommufd/vdevice: Add TSM map ioctl
KVM: arm64: CCA: register host tsm platform device
coco: host: arm64: CCA host platform device driver
coco: host: arm64: Create a PDEV with rmm
coco: host: arm64: Device communication support
coco: host: arm64: Stop and destroy the physical device
coco: host: arm64: set_pubkey support
coco: host: arm64: Add support for creating a virtual device
coco: host: arm64: Add support for virtual device communication
coco: host: arm64: Stop and destroy virtual device
coco: guest: arm64: Update arm CCA guest driver
arm64: CCA: Register guest tsm callback
cca: guest: arm64: Realm device lock support
KVM: arm64: Add exit handler related to device assignment
coco: host: arm64: add RSI_RDEV_GET_INSTANCE_ID related exit handler
coco: host: arm64: Add support for device communication exit handler
coco: guest: arm64: Add support for collecting interface reports
coco: host: arm64: Add support for realm host interface (RHI)
coco: guest: arm64: Add support for fetching interface report and
certificate chain from host
coco: guest: arm64: Add support for guest initiated TDI bind/unbind
KVM: arm64: CCA: handle dev mem map/unmap
coco: guest: arm64: Validate mmio range found in the interface report
coco: guest: arm64: Add Realm device start and stop support
KVM: arm64: CCA: enable DA in realm create parameters
coco: guest: arm64: Add support for fetching device measurements
coco: guest: arm64: Add support for fetching device info
Lukas Wunner (3):
X.509: Make certificate parser public
X.509: Parse Subject Alternative Name in certificates X.509: Move certificate length retrieval into new helper
arch/arm64/include/asm/kvm_rme.h | 3 +
arch/arm64/include/asm/mem_encrypt.h | 6 +-
arch/arm64/include/asm/rhi.h | 39 +
arch/arm64/include/asm/rmi_cmds.h | 173 ++++
arch/arm64/include/asm/rmi_smc.h | 210 ++++-
arch/arm64/include/asm/rsi.h | 5 +-
arch/arm64/include/asm/rsi_cmds.h | 129 +++
arch/arm64/include/asm/rsi_smc.h | 60 ++
arch/arm64/kernel/Makefile | 2 +-
arch/arm64/kernel/rhi.c | 35 +
arch/arm64/kernel/rsi.c | 26 +-
arch/arm64/kvm/mmu.c | 45 +
arch/arm64/kvm/rme-exit.c | 87 ++
arch/arm64/kvm/rme.c | 208 ++++-
arch/arm64/mm/mem_encrypt.c | 10 +
crypto/asymmetric_keys/x509_cert_parser.c | 9 +
crypto/asymmetric_keys/x509_loader.c | 38 +-
crypto/asymmetric_keys/x509_parser.h | 40 +-
drivers/iommu/iommufd/device.c | 54 ++
drivers/iommu/iommufd/iommufd_private.h | 7 +
drivers/iommu/iommufd/main.c | 13 +
drivers/iommu/iommufd/viommu.c | 178 +++-
drivers/pci/tsm.c | 229 ++++-
drivers/vfio/pci/vfio_pci_core.c | 20 +-
drivers/virt/coco/Kconfig | 5 +-
drivers/virt/coco/Makefile | 7 +-
drivers/virt/coco/arm-cca-guest/Kconfig | 10 +-
drivers/virt/coco/arm-cca-guest/Makefile | 3 +
.../{arm-cca-guest.c => arm-cca.c} | 175 +++-
drivers/virt/coco/arm-cca-guest/rsi-da.c | 576 ++++++++++++
drivers/virt/coco/arm-cca-guest/rsi-da.h | 73 ++
drivers/virt/coco/arm-cca-host/Kconfig | 17 +
drivers/virt/coco/arm-cca-host/Makefile | 5 +
drivers/virt/coco/arm-cca-host/arm-cca.c | 384 ++++++++
drivers/virt/coco/arm-cca-host/rmm-da.c | 857 ++++++++++++++++++
drivers/virt/coco/arm-cca-host/rmm-da.h | 108 +++
drivers/virt/coco/host/Kconfig | 6 -
drivers/virt/coco/host/Makefile | 6 -
drivers/virt/coco/{host => }/tsm-core.c | 27 +
include/keys/asymmetric-type.h | 2 +
include/keys/x509-parser.h | 55 ++
include/linux/device.h | 1 +
include/linux/iommufd.h | 4 +
include/linux/kvm_host.h | 1 +
include/linux/pci-tsm.h | 37 +-
include/linux/swiotlb.h | 4 +
include/linux/tsm.h | 29 +
include/uapi/linux/iommufd.h | 69 ++
48 files changed, 3887 insertions(+), 200 deletions(-)
create mode 100644 arch/arm64/include/asm/rhi.h
create mode 100644 arch/arm64/kernel/rhi.c
rename drivers/virt/coco/arm-cca-guest/{arm-cca-guest.c => arm-cca.c} (62%)
create mode 100644 drivers/virt/coco/arm-cca-guest/rsi-da.c
create mode 100644 drivers/virt/coco/arm-cca-guest/rsi-da.h
create mode 100644 drivers/virt/coco/arm-cca-host/Kconfig
create mode 100644 drivers/virt/coco/arm-cca-host/Makefile
create mode 100644 drivers/virt/coco/arm-cca-host/arm-cca.c
create mode 100644 drivers/virt/coco/arm-cca-host/rmm-da.c
create mode 100644 drivers/virt/coco/arm-cca-host/rmm-da.h
delete mode 100644 drivers/virt/coco/host/Kconfig
delete mode 100644 drivers/virt/coco/host/Makefile
rename drivers/virt/coco/{host => }/tsm-core.c (85%)
create mode 100644 include/keys/x509-parser.hSignature · 1 line
2.43.0
Aneesh Kumar K.V (Arm)
maintainerRFC PATCH 1/38 tsm: Add tsm_bind/unbind helpers
This will be later used by iommufd to bind a tdi to guest.
--- drivers/virt/coco/host/tsm-core.c | 18 ++++++++++++++++++ include/linux/tsm.h | 3 +++ 2 files changed, 21 insertions(+)
drivers/virt/coco/host/tsm-core.c +18 -0
@@ -116,6 +116,24 @@ void tsm_ide_stream_unregister(struct pci_ide *ide) }EXPORT_SYMBOL_GPL (tsm_ide_stream_unregister); +int tsm_bind (struct device *dev,struct kvm *kvm,u64 tdi_id) +{ +if (!dev_is_pci (dev)) +return -EINVAL ; + +return pci_tsm_bind (to_pci_dev (dev), kvm, tdi_id); +} +EXPORT_SYMBOL_GPL (tsm_bind); + +int tsm_unbind (struct device *dev) +{ +if (!dev_is_pci (dev)) +return -EINVAL ; + +return pci_tsm_unbind (to_pci_dev (dev)); +} +EXPORT_SYMBOL_GPL (tsm_unbind); +static void tsm_release (struct device *dev) {struct tsm_core_dev *core =container_of (dev,typeof (*core), dev);
include/linux/tsm.h +3 -0
@@ -118,6 +118,9 @@ struct tsm_core_dev *tsm_register(struct device *parent,void tsm_unregister (struct tsm_core_dev *tsm_core);struct pci_dev ;struct pci_ide ; +struct kvm ;int tsm_ide_stream_register (struct pci_dev *pdev,struct pci_ide *ide);void tsm_ide_stream_unregister (struct pci_ide *ide); +int tsm_bind (struct device *dev,struct kvm *kvm,u64 tdi_id); +int tsm_unbind (struct device *dev);#endif /* __TSM_H */
Signature · 1 line
2.43.0
Aneesh Kumar K.V (Arm)
maintainerRFC PATCH 2/38 tsm: Move tsm core outside the host directory
In reply to Aneesh Kumar K.V (Arm)A later patch will add guest changes that will also use the same infrastructure.
---
drivers/virt/coco/Kconfig | 3 ++-
drivers/virt/coco/Makefile | 6 ++++--
drivers/virt/coco/host/Kconfig | 6 ------
drivers/virt/coco/host/Makefile | 6 ------
drivers/virt/coco/{host => }/tsm-core.c | 0
5 files changed, 6 insertions(+), 15 deletions(-)
delete mode 100644 drivers/virt/coco/host/Kconfig
delete mode 100644 drivers/virt/coco/host/Makefile
rename drivers/virt/coco/{host => }/tsm-core.c (100%)drivers/virt/coco/Kconfig +2 -1
@@ -15,4 +15,5 @@ source "drivers/virt/coco/arm-cca-guest/Kconfig" source "drivers/virt/coco/guest/Kconfig" -source "drivers/virt/coco/host/Kconfig" +config TSM + tristate
drivers/virt/coco/Makefile +4 -2
@@ -2,10 +2,12 @@ # # Confidential computing related collateral # + obj-$(CONFIG_EFI_SECRET) += efi_secret/ obj-$(CONFIG_ARM_PKVM_GUEST) += pkvm-guest/ obj-$(CONFIG_SEV_GUEST) += sev-guest/ obj-$(CONFIG_INTEL_TDX_GUEST) += tdx-guest/ obj-$(CONFIG_ARM_CCA_GUEST) += arm-cca-guest/ -obj-$(CONFIG_TSM_REPORTS) += guest/ -obj-y += host/ + +obj-$(CONFIG_TSM) += tsm-core.o +obj-y += guest/
drivers/virt/coco/host/Kconfig deleted +0 -6
@@ -1,6 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only -# -# TSM (TEE Security Manager) Common infrastructure and host drivers -# -config TSM - tristate
drivers/virt/coco/host/Makefile deleted +0 -6
@@ -1,6 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only -# -# TSM (TEE Security Manager) Common infrastructure and host drivers - -obj-$(CONFIG_TSM) += tsm.o -tsm-y := tsm-core.o
drivers/virt/coco/tsm-core.c renamed from drivers/virt/coco/host/tsm-core.c +0 -0
Signature · 1 line
2.43.0
Aneesh Kumar K.V (Arm)
maintainerRFC PATCH 3/38 tsm: Move dsm_dev from pci_tdi to pci_tsm
In reply to Aneesh Kumar K.V (Arm)--- drivers/pci/tsm.c | 72 ++++++++++++++++++++++++----------------- include/linux/pci-tsm.h | 4 +-- 2 files changed, 45 insertions(+), 31 deletions(-)
drivers/pci/tsm.c +43 -29
@@ -415,15 +415,55 @@ static enum pci_tsm_type pci_tsm_type(struct pci_dev *pdev)return PCI_TSM_INVALID ; } +/* lookup the Device Security Manager (DSM) pf0 for @pdev */ +static struct pci_dev *dsm_dev_get (struct pci_dev *pdev) +{ +struct pci_dev *uport_pf0; + + struct pci_dev *pf0 __free(pci_dev_put) = pf0_dev_get(pdev); +if (!pf0) +return NULL ; + +if (pf0 == pdev) +return no_free_ptr (pf0); + +/* Check that @pf0 was not initialized as PCI_TSM_DOWNSTREAM */ +if (pf0->tsm && pf0->tsm->type ==PCI_TSM_PF0 ) +return no_free_ptr (pf0); + +/* +* For cases where a switch may be hosting TDISP services on +* behalf of downstream devices, check the first usptream port +* relative to this endpoint. +*/ +if (!pdev->dev.parent || !pdev->dev.parent->parent) +return NULL ; + + uport_pf0 =to_pci_dev (pdev->dev.parent->parent); +if (!uport_pf0->tsm) +return NULL ; +return pci_dev_get (uport_pf0); +} +/** * pci_tsm_initialize() - base 'struct pci_tsm' initialization * @pdev: The PCI device * @tsm: context to initialize */ -void pci_tsm_initialize (struct pci_dev *pdev,struct pci_tsm *tsm) +int pci_tsm_initialize (struct pci_dev *pdev,struct pci_tsm *tsm) { + struct pci_dev *dsm_dev __free(pci_dev_put) = dsm_dev_get(pdev); +if (!dsm_dev) +return -EINVAL ; + tsm->type =pci_tsm_type (pdev); tsm->pdev = pdev; +/* +* No reference needed because when we destroy +* dsm_dev all the tdis get destroyed before that. +*/ + tsm->dsm_dev = dsm_dev; +return 0 ; }EXPORT_SYMBOL_GPL (pci_tsm_initialize);
@@ -447,7 +487,8 @@ int pci_tsm_pf0_initialize(struct pci_dev *pdev, struct pci_tsm_pf0 *tsm) } tsm->state =PCI_TSM_INIT ; -pci_tsm_initialize (pdev, &tsm->tsm); +if (pci_tsm_initialize (pdev, &tsm->tsm)) +return -ENODEV ;return 0 ; }
@@ -612,32 +653,6 @@ int pci_tsm_doe_transfer(struct pci_dev *pdev, enum pci_doe_proto type, }EXPORT_SYMBOL_GPL (pci_tsm_doe_transfer); -/* lookup the Device Security Manager (DSM) pf0 for @pdev */ -static struct pci_dev *dsm_dev_get (struct pci_dev *pdev) -{ -struct pci_dev *uport_pf0; - - struct pci_dev *pf0 __free(pci_dev_put) = pf0_dev_get(pdev); -if (!pf0) -return NULL ; - -/* Check that @pf0 was not initialized as PCI_TSM_DOWNSTREAM */ -if (pf0->tsm && pf0->tsm->type ==PCI_TSM_PF0 ) -return no_free_ptr (pf0); - -/* -* For cases where a switch may be hosting TDISP services on -* behalf of downstream devices, check the first usptream port -* relative to this endpoint. -*/ -if (!pdev->dev.parent || !pdev->dev.parent->parent) -return NULL ; - - uport_pf0 =to_pci_dev (pdev->dev.parent->parent); -if (!uport_pf0->tsm) -return NULL ; -return pci_dev_get (uport_pf0); -}/* Only implement non-interruptible lock for now */ static struct mutex *tdi_ops_lock (struct pci_dev *pf0_dev)
@@ -695,7 +710,6 @@ int pci_tsm_bind(struct pci_dev *pdev, struct kvm *kvm, u64 tdi_id)return -ENXIO ; tdi->pdev = pdev; - tdi->dsm_dev = dsm_dev; tdi->kvm = kvm; pdev->tsm->tdi = tdi;
include/linux/pci-tsm.h +2 -2
@@ -38,7 +38,6 @@ enum pci_tsm_type {*/ struct pci_tdi {struct pci_dev *pdev; -struct pci_dev *dsm_dev;struct kvm *kvm; };
@@ -56,6 +55,7 @@ struct pci_tdi {*/ struct pci_tsm {struct pci_dev *pdev; +struct pci_dev *dsm_dev;enum pci_tsm_type type;struct pci_tdi *tdi; };
@@ -173,7 +173,7 @@ void pci_tsm_core_unregister(const struct pci_tsm_ops *ops);int pci_tsm_doe_transfer (struct pci_dev *pdev,enum pci_doe_proto type,const void *req,size_t req_sz,void *resp,size_t resp_sz); -void pci_tsm_initialize (struct pci_dev *pdev,struct pci_tsm *tsm); +int pci_tsm_initialize (struct pci_dev *pdev,struct pci_tsm *tsm);int pci_tsm_pf0_initialize (struct pci_dev *pdev,struct pci_tsm_pf0 *tsm);int pci_tsm_bind (struct pci_dev *pdev,struct kvm *kvm,u64 tdi_id);int pci_tsm_unbind (struct pci_dev *pdev);
Signature · 1 line
2.43.0
Aneesh Kumar K.V (Arm)
maintainerRFC PATCH 4/38 tsm: Support DMA Allocation from private memory
In reply to Aneesh Kumar K.V (Arm)Currently, we enforce the use of bounce buffers to ensure that memory accessed by non-secure devices is explicitly shared with the host [1]. However, for secure devices, this approach must be avoided.
To achieve this, we introduce a device flag that controls whether a bounce buffer allocation is required for the device. Additionally, this flag is used to manage the top IPA bit assignment for setting up protected/unprotected IPA aliases.
[1] commit fbf979a01375 ("arm64: Enforce bounce buffers for realm DMA")
based on changes from Alexey Kardashevskiy <[email protected]>
--- arch/arm64/include/asm/mem_encrypt.h | 6 +----- arch/arm64/mm/mem_encrypt.c | 10 ++++++++++ drivers/pci/tsm.c | 6 ++++++ include/linux/device.h | 1 + include/linux/swiotlb.h | 4 ++++ 5 files changed, 22 insertions(+), 5 deletions(-)
arch/arm64/include/asm/mem_encrypt.h +1 -5
@@ -15,14 +15,10 @@ int arm64_mem_crypt_ops_register(const struct arm64_mem_crypt_ops *ops);int set_memory_encrypted (unsigned long addr,int numpages);int set_memory_decrypted (unsigned long addr,int numpages); +bool force_dma_unencrypted (struct device *dev);int realm_register_memory_enc_ops (void ); -static inline bool force_dma_unencrypted (struct device *dev) -{ -return is_realm_world (); -} -/* * For Arm CCA guests, canonical addresses are "encrypted", so no changes * required for dma_addr_encrypted().
arch/arm64/mm/mem_encrypt.c +10 -0
@@ -17,6 +17,7 @@#include <linux/compiler.h> #include <linux/err.h> #include <linux/mm.h> +#include <linux/device.h> #include <asm/mem_encrypt.h>
@@ -48,3 +49,12 @@ int set_memory_decrypted(unsigned long addr, int numpages)return crypt_ops->decrypt (addr, numpages); }EXPORT_SYMBOL_GPL (set_memory_decrypted); + +bool force_dma_unencrypted (struct device *dev) +{ +if (dev->tdi_enabled) +return false; + +return is_realm_world (); +} +EXPORT_SYMBOL_GPL (force_dma_unencrypted);
drivers/pci/tsm.c +6 -0
@@ -120,6 +120,7 @@ static int pci_tsm_disconnect(struct pci_dev *pdev) tsm_ops->disconnect (pdev); tsm->state =PCI_TSM_INIT ; + pdev->dev.tdi_enabled = false;return 0 ; }
@@ -199,6 +200,8 @@ static int pci_tsm_accept(struct pci_dev *pdev)if (rc)return rc; tsm->state =PCI_TSM_ACCEPT ; + pdev->dev.tdi_enabled = true; +return 0 ; }
@@ -557,6 +560,9 @@ static void __pci_tsm_init(struct pci_dev *pdev)default :break ; } + +/* FIXME!! should this be default true and switch to false for TEE capable device */ + pdev->dev.tdi_enabled = false; }void pci_tsm_init (struct pci_dev *pdev)
include/linux/device.h +1 -0
@@ -688,6 +688,7 @@ struct device {#ifdef CONFIG_IOMMU_DMA bool dma_iommu:1;#endif + bool tdi_enabled:1; };/**
include/linux/swiotlb.h +4 -0
@@ -173,6 +173,10 @@ static inline bool is_swiotlb_force_bounce(struct device *dev) {struct io_tlb_mem *mem = dev->dma_io_tlb_mem; +if (dev->tdi_enabled) { +dev_warn_once (dev,"(TIO) Disable SWIOTLB" ); +return false; + }return mem && mem->force_bounce; }
Signature · 1 line
2.43.0
Aneesh Kumar K.V (Arm)
maintainerRFC PATCH 5/38 tsm: Don't overload connect
In reply to Aneesh Kumar K.V (Arm)We need separate handling in the guest while destroying the device. Hence switch to new callback lock and unlock. Hide the connect sysfs file in guest.
guest sysfs will now looks like
ls -al /sys/bus/pci/devices/0000:02:00.0/tsm/
total 0
drwxr-xr-x 2 root root 0 Jan 1 00:00 . drwxr-xr-x 7 root root 0 Jan 1 00:00 .. -rw-r--r-- 1 root root 4096 Jan 1 00:00 accept -rw-r--r-- 1 root root 4096 Jan 1 00:00 lock
--- drivers/pci/tsm.c | 136 ++++++++++++++++++++++++++++++++++------ include/linux/pci-tsm.h | 3 + 2 files changed, 121 insertions(+), 18 deletions(-)
drivers/pci/tsm.c +118 -18
@@ -125,18 +125,6 @@ static int pci_tsm_disconnect(struct pci_dev *pdev)return 0 ; } -/* -* TDISP locked state temporarily makes the device inaccessible, do not -* surprise live attached drivers -*/ -static int __driver_idle_connect (struct pci_dev *pdev) -{ -guard (device)(&pdev->dev); -if (pdev->dev.driver) -return -EBUSY ; -return tsm_ops->connect (pdev); -} -/* * When the registered ops support accept it indicates that this is a * TVM-side (guest) TSM operations structure. In this mode ->connect()
@@ -162,10 +150,7 @@ static int pci_tsm_connect(struct pci_dev *pdev)if (tsm->state >=PCI_TSM_CONNECT )return 0 ; -if (tvm_mode ()) - rc =__driver_idle_connect (pdev); -else - rc = tsm_ops->connect (pdev); + rc = tsm_ops->connect (pdev);if (rc)return rc; tsm->state =PCI_TSM_CONNECT ;
@@ -299,6 +284,99 @@ static ssize_t accept_show(struct device *dev, struct device_attribute *attr, }static DEVICE_ATTR_RW (accept); +static int pci_tsm_unlock (struct pci_dev *pdev) +{ +struct pci_tsm_pf0 *tsm =to_pci_tsm_pf0 (pdev->tsm); + + struct mutex *lock __free(tsm_ops_unlock) = tsm_ops_lock(tsm); +if (!lock) +return -EINTR ; + +if (tsm->state <PCI_TSM_INIT ) +return -ENXIO ; +if (tsm->state <PCI_TSM_LOCK ) +return 0 ; + + tsm_ops->unlock (pdev); + tsm->state =PCI_TSM_INIT ; + pdev->dev.tdi_enabled = false; + +return 0 ; +} + +/* +* TDISP locked state temporarily makes the device inaccessible, do not +* surprise live attached drivers +*/ +static int __driver_idle_lock (struct pci_dev *pdev) +{ +guard (device)(&pdev->dev); +if (pdev->dev.driver) +return -EBUSY ; +return tsm_ops->lock (pdev); +} + +static int pci_tsm_lock (struct pci_dev *pdev) +{ +struct pci_tsm_pf0 *tsm =to_pci_tsm_pf0 (pdev->tsm); +int rc; + + struct mutex *lock __free(tsm_ops_unlock) = tsm_ops_lock(tsm); +if (!lock) +return -EINTR ; + +if (tsm->state <PCI_TSM_INIT ) +return -ENXIO ; + + rc =__driver_idle_lock (pdev); +if (rc) +return rc; + tsm->state =PCI_TSM_CONNECT ; +return 0 ; +} + +static ssize_t lock_store (struct device *dev,struct device_attribute *attr, +const char *buf,size_t len) +{ +int rc; +bool connect; +struct pci_dev *pdev =to_pci_dev (dev); + + rc =kstrtobool (buf, &connect); +if (rc) +return rc; + + struct rw_semaphore *lock __free(tsm_read_unlock) = tsm_read_lock(); +if (!lock) +return -EINTR ; + +if (connect) + rc =pci_tsm_lock (pdev); +else + rc =pci_tsm_unlock (pdev); +if (rc) +return rc; +return len; +} + +static ssize_t lock_show (struct device *dev,struct device_attribute *attr, +char *buf) +{ +struct pci_dev *pdev =to_pci_dev (dev); +struct pci_tsm_pf0 *tsm; + + struct rw_semaphore *lock __free(tsm_read_unlock) = tsm_read_lock(); +if (!lock) +return -EINTR ; + +if (!pdev->tsm) +return -ENXIO ; + + tsm =to_pci_tsm_pf0 (pdev->tsm); +return sysfs_emit (buf,"%d\n" , tsm->state >=PCI_TSM_LOCK ); +} +static DEVICE_ATTR_RW (lock); +static umode_t pci_tsm_pf0_attr_visible (struct kobject *kobj,struct attribute *a,int n) {
@@ -306,6 +384,11 @@ static umode_t pci_tsm_pf0_attr_visible(struct kobject *kobj,/* Host context, filter out guest only attributes */ if (a == &dev_attr_accept.attr)return 0 ; +if (a == &dev_attr_lock.attr) +return 0 ; + }else { +if (a == &dev_attr_connect.attr) +return 0 ; }return a->mode;
@@ -325,6 +408,7 @@ DEFINE_SYSFS_GROUP_VISIBLE(pci_tsm_pf0);static struct attribute *pci_tsm_pf0_attrs[] = { &dev_attr_connect.attr, &dev_attr_accept.attr, + &dev_attr_lock.attr,NULL };
@@ -537,7 +621,8 @@ static void pci_tsm_pf0_init(struct pci_dev *pdev)return ; pdev->tsm =no_free_ptr (pci_tsm); -sysfs_update_group (&pdev->dev.kobj, &pci_tsm_auth_attr_group); +if (!tvm_mode ()) +sysfs_update_group (&pdev->dev.kobj, &pci_tsm_auth_attr_group);sysfs_update_group (&pdev->dev.kobj, &pci_tsm_pf0_attr_group);if (pci_tsm_owner_attr_group)sysfs_merge_group (&pdev->dev.kobj, pci_tsm_owner_attr_group);
@@ -602,6 +687,19 @@ static void pci_tsm_pf0_destroy(struct pci_dev *pdev)__pci_tsm_pf0_destroy (tsm); } +static void pci_tsm_guest_destroy (struct pci_dev *pdev) +{ +struct pci_tsm_pf0 *tsm =to_pci_tsm_pf0 (pdev->tsm); + +if (tsm->state >PCI_TSM_INIT ) +pci_tsm_unlock (pdev); + pdev->tsm =NULL ; +if (pci_tsm_owner_attr_group) +sysfs_unmerge_group (&pdev->dev.kobj, pci_tsm_owner_attr_group); +sysfs_update_group (&pdev->dev.kobj, &pci_tsm_pf0_attr_group); +__pci_tsm_pf0_destroy (tsm); +} +static void __pci_tsm_destroy (struct pci_dev *pdev) {struct pci_tsm *pci_tsm = pdev->tsm;
@@ -611,7 +709,9 @@ static void __pci_tsm_destroy(struct pci_dev *pdev)lockdep_assert_held_write (&pci_tsm_rwsem); -if (is_pci_tsm_pf0 (pdev)) { +if (tvm_mode ()) { +pci_tsm_guest_destroy (pdev); + }else if (is_pci_tsm_pf0 (pdev)) {pci_tsm_pf0_destroy (pdev); }else {__pci_tsm_unbind (pdev);
include/linux/pci-tsm.h +3 -0
@@ -11,6 +11,7 @@ enum pci_tsm_state {PCI_TSM_ERR =-1 ,PCI_TSM_INIT ,PCI_TSM_CONNECT , +PCI_TSM_LOCK , PCI_TSM_ACCEPT, };
@@ -153,6 +154,8 @@ struct pci_tsm_ops {void (*remove)(struct pci_tsm *tsm);int (*connect)(struct pci_dev *pdev);void (*disconnect)(struct pci_dev *pdev); +int (*lock)(struct pci_dev *pdev); +void (*unlock)(struct pci_dev *pdev);struct pci_tdi *(*bind)(struct pci_dev *pdev,struct pci_dev *pf0_dev,struct kvm *kvm,u64 tdi_id);void (*unbind)(struct pci_tdi *tdi);
Signature · 1 line
2.43.0
Aneesh Kumar K.V (Arm)
maintainerRFC PATCH 6/38 iommufd: Add and option to request for bar mapping with IORESOURCE_EXCLUSIVE
In reply to Aneesh Kumar K.V (Arm)--- drivers/iommu/iommufd/device.c | 54 +++++++++++++++++++++++++ drivers/iommu/iommufd/iommufd_private.h | 4 ++ drivers/iommu/iommufd/main.c | 3 ++ drivers/vfio/pci/vfio_pci_core.c | 9 ++++- include/linux/iommufd.h | 1 + include/uapi/linux/iommufd.h | 1 + 6 files changed, 71 insertions(+), 1 deletion(-)
drivers/iommu/iommufd/device.c +54 -0
@@ -1660,3 +1660,57 @@ int iommufd_get_hw_info(struct iommufd_ucmd *ucmd)iommufd_put_object (ucmd->ictx, &idev->obj);return rc; } + +static int iommufd_device_option_exclusive_ranges (struct iommu_option *cmd, +struct iommufd_device *idev) +{ +if (cmd->op ==IOMMU_OPTION_OP_GET ) { + cmd->val64 = idev->flags &IOMMUFD_DEVICE_EXCLUSIVE_RANGE ; +return 0 ; + } + +if (cmd->op ==IOMMU_OPTION_OP_SET ) { +if (cmd->val64 ==0 ) { + idev->flags &= ~IOMMUFD_DEVICE_EXCLUSIVE_RANGE ; +return 0 ; + }else if (cmd->val64 &IOMMUFD_DEVICE_EXCLUSIVE_RANGE ) { + idev->flags |=IOMMUFD_DEVICE_EXCLUSIVE_RANGE ; +return 0 ; + } +return -EINVAL ; + } +return -EOPNOTSUPP ; +} +bool iommufd_device_need_exclusive_range (struct iommufd_device *idev) +{ +return !!(idev->flags &IOMMUFD_DEVICE_EXCLUSIVE_RANGE ); +} + +int iommufd_device_option (struct iommufd_ucmd *ucmd) +{ +struct iommu_option *cmd = ucmd->cmd; +struct iommufd_device *idev; +int rc =0 ; + +if (cmd->__reserved) +return -EOPNOTSUPP ; + + idev =iommufd_get_device (ucmd, cmd->object_id); +if (IS_ERR (idev)) +return PTR_ERR (idev); + +mutex_lock (&idev->igroup->lock); + + +switch (cmd->option_id) { +case IOMMU_OPTION_EXCLUSIVE_RANGES : + rc =iommufd_device_option_exclusive_ranges (cmd, idev); +break ; +default : + rc = -EOPNOTSUPP ; + } + +mutex_unlock (&idev->igroup->lock); +iommufd_put_object (ucmd->ictx, &idev->obj); +return rc; +}
drivers/iommu/iommufd/iommufd_private.h +4 -0
@@ -346,6 +346,7 @@ int iommufd_ioas_change_process(struct iommufd_ucmd *ucmd);int iommufd_ioas_copy (struct iommufd_ucmd *ucmd);int iommufd_ioas_unmap (struct iommufd_ucmd *ucmd);int iommufd_ioas_option (struct iommufd_ucmd *ucmd); +int iommufd_device_option (struct iommufd_ucmd *ucmd);int iommufd_option_rlimit_mode (struct iommu_option *cmd,struct iommufd_ctx *ictx);
@@ -489,10 +490,13 @@ struct iommufd_device {/* always the physical device */ struct device *dev;bool enforce_cache_coherency; +unsigned long flags;struct iommufd_vdevice *vdev;bool destroying; }; +#define IOMMUFD_DEVICE_EXCLUSIVE_RANGE BIT(0) +static inline struct iommufd_device *iommufd_get_device (struct iommufd_ucmd *ucmd,u32 id) {
drivers/iommu/iommufd/main.c +3 -0
@@ -376,6 +376,9 @@ static int iommufd_option(struct iommufd_ucmd *ucmd)case IOMMU_OPTION_HUGE_PAGES : rc =iommufd_ioas_option (ucmd);break ; +case IOMMU_OPTION_EXCLUSIVE_RANGES : + rc =iommufd_device_option (ucmd); +break ;default :return -EOPNOTSUPP ; }
drivers/vfio/pci/vfio_pci_core.c +8 -1
@@ -1753,8 +1753,15 @@ int vfio_pci_core_mmap(struct vfio_device *core_vdev, struct vm_area_struct *vma* we need to request the region and the barmap tracks that. */ if (!vdev->barmap[index]) { - ret =pci_request_selected_regions (pdev, + +if (core_vdev->iommufd_device && +iommufd_device_need_exclusive_range (core_vdev->iommufd_device)) + ret =pci_request_selected_regions_exclusive (pdev, +1 << index,"vfio-pci" ); +else + ret =pci_request_selected_regions (pdev,1 << index,"vfio-pci" ); +if (ret)return ret;
include/linux/iommufd.h +1 -0
@@ -70,6 +70,7 @@ void iommufd_device_detach(struct iommufd_device *idev, ioasid_t pasid);struct iommufd_ctx *iommufd_device_to_ictx (struct iommufd_device *idev);u32 iommufd_device_to_id (struct iommufd_device *idev); +bool iommufd_device_need_exclusive_range (struct iommufd_device *idev);struct iommufd_access_ops {u8 needs_pin_pages :1 ;
include/uapi/linux/iommufd.h +1 -0
@@ -310,6 +310,7 @@ struct iommu_ioas_unmap {enum iommufd_option {IOMMU_OPTION_RLIMIT_MODE =0 ,IOMMU_OPTION_HUGE_PAGES =1 , +IOMMU_OPTION_EXCLUSIVE_RANGES =2 , };/**
Signature · 1 line
2.43.0
Aneesh Kumar K.V (Arm)
maintainerRFC PATCH 7/38 iommufd/viommu: Add support to associate viommu with kvm instance
In reply to Aneesh Kumar K.V (Arm)The associated kvm instance will be used in later patch by iommufd to bind a tdi to kvm.
--- drivers/iommu/iommufd/viommu.c | 45 +++++++++++++++++++++++++++++++++- include/linux/iommufd.h | 3 +++ include/uapi/linux/iommufd.h | 12 +++++++++ 3 files changed, 59 insertions(+), 1 deletion(-)
drivers/iommu/iommufd/viommu.c +44 -1
@@ -2,6 +2,36 @@/* Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES */ #include "iommufd_private.h" +#include "linux/tsm.h" + +#if IS_ENABLED (CONFIG_KVM ) +#include <linux/kvm_host.h> + +static int viommu_get_kvm (struct iommufd_viommu *viommu,int kvm_vm_fd) +{ +int rc = -EBADF ; +struct file *filp; + + filp =fget (kvm_vm_fd); + +if (!file_is_kvm (filp)) + goto err_out; + +/* hold the kvm reference via file descriptor */ + viommu->kvm_filp = filp; +return 0 ; +err_out: + viommu->kvm_filp =NULL ; +fput (filp); +return rc; +} + +static void viommu_put_kvm (struct iommufd_viommu *viommu) +{ +fput (viommu->kvm_filp); + viommu->kvm_filp =NULL ; +} +#endif void iommufd_viommu_destroy (struct iommufd_object *obj) {
@@ -12,6 +42,8 @@ void iommufd_viommu_destroy(struct iommufd_object *obj) viommu->ops->destroy (viommu);refcount_dec (&viommu->hwpt->common.obj.users);xa_destroy (&viommu->vdevs); + +viommu_put_kvm (viommu); }int iommufd_viommu_alloc_ioctl (struct iommufd_ucmd *ucmd)
@@ -29,7 +61,9 @@ int iommufd_viommu_alloc_ioctl(struct iommufd_ucmd *ucmd)size_t viommu_size;int rc; -if (cmd->flags || cmd->type ==IOMMU_VIOMMU_TYPE_DEFAULT ) +if (cmd->flags & ~IOMMU_VIOMMU_KVM_FD ) +return -EOPNOTSUPP ; +if (cmd->type ==IOMMU_VIOMMU_TYPE_DEFAULT )return -EOPNOTSUPP ; idev =iommufd_get_device (ucmd, cmd->dev_id);
@@ -100,8 +134,17 @@ int iommufd_viommu_alloc_ioctl(struct iommufd_ucmd *ucmd) goto out_put_hwpt; } +/* get the kvm details if specified. */ +if (cmd->flags &IOMMU_VIOMMU_KVM_FD ) { + rc =viommu_get_kvm (viommu, cmd->kvm_vm_fd); +if (rc) + goto out_put_hwpt; + } + cmd->out_viommu_id = viommu->obj.id; rc =iommufd_ucmd_respond (ucmd,sizeof (*cmd)); +if (rc) +viommu_put_kvm (viommu); out_put_hwpt:iommufd_put_object (ucmd->ictx, &hwpt_paging->common.obj);
include/linux/iommufd.h +3 -0
@@ -12,6 +12,7 @@#include <linux/refcount.h> #include <linux/types.h> #include <linux/xarray.h> +#include <linux/file.h> #include <uapi/linux/iommufd.h> struct device ;
@@ -58,6 +59,7 @@ struct iommufd_object {unsigned int id; }; +struct kvm ;struct iommufd_device *iommufd_device_bind (struct iommufd_ctx *ictx,struct device *dev,u32 *id);void iommufd_device_unbind (struct iommufd_device *idev);
@@ -102,6 +104,7 @@ struct iommufd_viommu {struct iommufd_ctx *ictx;struct iommu_device *iommu_dev;struct iommufd_hwpt_paging *hwpt; +struct file *kvm_filp;const struct iommufd_viommu_ops *ops;
include/uapi/linux/iommufd.h +12 -0
@@ -1023,6 +1023,17 @@ struct iommu_viommu_tegra241_cmdqv {__aligned_u64 out_vintf_mmap_length; }; +/** +* define IOMMU_VIOMMU_KVM_FD - Flag indicating a valid KVM VM file descriptor +* +* This flag must be set when allocating a viommu instance that will be +* associated with a specific KVM VM. When allocating a viommu instance for a +* KVM VM, this flag must be set to inform the initialization logic that +* @iommu_viommu_alloc::kvm_vm_fd is properly initialized. If this flag is not +* provided but @iommu_viommu_alloc::kvm_vm_fd field will be ignored. +*/ +#define IOMMU_VIOMMU_KVM_FD BIT(0) +/** * struct iommu_viommu_alloc - ioctl(IOMMU_VIOMMU_ALLOC) * @size: sizeof(struct iommu_viommu_alloc)
@@ -1057,6 +1068,7 @@ struct iommu_viommu_alloc {__u32 data_len;__u32 __reserved;__aligned_u64 data_uptr; +__u32 kvm_vm_fd; };#define IOMMU_VIOMMU_ALLOC _IO(IOMMUFD_TYPE, IOMMUFD_CMD_VIOMMU_ALLOC)
Signature · 1 line
2.43.0
Aneesh Kumar K.V (Arm)
maintainerRFC PATCH 8/38 iommufd/tsm: Add tsm_op iommufd ioctls
In reply to Aneesh Kumar K.V (Arm)Add operations bind and unbind used to bind a TDI to the secure guest.
--- drivers/iommu/iommufd/iommufd_private.h | 1 + drivers/iommu/iommufd/main.c | 3 ++ drivers/iommu/iommufd/viommu.c | 50 +++++++++++++++++++++++++ drivers/vfio/pci/vfio_pci_core.c | 10 +++++ include/uapi/linux/iommufd.h | 18 +++++++++ 5 files changed, 82 insertions(+)
drivers/iommu/iommufd/iommufd_private.h +1 -0
@@ -697,6 +697,7 @@ void iommufd_vdevice_destroy(struct iommufd_object *obj);void iommufd_vdevice_abort (struct iommufd_object *obj);int iommufd_hw_queue_alloc_ioctl (struct iommufd_ucmd *ucmd);void iommufd_hw_queue_destroy (struct iommufd_object *obj); +int iommufd_vdevice_tsm_op_ioctl (struct iommufd_ucmd *ucmd);static inline struct iommufd_vdevice *iommufd_get_vdevice (struct iommufd_ctx *ictx,u32 id)
drivers/iommu/iommufd/main.c +3 -0
@@ -410,6 +410,7 @@ union ucmd_buffer {struct iommu_veventq_alloc veventq;struct iommu_vfio_ioas vfio_ioas;struct iommu_viommu_alloc viommu; +struct iommu_vdevice_tsm_op tsm_op;#ifdef CONFIG_IOMMUFD_TEST struct iommu_test_cmd test;#endif
@@ -471,6 +472,8 @@ static const struct iommufd_ioctl_op iommufd_ioctl_ops[] = { __reserved),IOCTL_OP (IOMMU_VIOMMU_ALLOC ,iommufd_viommu_alloc_ioctl ,struct iommu_viommu_alloc ,out_viommu_id ), +IOCTL_OP (IOMMU_VDEVICE_TSM_OP ,iommufd_vdevice_tsm_op_ioctl , +struct iommu_vdevice_tsm_op ,vdevice_id ), #ifdef CONFIG_IOMMUFD_TEST IOCTL_OP(IOMMU_TEST_CMD, iommufd_test, struct iommu_test_cmd, last), #endif
drivers/iommu/iommufd/viommu.c +50 -0
59 lines not on this page · raw message
drivers/vfio/pci/vfio_pci_core.c +10 -0
@@ -694,6 +694,16 @@ void vfio_pci_core_close_device(struct vfio_device *core_vdev)#if IS_ENABLED (CONFIG_EEH )eeh_dev_release (vdev->pdev);#endif + +#if 0 +/* +* destroy vdevice which involves tsm unbind before we disable pci disable +* A MSE/BME clear will transition the device to error state. +*/ +if (core_vdev->iommufd_device) +iommufd_device_tombstone_vdevice (core_vdev->iommufd_device); +#endif +vfio_pci_core_disable (vdev);mutex_lock (&vdev->igate);
include/uapi/linux/iommufd.h +18 -0
@@ -57,6 +57,7 @@ enum {IOMMUFD_CMD_IOAS_CHANGE_PROCESS =0x92 ,IOMMUFD_CMD_VEVENTQ_ALLOC =0x93 ,IOMMUFD_CMD_HW_QUEUE_ALLOC =0x94 , + IOMMUFD_CMD_VDEVICE_TSM_OP = 0x95, };/**
@@ -1127,6 +1128,23 @@ enum iommu_veventq_flag { IOMMU_VEVENTQ_FLAG_LOST_EVENTS = (1U << 0), }; +/** +* struct iommu_vdevice_tsm_OP - ioctl(IOMMU_VDEVICE_TSM_OP) +* @size: sizeof(struct iommu_vdevice_tsm_OP) +* @op: Either TSM_BIND or TSM_UNBIMD +* @flags: Must be 0 +* @vdevice_id: Object handle for the vDevice. Returned from IOMMU_VDEVICE_ALLOC +*/ +struct iommu_vdevice_tsm_op { +__u32 size; +__u32 op; +__u32 flags; +__u32 vdevice_id; +}; +#define IOMMU_VDEVICE_TSM_OP _IO(IOMMUFD_TYPE, IOMMUFD_CMD_VDEVICE_TSM_OP) +#define IOMMU_VDEICE_TSM_BIND 0x1 +#define IOMMU_VDEICE_TSM_UNBIND 0x2 +/** * struct iommufd_vevent_header - Virtual Event Header for a vEVENTQ Status * @flags: Combination of enum iommu_veventq_flag
Signature · 1 line
2.43.0
Aneesh Kumar K.V (Arm)
maintainerRFC PATCH 9/38 iommufd/vdevice: Add TSM Guest request uAPI
In reply to Aneesh Kumar K.V (Arm)Add TSM Guest request uAPI against iommufd_vdevice to forward various TSM attestation & acceptance requests from guest to TSM driver/secure firmware. This uAPI takes function only after TSM Bind.
After a vPCI device is locked down by TSM Bind, CoCo VM should attest and accept the device in its TEE. These operations needs interaction with secure firmware and the device, but doesn't impact the device management from host's POV. It doesn't change the fact that host should not touch some part of the device (see TDISP spec) to keep the trusted assignment, and host could exit trusted assignment and roll back everything by TSM Unbind.
So the TSM Guest request becomes a passthrough channel for CoCo VM to exchange request/response blobs with TSM driver/secure firmware. The definition of this IOCTL illustates this idea.
Based on changes from: Alexey Kardashevskiy <[email protected]>
--- drivers/iommu/iommufd/iommufd_private.h | 1 + drivers/iommu/iommufd/main.c | 3 ++ drivers/iommu/iommufd/viommu.c | 40 +++++++++++++++++++++++++ drivers/pci/tsm.c | 15 ++++++++-- drivers/virt/coco/tsm-core.c | 9 ++++++ include/linux/pci-tsm.h | 30 ++----------------- include/linux/tsm.h | 23 ++++++++++++++ include/uapi/linux/iommufd.h | 28 +++++++++++++++++ 8 files changed, 120 insertions(+), 29 deletions(-)
drivers/iommu/iommufd/iommufd_private.h +1 -0
@@ -698,6 +698,7 @@ void iommufd_vdevice_abort(struct iommufd_object *obj);int iommufd_hw_queue_alloc_ioctl (struct iommufd_ucmd *ucmd);void iommufd_hw_queue_destroy (struct iommufd_object *obj);int iommufd_vdevice_tsm_op_ioctl (struct iommufd_ucmd *ucmd); +int iommufd_vdevice_tsm_guest_request_ioctl (struct iommufd_ucmd *ucmd);static inline struct iommufd_vdevice *iommufd_get_vdevice (struct iommufd_ctx *ictx,u32 id)
drivers/iommu/iommufd/main.c +3 -0
@@ -411,6 +411,7 @@ union ucmd_buffer {struct iommu_vfio_ioas vfio_ioas;struct iommu_viommu_alloc viommu;struct iommu_vdevice_tsm_op tsm_op; +struct iommu_vdevice_tsm_guest_request gr;#ifdef CONFIG_IOMMUFD_TEST struct iommu_test_cmd test;#endif
@@ -474,6 +475,8 @@ static const struct iommufd_ioctl_op iommufd_ioctl_ops[] = { struct iommu_viommu_alloc, out_viommu_id),IOCTL_OP (IOMMU_VDEVICE_TSM_OP ,iommufd_vdevice_tsm_op_ioctl ,struct iommu_vdevice_tsm_op ,vdevice_id ), +IOCTL_OP (IOMMU_VDEVICE_TSM_GUEST_REQUEST ,iommufd_vdevice_tsm_guest_request_ioctl , +struct iommu_vdevice_tsm_guest_request ,resp_uptr ), #ifdef CONFIG_IOMMUFD_TEST IOCTL_OP(IOMMU_TEST_CMD, iommufd_test, struct iommu_test_cmd, last), #endif
drivers/iommu/iommufd/viommu.c +40 -0
@@ -515,9 +515,49 @@ int iommufd_vdevice_tsm_op_ioctl(struct iommufd_ucmd *ucmd)iommufd_put_object (ucmd->ictx, &vdev->obj);return rc; } + +int iommufd_vdevice_tsm_guest_request_ioctl (struct iommufd_ucmd *ucmd) +{ +struct iommu_vdevice_tsm_guest_request *cmd = ucmd->cmd; +struct tsm_guest_req_info info = { + .type = cmd->type, + .type_info =u64_to_user_ptr (cmd->type_info_uptr), + .type_info_len = cmd->type_info_len, + .req =u64_to_user_ptr (cmd->req_uptr), + .req_len = cmd->req_len, + .resp =u64_to_user_ptr (cmd->resp_uptr), + .resp_len = cmd->resp_len, + }; +struct iommufd_vdevice *vdev; +int rc; + + vdev =container_of (iommufd_get_object (ucmd->ictx, cmd->vdevice_id, +IOMMUFD_OBJ_VDEVICE ), + struct iommufd_vdevice, obj); +if (IS_ERR (vdev)) +return PTR_ERR (vdev); + + rc =tsm_guest_req (vdev->idev->dev, &info); +if (rc) + goto err_out; + + cmd->resp_len = info.resp_len; + rc =iommufd_ucmd_respond (ucmd,sizeof (*cmd)); +err_out: +iommufd_put_object (ucmd->ictx, &vdev->obj); +return rc; +} +#else /* !CONFIG_TSM */ +int iommufd_vdevice_tsm_op_ioctl (struct iommufd_ucmd *ucmd) {return -ENODEV ; } + +int iommufd_vdevice_tsm_guest_request_ioctl (struct iommufd_ucmd *ucmd) +{ +return -ENODEV ; +} +#endif
drivers/pci/tsm.c +13 -2
@@ -861,7 +861,7 @@ int pci_tsm_unbind(struct pci_dev *pdev)EXPORT_SYMBOL_GPL (pci_tsm_unbind);/** -* pci_tsm_guest_req - VFIO/IOMMUFD helper to handle guest requests +* pci_tsm_guest_req - IOMMUFD helper to handle guest requests * @pdev: @pdev representing a bound tdi * @info: envelope for the request *
@@ -871,11 +871,12 @@ EXPORT_SYMBOL_GPL(pci_tsm_unbind);* posts to userspace (e.g. QEMU) that holds the host-to-guest RID * mapping. */ -int pci_tsm_guest_req (struct pci_dev *pdev,struct pci_tsm_guest_req_info *info) +static int __pci_tsm_guest_req (struct pci_dev *pdev,struct tsm_guest_req_info *info) {struct pci_tdi *tdi;int rc; +lockdep_assert_held_read (&pci_tsm_rwsem);if (!pdev->tsm)
@@ -899,4 +900,14 @@ int pci_tsm_guest_req(struct pci_dev *pdev, struct pci_tsm_guest_req_info *info)return 0 ; } + +int pci_tsm_guest_req (struct pci_dev *pdev,struct tsm_guest_req_info *info) +{ + struct rw_semaphore *lock __free(tsm_read_unlock) = tsm_read_lock(); +if (!lock) +return -EINTR ; + +return __pci_tsm_guest_req (pdev, info); +} +EXPORT_SYMBOL_GPL (pci_tsm_guest_req);
drivers/virt/coco/tsm-core.c +9 -0
@@ -134,6 +134,15 @@ int tsm_unbind(struct device *dev) }EXPORT_SYMBOL_GPL (tsm_unbind); +int tsm_guest_req (struct device *dev,struct tsm_guest_req_info *info) +{ +if (!dev_is_pci (dev)) +return -EINVAL ; + +return pci_tsm_guest_req (to_pci_dev (dev), info); +} +EXPORT_SYMBOL_GPL (tsm_guest_req); +static void tsm_release (struct device *dev) {struct tsm_core_dev *core =container_of (dev,typeof (*core), dev);
include/linux/pci-tsm.h +3 -27
@@ -108,31 +108,7 @@ static inline bool is_pci_tsm_pf0(struct pci_dev *pdev)return PCI_FUNC (pdev->devfn) ==0 ; } -enum pci_tsm_guest_req_type { -PCI_TSM_GUEST_REQ_TDXC , -}; - -/** -* struct pci_tsm_guest_req_info - parameter for pci_tsm_ops.guest_req() -* @type: identify the format of the following blobs -* @type_info: extra input/output info, e.g. firmware error code -* @type_info_len: the size of @type_info -* @req: request data buffer filled by guest -* @req_len: the size of @req filled by guest -* @resp: response data buffer filled by host -* @resp_len: for input, the size of @resp buffer filled by guest -* for output, the size of actual response data filled by host -*/ -struct pci_tsm_guest_req_info { -enum pci_tsm_guest_req_type type; -void *type_info; -size_t type_info_len; -void *req; -size_t req_len; -void *resp; -size_t resp_len; -}; - +struct tsm_guest_req_info ;/** * struct pci_tsm_ops - Low-level TSM-exported interface to the PCI core * @probe: probe/accept device for tsm operation, setup DSM context
@@ -160,7 +136,7 @@ struct pci_tsm_ops { struct kvm *kvm, u64 tdi_id);void (*unbind)(struct pci_tdi *tdi);int (*guest_req)(struct pci_dev *pdev, -struct pci_tsm_guest_req_info *info); +struct tsm_guest_req_info *info);int (*accept)(struct pci_dev *pdev); };
@@ -180,7 +156,7 @@ int pci_tsm_initialize(struct pci_dev *pdev, struct pci_tsm *tsm);int pci_tsm_pf0_initialize (struct pci_dev *pdev,struct pci_tsm_pf0 *tsm);int pci_tsm_bind (struct pci_dev *pdev,struct kvm *kvm,u64 tdi_id);int pci_tsm_unbind (struct pci_dev *pdev); -int pci_tsm_guest_req (struct pci_dev *pdev,struct pci_tsm_guest_req_info *info); +int pci_tsm_guest_req (struct pci_dev *pdev,struct tsm_guest_req_info *info);#else static inline int pci_tsm_core_register (const struct pci_tsm_ops *ops,const struct attribute_group *grp)
include/linux/tsm.h +23 -0
@@ -123,4 +123,27 @@ int tsm_ide_stream_register(struct pci_dev *pdev, struct pci_ide *ide);void tsm_ide_stream_unregister (struct pci_ide *ide);int tsm_bind (struct device *dev,struct kvm *kvm,u64 tdi_id);int tsm_unbind (struct device *dev); + +/** +* struct tsm_guest_req_info - parameter for pci_tsm_ops.guest_req() +* @type: identify the format of the following blobs +* @type_info: extra input/output info, e.g. firmware error code +* @type_info_len: the size of @type_info +* @req: request data buffer filled by guest +* @req_len: the size of @req filled by guest +* @resp: response data buffer filled by host +* @resp_len: for input, the size of @resp buffer filled by guest +* for output, the size of actual response data filled by host +*/ +struct tsm_guest_req_info { +u32 type; +void __user *type_info; +size_t type_info_len; +void __user *req; +size_t req_len; +void __user *resp; +size_t resp_len; +}; + +int tsm_guest_req (struct device *dev,struct tsm_guest_req_info *info);#endif /* __TSM_H */
include/uapi/linux/iommufd.h +28 -0
@@ -58,6 +58,7 @@ enum {IOMMUFD_CMD_VEVENTQ_ALLOC =0x93 ,IOMMUFD_CMD_HW_QUEUE_ALLOC =0x94 ,IOMMUFD_CMD_VDEVICE_TSM_OP =0x95 , + IOMMUFD_CMD_VDEVICE_TSM_GUEST_REQUEST = 0x96, };/**
@@ -1320,4 +1321,31 @@ struct iommu_hw_queue_alloc { __aligned_u64 length; };#define IOMMU_HW_QUEUE_ALLOC _IO(IOMMUFD_TYPE, IOMMUFD_CMD_HW_QUEUE_ALLOC) + +/** +* struct iommu_vdevice_tsm_guest_request - ioctl(IOMMU_VDEVICE_TSM_GUEST_REQUEST) +* @size: sizeof(struct iommu_vdevice_tsm_guest_request) +* @vdevice_id: vDevice ID the guest request is for +* @type: identify the format of the following blobs +* @type_info_len: the blob size for @type_info_uptr +* @req_len: the blob size for @req_uptr, filled by guest +* @resp_len: for input, the blob size for @resp_uptr, filled by guest +* for output, the size of actual response data, filled by host +* @type_info_uptr: extra input/output info, e.g. firmware error code +* @req_uptr: request data buffer filled by guest +* @resp_uptr: response data buffer filled by host +*/ +struct iommu_vdevice_tsm_guest_request { +__u32 size; +__u32 vdevice_id; +__u32 type; +__u32 type_info_len; +__u32 req_len; +__u32 resp_len; +__aligned_u64 type_info_uptr; +__aligned_u64 req_uptr; +__aligned_u64 resp_uptr; +}; +#define IOMMU_VDEVICE_TSM_GUEST_REQUEST _IO(IOMMUFD_TYPE, IOMMUFD_CMD_VDEVICE_TSM_GUEST_REQUEST) +#endif
Signature · 1 line
2.43.0
Aneesh Kumar K.V (Arm)
maintainerRFC PATCH 10/38 iommufd/vdevice: Add TSM map ioctl
In reply to Aneesh Kumar K.V (Arm)With passthrough devices, we need to make sure private memory is allocated and assigned to the secure guest before we can issue the DMA. For ARM RMM, we only need to map and the secure SMMU management is internal to RMM. For shared IPA, vfio/iommufd DMA MAP/UNMAP interface does the equivalent
--- arch/arm64/kvm/mmu.c | 45 +++++++++++++++++++++++++ drivers/iommu/iommufd/iommufd_private.h | 1 + drivers/iommu/iommufd/main.c | 4 +++ drivers/iommu/iommufd/viommu.c | 43 +++++++++++++++++++++++ include/linux/kvm_host.h | 1 + include/uapi/linux/iommufd.h | 10 ++++++ 6 files changed, 104 insertions(+)
arch/arm64/kvm/mmu.c +45 -0
@@ -1530,6 +1530,51 @@ static int realm_map_ipa(struct kvm *kvm, phys_addr_t ipa,return realm_map_protected (realm, ipa, pfn, map_size, memcache); } +int kvm_map_private_memory (struct kvm *kvm,phys_addr_t start_gpa, +phys_addr_t end_gpa) +{ +struct kvm_mmu_memory_cache cache = { .gfp_zero = __GFP_ZERO }; +struct kvm_s2_mmu *mmu = &kvm->arch.mmu; +struct kvm_memory_slot *memslot; +phys_addr_t addr; +struct page *page; +kvm_pfn_t pfn; +int ret =0 , idx; +gfn_t gfn; + + idx =srcu_read_lock (&kvm->srcu); +for (addr = start_gpa; addr < end_gpa; addr +=PAGE_SIZE ) { + + ret =kvm_mmu_topup_memory_cache (&cache, +kvm_mmu_cache_min_pages (mmu)); +if (ret) +break ; + + gfn = addr >>PAGE_SHIFT ; + + memslot =gfn_to_memslot (kvm, gfn); +if (!kvm_slot_can_be_private (memslot)) { + ret = -EINVAL ; +break ; + } +/* should we check if kvm_mem_is_private()? */ + ret =kvm_gmem_get_pfn (kvm, memslot, gfn, &pfn, &page,NULL ); +if (ret) +break ; + +/* should we hold kvm_fault_lock()? */ + ret =realm_map_ipa (kvm, addr, pfn,PAGE_SIZE ,KVM_PGTABLE_PROT_W , + &cache); +if (ret) { +put_page (page); +break ; + } + } +kvm_mmu_free_memory_cache (&cache); +srcu_read_unlock (&kvm->srcu, idx); +return ret; +} +static int private_memslot_fault (struct kvm_vcpu *vcpu,phys_addr_t fault_ipa,struct kvm_memory_slot *memslot)
drivers/iommu/iommufd/iommufd_private.h +1 -0
@@ -698,6 +698,7 @@ void iommufd_vdevice_abort(struct iommufd_object *obj);int iommufd_hw_queue_alloc_ioctl (struct iommufd_ucmd *ucmd);void iommufd_hw_queue_destroy (struct iommufd_object *obj);int iommufd_vdevice_tsm_op_ioctl (struct iommufd_ucmd *ucmd); +int iommufd_vdevice_tsm_map_ioctl (struct iommufd_ucmd *ucmd);int iommufd_vdevice_tsm_guest_request_ioctl (struct iommufd_ucmd *ucmd); static inline struct iommufd_vdevice *
drivers/iommu/iommufd/main.c +4 -0
@@ -411,6 +411,7 @@ union ucmd_buffer {struct iommu_vfio_ioas vfio_ioas;struct iommu_viommu_alloc viommu;struct iommu_vdevice_tsm_op tsm_op; +struct iommu_vdevice_tsm_map tsm_map;struct iommu_vdevice_tsm_guest_request gr;#ifdef CONFIG_IOMMUFD_TEST struct iommu_test_cmd test;
@@ -475,6 +476,9 @@ static const struct iommufd_ioctl_op iommufd_ioctl_ops[] = { struct iommu_viommu_alloc, out_viommu_id),IOCTL_OP (IOMMU_VDEVICE_TSM_OP ,iommufd_vdevice_tsm_op_ioctl ,struct iommu_vdevice_tsm_op ,vdevice_id ), +IOCTL_OP (IOMMU_VDEVICE_TSM_MAP ,iommufd_vdevice_tsm_map_ioctl , +struct iommu_vdevice_tsm_map ,vdevice_id ), +IOCTL_OP (IOMMU_VDEVICE_TSM_GUEST_REQUEST ,iommufd_vdevice_tsm_guest_request_ioctl ,struct iommu_vdevice_tsm_guest_request ,resp_uptr ), #ifdef CONFIG_IOMMUFD_TEST
drivers/iommu/iommufd/viommu.c +43 -0
55 lines not on this page · raw message
include/linux/kvm_host.h +1 -0
@@ -2602,4 +2602,5 @@ static inline int kvm_enable_virtualization(void) { return 0; }static inline void kvm_disable_virtualization (void ) { }#endif +int kvm_map_private_memory (struct kvm *kvm,phys_addr_t start_gpa,phys_addr_t end_gpa);#endif
include/uapi/linux/iommufd.h +10 -0
@@ -59,6 +59,7 @@ enum {IOMMUFD_CMD_HW_QUEUE_ALLOC =0x94 ,IOMMUFD_CMD_VDEVICE_TSM_OP =0x95 ,IOMMUFD_CMD_VDEVICE_TSM_GUEST_REQUEST =0x96 , + IOMMUFD_CMD_VDEVICE_TSM_MAP = 0x97, };/**
@@ -1146,6 +1147,15 @@ struct iommu_vdevice_tsm_op {#define IOMMU_VDEICE_TSM_BIND 0x1#define IOMMU_VDEICE_TSM_UNBIND 0x2 +struct iommu_vdevice_tsm_map { +__u32 size; +__u32 flags; +__u64 start_gpa; +__u64 end_gpa; +__u32 vdevice_id; +}; +#define IOMMU_VDEVICE_TSM_MAP _IO(IOMMUFD_TYPE, IOMMUFD_CMD_VDEVICE_TSM_MAP) +/** * struct iommufd_vevent_header - Virtual Event Header for a vEVENTQ Status * @flags: Combination of enum iommu_veventq_flag
Signature · 1 line
2.43.0
Aneesh Kumar K.V (Arm)
maintainerRFC PATCH 11/38 KVM: arm64: CCA: register host tsm platform device
In reply to Aneesh Kumar K.V (Arm)Register a platform device if the CCA DA feature is supported. A driver for this platform device will further drive the CCA DA workflow.
--- arch/arm64/include/asm/rmi_smc.h | 3 +++ arch/arm64/kvm/rme.c | 16 ++++++++++++++++ 2 files changed, 19 insertions(+)
arch/arm64/include/asm/rmi_smc.h +3 -0
@@ -12,6 +12,8 @@#include <linux/arm-smccc.h> +#define RMI_DEV_NAME "arm-rmi-dev" +#define SMC_RMI_CALL (func) \ ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL, \ ARM_SMCCC_SMC_64, \
@@ -87,6 +89,7 @@ enum rmi_ripas {#define RMI_FEATURE_REGISTER_0_GICV3_NUM_LRS GENMASK(37, 34)#define RMI_FEATURE_REGISTER_0_MAX_RECS_ORDER GENMASK(41, 38)#define RMI_FEATURE_REGISTER_0_Reserved GENMASK(63, 42) +#define RMI_FEATURE_REGISTER_0_DA BIT(42)#define RMI_REALM_PARAM_FLAG_LPA2 BIT(0)#define RMI_REALM_PARAM_FLAG_SVE BIT(1)
arch/arm64/kvm/rme.c +16 -0
@@ -4,6 +4,7 @@*/ #include <linux/kvm_host.h> +#include <linux/platform_device.h> #include <asm/kvm_emulate.h> #include <asm/kvm_mmu.h>
@@ -1724,6 +1725,18 @@ int kvm_init_realm_vm(struct kvm *kvm)return 0 ; } +static struct platform_device cca_host_dev = { + .name =RMI_DEV_NAME , + .id =PLATFORM_DEVID_NONE +}; + +static void rmm_tsm_init (void ) +{ +if (!platform_device_register (&cca_host_dev)) +pr_info ("CCA host DA platform device initialized.\n" ); + +} +void kvm_init_rme (void ) {if (PAGE_SIZE !=SZ_4K )
@@ -1737,6 +1750,9 @@ void kvm_init_rme(void)if (WARN_ON (rmi_features (0 , &rmm_feat_reg0)))return ; +if (rme_has_feature (RMI_FEATURE_REGISTER_0_DA )) +rmm_tsm_init (); +if (rme_vmid_init ())return ;
Signature · 1 line
2.43.0
Aneesh Kumar K.V (Arm)
maintainerRFC PATCH 12/38 coco: host: arm64: CCA host platform device driver
In reply to Aneesh Kumar K.V (Arm)This driver registers the pci_tsm_ops with tsm subsystem.
--- drivers/virt/coco/Kconfig | 2 + drivers/virt/coco/Makefile | 1 + drivers/virt/coco/arm-cca-host/Kconfig | 12 ++ drivers/virt/coco/arm-cca-host/Makefile | 5 + drivers/virt/coco/arm-cca-host/arm-cca.c | 209 +++++++++++++++++++++++ drivers/virt/coco/arm-cca-host/rmm-da.h | 29 ++++ 6 files changed, 258 insertions(+) create mode 100644 drivers/virt/coco/arm-cca-host/Kconfig create mode 100644 drivers/virt/coco/arm-cca-host/Makefile create mode 100644 drivers/virt/coco/arm-cca-host/arm-cca.c create mode 100644 drivers/virt/coco/arm-cca-host/rmm-da.h
drivers/virt/coco/Kconfig +2 -0
@@ -15,5 +15,7 @@ source "drivers/virt/coco/arm-cca-guest/Kconfig" source "drivers/virt/coco/guest/Kconfig" +source "drivers/virt/coco/arm-cca-host/Kconfig" + config TSM tristate
drivers/virt/coco/Makefile +1 -0
@@ -11,3 +11,4 @@ obj-$(CONFIG_ARM_CCA_GUEST) += arm-cca-guest/ obj-$(CONFIG_TSM) += tsm-core.o obj-y += guest/ +obj-$(CONFIG_ARM_CCA_HOST) += arm-cca-host/
drivers/virt/coco/arm-cca-host/Kconfig new file +12 -0
@@ -0,0 +1,12 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# TSM (TEE Security Manager) host drivers +# +config ARM_CCA_HOST + tristate "Arm CCA Host driver" + depends on ARM64 + depends on PCI_TSM + select TSM + + help + The driver provides TSM backend for ARM CCA
drivers/virt/coco/arm-cca-host/Makefile new file +5 -0
@@ -0,0 +1,5 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +obj-$(CONFIG_ARM_CCA_HOST) += arm-cca-host.o + +arm-cca-host-$(CONFIG_TSM) += arm-cca.o
drivers/virt/coco/arm-cca-host/arm-cca.c new file +209 -0
209 lines not on this page · raw message
drivers/virt/coco/arm-cca-host/rmm-da.h new file +29 -0
@@ -0,0 +1,29 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* +* Copyright (C) 2024 ARM Ltd. +*/ + +#ifndef RMM_DA_H_ +#define RMM_DA_H_ + +#include <linux/pci.h> +#include <linux/pci-ide.h> +#include <linux/pci-tsm.h> +#include <asm/rmi_smc.h> + +struct cca_host_dsc_pf0 { +struct pci_tsm_pf0 pci; +struct pci_ide *sel_stream; +}; + +static inline struct cca_host_dsc_pf0 *to_cca_dsc_pf0 (struct pci_dev *pdev) +{ +struct pci_tsm *tsm = pdev->tsm; + +if (!tsm || pdev->is_virtfn || !is_pci_tsm_pf0 (pdev)) +return NULL ; + + return container_of(tsm, struct cca_host_dsc_pf0, pci.tsm); +} + +#endif
Signature · 1 line
2.43.0
Aneesh Kumar K.V (Arm)
maintainerRFC PATCH 13/38 coco: host: arm64: Create a PDEV with rmm
In reply to Aneesh Kumar K.V (Arm)Create the realm physical device with RMM.
--- arch/arm64/include/asm/rmi_cmds.h | 31 +++++ arch/arm64/include/asm/rmi_smc.h | 72 ++++++++++- drivers/virt/coco/arm-cca-host/Makefile | 2 +- drivers/virt/coco/arm-cca-host/arm-cca.c | 10 +- drivers/virt/coco/arm-cca-host/rmm-da.c | 150 +++++++++++++++++++++++ drivers/virt/coco/arm-cca-host/rmm-da.h | 5 + 6 files changed, 267 insertions(+), 3 deletions(-) create mode 100644 drivers/virt/coco/arm-cca-host/rmm-da.c
arch/arm64/include/asm/rmi_cmds.h +31 -0
@@ -505,4 +505,35 @@ static inline int rmi_rtt_unmap_unprotected(unsigned long rd, return res.a0; } +static inline unsigned long rmi_pdev_aux_count (unsigned long flags,u64 *aux_count) +{ +struct arm_smccc_res res; + +arm_smccc_1_1_invoke (SMC_RMI_PDEV_AUX_COUNT , flags, &res); + + *aux_count = res.a1; +return res.a0; +} + +static inline unsigned long rmi_pdev_create (unsigned long pdev_phys, +unsigned long pdev_params_phys) +{ +struct arm_smccc_res res; + +arm_smccc_1_1_invoke (SMC_RMI_PDEV_CREATE , + pdev_phys, pdev_params_phys, &res); + +return res.a0; +} + +static inline unsigned long rmi_pdev_get_state (unsigned long pdev_phys,unsigned long *state) +{ +struct arm_smccc_res res; + +arm_smccc_1_1_invoke (SMC_RMI_PDEV_GET_STATE , pdev_phys, &res); + + *state = res.a1; +return res.a0; +} +#endif /* __ASM_RMI_CMDS_H */
arch/arm64/include/asm/rmi_smc.h +71 -1
88 lines not on this page · raw message
drivers/virt/coco/arm-cca-host/Makefile +1 -1
@@ -2,4 +2,4 @@ # obj-$(CONFIG_ARM_CCA_HOST) += arm-cca-host.o -arm-cca-host-$(CONFIG_TSM) += arm-cca.o +arm-cca-host-$(CONFIG_TSM) += arm-cca.o rmm-da.o
drivers/virt/coco/arm-cca-host/arm-cca.c +9 -1
@@ -124,7 +124,15 @@ static int cca_tsm_connect(struct pci_dev *pdev) rc =tsm_ide_stream_register (pdev, ide);if (rc) goto err_tsm; - +/* +* Take a module reference so that we won't call unregister +* without rme_unasign_device +*/ +if (!try_module_get (THIS_MODULE )) { + rc = -ENXIO ; + goto err_tsm; + } +rme_asign_device (pdev);/* * Once ide is setup enable the stream at endpoint * Root port will be done by RMM
drivers/virt/coco/arm-cca-host/rmm-da.c new file +150 -0
150 lines not on this page · raw message
drivers/virt/coco/arm-cca-host/rmm-da.h +5 -0
@@ -14,6 +14,10 @@struct cca_host_dsc_pf0 {struct pci_tsm_pf0 pci;struct pci_ide *sel_stream; + +void *rmm_pdev; +int num_aux; +void *aux[MAX_PDEV_AUX_GRANULES ]; };static inline struct cca_host_dsc_pf0 *to_cca_dsc_pf0 (struct pci_dev *pdev)
@@ -26,4 +30,5 @@ static inline struct cca_host_dsc_pf0 *to_cca_dsc_pf0(struct pci_dev *pdev) return container_of(tsm, struct cca_host_dsc_pf0, pci.tsm); } +int rme_asign_device (struct pci_dev *pdev);#endif
Signature · 1 line
2.43.0
Aneesh Kumar K.V (Arm)
maintainerRFC PATCH 14/38 coco: host: arm64: Device communication support
In reply to Aneesh Kumar K.V (Arm)Add helpers for device communication from RMM
--- arch/arm64/include/asm/rmi_cmds.h | 11 ++ arch/arm64/include/asm/rmi_smc.h | 49 ++++++ drivers/virt/coco/arm-cca-host/arm-cca.c | 45 ++++++ drivers/virt/coco/arm-cca-host/rmm-da.c | 198 +++++++++++++++++++++++ drivers/virt/coco/arm-cca-host/rmm-da.h | 41 +++++ 5 files changed, 344 insertions(+)
arch/arm64/include/asm/rmi_cmds.h +11 -0
@@ -536,4 +536,15 @@ static inline unsigned long rmi_pdev_get_state(unsigned long pdev_phys, unsigned return res.a0; } +static inline unsigned long rmi_pdev_communicate (unsigned long pdev_phys, +unsigned long pdev_comm_data_phys) +{ +struct arm_smccc_res res; + +arm_smccc_1_1_invoke (SMC_RMI_PDEV_COMMUNICATE , + pdev_phys, pdev_comm_data_phys, &res); + +return res.a0; +} +#endif /* __ASM_RMI_CMDS_H */
arch/arm64/include/asm/rmi_smc.h +49 -0
59 lines not on this page · raw message
drivers/virt/coco/arm-cca-host/arm-cca.c +45 -0
63 lines not on this page · raw message
drivers/virt/coco/arm-cca-host/rmm-da.c +198 -0
201 lines not on this page · raw message
drivers/virt/coco/arm-cca-host/rmm-da.h +41 -0
61 lines not on this page · raw message
Signature · 1 line
2.43.0
Aneesh Kumar K.V (Arm)
maintainerRFC PATCH 15/38 coco: host: arm64: Stop and destroy the physical device
In reply to Aneesh Kumar K.V (Arm)Add support for stopping and destroying physical devices.
--- arch/arm64/include/asm/rmi_cmds.h | 18 ++++++++++++++++++ arch/arm64/include/asm/rmi_smc.h | 2 ++ drivers/virt/coco/arm-cca-host/arm-cca.c | 3 +++ drivers/virt/coco/arm-cca-host/rmm-da.c | 21 +++++++++++++++++++++ drivers/virt/coco/arm-cca-host/rmm-da.h | 1 + 5 files changed, 45 insertions(+)
arch/arm64/include/asm/rmi_cmds.h +18 -0
@@ -547,4 +547,22 @@ static inline unsigned long rmi_pdev_communicate(unsigned long pdev_phys, return res.a0; } +static inline unsigned long rmi_pdev_stop (unsigned long pdev_phys) +{ +struct arm_smccc_res res; + +arm_smccc_1_1_invoke (SMC_RMI_PDEV_STOP , pdev_phys, &res); + +return res.a0; +} + +static inline unsigned long rmi_pdev_destroy (unsigned long pdev_phys) +{ +struct arm_smccc_res res; + +arm_smccc_1_1_invoke (SMC_RMI_PDEV_DESTROY , pdev_phys, &res); + +return res.a0; +} +#endif /* __ASM_RMI_CMDS_H */
arch/arm64/include/asm/rmi_smc.h +2 -0
@@ -49,7 +49,9 @@#define SMC_RMI_PDEV_COMMUNICATE SMC_RMI_CALL(0x0175)#define SMC_RMI_PDEV_CREATE SMC_RMI_CALL(0x0176) +#define SMC_RMI_PDEV_DESTROY SMC_RMI_CALL(0x0177)#define SMC_RMI_PDEV_GET_STATE SMC_RMI_CALL(0x0178) +#define SMC_RMI_PDEV_STOP SMC_RMI_CALL(0x017c)#define RMI_ABI_MAJOR_VERSION 1#define RMI_ABI_MINOR_VERSION 0
drivers/virt/coco/arm-cca-host/arm-cca.c +3 -0
@@ -210,12 +210,15 @@ static void cca_tsm_disconnect(struct pci_dev *pdev) ide = dsc_pf0->sel_stream; dsc_pf0->sel_stream =NULL ;pci_ide_stream_disable (pdev, ide); +rme_unassign_device (pdev); +module_put (THIS_MODULE );tsm_ide_stream_unregister (ide);pci_ide_stream_teardown (rp, ide);pci_ide_stream_teardown (pdev, ide);pci_ide_stream_unregister (ide);clear_bit (ide->stream_id, cca_stream_ids);pci_ide_stream_free (ide); +free_dev_communication_buffers (&dsc_pf0->comm_data); }static const struct pci_tsm_ops cca_pci_ops = {
drivers/virt/coco/arm-cca-host/rmm-da.c +21 -0
@@ -346,3 +346,24 @@ int schedule_rme_ide_setup(struct pci_dev *pdev)return 0 ; } + +void rme_unassign_device (struct pci_dev *pdev) +{ +unsigned long ret; +unsigned long state; +phys_addr_t rmm_pdev_phys; +struct cca_host_dsc_pf0 *dsc_pf0; + + dsc_pf0 =to_cca_dsc_pf0 (pdev); + rmm_pdev_phys =virt_to_phys (dsc_pf0->rmm_pdev); + ret =rmi_pdev_stop (rmm_pdev_phys); +if (WARN_ON (ret !=RMI_SUCCESS )) +return ; + + state =do_pdev_communicate (pdev->tsm,RMI_PDEV_STOPPED ); +/* ignore the error state and destroy the device */ +WARN_ON (state !=RMI_PDEV_STOPPED ); + ret =rmi_pdev_destroy (rmm_pdev_phys); +if (WARN_ON (ret !=RMI_SUCCESS )) +return ; +}
drivers/virt/coco/arm-cca-host/rmm-da.h +1 -0
@@ -71,5 +71,6 @@ static inline struct cca_host_comm_data *to_cca_comm_data(struct pci_dev *pdev) }int rme_asign_device (struct pci_dev *pdev); +void rme_unassign_device (struct pci_dev *pdev);int schedule_rme_ide_setup (struct pci_dev *pdev);#endif
Signature · 1 line
2.43.0
Aneesh Kumar K.V (Arm)
maintainerRFC PATCH 16/38 X.509: Make certificate parser public
In reply to Aneesh Kumar K.V (Arm)From: Lukas Wunner <[email protected]>
The upcoming support for PCI device authentication with CMA-SPDM (PCIe r6.1 sec 6.31) requires validating the Subject Alternative Name in X.509 certificates.
High-level functions for X.509 parsing such as key_create_or_update() throw away the internal, low-level struct x509_certificate after extracting the struct public_key and public_key_signature from it. The Subject Alternative Name is thus inaccessible when using those functions.
Afford CMA-SPDM access to the Subject Alternative Name by making struct x509_certificate public, together with the functions for parsing an X.509 certificate into such a struct and freeing such a struct.
The private header file x509_parser.h previously included <linux/time.h> for the definition of time64_t. That definition was since moved to <linux/time64.h> by commit 361a3bf00582 ("time64: Add time64.h header and define struct timespec64"), so adjust the #include directive as part of the move to the new public header file <keys/x509-parser.h>.
No functional change intended.
--- crypto/asymmetric_keys/x509_parser.h | 40 +-------------------- include/keys/x509-parser.h | 53 ++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+), 39 deletions(-) create mode 100644 include/keys/x509-parser.h
crypto/asymmetric_keys/x509_parser.h +1 -39
@@ -5,49 +5,11 @@* Written by David Howells ([email protected]) */ -#include <linux/cleanup.h> -#include <linux/time.h> -#include <crypto/public_key.h> -#include <keys/asymmetric-type.h> - -struct x509_certificate { -struct x509_certificate *next; -struct x509_certificate *signer;/* Certificate that signed this one */ -struct public_key *pub;/* Public key details */ -struct public_key_signature *sig;/* Signature parameters */ -char *issuer;/* Name of certificate issuer */ -char *subject;/* Name of certificate subject */ -struct asymmetric_key_id *id;/* Issuer + Serial number */ -struct asymmetric_key_id *skid;/* Subject + subjectKeyId (optional) */ -time64_t valid_from; -time64_t valid_to; -const void *tbs;/* Signed data */ -unsigned tbs_size;/* Size of signed data */ -unsigned raw_sig_size;/* Size of signature */ -const void *raw_sig;/* Signature data */ -const void *raw_serial;/* Raw serial number in ASN.1 */ -unsigned raw_serial_size; -unsigned raw_issuer_size; -const void *raw_issuer;/* Raw issuer name in ASN.1 */ -const void *raw_subject;/* Raw subject name in ASN.1 */ -unsigned raw_subject_size; -unsigned raw_skid_size; -const void *raw_skid;/* Raw subjectKeyId in ASN.1 */ -unsigned index; -bool seen;/* Infinite recursion prevention */ -bool verified; -bool self_signed;/* T if self-signed (check unsupported_sig too) */ -bool unsupported_sig;/* T if signature uses unsupported crypto */ -bool blacklisted; -}; +#include <keys/x509-parser.h> /* * x509_cert_parser.c */ -extern void x509_free_certificate (struct x509_certificate *cert); -DEFINE_FREE(x509_free_certificate, struct x509_certificate *, - if (!IS_ERR(_T)) x509_free_certificate(_T)) -extern struct x509_certificate *x509_cert_parse (const void *data,size_t datalen);extern int x509_decode_time (time64_t *_t,size_t hdrlen,unsigned char tag,const unsigned char *value,size_t vlen);
include/keys/x509-parser.h new file +53 -0
53 lines not on this page · raw message
Signature · 1 line
2.43.0
Aneesh Kumar K.V (Arm)
maintainerRFC PATCH 17/38 X.509: Parse Subject Alternative Name in certificates
In reply to Aneesh Kumar K.V (Arm)From: Lukas Wunner <[email protected]>
The upcoming support for PCI device authentication with CMA-SPDM (PCIe r6.1 sec 6.31) requires validating the Subject Alternative Name in X.509 certificates.
Store a pointer to the Subject Alternative Name upon parsing for consumption by CMA-SPDM.
--- crypto/asymmetric_keys/x509_cert_parser.c | 9 +++++++++ include/keys/x509-parser.h | 2 ++ 2 files changed, 11 insertions(+)
crypto/asymmetric_keys/x509_cert_parser.c +9 -0
@@ -571,6 +571,15 @@ int x509_process_extension(void *context, size_t hdrlen,return 0 ; } +if (ctx->last_oid == OID_subjectAltName) { +if (ctx->cert->raw_san) +return -EBADMSG ; + + ctx->cert->raw_san = v; + ctx->cert->raw_san_size = vlen; +return 0 ; + } +if (ctx->last_oid == OID_keyUsage) {/* * Get hold of the keyUsage bit string
include/keys/x509-parser.h +2 -0
@@ -36,6 +36,8 @@ struct x509_certificate {unsigned raw_subject_size;unsigned raw_skid_size;const void *raw_skid;/* Raw subjectKeyId in ASN.1 */ +const void *raw_san;/* Raw subjectAltName in ASN.1 */ +unsigned raw_san_size;unsigned index;bool seen;/* Infinite recursion prevention */ bool verified;
Signature · 1 line
2.43.0
Aneesh Kumar K.V (Arm)
maintainerRFC PATCH 18/38 X.509: Move certificate length retrieval into new helper
In reply to Aneesh Kumar K.V (Arm)From: Lukas Wunner <[email protected]>
The upcoming in-kernel SPDM library (Security Protocol and Data Model,
https://www.dmtf.org/dsp/DSP0274) needs to retrieve the length from ASN.1 DER-encoded X.509 certificates.
Such code already exists in x509_load_certificate_list(), so move it into a new helper for reuse by SPDM.
Export the helper so that SPDM can be tristate. (Some upcoming users of the SPDM libray may be modular, such as SCSI and ATA.)
No functional change intended.
--- crypto/asymmetric_keys/x509_loader.c | 38 +++++++++++++++++++--------- include/keys/asymmetric-type.h | 2 ++ 2 files changed, 28 insertions(+), 12 deletions(-)
crypto/asymmetric_keys/x509_loader.c +26 -12
@@ -4,28 +4,42 @@#include <linux/key.h> #include <keys/asymmetric-type.h> +ssize_t x509_get_certificate_length (const u8 *p,unsigned long buflen) +{ +ssize_t plen; + +/* Each cert begins with an ASN.1 SEQUENCE tag and must be more +* than 256 bytes in size. +*/ +if (buflen <4 ) +return -EINVAL ; + +if (p[0 ] !=0x30 && + p[1 ] !=0x82 ) +return -EINVAL ; + + plen = (p[2 ] <<8 ) | p[3 ]; + plen +=4 ; +if (plen > buflen) +return -EINVAL ; + +return plen; +} +EXPORT_SYMBOL_GPL (x509_get_certificate_length); +int x509_load_certificate_list (const u8 cert_list[],const unsigned long list_size,const struct key *keyring) {key_ref_t key;const u8 *p, *end; -size_t plen; +ssize_t plen; p = cert_list; end = p + list_size;while (p < end) { -/* Each cert begins with an ASN.1 SEQUENCE tag and must be more -* than 256 bytes in size. -*/ -if (end - p <4 ) - goto dodgy_cert; -if (p[0 ] !=0x30 && - p[1 ] !=0x82 ) - goto dodgy_cert; - plen = (p[2 ] <<8 ) | p[3 ]; - plen +=4 ; -if (plen > end - p) + plen =x509_get_certificate_length (p, end - p); +if (plen <0 ) goto dodgy_cert; key = key_create_or_update(make_key_ref(keyring, 1),
include/keys/asymmetric-type.h +2 -0
@@ -84,6 +84,8 @@ extern struct key *find_asymmetric_key(struct key *keyring,const struct asymmetric_key_id *id_2, bool partial); +ssize_t x509_get_certificate_length (const u8 *p,unsigned long buflen); +int x509_load_certificate_list (const u8 cert_list[],const unsigned long list_size,const struct key *keyring);
Signature · 1 line
2.43.0
Aneesh Kumar K.V (Arm)
maintainerRFC PATCH 19/38 coco: host: arm64: set_pubkey support
In reply to Aneesh Kumar K.V (Arm)Add changes to share the device's public key with the RMM.
--- arch/arm64/include/asm/rmi_cmds.h | 9 ++ arch/arm64/include/asm/rmi_smc.h | 18 +++ drivers/virt/coco/arm-cca-host/Kconfig | 4 + drivers/virt/coco/arm-cca-host/rmm-da.c | 150 ++++++++++++++++++++++++ drivers/virt/coco/arm-cca-host/rmm-da.h | 1 + 5 files changed, 182 insertions(+)
arch/arm64/include/asm/rmi_cmds.h +9 -0
@@ -565,4 +565,13 @@ static inline unsigned long rmi_pdev_destroy(unsigned long pdev_phys) return res.a0; } +static inline unsigned long rmi_pdev_set_pubkey (unsigned long pdev_phys,unsigned long key_phys) +{ +struct arm_smccc_res res; + +arm_smccc_1_1_invoke (SMC_RMI_PDEV_SET_PUBKEY , pdev_phys, key_phys, &res); + +return res.a0; +} +#endif /* __ASM_RMI_CMDS_H */
arch/arm64/include/asm/rmi_smc.h +18 -0
@@ -51,6 +51,7 @@#define SMC_RMI_PDEV_CREATE SMC_RMI_CALL(0x0176)#define SMC_RMI_PDEV_DESTROY SMC_RMI_CALL(0x0177)#define SMC_RMI_PDEV_GET_STATE SMC_RMI_CALL(0x0178) +#define SMC_RMI_PDEV_SET_PUBKEY SMC_RMI_CALL(0x017b)#define SMC_RMI_PDEV_STOP SMC_RMI_CALL(0x017c)#define RMI_ABI_MAJOR_VERSION 1
@@ -389,4 +390,21 @@ struct rmi_dev_comm_data { }; }; +#define RMI_SIG_RSASSA_3072 0 +#define RMI_SIG_ECDSA_P256 1 +#define RMI_SIG_ECDSA_P384 2 + +struct rmi_public_key_params { +union { +struct { +u8 public_key[1024 ]; +u8 metadata[1024 ]; +u64 public_key_len; +u64 metadata_len; +u8 rmi_signature_algorithm; + } __packed; +u8 padding[0x1000 ]; + }; +}; +#endif /* __ASM_RMI_SMC_H */
drivers/virt/coco/arm-cca-host/Kconfig +4 -0
@@ -6,7 +6,11 @@ config ARM_CCA_HOST tristate "Arm CCA Host driver" depends on ARM64 depends on PCI_TSM + select KEYS + select X509_CERTIFICATE_PARSER select TSM + select CRYPTO_ECDSA + select CRYPTO_RSA help The driver provides TSM backend for ARM CCA
drivers/virt/coco/arm-cca-host/rmm-da.c +150 -0
174 lines not on this page · raw message
drivers/virt/coco/arm-cca-host/rmm-da.h +1 -0
@@ -33,6 +33,7 @@ struct cca_host_dsc_pf0 {int num_aux;void *aux[MAX_PDEV_AUX_GRANULES ]; +uint8_t rmi_signature_algorithm;struct {struct cache_object cache;
Signature · 1 line
2.43.0
Aneesh Kumar K.V (Arm)
maintainerRFC PATCH 20/38 coco: host: arm64: Add support for creating a virtual device
In reply to Aneesh Kumar K.V (Arm)Changes to support the creation of virtual device objects with RMM.
--- arch/arm64/include/asm/rmi_cmds.h | 13 +++++ arch/arm64/include/asm/rmi_smc.h | 30 +++++++++++ drivers/virt/coco/arm-cca-host/Kconfig | 1 + drivers/virt/coco/arm-cca-host/arm-cca.c | 30 +++++++++++ drivers/virt/coco/arm-cca-host/rmm-da.c | 67 ++++++++++++++++++++++++ drivers/virt/coco/arm-cca-host/rmm-da.h | 17 ++++++ 6 files changed, 158 insertions(+)
arch/arm64/include/asm/rmi_cmds.h +13 -0
@@ -574,4 +574,17 @@ static inline unsigned long rmi_pdev_set_pubkey(unsigned long pdev_phys, unsigne return res.a0; } +static inline unsigned long rmi_vdev_create (unsigned long rd, +unsigned long pdev_phys, +unsigned long vdev_phys, +unsigned long vdev_params_phys) +{ +struct arm_smccc_res res; + +arm_smccc_1_1_invoke (SMC_RMI_VDEV_CREATE , rd, pdev_phys, + vdev_phys, vdev_params_phys, &res); + +return res.a0; +} +#endif /* __ASM_RMI_CMDS_H */
arch/arm64/include/asm/rmi_smc.h +30 -0
@@ -53,6 +53,8 @@#define SMC_RMI_PDEV_GET_STATE SMC_RMI_CALL(0x0178)#define SMC_RMI_PDEV_SET_PUBKEY SMC_RMI_CALL(0x017b)#define SMC_RMI_PDEV_STOP SMC_RMI_CALL(0x017c) +#define SMC_RMI_VDEV_CREATE SMC_RMI_CALL(0x0187) +#define RMI_ABI_MAJOR_VERSION 1#define RMI_ABI_MINOR_VERSION 0
@@ -407,4 +409,32 @@ struct rmi_public_key_params { }; }; +enum rmi_vdev_state { +RMI_VDEV_READY , +RMI_VDEV_COMMUNICATING , +RMI_VDEV_STOPPING , +RMI_VDEV_STOPPED , +RMI_VDEV_ERROR , +}; + +#define MAX_VDEV_AUX_GRANULES 32 + +struct rmi_vdev_params { +union { +struct { +u64 flags; +u64 vdev_id; +u64 tdi_id; +u64 num_aux; + }; +u8 padding1[0x100 ]; + }; +union {/* 0x100 */ +struct { +unsigned long aux[MAX_VDEV_AUX_GRANULES ]; + }; +u8 padding2[0x900 ]; + }; +}; +#endif /* __ASM_RMI_SMC_H */
drivers/virt/coco/arm-cca-host/Kconfig +1 -0
@@ -6,6 +6,7 @@ config ARM_CCA_HOST tristate "Arm CCA Host driver" depends on ARM64 depends on PCI_TSM + depends on KVM select KEYS select X509_CERTIFICATE_PARSER select TSM
drivers/virt/coco/arm-cca-host/arm-cca.c +30 -0
@@ -10,6 +10,8 @@#include <linux/pci.h> #include <linux/tsm.h> #include <linux/vmalloc.h> +#include <linux/kvm_host.h> +#include <linux/pci.h> #include "rmm-da.h"
@@ -221,11 +223,39 @@ static void cca_tsm_disconnect(struct pci_dev *pdev)free_dev_communication_buffers (&dsc_pf0->comm_data); } +static struct pci_tdi *cca_tsm_bind (struct pci_dev *pdev,struct pci_dev *pf0_dev, +struct kvm *kvm,u64 tdi_id) +{ +void *rmm_vdev; + struct cca_host_tdi *host_tdi __free(kfree) = NULL; +struct realm *realm = &kvm->arch.realm; + +if (pdev->is_virtfn) +return ERR_PTR (-ENXIO ); + +if (!try_module_get (THIS_MODULE )) +return ERR_PTR (-ENXIO ); + + host_tdi =kmalloc (sizeof (struct cca_host_tdi ),GFP_KERNEL ); +if (!host_tdi) +return ERR_PTR (-ENOMEM ); + + rmm_vdev =rme_create_vdev (realm, pdev, pf0_dev, tdi_id); +if (!IS_ERR_OR_NULL (rmm_vdev)) { + host_tdi->rmm_vdev = rmm_vdev; +return &no_free_ptr (host_tdi)->tdi; + } + +module_put (THIS_MODULE ); +return rmm_vdev; +} +static const struct pci_tsm_ops cca_pci_ops = { .probe = cca_tsm_pci_probe, .remove = cca_tsm_pci_remove, .connect = cca_tsm_connect, .disconnect = cca_tsm_disconnect, + .bind = cca_tsm_bind, };static void cca_tsm_remove (void *tsm_core)
drivers/virt/coco/arm-cca-host/rmm-da.c +67 -0
76 lines not on this page · raw message
drivers/virt/coco/arm-cca-host/rmm-da.h +17 -0
@@ -45,6 +45,11 @@ struct cca_host_dsc_pf0 {struct cache_object vca; }; +struct cca_host_tdi { +struct pci_tdi tdi; +void *rmm_vdev; +}; +#define PDEV_COMMUNICATE 0x1static inline struct cca_host_dsc_pf0 *to_cca_dsc_pf0 (struct pci_dev *pdev) {
@@ -71,7 +76,19 @@ static inline struct cca_host_comm_data *to_cca_comm_data(struct pci_dev *pdev)return NULL ; } +static inline struct cca_host_tdi *to_cca_host_tdi (struct pci_dev *pdev) +{ +struct pci_tsm *tsm = pdev->tsm; + +if (!tsm || !tsm->tdi) +return NULL ; + + return container_of(tsm->tdi, struct cca_host_tdi, tdi); +} +int rme_asign_device (struct pci_dev *pdev);void rme_unassign_device (struct pci_dev *pdev);int schedule_rme_ide_setup (struct pci_dev *pdev); +void *rme_create_vdev (struct realm *realm,struct pci_dev *pdev, +struct pci_dev *pf0_dev,u32 guest_rid);#endif
Signature · 1 line
2.43.0
Aneesh Kumar K.V (Arm)
maintainerRFC PATCH 21/38 coco: host: arm64: Add support for virtual device communication
In reply to Aneesh Kumar K.V (Arm)Add support for vdev_communicate with RMM.
--- arch/arm64/include/asm/rmi_cmds.h | 22 ++++++++++++++++++++++ arch/arm64/include/asm/rmi_smc.h | 2 ++ drivers/virt/coco/arm-cca-host/rmm-da.c | 21 +++++++++++++++++++-- drivers/virt/coco/arm-cca-host/rmm-da.h | 2 ++ 4 files changed, 45 insertions(+), 2 deletions(-)
arch/arm64/include/asm/rmi_cmds.h +22 -0
@@ -587,4 +587,26 @@ static inline unsigned long rmi_vdev_create(unsigned long rd, return res.a0; } +static inline unsigned long rmi_vdev_communicate (unsigned long pdev_phys, +unsigned long vdev_phys, +unsigned long vdev_comm_data_phys) +{ +struct arm_smccc_res res; + +arm_smccc_1_1_invoke (SMC_RMI_VDEV_COMMUNICATE , + pdev_phys, vdev_phys, vdev_comm_data_phys, &res); + +return res.a0; +} + +static inline unsigned long rmi_vdev_get_state (unsigned long vdev_phys,unsigned long *state) +{ +struct arm_smccc_res res; + +arm_smccc_1_1_invoke (SMC_RMI_VDEV_GET_STATE , vdev_phys, &res); + + *state = res.a1; +return res.a0; +} +#endif /* __ASM_RMI_CMDS_H */
arch/arm64/include/asm/rmi_smc.h +2 -0
@@ -53,7 +53,9 @@#define SMC_RMI_PDEV_GET_STATE SMC_RMI_CALL(0x0178)#define SMC_RMI_PDEV_SET_PUBKEY SMC_RMI_CALL(0x017b)#define SMC_RMI_PDEV_STOP SMC_RMI_CALL(0x017c) +#define SMC_RMI_VDEV_COMMUNICATE SMC_RMI_CALL(0x0186)#define SMC_RMI_VDEV_CREATE SMC_RMI_CALL(0x0187) +#define SMC_RMI_VDEV_GET_STATE SMC_RMI_CALL(0x0189)#define RMI_ABI_MAJOR_VERSION 1
drivers/virt/coco/arm-cca-host/rmm-da.c +19 -2
@@ -207,8 +207,14 @@ static int __do_dev_communicate(int type, struct pci_tsm *tsm)if (type ==PDEV_COMMUNICATE ) ret =rmi_pdev_communicate (virt_to_phys (dsc_pf0->rmm_pdev),virt_to_phys (comm_data->io_params)); -else - ret =RMI_ERROR_INPUT ; +else { + struct cca_host_tdi *host_tdi = container_of(tsm->tdi, struct cca_host_tdi, tdi); + + ret =rmi_vdev_communicate (virt_to_phys (dsc_pf0->rmm_pdev), +virt_to_phys (host_tdi->rmm_vdev), +virt_to_phys (comm_data->io_params)); + } +if (ret !=RMI_SUCCESS ) {pr_err ("pdev communicate error\n" );return ret;
@@ -299,6 +305,12 @@ static int do_dev_communicate(int type, struct pci_tsm *tsm, int target_state) ret =rmi_pdev_get_state (virt_to_phys (dsc_pf0->rmm_pdev), &state); error_state =RMI_PDEV_ERROR ; + }else { + struct cca_host_tdi *host_tdi = container_of(tsm->tdi, struct cca_host_tdi, tdi); + + ret =rmi_vdev_get_state (virt_to_phys (host_tdi->rmm_vdev), + &state); + error_state =RMI_VDEV_ERROR ; }if (ret !=0 ) {pr_err ("Get dev state error\n" );
@@ -584,3 +596,8 @@ void *rme_create_vdev(struct realm *realm, struct pci_dev *pdev, err_out:return ERR_PTR (ret); } + +static int __maybe_unused do_vdev_communicate (struct pci_tsm *tsm,int target_state) +{ +return do_dev_communicate (VDEV_COMMUNICATE , tsm, target_state); +}
drivers/virt/coco/arm-cca-host/rmm-da.h +2 -0
@@ -51,6 +51,8 @@ struct cca_host_tdi { };#define PDEV_COMMUNICATE 0x1 +#define VDEV_COMMUNICATE 0x2 +static inline struct cca_host_dsc_pf0 *to_cca_dsc_pf0 (struct pci_dev *pdev) {struct pci_tsm *tsm = pdev->tsm;
Signature · 1 line
2.43.0
Aneesh Kumar K.V (Arm)
maintainerRFC PATCH 22/38 coco: host: arm64: Stop and destroy virtual device
In reply to Aneesh Kumar K.V (Arm)Add support for vdev_stop and vdev_destroy.
--- arch/arm64/include/asm/rmi_cmds.h | 21 ++++++++ arch/arm64/include/asm/rmi_smc.h | 3 +- drivers/virt/coco/arm-cca-host/arm-cca.c | 10 ++++ drivers/virt/coco/arm-cca-host/rmm-da.c | 61 +++++++++++++++++++++++- drivers/virt/coco/arm-cca-host/rmm-da.h | 2 + 5 files changed, 95 insertions(+), 2 deletions(-)
arch/arm64/include/asm/rmi_cmds.h +21 -0
@@ -609,4 +609,25 @@ static inline unsigned long rmi_vdev_get_state(unsigned long vdev_phys, unsigned return res.a0; } +static inline unsigned long rmi_vdev_stop (unsigned long vdev_phys) +{ +struct arm_smccc_res res; + +arm_smccc_1_1_invoke (SMC_RMI_VDEV_STOP , vdev_phys, &res); + +return res.a0; +} + +static inline unsigned long rmi_vdev_destroy (unsigned long rd, +unsigned long pdev_phys, +unsigned long vdev_phys) +{ +struct arm_smccc_res res; + +arm_smccc_1_1_invoke (SMC_RMI_VDEV_DESTROY , rd, pdev_phys, vdev_phys, &res); + +return res.a0; +} + +#endif /* __ASM_RMI_CMDS_H */
arch/arm64/include/asm/rmi_smc.h +2 -1
@@ -55,8 +55,9 @@#define SMC_RMI_PDEV_STOP SMC_RMI_CALL(0x017c)#define SMC_RMI_VDEV_COMMUNICATE SMC_RMI_CALL(0x0186)#define SMC_RMI_VDEV_CREATE SMC_RMI_CALL(0x0187) +#define SMC_RMI_VDEV_DESTROY SMC_RMI_CALL(0x0188)#define SMC_RMI_VDEV_GET_STATE SMC_RMI_CALL(0x0189) - +#define SMC_RMI_VDEV_STOP SMC_RMI_CALL(0x018A)#define RMI_ABI_MAJOR_VERSION 1#define RMI_ABI_MINOR_VERSION 0
drivers/virt/coco/arm-cca-host/arm-cca.c +10 -0
@@ -250,12 +250,22 @@ static struct pci_tdi *cca_tsm_bind(struct pci_dev *pdev, struct pci_dev *pf0_dereturn rmm_vdev; } +static void cca_tsm_unbind (struct pci_tdi *tdi) +{ +struct realm *realm = &tdi->kvm->arch.realm; + +rme_unbind_vdev (realm, tdi->pdev, tdi->pdev->tsm->dsm_dev); + +module_put (THIS_MODULE ); +} +static const struct pci_tsm_ops cca_pci_ops = { .probe = cca_tsm_pci_probe, .remove = cca_tsm_pci_remove, .connect = cca_tsm_connect, .disconnect = cca_tsm_disconnect, .bind = cca_tsm_bind, + .unbind = cca_tsm_unbind, };static void cca_tsm_remove (void *tsm_core)
drivers/virt/coco/arm-cca-host/rmm-da.c +60 -1
67 lines not on this page · raw message
drivers/virt/coco/arm-cca-host/rmm-da.h +2 -0
@@ -93,4 +93,6 @@ void rme_unassign_device(struct pci_dev *pdev);int schedule_rme_ide_setup (struct pci_dev *pdev);void *rme_create_vdev (struct realm *realm,struct pci_dev *pdev,struct pci_dev *pf0_dev,u32 guest_rid); +void rme_unbind_vdev (struct realm *realm,struct pci_dev *pdev, +struct pci_dev *pf0_dev);#endif
Signature · 1 line
2.43.0
Aneesh Kumar K.V (Arm)
maintainerRFC PATCH 23/38 coco: guest: arm64: Update arm CCA guest driver
In reply to Aneesh Kumar K.V (Arm)This patch includes renaming changes to simplify the registration of a TSM backend in the next patch. There are no functional changes in this update.
---
arch/arm64/include/asm/rsi.h | 2 +-
arch/arm64/kernel/rsi.c | 2 +-
drivers/virt/coco/arm-cca-guest/Makefile | 3 ++
.../{arm-cca-guest.c => arm-cca.c} | 52 +++++++++----------
4 files changed, 29 insertions(+), 30 deletions(-)
rename drivers/virt/coco/arm-cca-guest/{arm-cca-guest.c => arm-cca.c} (86%)arch/arm64/include/asm/rsi.h +1 -1
@@ -10,7 +10,7 @@#include <linux/jump_label.h> #include <asm/rsi_cmds.h> -#define RSI_PDEV_NAME "arm-cca-dev" +#define RSI_DEV_NAME "arm-rsi-dev"DECLARE_STATIC_KEY_FALSE (rsi_present);
arch/arm64/kernel/rsi.c +1 -1
@@ -142,7 +142,7 @@ void __init arm64_rsi_init(void) }static struct platform_device rsi_dev = { - .name =RSI_PDEV_NAME , + .name =RSI_DEV_NAME , .id =PLATFORM_DEVID_NONE };
drivers/virt/coco/arm-cca-guest/Makefile +3 -0
@@ -1,2 +1,5 @@ # SPDX-License-Identifier: GPL-2.0-only +# obj-$(CONFIG_ARM_CCA_GUEST) += arm-cca-guest.o + +arm-cca-guest-$(CONFIG_TSM) += arm-cca.o
drivers/virt/coco/arm-cca-guest/arm-cca.c renamed from drivers/virt/coco/arm-cca-guest/arm-cca-guest.c +24 -28
82 lines not on this page · raw message
Signature · 1 line
2.43.0
Aneesh Kumar K.V (Arm)
maintainerRFC PATCH 24/38 arm64: CCA: Register guest tsm callback
In reply to Aneesh Kumar K.V (Arm)Register the TSM callback if the DA feature is supported by RSI.
Additionally, adjust the build order so that the TSM class is created before the arm-cca-guest driver initialization.
--- arch/arm64/include/asm/rsi.h | 3 + arch/arm64/include/asm/rsi_cmds.h | 18 ++++++ arch/arm64/include/asm/rsi_smc.h | 1 + arch/arm64/kernel/rsi.c | 24 ++++++-- drivers/virt/coco/Makefile | 2 +- drivers/virt/coco/arm-cca-guest/Kconfig | 8 ++- drivers/virt/coco/arm-cca-guest/arm-cca.c | 71 ++++++++++++++++++++++- drivers/virt/coco/arm-cca-guest/rsi-da.h | 27 +++++++++ 8 files changed, 144 insertions(+), 10 deletions(-) create mode 100644 drivers/virt/coco/arm-cca-guest/rsi-da.h
arch/arm64/include/asm/rsi.h +3 -0
@@ -67,4 +67,7 @@ static inline int rsi_set_memory_range_shared(phys_addr_t start,return rsi_set_memory_range (start, end,RSI_RIPAS_EMPTY , RSI_CHANGE_DESTROYED); } + +bool rsi_has_da_feature (void ); +#endif /* __ASM_RSI_H_ */
arch/arm64/include/asm/rsi_cmds.h +18 -0
@@ -159,4 +159,22 @@ static inline unsigned long rsi_attestation_token_continue(phys_addr_t granule, return res.a0; } +/** +* rsi_features() - Read feature register +* @index: Feature register index +* @out: Feature register value is written to this pointer +* +* Return: RSI return code +*/ +static inline int rsi_features (unsigned long index,unsigned long *out) +{ +struct arm_smccc_res res; + +arm_smccc_1_1_invoke (SMC_RSI_FEATURES , index, &res); + +if (out) + *out = res.a1; +return res.a0; +} +#endif /* __ASM_RSI_CMDS_H */
arch/arm64/include/asm/rsi_smc.h +1 -0
@@ -53,6 +53,7 @@*/ #define SMC_RSI_ABI_VERSION SMC_RSI_FID(0x190) +#define RSI_FEATURE_REGISTER_0_DA BIT(0)/* * Read feature register. *
arch/arm64/kernel/rsi.c +18 -6
@@ -15,6 +15,7 @@#include <asm/rsi.h> static struct realm_config config; +static unsigned long rsi_feat_reg0;unsigned long prot_ns_shared;EXPORT_SYMBOL (prot_ns_shared);
@@ -22,6 +23,12 @@ EXPORT_SYMBOL(prot_ns_shared);DEFINE_STATIC_KEY_FALSE_RO (rsi_present);EXPORT_SYMBOL (rsi_present); +bool rsi_has_da_feature (void ) +{ +return !!u64_get_bits (rsi_feat_reg0,RSI_FEATURE_REGISTER_0_DA ); +} +EXPORT_SYMBOL_GPL (rsi_has_da_feature); +bool cc_platform_has (enum cc_attr attr) {switch (attr) {
@@ -128,6 +135,10 @@ void __init arm64_rsi_init(void)return ;if (WARN_ON (rsi_get_realm_config (&config)))return ; + +if (WARN_ON (rsi_features (0 , &rsi_feat_reg0))) +return ; + prot_ns_shared =BIT (config.ipa_bits -1 );if (arm64_ioremap_prot_hook_register (realm_ioremap_hook))
@@ -141,17 +152,18 @@ void __init arm64_rsi_init(void)static_branch_enable (&rsi_present); } -static struct platform_device rsi_dev = { +static struct platform_device cca_guest_dev = { .name =RSI_DEV_NAME , .id =PLATFORM_DEVID_NONE }; -static int __init arm64_create_dummy_rsi_dev (void ) +static int __init arm64_create_cca_guest_dev (void ) { -if (is_realm_world () && -platform_device_register (&rsi_dev)) -pr_err ("failed to register rsi platform device\n" ); +if (is_realm_world ()) { +if (!platform_device_register (&cca_guest_dev)) +pr_info ("CCA guest platform device registered.\n" ); + }return 0 ; } -arch_initcall (arm64_create_dummy_rsi_dev) +device_initcall (arm64_create_cca_guest_dev)
drivers/virt/coco/Makefile +1 -1
@@ -7,8 +7,8 @@ obj-$(CONFIG_EFI_SECRET) += efi_secret/ obj-$(CONFIG_ARM_PKVM_GUEST) += pkvm-guest/ obj-$(CONFIG_SEV_GUEST) += sev-guest/ obj-$(CONFIG_INTEL_TDX_GUEST) += tdx-guest/ -obj-$(CONFIG_ARM_CCA_GUEST) += arm-cca-guest/ obj-$(CONFIG_TSM) += tsm-core.o obj-y += guest/ +obj-$(CONFIG_ARM_CCA_GUEST) += arm-cca-guest/ obj-$(CONFIG_ARM_CCA_HOST) += arm-cca-host/
drivers/virt/coco/arm-cca-guest/Kconfig +7 -1
@@ -1,10 +1,16 @@ +# SPDX-License-Identifier: GPL-2.0-only +# + config ARM_CCA_GUEST tristate "Arm CCA Guest driver" depends on ARM64 + depends on PCI_TSM select TSM_REPORTS + select TSM help - The driver provides userspace interface to request and + The driver provides userspace interface to request an attestation report from the Realm Management Monitor(RMM). + If the DA feature is supported, it also register with TSM framework. If you choose 'M' here, this module will be called arm-cca-guest.
drivers/virt/coco/arm-cca-guest/arm-cca.c +69 -2
98 lines not on this page · raw message
drivers/virt/coco/arm-cca-guest/rsi-da.h new file +27 -0
@@ -0,0 +1,27 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* +* Copyright (C) 2025 ARM Ltd. +*/ + +#ifndef RSI_DA_H_ +#define RSI_DA_H_ + +#include <linux/pci.h> +#include <linux/pci-tsm.h> +#include <asm/rsi_smc.h> + + +struct cca_guest_dsc { +struct pci_tsm_pf0 pci; +}; + +static inline struct cca_guest_dsc *to_cca_guest_dsc (struct pci_dev *pdev) +{ +struct pci_tsm *tsm = pdev->tsm; + +if (!tsm) +return NULL ; + return container_of(tsm, struct cca_guest_dsc, pci.tsm); +} + +#endif
Signature · 1 line
2.43.0
Aneesh Kumar K.V (Arm)
maintainerRFC PATCH 25/38 cca: guest: arm64: Realm device lock support
In reply to Aneesh Kumar K.V (Arm)Writing 1 to 'tsm/lock' will initiate the TDISP lock sequence.
--- arch/arm64/include/asm/rsi_cmds.h | 32 +++++++++++++- arch/arm64/include/asm/rsi_smc.h | 5 +++ drivers/virt/coco/arm-cca-guest/Makefile | 2 +- drivers/virt/coco/arm-cca-guest/arm-cca.c | 18 ++++++++ drivers/virt/coco/arm-cca-guest/rsi-da.c | 52 +++++++++++++++++++++++ drivers/virt/coco/arm-cca-guest/rsi-da.h | 3 +- 6 files changed, 108 insertions(+), 4 deletions(-) create mode 100644 drivers/virt/coco/arm-cca-guest/rsi-da.c
arch/arm64/include/asm/rsi_cmds.h +30 -2
@@ -172,8 +172,36 @@ static inline int rsi_features(unsigned long index, unsigned long *out)arm_smccc_1_1_invoke (SMC_RSI_FEATURES , index, &res); -if (out) - *out = res.a1; + *out = res.a1; +return res.a0; +} + +static inline unsigned long rsi_rdev_get_instance_id (unsigned long vdev_id,unsigned long *inst_id) +{ +struct arm_smccc_res res; + +arm_smccc_1_1_invoke (SMC_RSI_RDEV_GET_INSTANCE_ID , vdev_id, &res); + + *inst_id = res.a1; +return res.a0; +} + +static inline unsigned long __rsi_rdev_lock (unsigned long vdev_id,unsigned long inst_id) +{ +struct arm_smccc_res res; + +arm_smccc_1_1_invoke (SMC_RSI_RDEV_LOCK , vdev_id, inst_id, &res); + +return res.a0; +} + + +static inline unsigned long rsi_rdev_continue (unsigned long vdev_id,unsigned long inst_id) +{ +struct arm_smccc_res res; + +arm_smccc_1_1_invoke (SMC_RSI_RDEV_CONTINUE , vdev_id, inst_id, &res); +return res.a0; }
arch/arm64/include/asm/rsi_smc.h +5 -0
@@ -191,4 +191,9 @@ struct realm_config {*/ #define SMC_RSI_HOST_CALL SMC_RSI_FID(0x199) +#define SMC_RSI_RDEV_GET_INSTANCE_ID SMC_RSI_FID(0x19c) +#define SMC_RSI_RDEV_CONTINUE SMC_RSI_FID(0x1a4) + +#define SMC_RSI_RDEV_LOCK SMC_RSI_FID(0x1a9) +#endif /* __ASM_RSI_SMC_H_ */
drivers/virt/coco/arm-cca-guest/Makefile +1 -1
@@ -2,4 +2,4 @@ # obj-$(CONFIG_ARM_CCA_GUEST) += arm-cca-guest.o -arm-cca-guest-$(CONFIG_TSM) += arm-cca.o +arm-cca-guest-$(CONFIG_TSM) += arm-cca.o rsi-da.o
drivers/virt/coco/arm-cca-guest/arm-cca.c +18 -0
@@ -220,9 +220,27 @@ static void cca_tsm_pci_remove(struct pci_tsm *tsm)kfree (cca_dsc); } +static int cca_tsm_lock (struct pci_dev *pdev) +{ +unsigned long ret; + + ret =rsi_device_lock (pdev); +if (ret) { +pci_err (pdev,"failed to lock the device (%lu)\n" , ret); +return -EIO ; + } +return 0 ; +} + +static void cca_tsm_unlock (struct pci_dev *pdev) +{ +} +static const struct pci_tsm_ops cca_pci_ops = { .probe = cca_tsm_pci_probe, .remove = cca_tsm_pci_remove, + .lock = cca_tsm_lock, + .unlock = cca_tsm_unlock, };static void cca_tsm_unregister (void *tsm)
drivers/virt/coco/arm-cca-guest/rsi-da.c new file +52 -0
52 lines not on this page · raw message
drivers/virt/coco/arm-cca-guest/rsi-da.h +2 -1
@@ -10,9 +10,9 @@#include <linux/pci-tsm.h> #include <asm/rsi_smc.h> -struct cca_guest_dsc {struct pci_tsm_pf0 pci; +unsigned long instance_id; };static inline struct cca_guest_dsc *to_cca_guest_dsc (struct pci_dev *pdev)
@@ -24,4 +24,5 @@ static inline struct cca_guest_dsc *to_cca_guest_dsc(struct pci_dev *pdev) return container_of(tsm, struct cca_guest_dsc, pci.tsm); } +int rsi_device_lock (struct pci_dev *pdev);#endif
Signature · 1 line
2.43.0
Aneesh Kumar K.V (Arm)
maintainerRFC PATCH 26/38 KVM: arm64: Add exit handler related to device assignment
In reply to Aneesh Kumar K.V (Arm)Different RSI calls related to DA result in REC exits. Add a facility to register handlers for handling these REC exits.
--- arch/arm64/include/asm/kvm_rme.h | 3 ++ arch/arm64/include/asm/rmi_smc.h | 14 +++++++- arch/arm64/kvm/rme-exit.c | 60 ++++++++++++++++++++++++++++++++ 3 files changed, 76 insertions(+), 1 deletion(-)
arch/arm64/include/asm/kvm_rme.h +3 -0
@@ -136,4 +136,7 @@ static inline bool kvm_realm_is_private_address(struct realm *realm, return !(addr & BIT(realm->ia_bits - 1)); } +extern int (*realm_exit_vdev_req_handler)(struct realm_rec *rec); +extern int (*realm_exit_vdev_comm_handler)(struct realm_rec *rec); +extern int (*realm_exit_dev_mem_map_handler)(struct realm_rec *rec);#endif /* __ASM_KVM_RME_H */
arch/arm64/include/asm/rmi_smc.h +13 -1
@@ -219,6 +219,9 @@ struct rec_enter {#define RMI_EXIT_RIPAS_CHANGE 0x04#define RMI_EXIT_HOST_CALL 0x05#define RMI_EXIT_SERROR 0x06 +#define RMI_EXIT_VDEV_REQUEST 0x08 +#define RMI_EXIT_VDEV_COMM 0x09 +#define RMI_EXIT_DEV_MEM_MAP 0x0a struct rec_exit { union { /* 0x000 */
@@ -264,7 +267,16 @@ struct rec_exit {u8 padding5[0x100 ]; };union {/* 0x600 */ -u16 imm; +struct { +u16 imm; +u8 padding[6 ]; +u64 plane; +u64 vdev; +u64 vdev_action; +u64 dev_mem_base; +u64 dev_mem_top; +u64 dev_mem_pa; + };u8 padding6[0x100 ]; };union {/* 0x700 */
arch/arm64/kvm/rme-exit.c +60 -0
72 lines not on this page · raw message
Signature · 1 line
2.43.0
Aneesh Kumar K.V (Arm)
maintainerRFC PATCH 27/38 coco: host: arm64: add RSI_RDEV_GET_INSTANCE_ID related exit handler
In reply to Aneesh Kumar K.V (Arm)Mapping the VDEV object that matches a specified virtual device ID results in a REC exit, which is handled by the VDEV request exit handler.
--- arch/arm64/include/asm/rmi_cmds.h | 8 +++++ arch/arm64/include/asm/rmi_smc.h | 9 ++++-- drivers/virt/coco/arm-cca-host/arm-cca.c | 8 ++++- drivers/virt/coco/arm-cca-host/rmm-da.c | 39 ++++++++++++++++++++++++ drivers/virt/coco/arm-cca-host/rmm-da.h | 2 ++ 5 files changed, 63 insertions(+), 3 deletions(-)
arch/arm64/include/asm/rmi_cmds.h +8 -0
@@ -629,5 +629,13 @@ static inline unsigned long rmi_vdev_destroy(unsigned long rd, return res.a0; } +static inline unsigned long rmi_vdev_complete (unsigned long rec_phys,unsigned long vdev_phys) +{ +struct arm_smccc_res res; + +arm_smccc_1_1_invoke (SMC_RMI_VDEV_COMPLETE , rec_phys, vdev_phys, &res); + +return res.a0; +}#endif /* __ASM_RMI_CMDS_H */
arch/arm64/include/asm/rmi_smc.h +7 -2
@@ -57,7 +57,8 @@#define SMC_RMI_VDEV_CREATE SMC_RMI_CALL(0x0187)#define SMC_RMI_VDEV_DESTROY SMC_RMI_CALL(0x0188)#define SMC_RMI_VDEV_GET_STATE SMC_RMI_CALL(0x0189) -#define SMC_RMI_VDEV_STOP SMC_RMI_CALL(0x018A) +#define SMC_RMI_VDEV_STOP SMC_RMI_CALL(0x018a) +#define SMC_RMI_VDEV_COMPLETE SMC_RMI_CALL(0x018e)#define RMI_ABI_MAJOR_VERSION 1#define RMI_ABI_MINOR_VERSION 0
@@ -262,7 +263,11 @@ struct rec_exit {struct {u64 ripas_base;u64 ripas_top; -u64 ripas_value; +u8 ripas_value; +u8 padding8[15 ]; +u64 s2ap_base; +u64 s2ap_top; +u64 vdev_id; };u8 padding5[0x100 ]; };
drivers/virt/coco/arm-cca-host/arm-cca.c +7 -1
@@ -275,13 +275,19 @@ static void cca_tsm_remove(void *tsm_core)static int cca_tsm_probe (struct platform_device *pdev) { +int rc;struct tsm_core_dev *tsm_core; tsm_core =tsm_register (&pdev->dev,NULL , &cca_pci_ops);if (IS_ERR (tsm_core))return PTR_ERR (tsm_core); -return devm_add_action_or_reset (&pdev->dev, cca_tsm_remove, tsm_core); + rc =devm_add_action_or_reset (&pdev->dev, cca_tsm_remove, tsm_core); +if (rc) +return rc; + +rme_register_exit_handlers (); +return 0 ; }static const struct platform_device_id arm_cca_host_id_table[] = {
drivers/virt/coco/arm-cca-host/rmm-da.c +39 -0
@@ -660,3 +660,42 @@ void rme_unbind_vdev(struct realm *realm, struct pci_dev *pdev, struct pci_dev *return ; } } + +static struct pci_tsm *find_pci_tsm_from_vdev_id (unsigned long vdev_id) +{ +struct pci_dev *pdev =NULL ; +struct cca_host_tdi *host_tdi; + +for_each_pci_dev (pdev) { + host_tdi =to_cca_host_tdi (pdev); +if (!host_tdi) +continue ; +if (host_tdi->vdev_id == vdev_id) +return pdev->tsm; + } +return NULL ; +} + +static int rme_exit_vdev_req_handler (struct realm_rec *rec) +{ +struct cca_host_tdi *host_tdi =NULL ; +unsigned long vdev_id = rec->run->exit.vdev_id; +struct pci_tsm *tsm =find_pci_tsm_from_vdev_id (vdev_id); +phys_addr_t rec_phys =virt_to_phys (rec->rec_page); + +if (tsm) + host_tdi =to_cca_host_tdi (tsm->pdev); + +if (host_tdi) +rmi_vdev_complete (rec_phys,virt_to_phys (host_tdi->rmm_vdev)); +/* +* Return back to the guest without calling vdev complete. +* The Realm will treat that as an error. +*/ +return 1 ; +} + +void rme_register_exit_handlers (void ) +{ + realm_exit_vdev_req_handler = rme_exit_vdev_req_handler; +}
drivers/virt/coco/arm-cca-host/rmm-da.h +2 -0
@@ -48,6 +48,7 @@ struct cca_host_dsc_pf0 {struct cca_host_tdi {struct pci_tdi tdi;void *rmm_vdev; +unsigned long vdev_id; };#define PDEV_COMMUNICATE 0x1
@@ -95,4 +96,5 @@ void *rme_create_vdev(struct realm *realm, struct pci_dev *pdev, struct pci_dev *pf0_dev, u32 guest_rid);void rme_unbind_vdev (struct realm *realm,struct pci_dev *pdev,struct pci_dev *pf0_dev); +void rme_register_exit_handlers (void );#endif
Signature · 1 line
2.43.0
Aneesh Kumar K.V (Arm)
maintainerRFC PATCH 28/38 coco: host: arm64: Add support for device communication exit handler
In reply to Aneesh Kumar K.V (Arm)Different RSI calls that require device communication result in a REC exit, which is handled by the device communication exit handler.
--- arch/arm64/include/asm/rmi_smc.h | 6 ++ drivers/virt/coco/arm-cca-host/arm-cca.c | 1 + drivers/virt/coco/arm-cca-host/rmm-da.c | 75 ++++++++++++++++++++++++ drivers/virt/coco/arm-cca-host/rmm-da.h | 4 ++ 4 files changed, 86 insertions(+)
arch/arm64/include/asm/rmi_smc.h +6 -0
@@ -457,4 +457,10 @@ struct rmi_vdev_params { }; }; +#define RMI_VDEV_ACTION_GET_INTERFACE_REPORT 0x0 +#define RMI_VDEV_ACTION_GET_MEASUREMENTS 0x1 +#define RMI_VDEV_ACTION_LOCK 0x2 +#define RMI_VDEV_ACTION_START 0x3 +#define RMI_VDEV_ACTION_STOP 0x4 +#endif /* __ASM_RMI_SMC_H */
drivers/virt/coco/arm-cca-host/arm-cca.c +1 -0
@@ -243,6 +243,7 @@ static struct pci_tdi *cca_tsm_bind(struct pci_dev *pdev, struct pci_dev *pf0_de rmm_vdev =rme_create_vdev (realm, pdev, pf0_dev, tdi_id);if (!IS_ERR_OR_NULL (rmm_vdev)) { host_tdi->rmm_vdev = rmm_vdev; + host_tdi->vdev_id = tdi_id;return &no_free_ptr (host_tdi)->tdi; }
drivers/virt/coco/arm-cca-host/rmm-da.c +75 -0
98 lines not on this page · raw message
drivers/virt/coco/arm-cca-host/rmm-da.h +4 -0
@@ -22,6 +22,8 @@ struct cca_host_comm_data {void *resp_buff;void *req_buff;struct rmi_dev_comm_data *io_params; + +bool vdev_comm_active; };struct cca_host_dsc_pf0 {
@@ -43,6 +45,8 @@ struct cca_host_dsc_pf0 {bool valid; } cert_chain;struct cache_object vca; +struct cache_object interface_report; +struct cache_object measurements; };struct cca_host_tdi {
Signature · 1 line
2.43.0
Aneesh Kumar K.V (Arm)
maintainerRFC PATCH 29/38 coco: guest: arm64: Add support for collecting interface reports
In reply to Aneesh Kumar K.V (Arm)Support collecting interface reports using RSI calls.
--- arch/arm64/include/asm/rsi_cmds.h | 14 ++++++++++ arch/arm64/include/asm/rsi_smc.h | 2 ++ drivers/virt/coco/arm-cca-guest/rsi-da.c | 34 ++++++++++++++++++++++++ 3 files changed, 50 insertions(+)
arch/arm64/include/asm/rsi_cmds.h +14 -0
@@ -205,4 +205,18 @@ static inline unsigned long rsi_rdev_continue(unsigned long vdev_id, unsigned lo return res.a0; } +static inline unsigned long __rsi_rdev_get_interface_report (unsigned long vdev_id, +unsigned long inst_id, +unsigned long version_max, +unsigned long *version) +{ +struct arm_smccc_res res; + +arm_smccc_1_1_invoke (SMC_RSI_RDEV_GET_INTERFACE_REPORT , + vdev_id, inst_id, version_max, &res); + + *version = res.a1; +return res.a0; +} +#endif /* __ASM_RSI_CMDS_H */
arch/arm64/include/asm/rsi_smc.h +2 -0
@@ -194,6 +194,8 @@ struct realm_config {#define SMC_RSI_RDEV_GET_INSTANCE_ID SMC_RSI_FID(0x19c)#define SMC_RSI_RDEV_CONTINUE SMC_RSI_FID(0x1a4) +#define SMC_RSI_RDEV_GET_INTERFACE_REPORT SMC_RSI_FID(0x1a6) +#define SMC_RSI_RDEV_LOCK SMC_RSI_FID(0x1a9)#endif /* __ASM_RSI_SMC_H_ */
drivers/virt/coco/arm-cca-guest/rsi-da.c +34 -0
@@ -29,9 +29,31 @@ static inline unsigned long rsi_rdev_lock(struct pci_dev *pdev,return RSI_SUCCESS ; } +static inline unsigned long +rsi_rdev_get_interface_report (struct pci_dev *pdev,unsigned long vdev_id, +unsigned long inst_id,unsigned long version_max, +unsigned long *version) +{ +unsigned long ret; + + ret =__rsi_rdev_get_interface_report (vdev_id, inst_id, version_max, version); +if (ret !=RSI_SUCCESS ) +return ret; + +do { + ret =rsi_rdev_continue (vdev_id, inst_id); + }while (ret ==RSI_INCOMPLETE ); +if (ret !=RSI_SUCCESS ) { +pci_err (pdev,"failed to communicate with the device (%lu)\n" , ret); +return ret; + } +return RSI_SUCCESS ; +} +int rsi_device_lock (struct pci_dev *pdev) {unsigned long ret; +unsigned long tdisp_version;struct cca_guest_dsc *dsm =to_cca_guest_dsc (pdev);int vdev_id = (pci_domain_nr (pdev->bus) <<16 ) |PCI_DEVID (pdev->bus->number, pdev->devfn);
@@ -48,5 +70,17 @@ int rsi_device_lock(struct pci_dev *pdev)return -EIO ; } + ret =rsi_rdev_get_interface_report (pdev, vdev_id, dsm->instance_id, +PCI_TDISP_MESSAGE_VERSION_10 , &tdisp_version); +if (ret !=RSI_SUCCESS ) { +pci_err (pdev,"failed to get interface report (%lu)\n" , ret); +return -EIO ; + } + +if (tdisp_version !=PCI_TDISP_MESSAGE_VERSION_10 ) { +pci_err (pdev,"unknown TDISP version (%lu)\n" , tdisp_version); +return -EOPNOTSUPP ; + } +return ret; }
Signature · 1 line
2.43.0
Aneesh Kumar K.V (Arm)
maintainerRFC PATCH 30/38 coco: host: arm64: Add support for realm host interface (RHI)
In reply to Aneesh Kumar K.V (Arm)Device assignment-related RHI calls result in a REC exit, which is handled by the tsm guest_request callback.
--- arch/arm64/include/asm/rhi.h | 32 ++++++++++ drivers/virt/coco/arm-cca-host/arm-cca.c | 68 ++++++++++++++++++++ drivers/virt/coco/arm-cca-host/rmm-da.c | 81 ++++++++++++++++++++++++ drivers/virt/coco/arm-cca-host/rmm-da.h | 4 ++ include/linux/tsm.h | 3 + 5 files changed, 188 insertions(+) create mode 100644 arch/arm64/include/asm/rhi.h
arch/arm64/include/asm/rhi.h new file +32 -0
@@ -0,0 +1,32 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* +* Copyright (C) 2024 ARM Ltd. +*/ + +#ifndef __ASM_RHI_H_ +#define __ASM_RHI_H_ + +#define SMC_RHI_CALL (func) \ + ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL, \ + ARM_SMCCC_SMC_64, \ + ARM_SMCCC_OWNER_STANDARD_HYP,\ + (func)) + + +#define RHI_DA_FEATURES SMC_RHI_CALL(0x004d) +#define RHI_DA_OBJECT_SIZE SMC_RHI_CALL(0x004e) +#define RHI_DA_OBJECT_READ SMC_RHI_CALL(0x004f) + +#define RHI_DA_OBJECT_CERTIFICATE 0x1 +#define RHI_DA_OBJECT_MEASUREMENT 0x2 +#define RHI_DA_OBJECT_INTERFACE_REPORT 0x3 +#define RHI_DA_OBJECT_VCA 0x4 + + +#define RHI_DA_SUCCESS 0x1 +#define RHI_ERROR_INVALID_VDEV_ID 0x2 +#define RHI_ERROR_INVALID_DA_OBJECT_TYPE 0x3 +#define RHI_ERROR_DATA_NOT_AVAILABLE 0x4 +#define RHI_ERROR_INVALID_OFFSET 0x5 +#define RHI_ERROR_INVALID_ADDR 0x6 +#endif
drivers/virt/coco/arm-cca-host/arm-cca.c +68 -0
80 lines not on this page · raw message
drivers/virt/coco/arm-cca-host/rmm-da.c +81 -0
94 lines not on this page · raw message
drivers/virt/coco/arm-cca-host/rmm-da.h +4 -0
@@ -10,6 +10,7 @@#include <linux/pci-ide.h> #include <linux/pci-tsm.h> #include <asm/rmi_smc.h> +#include <asm/rhi.h> #define MAX_CACHE_OBJ_SIZE 4096struct cache_object {
@@ -101,4 +102,7 @@ void *rme_create_vdev(struct realm *realm, struct pci_dev *pdev,void rme_unbind_vdev (struct realm *realm,struct pci_dev *pdev,struct pci_dev *pf0_dev);void rme_register_exit_handlers (void ); +int rme_get_da_object_size (struct pci_dev *pdev,int type); +int rme_da_object_read (struct pci_dev *pdev,int type,unsigned long offset, +unsigned long max_len,void __user *user_buf);#endif
include/linux/tsm.h +3 -0
@@ -145,5 +145,8 @@ struct tsm_guest_req_info { size_t resp_len; }; +#define ARM_CCA_DA_OBJECT_SIZE 0x1 +#define ARM_CCA_DA_OBJECT_READ 0x2 +int tsm_guest_req (struct device *dev,struct tsm_guest_req_info *info);#endif /* __TSM_H */
Signature · 1 line
2.43.0
Aneesh Kumar K.V (Arm)
maintainerRFC PATCH 31/38 coco: guest: arm64: Add support for fetching interface report and certificate chain from host
In reply to Aneesh Kumar K.V (Arm)Fetch interface report and certificate chain from the host using RHI calls.
--- arch/arm64/include/asm/rsi_cmds.h | 9 ++ arch/arm64/include/asm/rsi_smc.h | 6 ++ drivers/virt/coco/arm-cca-guest/rsi-da.c | 131 +++++++++++++++++++++++ drivers/virt/coco/arm-cca-guest/rsi-da.h | 5 + 4 files changed, 151 insertions(+)
arch/arm64/include/asm/rsi_cmds.h +9 -0
@@ -219,4 +219,13 @@ static inline unsigned long __rsi_rdev_get_interface_report(unsigned long vdev_i return res.a0; } +static inline unsigned long rsi_host_call (phys_addr_t addr) +{ +struct arm_smccc_res res; + +arm_smccc_1_1_invoke (SMC_RSI_HOST_CALL , addr, &res); + +return res.a0; +} +#endif /* __ASM_RSI_CMDS_H */
arch/arm64/include/asm/rsi_smc.h +6 -0
@@ -183,6 +183,12 @@ struct realm_config {*/ #define SMC_RSI_IPA_STATE_GET SMC_RSI_FID(0x198) +struct rsi_host_call { +u16 imm; +u8 padding[6 ]; +u64 gprs[31 ]; +}; +/* * Make a Host call. *
drivers/virt/coco/arm-cca-guest/rsi-da.c +131 -0
148 lines not on this page · raw message
drivers/virt/coco/arm-cca-guest/rsi-da.h +5 -0
@@ -9,10 +9,15 @@#include <linux/pci.h> #include <linux/pci-tsm.h> #include <asm/rsi_smc.h> +#include <asm/rhi.h> struct cca_guest_dsc {struct pci_tsm_pf0 pci;unsigned long instance_id; +void *interface_report; +int interface_report_size; +void *certificate; +int certificate_size; };static inline struct cca_guest_dsc *to_cca_guest_dsc (struct pci_dev *pdev)
Signature · 1 line
2.43.0
Aneesh Kumar K.V (Arm)
maintainerRFC PATCH 32/38 coco: guest: arm64: Add support for guest initiated TDI bind/unbind
In reply to Aneesh Kumar K.V (Arm)Add RHI for VDEV_SET_TDI_STATE
Note: This is not part of RHI spec. This is a POC implementation and will be later switced to correct interface defined by RHI.
--- arch/arm64/include/asm/rhi.h | 7 +++++ arch/arm64/kernel/Makefile | 2 +- arch/arm64/kernel/rhi.c | 35 +++++++++++++++++++++++ drivers/virt/coco/arm-cca-guest/arm-cca.c | 22 ++++++++++++-- drivers/virt/coco/arm-cca-host/arm-cca.c | 8 ++++-- 5 files changed, 69 insertions(+), 5 deletions(-) create mode 100644 arch/arm64/kernel/rhi.c
arch/arm64/include/asm/rhi.h +7 -0
@@ -16,6 +16,7 @@#define RHI_DA_FEATURES SMC_RHI_CALL(0x004d)#define RHI_DA_OBJECT_SIZE SMC_RHI_CALL(0x004e)#define RHI_DA_OBJECT_READ SMC_RHI_CALL(0x004f) +#define RHI_DA_VDEV_SET_TDI_STATE SMC_RHI_CALL(0x0052)#define RHI_DA_OBJECT_CERTIFICATE 0x1#define RHI_DA_OBJECT_MEASUREMENT 0x2
@@ -29,4 +30,10 @@#define RHI_ERROR_DATA_NOT_AVAILABLE 0x4#define RHI_ERROR_INVALID_OFFSET 0x5#define RHI_ERROR_INVALID_ADDR 0x6 + +#define RHI_DA_TDI_CONFIG_UNLOCKED 0x0 +#define RHI_DA_TDI_CONFIG_LOCKED 0x1 +#define RHI_DA_TDI_CONFIG_RUN 0x2 +long rhi_da_vdev_set_tdi_state (unsigned long vdev_id,unsigned long target_state); +#endif
arch/arm64/kernel/Makefile +1 -1
@@ -34,7 +34,7 @@ obj-y := debug-monitors.o entry.o irq.o fpsimd.o \ cpufeature.o alternative.o cacheinfo.o \ smp.o smp_spin_table.o topology.o smccc-call.o \ syscall.o proton-pack.o idle.o patching.o pi/ \ - rsi.o jump_label.o + rsi.o jump_label.o rhi.o obj-$(CONFIG_COMPAT) += sys32.o signal32.o \ sys_compat.o
arch/arm64/kernel/rhi.c new file +35 -0
@@ -0,0 +1,35 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* +* Copyright (C) 2025 ARM Ltd. +*/ + +#include <asm/memory.h> +#include <asm/string.h> +#include <asm/rsi.h> +#include <asm/rhi.h> + +#include <linux/slab.h> + +long rhi_da_vdev_set_tdi_state (unsigned long guest_rid,unsigned long target_state) +{ +long ret; +struct rsi_host_call *rhi_call; + + rhi_call =kmalloc (sizeof (struct rsi_host_call ),GFP_KERNEL ); +if (!rhi_call) +return -ENOMEM ; + + rhi_call->imm =0 ; + rhi_call->gprs[0 ] =RHI_DA_VDEV_SET_TDI_STATE ; + rhi_call->gprs[1 ] = guest_rid; + rhi_call->gprs[2 ] = target_state; + + ret =rsi_host_call (virt_to_phys (rhi_call)); +if (ret !=RSI_SUCCESS ) + ret = -EIO ; +else + ret = rhi_call->gprs[0 ]; + +kfree (rhi_call); +return ret; +}
drivers/virt/coco/arm-cca-guest/arm-cca.c +20 -2
@@ -222,11 +222,20 @@ static void cca_tsm_pci_remove(struct pci_tsm *tsm)static int cca_tsm_lock (struct pci_dev *pdev) { -unsigned long ret; +long ret; +int vdev_id = (pci_domain_nr (pdev->bus) <<16 ) | +PCI_DEVID (pdev->bus->number, pdev->devfn); + ret =rhi_da_vdev_set_tdi_state (vdev_id,RHI_DA_TDI_CONFIG_LOCKED ); +if (ret) { +pci_err (pdev,"failed to TSM bind the device (%ld)\n" , ret); +return -EIO ; + } + +/* This will be done by above rhi in later spec */ ret =rsi_device_lock (pdev);if (ret) { -pci_err (pdev,"failed to lock the device (%lu)\n" , ret); +pci_err (pdev,"failed to lock the device (%ld)\n" , ret);return -EIO ; }return 0 ;
@@ -234,6 +243,15 @@ static int cca_tsm_lock(struct pci_dev *pdev)static void cca_tsm_unlock (struct pci_dev *pdev) { +long ret; +int vdev_id = (pci_domain_nr (pdev->bus) <<16 ) | +PCI_DEVID (pdev->bus->number, pdev->devfn); + + ret =rhi_da_vdev_set_tdi_state (vdev_id,RHI_DA_TDI_CONFIG_UNLOCKED ); +if (ret) { +pci_err (pdev,"failed to TSM unbind the device (%ld)\n" , ret); +return ; + } }static const struct pci_tsm_ops cca_pci_ops = {
drivers/virt/coco/arm-cca-host/arm-cca.c +6 -2
@@ -254,9 +254,13 @@ static struct pci_tdi *cca_tsm_bind(struct pci_dev *pdev, struct pci_dev *pf0_destatic void cca_tsm_unbind (struct pci_tdi *tdi) {struct realm *realm = &tdi->kvm->arch.realm; - +/* +* FIXME!! +* All the related DEV RIPAS regions should be unmapped by now. +* For now we handle them during stage2 teardown. There is no +* bound IPA address available here. Possibly dmabuf can help +*/ rme_unbind_vdev (realm, tdi->pdev, tdi->pdev->tsm->dsm_dev); -module_put (THIS_MODULE ); }
Signature · 1 line
2.43.0
Aneesh Kumar K.V (Arm)
maintainerRFC PATCH 33/38 KVM: arm64: CCA: handle dev mem map/unmap
In reply to Aneesh Kumar K.V (Arm)Handle VM exit on DEV_MEM_MAP
--- arch/arm64/include/asm/rmi_cmds.h | 40 +++++++ arch/arm64/include/asm/rmi_smc.h | 5 + arch/arm64/kvm/rme-exit.c | 39 +++++- arch/arm64/kvm/rme.c | 190 ++++++++++++++++++++++++++++-- drivers/vfio/pci/vfio_pci_core.c | 1 + 5 files changed, 262 insertions(+), 13 deletions(-)
arch/arm64/include/asm/rmi_cmds.h +40 -0
@@ -638,4 +638,44 @@ static inline unsigned long rmi_vdev_complete(unsigned long rec_phys, unsigned lreturn res.a0; } +static inline int rmi_rtt_dev_mem_validate (unsigned long rd,unsigned long rec, +unsigned long base,unsigned long top, +unsigned long *out_top) +{ +struct arm_smccc_res res; + +arm_smccc_1_1_invoke (SMC_RMI_RTT_DEV_MEM_VALIDATE , rd, rec, base, top, &res); + +if (out_top) + *out_top = res.a1; + +return res.a0; +} + +static inline int rmi_dev_mem_map (unsigned long rd,unsigned long ipa, +unsigned long level,unsigned long pa) +{ +struct arm_smccc_res res; + +arm_smccc_1_1_invoke (SMC_RMI_DEV_MEM_MAP , rd, ipa, level, pa, &res); + +return res.a0; +} + +static inline int rmi_dev_mem_unmap (unsigned long rd,unsigned long ipa, +unsigned long level,unsigned long *out_pa, +unsigned long *out_ipa) +{ +struct arm_smccc_res res; + +arm_smccc_1_1_invoke (SMC_RMI_DEV_MEM_UNMAP , rd, ipa, level, &res); + +if (out_pa) + *out_pa = res.a1; +if (out_ipa) + *out_ipa = res.a2; + +return res.a0; +} +#endif /* __ASM_RMI_CMDS_H */
arch/arm64/include/asm/rmi_smc.h +5 -0
@@ -39,6 +39,7 @@#define SMC_RMI_RTT_READ_ENTRY SMC_RMI_CALL(0x0161)#define SMC_RMI_RTT_UNMAP_UNPROTECTED SMC_RMI_CALL(0x0162) +#define SMC_RMI_RTT_DEV_MEM_VALIDATE SMC_RMI_CALL(0x0163)#define SMC_RMI_PSCI_COMPLETE SMC_RMI_CALL(0x0164)#define SMC_RMI_FEATURES SMC_RMI_CALL(0x0165)
@@ -47,6 +48,9 @@#define SMC_RMI_RTT_INIT_RIPAS SMC_RMI_CALL(0x0168)#define SMC_RMI_RTT_SET_RIPAS SMC_RMI_CALL(0x0169) +#define SMC_RMI_DEV_MEM_MAP SMC_RMI_CALL(0x0172) +#define SMC_RMI_DEV_MEM_UNMAP SMC_RMI_CALL(0x0173) +#define SMC_RMI_PDEV_COMMUNICATE SMC_RMI_CALL(0x0175)#define SMC_RMI_PDEV_CREATE SMC_RMI_CALL(0x0176)#define SMC_RMI_PDEV_DESTROY SMC_RMI_CALL(0x0177)
@@ -84,6 +88,7 @@ enum rmi_ripas {RMI_EMPTY =0 ,RMI_RAM =1 ,RMI_DESTROYED =2 , + RMI_DEV = 3, };#define RMI_NO_MEASURE_CONTENT 0
arch/arm64/kvm/rme-exit.c +33 -6
@@ -170,17 +170,44 @@ EXPORT_SYMBOL_GPL(realm_exit_dev_mem_map_handler);static int rec_exit_dev_mem_map (struct kvm_vcpu *vcpu) {int ret; +struct kvm *kvm = vcpu->kvm; +struct realm *realm = &kvm->arch.realm;struct realm_rec *rec = &vcpu->arch.rec; +unsigned long base = rec->run->exit.dev_mem_base; +unsigned long top = rec->run->exit.dev_mem_top; + +if (!kvm_realm_is_private_address (realm, base) || + !kvm_realm_is_private_address (realm, top -1 )) { +vcpu_err (vcpu,"Invalid DEV_MEM_VALIDATE for %#lx - %#lx\n" , base, top); +return -EINVAL ; + } +/* See if coco driver want to look at the dev mem_map request */ if (realm_exit_dev_mem_map_handler) { ret = (*realm_exit_dev_mem_map_handler)(rec); - }else { -kvm_pr_unimpl ("Unsupported exit reason: %u\n" , - rec->run->exit.exit_reason); - vcpu->run->exit_reason =KVM_EXIT_INTERNAL_ERROR ; - ret =0 ; +if (ret) +return ret; } -return ret; + +#if 0 +/* we don't need a memory fault exit for device mapping. +* 1. On enter to rec, we map the device memory using dev_mem_map +2. There is no fallocate, and we are not tracking this via memory attributes. +If we need a fault exit, we need to differentiate it in VMM so that we don't +map the private memory via tsm map ioctl. +*/ +/* +* Exit to VMM so that VMM can deny the validation, the actual +* validation response is done on next entry +*/ +kvm_prepare_memory_fault_exit (vcpu, base, top - base, false, false, + true); + +/* exit to hypervisor */ +return -EFAULT ; +#else +return 1 ; +#endif }static void update_arch_timer_irq_lines (struct kvm_vcpu *vcpu)
arch/arm64/kvm/rme.c +183 -7
243 lines not on this page · raw message
drivers/vfio/pci/vfio_pci_core.c +1 -0
@@ -1718,6 +1718,7 @@ static const struct vm_operations_struct vfio_pci_mmap_ops = {#endif }; +/* FIXME!! don't allow mmap once the device is TDISP locked and we did dev mem_map. */ int vfio_pci_core_mmap (struct vfio_device *core_vdev,struct vm_area_struct *vma) { struct vfio_pci_core_device *vdev =
Signature · 1 line
2.43.0
Aneesh Kumar K.V (Arm)
maintainerRFC PATCH 34/38 coco: guest: arm64: Validate mmio range found in the interface report
In reply to Aneesh Kumar K.V (Arm)This starts the sequence to transition the realm device to the TDISP RUN state by writing 1 to 'tsm/accept'.
--- arch/arm64/include/asm/rsi_cmds.h | 18 +++ arch/arm64/include/asm/rsi_smc.h | 4 + drivers/virt/coco/arm-cca-guest/arm-cca.c | 3 + drivers/virt/coco/arm-cca-guest/rsi-da.c | 132 ++++++++++++++++++++++ drivers/virt/coco/arm-cca-guest/rsi-da.h | 25 ++++ 5 files changed, 182 insertions(+)
arch/arm64/include/asm/rsi_cmds.h +18 -0
@@ -228,4 +228,22 @@ static inline unsigned long rsi_host_call(phys_addr_t addr) return res.a0; } +static inline unsigned long +rsi_rdev_validate_mapping (unsigned long vdev_id,unsigned long inst_id, +phys_addr_t start_ipa,phys_addr_t end_ipa, +phys_addr_t io_pa,phys_addr_t *next_ipa,unsigned long flags) +{ +struct arm_smccc_res res; + +arm_smccc_1_1_invoke (SMC_RSI_RDEV_VALIDATE_MAPPING , vdev_id, + inst_id, start_ipa, end_ipa, io_pa, flags, &res); + *next_ipa = res.a1; + +if (res.a2 !=RSI_ACCEPT ) +return -EPERM ; + +if (res.a0 !=RSI_SUCCESS ) +return -EINVAL ; +return 0 ; +}#endif /* __ASM_RSI_CMDS_H */
arch/arm64/include/asm/rsi_smc.h +4 -0
@@ -204,4 +204,8 @@ struct rsi_host_call {#define SMC_RSI_RDEV_LOCK SMC_RSI_FID(0x1a9) +#define RSI_DEV_MEM_COHERENT BIT(0) +#define RSI_DEV_MEM_LIMITED_ORDER BIT(1) +#define SMC_RSI_RDEV_VALIDATE_MAPPING SMC_RSI_FID(0x1ac) +#endif /* __ASM_RSI_SMC_H_ */
drivers/virt/coco/arm-cca-guest/arm-cca.c +3 -0
@@ -247,6 +247,9 @@ static void cca_tsm_unlock(struct pci_dev *pdev)int vdev_id = (pci_domain_nr (pdev->bus) <<16 ) |PCI_DEVID (pdev->bus->number, pdev->devfn); +/* invalidate dev mapping based on interface report */ +rsi_update_interface_report (pdev, false); + ret =rhi_da_vdev_set_tdi_state (vdev_id,RHI_DA_TDI_CONFIG_UNLOCKED );if (ret) {pci_err (pdev,"failed to TSM unbind the device (%ld)\n" , ret);
drivers/virt/coco/arm-cca-guest/rsi-da.c +132 -0
135 lines not on this page · raw message
drivers/virt/coco/arm-cca-guest/rsi-da.h +25 -0
@@ -11,6 +11,28 @@#include <asm/rsi_smc.h> #include <asm/rhi.h> +struct pci_tdisp_device_interface_report { +u16 interface_info; +u16 reserved; +u16 msi_x_message_control; +u16 lnr_control; +u32 tph_control; +u32 mmio_range_count; +} __packed; + +struct pci_tdisp_mmio_range { +u64 first_page; +u32 num_pages; +u32 range_attributes; +} __packed; + +#define TSM_INTF_REPORT_MMIO_MSIX_TABLE BIT(0) +#define TSM_INTF_REPORT_MMIO_PBA BIT(1) +#define TSM_INTF_REPORT_MMIO_IS_NON_TEE BIT(2) +#define TSM_INTF_REPORT_MMIO_IS_UPDATABLE BIT(3) +#define TSM_INTF_REPORT_MMIO_RESERVED GENMASK(15, 4) +#define TSM_INTF_REPORT_MMIO_RANGE_ID GENMASK(31, 16) +struct cca_guest_dsc {struct pci_tsm_pf0 pci;unsigned long instance_id;
@@ -29,5 +51,8 @@ static inline struct cca_guest_dsc *to_cca_guest_dsc(struct pci_dev *pdev) return container_of(tsm, struct cca_guest_dsc, pci.tsm); } +int rsi_update_interface_report (struct pci_dev *pdev,bool validate);int rsi_device_lock (struct pci_dev *pdev); +int rsi_device_start (struct pci_dev *pdev); +#endif
Signature · 1 line
2.43.0
Aneesh Kumar K.V (Arm)
maintainerRFC PATCH 35/38 coco: guest: arm64: Add Realm device start and stop support
In reply to Aneesh Kumar K.V (Arm)Writing 1 to 'tsm/acceept' will initiate the TDISP RUN sequence.
--- arch/arm64/include/asm/rsi_cmds.h | 19 +++++++ arch/arm64/include/asm/rsi_smc.h | 2 + drivers/virt/coco/arm-cca-guest/arm-cca.c | 15 ++++++ drivers/virt/coco/arm-cca-guest/rsi-da.c | 60 +++++++++++++++++++++++ drivers/virt/coco/arm-cca-guest/rsi-da.h | 2 +- 5 files changed, 97 insertions(+), 1 deletion(-)
arch/arm64/include/asm/rsi_cmds.h +19 -0
@@ -246,4 +246,23 @@ rsi_rdev_validate_mapping(unsigned long vdev_id, unsigned long inst_id,return -EINVAL ; return 0; } + +static inline unsigned long __rsi_rdev_start (unsigned long vdev_id,unsigned long inst_id) +{ +struct arm_smccc_res res; + +arm_smccc_1_1_invoke (SMC_RSI_RDEV_START , vdev_id, inst_id, &res); + +return res.a0; +} + +static inline unsigned long __rsi_rdev_stop (unsigned long vdev_id,unsigned long inst_id) +{ +struct arm_smccc_res res; + +arm_smccc_1_1_invoke (SMC_RSI_RDEV_STOP , vdev_id, inst_id, &res); + +return res.a0; +} +#endif /* __ASM_RSI_CMDS_H */
arch/arm64/include/asm/rsi_smc.h +2 -0
@@ -203,6 +203,8 @@ struct rsi_host_call {#define SMC_RSI_RDEV_GET_INTERFACE_REPORT SMC_RSI_FID(0x1a6)#define SMC_RSI_RDEV_LOCK SMC_RSI_FID(0x1a9) +#define SMC_RSI_RDEV_START SMC_RSI_FID(0x1aa) +#define SMC_RSI_RDEV_STOP SMC_RSI_FID(0x1ab)#define RSI_DEV_MEM_COHERENT BIT(0)#define RSI_DEV_MEM_LIMITED_ORDER BIT(1)
drivers/virt/coco/arm-cca-guest/arm-cca.c +15 -0
@@ -250,6 +250,8 @@ static void cca_tsm_unlock(struct pci_dev *pdev)/* invalidate dev mapping based on interface report */ rsi_update_interface_report (pdev, false); +rsi_device_stop (pdev); + ret =rhi_da_vdev_set_tdi_state (vdev_id,RHI_DA_TDI_CONFIG_UNLOCKED );if (ret) {pci_err (pdev,"failed to TSM unbind the device (%ld)\n" , ret);
@@ -257,11 +259,24 @@ static void cca_tsm_unlock(struct pci_dev *pdev) } } +static int cca_tsm_accept (struct pci_dev *pdev) +{ +int ret; + + ret =rsi_device_start (pdev); +if (ret) { +pci_err (pdev,"failed to transition the device to run state (%d)\n" , ret); +return -EIO ; + } +return 0 ; +} +static const struct pci_tsm_ops cca_pci_ops = { .probe = cca_tsm_pci_probe, .remove = cca_tsm_pci_remove, .lock = cca_tsm_lock, .unlock = cca_tsm_unlock, + .accept = cca_tsm_accept, };static void cca_tsm_unregister (void *tsm)
drivers/virt/coco/arm-cca-guest/rsi-da.c +60 -0
77 lines not on this page · raw message
drivers/virt/coco/arm-cca-guest/rsi-da.h +1 -1
@@ -54,5 +54,5 @@ static inline struct cca_guest_dsc *to_cca_guest_dsc(struct pci_dev *pdev)int rsi_update_interface_report (struct pci_dev *pdev,bool validate);int rsi_device_lock (struct pci_dev *pdev);int rsi_device_start (struct pci_dev *pdev); - +int rsi_device_stop (struct pci_dev *pdev);#endif
Signature · 1 line
2.43.0
Aneesh Kumar K.V (Arm)
maintainerRFC PATCH 36/38 KVM: arm64: CCA: enable DA in realm create parameters
In reply to Aneesh Kumar K.V (Arm)Now that we have all the required steps for DA in-place, enable DA while creating ralm.
--- arch/arm64/include/asm/rmi_smc.h | 1 + arch/arm64/kvm/rme.c | 2 ++ 2 files changed, 3 insertions(+)
arch/arm64/include/asm/rmi_smc.h +1 -0
@@ -112,6 +112,7 @@ enum rmi_ripas {#define RMI_REALM_PARAM_FLAG_LPA2 BIT(0)#define RMI_REALM_PARAM_FLAG_SVE BIT(1)#define RMI_REALM_PARAM_FLAG_PMU BIT(2) +#define RMI_REALM_PARAM_FLAG_DA BIT(3)/* * Note many of these fields are smaller than u64 but all fields have u64
arch/arm64/kvm/rme.c +2 -0
@@ -695,6 +695,8 @@ static int realm_create_rd(struct kvm *kvm)if (r) goto out_undelegate_tables; +/* For now default enable DA */ + params->flags =RMI_REALM_PARAM_FLAG_DA ; params_phys =virt_to_phys (params);if (rmi_realm_create (rd_phys, params_phys)) {
Signature · 1 line
2.43.0
Aneesh Kumar K.V (Arm)
maintainerRFC PATCH 37/38 coco: guest: arm64: Add support for fetching device measurements
In reply to Aneesh Kumar K.V (Arm)Fetch device measurements using RSI_RDEV_GET_MEASUREMENTS.
--- arch/arm64/include/asm/rsi_cmds.h | 11 +++++++ arch/arm64/include/asm/rsi_smc.h | 16 ++++++++++ drivers/virt/coco/arm-cca-guest/rsi-da.c | 39 ++++++++++++++++++++++++ drivers/virt/coco/arm-cca-guest/rsi-da.h | 2 ++ 4 files changed, 68 insertions(+)
arch/arm64/include/asm/rsi_cmds.h +11 -0
@@ -265,4 +265,15 @@ static inline unsigned long __rsi_rdev_stop(unsigned long vdev_id, unsigned long return res.a0; } +static inline unsigned long __rsi_rdev_get_measurements (unsigned long vdev_id, +unsigned long inst_id, +phys_addr_t meas) +{ +struct arm_smccc_res res; + +arm_smccc_1_1_invoke (SMC_RSI_RDEV_GET_MEASUREMENTS , vdev_id, inst_id, meas, &res); + +return res.a0; +} +#endif /* __ASM_RSI_CMDS_H */
arch/arm64/include/asm/rsi_smc.h +16 -0
@@ -202,6 +202,22 @@ struct rsi_host_call {#define SMC_RSI_RDEV_GET_INTERFACE_REPORT SMC_RSI_FID(0x1a6) +#define RSI_DEV_MEASURE_ALL BIT(0) +#define RSI_DEV_MEASURE_SIGNED BIT(1) +#define RSI_DEV_MEASURE_RAW BIT(2) + +struct rsi_device_measurements_params { +union { +struct { +u64 flags; +u8 indices[32 ]; +u8 nounce[32 ]; + }; +u8 padding[0x100 ]; + }; +}; + +#define SMC_RSI_RDEV_GET_MEASUREMENTS SMC_RSI_FID(0x1a7)#define SMC_RSI_RDEV_LOCK SMC_RSI_FID(0x1a9)#define SMC_RSI_RDEV_START SMC_RSI_FID(0x1aa)#define SMC_RSI_RDEV_STOP SMC_RSI_FID(0x1ab)
drivers/virt/coco/arm-cca-guest/rsi-da.c +39 -0
@@ -166,10 +166,31 @@ static long rhi_get_report(int vdev_id, int da_object_type, void **report, int *return ret; } +static inline unsigned long +rsi_rdev_get_measurements (struct pci_dev *pdev,unsigned long vdev_id, +unsigned long inst_id,phys_addr_t meas) +{ +unsigned long ret; + + ret =__rsi_rdev_get_measurements (vdev_id, inst_id, meas); +if (ret !=RSI_SUCCESS ) +return ret; + +do { + ret =rsi_rdev_continue (vdev_id, inst_id); + }while (ret ==RSI_INCOMPLETE ); +if (ret !=RSI_SUCCESS ) { +pci_err (pdev,"failed to communicate with the device (%lu)\n" , ret); +return ret; + } +return RSI_SUCCESS ; +} +int rsi_device_lock (struct pci_dev *pdev) {unsigned long ret;unsigned long tdisp_version; +struct rsi_device_measurements_params *rsi_dev_meas;struct cca_guest_dsc *dsm =to_cca_guest_dsc (pdev);int vdev_id = (pci_domain_nr (pdev->bus) <<16 ) |PCI_DEVID (pdev->bus->number, pdev->devfn);
@@ -198,6 +219,17 @@ int rsi_device_lock(struct pci_dev *pdev)return -EOPNOTSUPP ; } + rsi_dev_meas = (struct rsi_device_measurements_params *)__get_free_page (GFP_KERNEL ); + rsi_dev_meas->flags =RSI_DEV_MEASURE_ALL ; + ret =rsi_rdev_get_measurements (pdev, vdev_id, dsm->instance_id, +virt_to_phys (rsi_dev_meas)); + +free_page ((unsigned long )rsi_dev_meas); +if (ret !=RSI_SUCCESS ) { +pci_err (pdev,"failed to get device measurement (%lu)\n" , ret); +return -EIO ; + } +/* Now make a host call to copy the interface report to guest. */ ret =rhi_get_report (vdev_id,RHI_DA_OBJECT_INTERFACE_REPORT , &dsm->interface_report, &dsm->interface_report_size);
@@ -213,6 +245,13 @@ int rsi_device_lock(struct pci_dev *pdev)return -EIO ; } + ret =rhi_get_report (vdev_id,RHI_DA_OBJECT_MEASUREMENT , + &dsm->measurements, &dsm->measurements_size); +if (ret) { +pci_err (pdev,"failed to get device certificate from the host (%lu)\n" , ret); +return -EIO ; + } +return ret; } static inline unsigned long rsi_rdev_start(struct pci_dev *pdev,
drivers/virt/coco/arm-cca-guest/rsi-da.h +2 -0
@@ -40,6 +40,8 @@ struct cca_guest_dsc {int interface_report_size;void *certificate;int certificate_size; +void *measurements; +int measurements_size; };static inline struct cca_guest_dsc *to_cca_guest_dsc (struct pci_dev *pdev)
Signature · 1 line
2.43.0
Aneesh Kumar K.V (Arm)
maintainerRFC PATCH 38/38 coco: guest: arm64: Add support for fetching device info
In reply to Aneesh Kumar K.V (Arm)RSI_RDEV_GET_INFO returns different digest hash values, which can be compared with host cached values to ensure the host didn't tamper with the cached data.
--- arch/arm64/include/asm/rsi_cmds.h | 12 +++ arch/arm64/include/asm/rsi_smc.h | 24 +++++ drivers/virt/coco/arm-cca-guest/Kconfig | 2 + drivers/virt/coco/arm-cca-guest/rsi-da.c | 128 +++++++++++++++++++++++ drivers/virt/coco/arm-cca-guest/rsi-da.h | 13 +++ 5 files changed, 179 insertions(+)
arch/arm64/include/asm/rsi_cmds.h +12 -0
@@ -276,4 +276,16 @@ static inline unsigned long __rsi_rdev_get_measurements(unsigned long vdev_id, return res.a0; } +static inline unsigned long rsi_rdev_get_info (unsigned long vdev_id, +unsigned long inst_id, +unsigned long digest_phys) +{ +struct arm_smccc_res res; + +arm_smccc_1_1_invoke (SMC_RSI_RDEV_GET_INFO , + vdev_id, inst_id, digest_phys, &res); + +return res.a0; +} +#endif /* __ASM_RSI_CMDS_H */
arch/arm64/include/asm/rsi_smc.h +24 -0
@@ -125,6 +125,9 @@ #ifndef __ASSEMBLY__ +#define RSI_HASH_SHA_256 0 +#define RSI_HASH_SHA_512 1 + struct realm_config { union { struct {
@@ -200,6 +203,27 @@ struct rsi_host_call {#define SMC_RSI_RDEV_GET_INSTANCE_ID SMC_RSI_FID(0x19c)#define SMC_RSI_RDEV_CONTINUE SMC_RSI_FID(0x1a4) +struct rsi_device_info { +union { +struct { +u64 flags; +u64 attest_type; +u64 cert_id; +u8 hash_algo; + }; +u8 padding[0x40 ]; + }; +union {/* 0x40 */ +struct { +u8 cert_digest[0x40 ]; +u8 meas_digest[0x40 ]; +u8 report_digest[0x40 ]; + }; +u8 padding2[0x200 -0x40 ]; + }; +}; + +#define SMC_RSI_RDEV_GET_INFO SMC_RSI_FID(0x1a5)#define SMC_RSI_RDEV_GET_INTERFACE_REPORT SMC_RSI_FID(0x1a6)#define RSI_DEV_MEASURE_ALL BIT(0)
drivers/virt/coco/arm-cca-guest/Kconfig +2 -0
@@ -5,6 +5,8 @@ config ARM_CCA_GUEST tristate "Arm CCA Guest driver" depends on ARM64 depends on PCI_TSM + select CRYPTO_SHA256 + select CRYPTO_SHA512 select TSM_REPORTS select TSM help
drivers/virt/coco/arm-cca-guest/rsi-da.c +128 -0
153 lines not on this page · raw message
drivers/virt/coco/arm-cca-guest/rsi-da.h +13 -0
@@ -10,6 +10,7 @@#include <linux/pci-tsm.h> #include <asm/rsi_smc.h> #include <asm/rhi.h> +#include <crypto/sha2.h> struct pci_tdisp_device_interface_report {u16 interface_info;
@@ -33,6 +34,17 @@ struct pci_tdisp_mmio_range {#define TSM_INTF_REPORT_MMIO_RESERVED GENMASK(15, 4)#define TSM_INTF_REPORT_MMIO_RANGE_ID GENMASK(31, 16) +struct dsm_device_info { +u64 flags; +u64 attest_type; +u64 cert_id; +u64 hash_algo; +u8 cert_digest[SHA512_DIGEST_SIZE ]; +u8 meas_digest[SHA512_DIGEST_SIZE ]; +u8 report_digest[SHA512_DIGEST_SIZE ]; +}; + +struct cca_guest_dsc {struct pci_tsm_pf0 pci;unsigned long instance_id;
@@ -42,6 +54,7 @@ struct cca_guest_dsc {int certificate_size;void *measurements;int measurements_size; +struct dsm_device_info dev_info; };static inline struct cca_guest_dsc *to_cca_guest_dsc (struct pci_dev *pdev)
Signature · 1 line
2.43.0
Re: [RFC PATCH v1 06/38] iommufd: Add and option to request for bar mapping with IORESOURCE_EXCLUSIVE
In reply to Aneesh Kumar K.V (Arm)Quoted 1 line · Aneesh Kumar K.V (Arm)
Signed-off-by Aneesh Kumar K.V (Arm) <[email protected]>
Why would we need this?
I can sort of understand why Intel would need it due to their issues with MCE, but ARM shouldn't care either way, should it?
But also why is it an iommufd option? That doesn't seem right..
Jason
Re: [RFC PATCH v1 07/38] iommufd/viommu: Add support to associate viommu with kvm instance
In reply to Aneesh Kumar K.V (Arm)Quoted 18 lines · Aneesh Kumar K.V (Arm)
drivers/iommu/iommufd/viommu.c +18
-0+#if IS_ENABLED (CONFIG_KVM ) +#include <linux/kvm_host.h> + +static int viommu_get_kvm (struct iommufd_viommu *viommu,int kvm_vm_fd) +{ +int rc = -EBADF ; +struct file *filp; + + filp =fget (kvm_vm_fd); + +if (!file_is_kvm (filp)) + goto err_out; + +/* hold the kvm reference via file descriptor */ + viommu->kvm_filp = filp; +return 0 ; +err_out: + viommu->kvm_filp =NULL ;
drivers/iommu/iommufd/viommu.c line 34
Quoted 10 lines
drivers/iommu/iommufd/viommu.c +10
-0+fput (filp); + return rc; +} + +static void viommu_put_kvm (struct iommufd_viommu *viommu) +{ +fput (viommu->kvm_filp); + viommu->kvm_filp =NULL ; +} +#endif
Missing stub functions for !CONFIG_KVM?
Looks like an OK design otherwise
include/uapi/linux/iommufd.h struct iommu_viommu_alloc { · line 1071
Quoted 5 lines
include/uapi/linux/iommufd.h +1
-0@@ -1057,6 +1068,7 @@ struct iommu_viommu_alloc {__u32 data_len;__u32 __reserved;__aligned_u64 data_uptr; +__u32 kvm_vm_fd;
fds are __s32, they are signed numbers.
Jason
Re: [RFC PATCH v1 10/38] iommufd/vdevice: Add TSM map ioctl
In reply to Aneesh Kumar K.V (Arm)Quoted 5 lines · Aneesh Kumar K.V (Arm)
With passthrough devices, we need to make sure private memory is allocated and assigned to the secure guest before we can issue the DMA. For ARM RMM, we only need to map and the secure SMMU management is internal to RMM. For shared IPA, vfio/iommufd DMA MAP/UNMAP interface does the equivalent
I'm not really sure what this is about? It is about getting KVM to pin all the memory and commit it to the RMM so it can be used for DMA?
But it looks really strange to have an iommufd ioctl that just calls a KVM function. Feeling this should be a KVM function, or a guestmfd behavior??
I was kind of thinking it would be nice to have a guestmemfd mode that was "pinned", meaning the memory is allocated and remains almost always mapped into the TSM's page tables automatically. VFIO using guests would set things this way.
Jason
Re: [RFC PATCH v1 04/38] tsm: Support DMA Allocation from private memory
In reply to Aneesh Kumar K.V (Arm)arch/arm64/mm/mem_encrypt.c int set_memory_decrypted(unsigned long addr, int numpages) · line 56
Quoted 9 lines · Aneesh Kumar K.V (Arm)
arch/arm64/mm/mem_encrypt.c +5
-0@@ -48,3 +49,12 @@ int set_memory_decrypted(unsigned long addr, int numpages)return crypt_ops->decrypt (addr, numpages); }EXPORT_SYMBOL_GPL (set_memory_decrypted); + +bool force_dma_unencrypted (struct device *dev) +{ +if (dev->tdi_enabled) +return false;
Is this OK? I see code like this:
static inline dma_addr_t phys_to_dma_direct(struct device *dev,
phys_addr_t phys)
{
if (force_dma_unencrypted(dev))
return phys_to_dma_unencrypted(dev, phys);
return phys_to_dma(dev, phys);What are the ARM rules for generating dma addreses?
1) Device is T=0, memory is unencrypted, call dma_addr_unencrypted() and do "top bit IBA set"
2) Device is T=1, memory is encrypted, use the phys_to_dma() normally
3) Device it T=1, memory is uncrypted, use the phys_to_dma() normally??? Seems odd, I would have guessed the DMA address sould be the same as case #1?
Can you document this in a comment?
include/linux/device.h struct device { · line 692
Quoted 10 lines
include/linux/device.h +1
-0@@ -688,6 +688,7 @@ struct device {#ifdef CONFIG_IOMMU_DMA bool dma_iommu:1;#endif + bool tdi_enabled:1; };
I would give the dev->tdi_enabled a clearer name, maybe dev->encrypted_dma_supported ?
Also need to think carefully of a bitfield is OK here, we can't locklessly change a bitfield so need to audit that all members are set under, probably, the device lock or some other single threaded hand waving. It seems believable it is like that but should be checked out, and add a lockdep if it relies on the device lock.
Jason
Aneesh Kumar K.V
maintainerRe: [RFC PATCH v1 04/38] tsm: Support DMA Allocation from private memory
Quoted 25 lines · Jason Gunthorpe
arch/arm64/mm/mem_encrypt.c int set_memory_decrypted(unsigned long addr, int numpages) · line 56
Quoted 9 lines · Aneesh Kumar K.V (Arm)
arch/arm64/mm/mem_encrypt.c +5
-0@@ -48,3 +49,12 @@ int set_memory_decrypted(unsigned long addr, int numpages)return crypt_ops->decrypt (addr, numpages); }EXPORT_SYMBOL_GPL (set_memory_decrypted); + +bool force_dma_unencrypted (struct device *dev) +{ +if (dev->tdi_enabled) +return false;Is this OK? I see code like this:
static inline
dma_addr_tphys_to_dma_direct(struct device *dev,
phys_addr_tphys){ if (force_dma_unencrypted(dev)) return phys_to_dma_unencrypted(dev, phys); return phys_to_dma(dev, phys);What are the ARM rules for generating dma addreses?
1) Device is T=0, memory is unencrypted, call
dma_addr_unencrypted()and do "top bit IBA set"
Quoted 8 lines
2) Device is T=1, memory is encrypted, use the
phys_to_dma()normally3) Device it T=1, memory is uncrypted, use the
phys_to_dma()normally??? Seems odd, I would have guessed the DMA address sould be the same as case #1?Can you document this in a comment?
If a device is operating in secure mode (T=1), it is currently assumed that only access to private (encrypted) memory is supported. It is unclear whether devices would need to perform DMA to shared (unencrypted) memory while operating in this mode, as TLPs with T=1 are generally expected to target private memory.
Based on this assumption, T=1 devices will always access private/encrypted memory, while T=0 devices will be restricted to shared/unencrypted memory.
Quoted 15 lines
include/linux/device.h struct device { · line 692
Quoted 11 lines
include/linux/device.h +1
-0@@ -688,6 +688,7 @@ struct device {#ifdef CONFIG_IOMMU_DMA bool dma_iommu:1;#endif + bool tdi_enabled:1; };I would give the
dev->tdi_enableda clearer name, maybedev->encrypted_dma_supported?
Quoted 6 lines
Also need to think carefully of a bitfield is OK here, we can't locklessly change a bitfield so need to audit that all members are set under, probably, the device lock or some other single threaded hand waving. It seems believable it is like that but should be checked out, and add a lockdep if it relies on the device lock.
Will check and update the patch.
-aneesh
Aneesh Kumar K.V
maintainerRe: [RFC PATCH v1 06/38] iommufd: Add and option to request for bar mapping with IORESOURCE_EXCLUSIVE
In reply to Jason GunthorpeQuoted 11 lines · Jason Gunthorpe
Quoted 1 line · Aneesh Kumar K.V (Arm)
Signed-off-by Aneesh Kumar K.V (Arm) <[email protected]>Why would we need this?
I can sort of understand why Intel would need it due to their issues with MCE, but ARM shouldn't care either way, should it?
But also why is it an iommufd option? That doesn't seem right..
Jason
This is based on our previous discussion https://lore.kernel.org/all/[email protected]
IIUC, we intend to request the resource in exclusive mode for secure guests—regardless of whether the platform is Intel or ARM. Could you help clarify the MCE issue observed on Intel platforms in this context?
-aneesh
Aneesh Kumar K.V
maintainerRe: [RFC PATCH v1 07/38] iommufd/viommu: Add support to associate viommu with kvm instance
In reply to Jason GunthorpeQuoted 35 lines · Jason Gunthorpe
Quoted 18 lines · Aneesh Kumar K.V (Arm)
drivers/iommu/iommufd/viommu.c +18
-0+#if IS_ENABLED (CONFIG_KVM ) +#include <linux/kvm_host.h> + +static int viommu_get_kvm (struct iommufd_viommu *viommu,int kvm_vm_fd) +{ +int rc = -EBADF ; +struct file *filp; + + filp =fget (kvm_vm_fd); + +if (!file_is_kvm (filp)) + goto err_out; + +/* hold the kvm reference via file descriptor */ + viommu->kvm_filp = filp; +return 0 ; +err_out: + viommu->kvm_filp =NULL ;drivers/iommu/iommufd/viommu.c line 34
Quoted 10 lines
drivers/iommu/iommufd/viommu.c +10
-0+fput (filp); + return rc; +} + +static void viommu_put_kvm (struct iommufd_viommu *viommu) +{ +fput (viommu->kvm_filp); + viommu->kvm_filp =NULL ; +} +#endif Missing stub functions for !CONFIG_KVM?
Looks like an OK design otherwise
Quoted 8 lines
include/uapi/linux/iommufd.h struct iommu_viommu_alloc { · line 1071
Quoted 5 lines
include/uapi/linux/iommufd.h +1
-0@@ -1057,6 +1068,7 @@ struct iommu_viommu_alloc {__u32 data_len;__u32 __reserved;__aligned_u64 data_uptr; +__u32 kvm_vm_fd;fds are
__s32, they are signed numbers.
Thanks for the review comments. I’ll update the patch with the suggested changes.
-aneesh
Aneesh Kumar K.V
maintainerRe: [RFC PATCH v1 10/38] iommufd/vdevice: Add TSM map ioctl
In reply to Jason GunthorpeQuoted 10 lines · Jason Gunthorpe
Quoted 5 lines · Aneesh Kumar K.V (Arm)
With passthrough devices, we need to make sure private memory is allocated and assigned to the secure guest before we can issue the DMA. For ARM RMM, we only need to map and the secure SMMU management is internal to RMM. For shared IPA, vfio/iommufd DMA MAP/UNMAP interface does the equivalent
I'm not really sure what this is about? It is about getting KVM to pin all the memory and commit it to the RMM so it can be used for DMA?
That is correct.
Quoted 5 lines
But it looks really strange to have an iommufd ioctl that just calls a KVM function. Feeling this should be a KVM function, or a guestmfd behavior??
This functionality is equivalent to IOMMU_IOAS_MAP, but in the presence of firmware like RMM, we also need to supply the realm descriptor associated with the KVM instance.
Initially, I attempted to handle this within the map_pages callback in iommu_domain_ops, but that path lacks any awareness of the associated KVM context, making it unsuitable for this purpose.
Quoted 5 lines
I was kind of thinking it would be nice to have a guestmemfd mode that was "pinned", meaning the memory is allocated and remains almost always mapped into the TSM's page tables automatically. VFIO using guests would set things this way.
We need to allocate and free these pages dynamically as they are converted between private and shared states.
-aneesh
Re: [RFC PATCH v1 06/38] iommufd: Add and option to request for bar mapping with IORESOURCE_EXCLUSIVE
In reply to Aneesh Kumar K.VQuoted 7 lines · Aneesh Kumar K.V
Quoted 5 lines · Jason Gunthorpe
Quoted 1 line · Aneesh Kumar K.V (Arm)
Signed-off-by Aneesh Kumar K.V (Arm) <[email protected]>Why would we need this?
Quoted 8 lines
Quoted 6 lines
I can sort of understand why Intel would need it due to their issues with MCE, but ARM shouldn't care either way, should it?
But also why is it an iommufd option? That doesn't seem right..
Jason
This is based on our previous discussion https://lore.kernel.org/all/[email protected]
I suggested a global option, this is a per-device option, and that especially seems wrong for iommufd. If it is per-device that is vfio, if it is global then vfio can pick it up during the early phases of opening the device.
Quoted 3 lines
IIUC, we intend to request the resource in exclusive mode for secure guests—regardless of whether the platform is Intel or ARM. Could you help clarify the MCE issue observed on Intel platforms in this context?
As I understand it Intel MCEs if the non-secure side ever reads from secure'd address space. So there is alot of emphasis there to ensure there are no CPU mappings.
Jason
Re: [RFC PATCH v1 10/38] iommufd/vdevice: Add TSM map ioctl
In reply to Aneesh Kumar K.VQuoted 7 lines · Aneesh Kumar K.V
Quoted 4 lines
But it looks really strange to have an iommufd ioctl that just calls a KVM function. Feeling this should be a KVM function, or a guestmfd behavior??
This functionality is equivalent to
IOMMU_IOAS_MAP, but in the presence of firmware like RMM, we also need to supply the realm descriptor associated with the KVM instance.
There is no IOAS here because the secure world is using the KVM page table. Since KVM owns this I don't see why iommufd should be invovled in any way.
You need KVM to push the guestmemfd to the RMM and pin all the memory.
Quoted 7 lines
Quoted 4 lines
I was kind of thinking it would be nice to have a guestmemfd mode that was "pinned", meaning the memory is allocated and remains almost always mapped into the TSM's page tables automatically. VFIO using guests would set things this way.
We need to allocate and free these pages dynamically as they are converted between private and shared states.
That's still within guestmemfd's area of concern and it can immediately pin on state changes.
Jason
Re: [RFC PATCH v1 04/38] tsm: Support DMA Allocation from private memory
In reply to Aneesh Kumar K.VQuoted 33 lines · Aneesh Kumar K.V
Quoted 31 lines · Jason Gunthorpe
arch/arm64/mm/mem_encrypt.c int set_memory_decrypted(unsigned long addr, int numpages) · line 56
Quoted 9 lines · Aneesh Kumar K.V (Arm)
arch/arm64/mm/mem_encrypt.c +5
-0@@ -48,3 +49,12 @@ int set_memory_decrypted(unsigned long addr, int numpages)return crypt_ops->decrypt (addr, numpages); }EXPORT_SYMBOL_GPL (set_memory_decrypted); + +bool force_dma_unencrypted (struct device *dev) +{ +if (dev->tdi_enabled) +return false;Is this OK? I see code like this:
static inline
dma_addr_tphys_to_dma_direct(struct device *dev,
phys_addr_tphys){ if (force_dma_unencrypted(dev)) return phys_to_dma_unencrypted(dev, phys); return phys_to_dma(dev, phys);What are the ARM rules for generating dma addreses?
1) Device is T=0, memory is unencrypted, call
dma_addr_unencrypted()and do "top bit IBA set"2) Device is T=1, memory is encrypted, use the
phys_to_dma()normally3) Device it T=1, memory is uncrypted, use the
phys_to_dma()normally??? Seems odd, I would have guessed the DMA address sould be the same as case #1?
Quoted 5 lines
Quoted 2 lines
Can you document this in a comment?
If a device is operating in secure mode (T=1), it is currently assumed that only access to private (encrypted) memory is supported.
No, this is no how the PCI specs were written as far as I understand. The XT bit thing is supposed to add more fine grained device side control over what memory the DMA can target. T alone does not do that.
Quoted 3 lines
It is unclear whether devices would need to perform DMA to shared (unencrypted) memory while operating in this mode, as TLPs with T=1 are generally expected to target private memory.
PCI SIG supports it, kernel should support it.
Jason
Re: [RFC PATCH v1 08/38] iommufd/tsm: Add tsm_op iommufd ioctls
In reply to Aneesh Kumar K.V (Arm)Quoted 3 lines · On Mon, 28 Jul 2025 19:21:45 +0530 "Aneesh Kumar K.V (Arm)"…
Add operations bind and unbind used to bind a TDI to the secure guest.
Signed-off-by Aneesh Kumar K.V (Arm) <[email protected]>
Hi Aneesh,
I'm mostly reading this to get head around it rather than fully review at this point.
A few things inline though that I noticed whilst doing so.
Jonathan
Quoted 26 lines
--- drivers/iommu/iommufd/iommufd_private.h | 1 + drivers/iommu/iommufd/main.c | 3 ++ drivers/iommu/iommufd/viommu.c | 50 +++++++++++++++++++++++++ drivers/vfio/pci/vfio_pci_core.c | 10 +++++ include/uapi/linux/iommufd.h | 18 +++++++++ 5 files changed, 82 insertions(+)drivers/iommu/iommufd/iommufd_private.h +1
-0@@ -697,6 +697,7 @@ void iommufd_vdevice_destroy(struct iommufd_object *obj);void iommufd_vdevice_abort (struct iommufd_object *obj);int iommufd_hw_queue_alloc_ioctl (struct iommufd_ucmd *ucmd);void iommufd_hw_queue_destroy (struct iommufd_object *obj); +int iommufd_vdevice_tsm_op_ioctl (struct iommufd_ucmd *ucmd);static inline struct iommufd_vdevice *iommufd_get_vdevice (struct iommufd_ctx *ictx,u32 id)drivers/iommu/iommufd/viommu.c +0
-0
drivers/iommu/iommufd/viommu.c void iommufd_vdevice_abort(struct iommufd_object *obj) · line 165
Quoted 4 lines
drivers/iommu/iommufd/viommu.c +1
-0@@ -162,6 +162,9 @@ void iommufd_vdevice_abort(struct iommufd_object *obj)lockdep_assert_held (&idev->igroup->lock); +#ifdef CONFIG_TSM
Can we use stubs for some of this stuff so we don't need ifdefs in as many places.
drivers/iommu/iommufd/viommu.c int iommufd_hw_queue_alloc_ioctl(struct iommufd_ucmd *ucmd) · line 479
Quoted 12 lines
drivers/iommu/iommufd/viommu.c +5
-0+tsm_unbind (idev->dev); +#endif if (vdev->destroy) vdev->destroy (vdev);/* xa_cmpxchg is okay to fail if alloc failed xa_cmpxchg previously */ @@ -471,3 +474,50 @@ int iommufd_hw_queue_alloc_ioctl(struct iommufd_ucmd *ucmd)iommufd_put_object (ucmd->ictx, &viommu->obj);return rc; } + +#ifdef CONFIG_TSM +int iommufd_vdevice_tsm_op_ioctl (struct iommufd_ucmd *ucmd)
Might want to split this out to a separate c file and use stubs in a header to keep the code clean here.
Quoted 9 lines
drivers/iommu/iommufd/viommu.c +9
-0+{ +struct iommu_vdevice_tsm_op *cmd = ucmd->cmd; +struct iommufd_vdevice *vdev; +struct kvm *kvm; +int rc = -ENODEV ; + +if (cmd->flags) +return -EOPNOTSUPP ; +
drivers/iommu/iommufd/viommu.c int iommufd_hw_queue_alloc_ioctl(struct iommufd_ucmd *ucmd) · line 498
Quoted 10 lines
drivers/iommu/iommufd/viommu.c +10
-0+ vdev =container_of (iommufd_get_object (ucmd->ictx, cmd->vdevice_id, +IOMMUFD_OBJ_VDEVICE ), + struct iommufd_vdevice, obj); +if (IS_ERR (vdev)) +return PTR_ERR (vdev); + + kvm = vdev->viommu->kvm_filp->private_data; +if (kvm) { +/* +* tsm layer will make take care of parallel calls to tsm_bind/unbind
Wrap comment to say under 80 chars. Or if file goes higher, use a single line comment.
tsm layer will take care ...
(stray 'make')
drivers/iommu/iommufd/viommu.c int iommufd_hw_queue_alloc_ioctl(struct iommufd_ucmd *ucmd) · line 506
Quoted 8 lines
drivers/iommu/iommufd/viommu.c +8
-0+*/ +if (cmd->op ==IOMMU_VDEICE_TSM_BIND ) + rc =tsm_bind (vdev->idev->dev, kvm, vdev->virt_id); +else if (cmd->op ==IOMMU_VDEICE_TSM_UNBIND ) + rc =tsm_unbind (vdev->idev->dev); + +if (rc) { + rc = -ENODEV ;
If we want to eat an error code coming from elsewhere, maybe a comment on why?
drivers/iommu/iommufd/viommu.c int iommufd_hw_queue_alloc_ioctl(struct iommufd_ucmd *ucmd) · line 510
Quoted 4 lines
drivers/iommu/iommufd/viommu.c +4
-0+ goto out_put_vdev; + } + }else { + goto out_put_vdev;
If this always skips the next line, does that imply that line should have been under if (kvm)? Maybe this makes more sense in later patches - if so ignore this comment.
Quoted 18 lines
drivers/iommu/iommufd/viommu.c +13
-0+ } + rc =iommufd_ucmd_respond (ucmd,sizeof (*cmd)); + +out_put_vdev: +iommufd_put_object (ucmd->ictx, &vdev->obj); +return rc; +} +#else /* !CONFIG_TSM */ +int iommufd_vdevice_tsm_op_ioctl (struct iommufd_ucmd *ucmd) +{ +return -ENODEV ; +} +#endif drivers/vfio/pci/vfio_pci_core.c +0
-0
drivers/vfio/pci/vfio_pci_core.c void vfio_pci_core_close_device(struct vfio_device *core_vdev) · line 698
Quoted 5 lines
drivers/vfio/pci/vfio_pci_core.c +2
-0@@ -694,6 +694,16 @@ void vfio_pci_core_close_device(struct vfio_device *core_vdev)#if IS_ENABLED (CONFIG_EEH )eeh_dev_release (vdev->pdev);#endif + +#if 0
If you really need to do this, add a comment on why if 0
Quoted 19 lines
drivers/vfio/pci/vfio_pci_core.c +8
-0+/* +* destroy vdevice which involves tsm unbind before we disable pci disable +* A MSE/BME clear will transition the device to error state. +*/ +if (core_vdev->iommufd_device) +iommufd_device_tombstone_vdevice (core_vdev->iommufd_device); +#endif +vfio_pci_core_disable (vdev);mutex_lock (&vdev->igate);include/uapi/linux/iommufd.h +0
-0@@ -57,6 +57,7 @@ enum {/**
include/uapi/linux/iommufd.h enum iommu_veventq_flag { · line 1133
Quoted 6 lines
include/uapi/linux/iommufd.h +3
-0@@ -1127,6 +1128,23 @@ enum iommu_veventq_flag { IOMMU_VEVENTQ_FLAG_LOST_EVENTS = (1U << 0), }; +/** +* struct iommu_vdevice_tsm_OP - ioctl(IOMMU_VDEVICE_TSM_OP) +* @size: sizeof(struct iommu_vdevice_tsm_OP)
_op I guess?
Quoted 17 lines
include/uapi/linux/iommufd.h +14
-0+* @op: Either TSM_BIND or TSM_UNBIMD +* @flags: Must be 0 +* @vdevice_id: Object handle for the vDevice. Returned from IOMMU_VDEVICE_ALLOC +*/ +struct iommu_vdevice_tsm_op { +__u32 size; +__u32 op; +__u32 flags; +__u32 vdevice_id; +}; +#define IOMMU_VDEVICE_TSM_OP _IO(IOMMUFD_TYPE, IOMMUFD_CMD_VDEVICE_TSM_OP) +#define IOMMU_VDEICE_TSM_BIND 0x1 +#define IOMMU_VDEICE_TSM_UNBIND 0x2 +/** * struct iommufd_vevent_header - Virtual Event Header for a vEVENTQ Status * @flags: Combination of enum iommu_veventq_flag
Re: [RFC PATCH v1 07/38] iommufd/viommu: Add support to associate viommu with kvm instance
In reply to Aneesh Kumar K.V (Arm)Quoted 28 lines · On Mon, 28 Jul 2025 19:21:44 +0530 "Aneesh Kumar K.V (Arm)"…
The associated kvm instance will be used in later patch by iommufd to bind a tdi to kvm.
Signed-off-by Aneesh Kumar K.V (Arm) <[email protected]>--- drivers/iommu/iommufd/viommu.c | 45 +++++++++++++++++++++++++++++++++- include/linux/iommufd.h | 3 +++ include/uapi/linux/iommufd.h | 12 +++++++++ 3 files changed, 59 insertions(+), 1 deletion(-)drivers/iommu/iommufd/viommu.c +10
-0@@ -2,6 +2,36 @@/* Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES */ #include "iommufd_private.h" +#include "linux/tsm.h" + +#if IS_ENABLED (CONFIG_KVM ) +#include <linux/kvm_host.h> + +static int viommu_get_kvm (struct iommufd_viommu *viommu,int kvm_vm_fd) +{ +int rc = -EBADF ; +struct file *filp; +
drivers/iommu/iommufd/viommu.c line 24
Quoted 10 lines
drivers/iommu/iommufd/viommu.c +10
-0+ filp =fget (kvm_vm_fd); + +if (!file_is_kvm (filp)) + goto err_out; + +/* hold the kvm reference via file descriptor */ + viommu->kvm_filp = filp; +return 0 ; +err_out: + viommu->kvm_filp =NULL ;
Is this to undo side effects from this function on error?
kvm_filp is only set after all error paths so maybe this isn't needed?
If this isn't needed then use __free(fput) and no_free_ptr() to deal with filp more simply and in teh erorr path can just return -EBADF directly rather than the goto.
Or are we avoiding that stuff in iommufd?
Quoted 10 lines
drivers/iommu/iommufd/viommu.c +10
-0+fput (filp); + return rc; +} + +static void viommu_put_kvm (struct iommufd_viommu *viommu) +{ +fput (viommu->kvm_filp); + viommu->kvm_filp =NULL ; +} +#endif
Re: [RFC PATCH v1 11/38] KVM: arm64: CCA: register host tsm platform device
In reply to Aneesh Kumar K.V (Arm)Quoted 4 lines · On Mon, 28 Jul 2025 19:21:48 +0530 "Aneesh Kumar K.V (Arm)"…
Register a platform device if the CCA DA feature is supported. A driver for this platform device will further drive the CCA DA workflow.
Signed-off-by Aneesh Kumar K.V (Arm) <[email protected]>
Few trivial things
Quoted 13 lines
arch/arm64/kvm/rme.c +1
-0@@ -4,6 +4,7 @@*/ #include <linux/kvm_host.h> +#include <linux/platform_device.h> #include <asm/kvm_emulate.h> #include <asm/kvm_mmu.h>
arch/arm64/kvm/rme.c int kvm_init_realm_vm(struct kvm *kvm) · line 1728
Quoted 4 lines
arch/arm64/kvm/rme.c +1
-0@@ -1724,6 +1725,18 @@ int kvm_init_realm_vm(struct kvm *kvm)return 0 ; } +static struct platform_device cca_host_dev = {
Hmm. Greg is getting increasingly (and correctly in my view) grumpy with platform devices being registered with no underlying resources etc as glue layers. Maybe some of that will come later.
arch/arm64/kvm/rme.c int kvm_init_realm_vm(struct kvm *kvm) · line 1730
Quoted 2 lines
arch/arm64/kvm/rme.c +2
-0+ .name = RMI_DEV_NAME, + .id =PLATFORM_DEVID_NONE
Add trailing comma. More than possible something else will be added after this.
arch/arm64/kvm/rme.c int kvm_init_realm_vm(struct kvm *kvm) · line 1736
Quoted 6 lines
arch/arm64/kvm/rme.c +6
-0+}; + +static void rmm_tsm_init (void ) +{ +if (!platform_device_register (&cca_host_dev)) +pr_info ("CCA host DA platform device initialized.\n" );
Noisy as we should be able to tell that in a bunch of other ways.
arch/arm64/kvm/rme.c int kvm_init_realm_vm(struct kvm *kvm) · line 1737
Quoted 1 line
arch/arm64/kvm/rme.c +1
-0+
Excess blank line.
Quoted 1 line
arch/arm64/kvm/rme.c +1
-0+}
Re: [RFC PATCH v1 12/38] coco: host: arm64: CCA host platform device driver
In reply to Aneesh Kumar K.V (Arm)Quoted 3 lines · On Mon, 28 Jul 2025 19:21:49 +0530 "Aneesh Kumar K.V (Arm)"…
This driver registers the
pci_tsm_opswith tsm subsystem.Signed-off-by Aneesh Kumar K.V (Arm) <[email protected]>
Hi Aneesh,
For this main comment is around use of __free.. Dan wrote up guidance and added to cleanup.h after many email threads kept running into same issues and Linus added his requirements for that stuff to be acceptable.
Anyhow, easy to fix - comments inline.
Quoted 8 lines
drivers/virt/coco/arm-cca-host/Kconfig new file +2
-0@@ -0,0 +1,12 @@ +# SPDX-License-Identifier: GPL-2.0-only +#
drivers/virt/coco/arm-cca-host/Kconfig line 12
Quoted 10 lines
drivers/virt/coco/arm-cca-host/Kconfig new file +10
-0+# TSM (TEE Security Manager) host drivers +# +config ARM_CCA_HOST + tristate "Arm CCA Host driver" + depends on ARM64 + depends on PCI_TSM + select TSM + + help + The driver provides TSM backend for ARM CCA
That's going to make for grumpy checkpatch! More help.
Quoted 8 lines
drivers/virt/coco/arm-cca-host/arm-cca.c new file +2
-0@@ -0,0 +1,209 @@ +// SPDX-License-Identifier: GPL-2.0-only +/*
drivers/virt/coco/arm-cca-host/arm-cca.c line 12
Quoted 10 lines
drivers/virt/coco/arm-cca-host/arm-cca.c new file +10
-0+* Copyright (C) 2025 ARM Ltd. +*/ + +#include <linux/platform_device.h> +#include <linux/pci-tsm.h> +#include <linux/pci-ide.h> +#include <linux/module.h> +#include <linux/pci.h> +#include <linux/tsm.h> +#include <linux/vmalloc.h>
cleanup.h and maybe others missing. Basically follow include what you use principles (flexed a little for headers that are front ends to others).
Quoted 5 lines
drivers/virt/coco/arm-cca-host/arm-cca.c new file +5
-0+ +#include "rmm-da.h" + +/* Number of streams that we can support at the hostbridge level */ +#define CCA_HB_PLATFORM_STREAMS 4
drivers/virt/coco/arm-cca-host/arm-cca.c line 27
Quoted 10 lines
drivers/virt/coco/arm-cca-host/arm-cca.c new file +10
-0+ +/* Total number of stream id supported at root port level */ +#define MAX_STREAM_ID 256 + +DEFINE_FREE(vfree, void *, if (!IS_ERR_OR_NULL(_T)) vfree(_T)) +static struct pci_tsm *cca_tsm_pci_probe (struct pci_dev *pdev) +{ +int rc; +struct pci_host_bridge *hb; + struct cca_host_dsc_pf0 *dsc_pf0 __free(vfree) = NULL;
Read the stuff in cleanup.h and work out why this needs changing to be inline below and not use this NULL pattern here (unless you like grumpy Linus ;)
Note that with the err_out, even if you do that you'll still be breaking with the guidance doc (and actually causing undefined behavior :) Get rid of those gotos if you want to use __free()
Quoted 24 lines
drivers/virt/coco/arm-cca-host/arm-cca.c new file +24
-0+ +if (pdev->is_virtfn) +return NULL ; + +if (!is_pci_tsm_pf0 (pdev)) { +struct pci_tsm *tsm =kzalloc (sizeof (*tsm),GFP_KERNEL ); + +if (!tsm) + goto err_out; + +pci_tsm_initialize (pdev, tsm); +return tsm; + } + +if (!pdev->ide_cap) + goto err_out; + + dsc_pf0 =vcalloc (sizeof (*dsc_pf0),GFP_KERNEL ); +if (!dsc_pf0) + goto err_out; + + rc =pci_tsm_pf0_initialize (pdev, &dsc_pf0->pci); +if (rc) +return NULL ;
drivers/virt/coco/arm-cca-host/arm-cca.c line 61
Quoted 10 lines
drivers/virt/coco/arm-cca-host/arm-cca.c new file +10
-0+/* +* FIXME!! +* update the hostbridge details. This should go into +* some host bridge probe/init routine. +* than the selective index supported by the endpoint +*/ + hb =pci_find_host_bridge (pdev->bus); +pci_ide_init_nr_streams (hb,CCA_HB_PLATFORM_STREAMS ); + +pci_info (pdev,"tsm enabled\n" );
Ok. RFC I guess. Still pci_dbg()
drivers/virt/coco/arm-cca-host/arm-cca.c line 64
Quoted 3 lines
drivers/virt/coco/arm-cca-host/arm-cca.c new file +3
-0+return &no_free_ptr (dsc_pf0)->pci.tsm; + +err_out:
Why? Random mix of direct returns of NULL above and goto here.
Quoted 36 lines
drivers/virt/coco/arm-cca-host/arm-cca.c new file +35
-0+return NULL ; +} + +static void cca_tsm_pci_remove (struct pci_tsm *tsm) +{ +struct pci_dev *pdev = tsm->pdev; +struct cca_host_dsc_pf0 *dsc_pf0; + +if (WARN_ON (pdev->is_virtfn)) +return ; + +if (!is_pci_tsm_pf0 (pdev)) { + +pci_dbg (tsm->pdev,"tsm disabled\n" ); +kfree (pdev->tsm); +return ; + } + + dsc_pf0 =to_cca_dsc_pf0 (pdev); +pci_dbg (tsm->pdev,"tsm disabled\n" ); +vfree (dsc_pf0); +} + +/* per root port unique with multiple restrictions. For now global */ +static DECLARE_BITMAP(cca_stream_ids, MAX_STREAM_ID); + +static void cca_tsm_disconnect (struct pci_dev *pdev) +{ +struct pci_dev *rp =pcie_find_root_port (pdev); +struct cca_host_dsc_pf0 *dsc_pf0; +struct pci_ide *ide; + +if (WARN_ON (!is_pci_tsm_pf0 (pdev))) +return ; +
drivers/virt/coco/arm-cca-host/arm-cca.c line 165
Quoted 10 lines
drivers/virt/coco/arm-cca-host/arm-cca.c new file +10
-0+ dsc_pf0 =to_cca_dsc_pf0 (pdev); + ide = dsc_pf0->sel_stream; + dsc_pf0->sel_stream =NULL ; +pci_ide_stream_disable (pdev, ide); +tsm_ide_stream_unregister (ide); +pci_ide_stream_teardown (rp, ide); +pci_ide_stream_teardown (pdev, ide); +pci_ide_stream_unregister (ide); +clear_bit (ide->stream_id, cca_stream_ids); +pci_ide_stream_free (ide);
Ordering subtly different from error path above.
If there is a reason for that add a comment.
Quoted 6 lines
drivers/virt/coco/arm-cca-host/arm-cca.c new file +5
-0+} +static void cca_tsm_remove (void *tsm_core) +{ +tsm_unregister (tsm_core); +}
drivers/virt/coco/arm-cca-host/arm-cca.c line 188
Quoted 10 lines
drivers/virt/coco/arm-cca-host/arm-cca.c new file +10
-0+ +static int cca_tsm_probe (struct platform_device *pdev) +{ +struct tsm_core_dev *tsm_core; + + tsm_core =tsm_register (&pdev->dev,NULL , &cca_pci_ops); +if (IS_ERR (tsm_core)) +return PTR_ERR (tsm_core); + +return devm_add_action_or_reset (&pdev->dev, cca_tsm_remove, tsm_core);
So this makes two with the one in Dan's test code. devm_tsm_register() seems to be a useful generic thing to add (implementation being exactly what you have here.
drivers/virt/coco/arm-cca-host/arm-cca.c line 192
Quoted 4 lines
drivers/virt/coco/arm-cca-host/arm-cca.c new file +4
-0+} + +static const struct platform_device_id arm_cca_host_id_table[] = { + {RMI_DEV_NAME ,0 },
Space before } and don't provide data until there is a use for it.
{ RMI_DEV_NAME }drivers/virt/coco/arm-cca-host/arm-cca.c line 196
Quoted 4 lines
drivers/virt/coco/arm-cca-host/arm-cca.c new file +4
-0+ { } +}; +MODULE_DEVICE_TABLE (platform, arm_cca_host_id_table); +
Consistency on spacing. I'd go for just 1 blank line for separation of things.
Quoted 13 lines
drivers/virt/coco/arm-cca-host/arm-cca.c new file +13
-0+ +static struct platform_driver cca_tsm_platform_driver = { + .probe = cca_tsm_probe, + .id_table = arm_cca_host_id_table, + .driver = { + .name ="cca_tsm" , + }, +}; + +MODULE_IMPORT_NS ("PCI_IDE" ); +module_platform_driver (cca_tsm_platform_driver); +MODULE_DESCRIPTION ("ARM CCA Host TSM driver" ); +MODULE_LICENSE ("GPL" );
Re: [RFC PATCH v1 07/38] iommufd/viommu: Add support to associate viommu with kvm instance
In reply to Jonathan CameronQuoted 44 lines · Jonathan Cameron
Quoted 28 lines · On Mon, 28 Jul 2025 19:21:44 +0530 "Aneesh Kumar K.V (Arm)"…
The associated kvm instance will be used in later patch by iommufd to bind a tdi to kvm.
Signed-off-by Aneesh Kumar K.V (Arm) <[email protected]>--- drivers/iommu/iommufd/viommu.c | 45 +++++++++++++++++++++++++++++++++- include/linux/iommufd.h | 3 +++ include/uapi/linux/iommufd.h | 12 +++++++++ 3 files changed, 59 insertions(+), 1 deletion(-)drivers/iommu/iommufd/viommu.c +10
-0@@ -2,6 +2,36 @@/* Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES */ #include "iommufd_private.h" +#include "linux/tsm.h" + +#if IS_ENABLED (CONFIG_KVM ) +#include <linux/kvm_host.h> + +static int viommu_get_kvm (struct iommufd_viommu *viommu,int kvm_vm_fd) +{ +int rc = -EBADF ; +struct file *filp; +drivers/iommu/iommufd/viommu.c line 24
Quoted 10 lines
drivers/iommu/iommufd/viommu.c +10
-0+ filp =fget (kvm_vm_fd); + +if (!file_is_kvm (filp)) + goto err_out; + +/* hold the kvm reference via file descriptor */ + viommu->kvm_filp = filp; +return 0 ; +err_out: + viommu->kvm_filp =NULL ;Is this to undo side effects from this function on error?
Quoted 2 lines
kvm_filpis only set after all error paths so maybe this isn't needed?
Looks like you are right to me
Quoted 5 lines
If this isn't needed then use
__free(fput)andno_free_ptr()to deal with filp more simply and in teh erorr path can just return-EBADFdirectly rather than the goto.Or are we avoiding that stuff in iommufd?
Nope, gentle obvious use is fine :)
Jason
Re: [RFC PATCH v1 11/38] KVM: arm64: CCA: register host tsm platform device
In reply to Jonathan CameronQuoted 4 lines · Jonathan Cameron
Quoted 1 line
+static struct
platform_devicecca_host_dev= {Hmm. Greg is getting increasingly (and correctly in my view) grumpy with platform devices being registered with no underlying resources etc as glue layers. Maybe some of that will come later.
Is faux_device a better choice? I admit to not knowing entirely what it is for..
But alternatively, why do we need a dummy "hw" struct device at all? Typically a subsystem like TSM should be structured to create its own struct devices..
I would expect this to just call 'register tsm' ?
Jason
Re: [RFC PATCH v1 12/38] coco: host: arm64: CCA host platform device driver
In reply to Jonathan CameronQuoted 12 lines · Jonathan Cameron
Quoted 12 lines
drivers/virt/coco/arm-cca-host/arm-cca.c new file +11
-0+} +static void cca_tsm_remove (void *tsm_core) +{ +tsm_unregister (tsm_core); +} + +static int cca_tsm_probe (struct platform_device *pdev) +{ +struct tsm_core_dev *tsm_core; + + tsm_core =tsm_register (&pdev->dev,NULL , &cca_pci_ops);
Quoted 8 lines
drivers/virt/coco/arm-cca-host/arm-cca.c line 188
Quoted 4 lines
drivers/virt/coco/arm-cca-host/arm-cca.c new file +4
-0+if (IS_ERR (tsm_core)) +return PTR_ERR (tsm_core); + +return devm_add_action_or_reset (&pdev->dev, cca_tsm_remove, tsm_core);So this makes two with the one in Dan's test code.
devm_tsm_register()seems to be a useful generic thing to add (implementation being exactly what you have here.
Pelase no, this is insane, you have a probed driver with a probe/remove function pairing already. Why on earth would you use devm just to call a remove function :(
Just put tsm_unregister() in the normal driver remove like it is supposed to be done and use the drvdata to pass the tsm_core_dev pointer. It is easy and normal, look at fwctl for a very simple example.
devm is useful to solve complex things, these trivial things should be done normally..
Jason
Re: [RFC PATCH v1 06/38] iommufd: Add and option to request for bar mapping with IORESOURCE_EXCLUSIVE
In reply to Aneesh Kumar K.V (Arm)Quoted 10 lines
drivers/vfio/pci/vfio_pci_core.c +1
-1@@ -1753,8 +1753,15 @@ int vfio_pci_core_mmap(struct vfio_device *core_vdev, struct vm_area_struct *vma* we need to request the region and the barmap tracks that. */ if (!vdev->barmap[index]) { - ret = pci_request_selected_regions(pdev, +
drivers/vfio/pci/vfio_pci_core.c int vfio_pci_core_mmap(struct vfio_device *core_vdev, struct vm_area_struct *vma · line 1766
Quoted 10 lines
drivers/vfio/pci/vfio_pci_core.c +7
-0+if (core_vdev->iommufd_device && +iommufd_device_need_exclusive_range (core_vdev->iommufd_device)) + ret =pci_request_selected_regions_exclusive (pdev, +1 << index,"vfio-pci" ); +else + ret =pci_request_selected_regions (pdev,1 << index,"vfio-pci" ); +if (ret)return ret;
I did't get the idea.
The purpose of my original patch [1] is not to make VFIO choose between pci_request_regions_exclusive() or pci_request_regions(). It is mainly to prevent userspace mmap/read/write against a vfio_cdev FD. For example:
If pci_request_selected_regions() is succesfully executed on mmap(), later TSM Bind would fail on its pci_request_regions_exclusive(). It means userspace should not mmap otherwise you can't do private assignment. Vice versa, if you've done TSM Bind, you cannot mmap anymore.
The _exclusive is just a bonus that further prevents "/dev/mem and the sysfs MMIO access"
[1]: https://lore.kernel.org/all/[email protected]/
Thanks,
Yilun
Re: [RFC PATCH v1 06/38] iommufd: Add and option to request for bar mapping with IORESOURCE_EXCLUSIVE
In reply to Jason GunthorpeQuoted 11 lines · Jason Gunthorpe
Quoted 10 lines · Aneesh Kumar K.V
Quoted 8 lines · Jason Gunthorpe
Quoted 1 line · Aneesh Kumar K.V (Arm)
Signed-off-by Aneesh Kumar K.V (Arm) <[email protected]>Why would we need this?
I can sort of understand why Intel would need it due to their issues with MCE, but ARM shouldn't care either way, should it?
Quoted 8 lines
Quoted 5 lines
Quoted 3 lines
But also why is it an iommufd option? That doesn't seem right..
Jason
This is based on our previous discussion https://lore.kernel.org/all/[email protected]
I suggested a global option, this is a per-device option, and that especially seems wrong for iommufd. If it is per-device that is vfio,
I think this should be per-device. The original purpose of this pci_region_request_*() is to prevent further mmap/read/write against a vfio_cdev FD which would be used for private assignment. You shouldn't prevent all other devices from working with userspace APPs (e.g. DPDK) if there is one private assignment in system.
Quoted 9 lines
if it is global then vfio can pick it up during the early phases of opening the device.
Quoted 3 lines
IIUC, we intend to request the resource in exclusive mode for secure guests—regardless of whether the platform is Intel or ARM. Could you help clarify the MCE issue observed on Intel platforms in this context?
As I understand it Intel MCEs if the non-secure side ever reads from secure'd address space. So there is alot of emphasis there to ensure
Yeah, Intel TDX doesn't have a lower access control table for CC. So if host reads, the TLP sends and MCE happens.
Thanks,
Yilun
Quoted 3 lines
there are no CPU mappings.
Jason
Aneesh Kumar K.V
maintainerRe: [RFC PATCH v1 11/38] KVM: arm64: CCA: register host tsm platform device
In reply to Jason GunthorpeQuoted 7 lines · Jason Gunthorpe
Quoted 4 lines · Jonathan Cameron
Quoted 1 line
+static struct
platform_devicecca_host_dev= {Hmm. Greg is getting increasingly (and correctly in my view) grumpy with platform devices being registered with no underlying resources etc as glue layers. Maybe some of that will come later.
Quoted 9 lines
Is
faux_devicea better choice? I admit to not knowing entirely what it is for..But alternatively, why do we need a dummy "hw" struct device at all? Typically a subsystem like TSM should be structured to create its own struct devices..
I would expect this to just call 'register tsm' ?
The goal is to have tsm class device to be parented by the platform device.
# ls -al total 0 drwxr-xr-x 2 root root 0 Jan 13 06:07 . drwxr-xr-x 23 root root 0 Jan 1 00:00 .. lrwxrwxrwx 1 root root 0 Jan 13 06:07 tsm0 -> ../../devices/platform/arm-rmi-dev/tsm/tsm0 # pwd /sys/class/tsm
-aneesh
Aneesh Kumar K.V
maintainerRe: [RFC PATCH v1 12/38] coco: host: arm64: CCA host platform device driver
In reply to Jonathan CameronQuoted 3 lines · Jonathan Cameron
On Mon, 28 Jul 2025 19:21:49 +0530
"Aneesh Kumar K.V (Arm)" <[email protected]> wrote:
...
Quoted 20 lines
Quoted 5 lines
drivers/virt/coco/arm-cca-host/arm-cca.c new file +5
-0+ +#include "rmm-da.h" + +/* Number of streams that we can support at the hostbridge level */ +#define CCA_HB_PLATFORM_STREAMS 4drivers/virt/coco/arm-cca-host/arm-cca.c line 27
Quoted 10 lines
drivers/virt/coco/arm-cca-host/arm-cca.c new file +10
-0+ +/* Total number of stream id supported at root port level */ +#define MAX_STREAM_ID 256 + +DEFINE_FREE(vfree, void *, if (!IS_ERR_OR_NULL(_T)) vfree(_T)) +static struct pci_tsm *cca_tsm_pci_probe (struct pci_dev *pdev) +{ +int rc; +struct pci_host_bridge *hb; + struct cca_host_dsc_pf0 *dsc_pf0 __free(vfree) = NULL;Read the stuff in
cleanup.hand work out why this needs changing to be inline below and not use this NULL pattern here (unless you like grumpy Linus ;)
Quoted 5 lines
Note that with the
err_out, even if you do that you'll still be breaking with the guidance doc (and actually causing undefined behavior :) Get rid of those gotos if you want to use__free()
I’ve already fixed up similar cases by removing the goto based on cleanup.h docs in other functions.I must have missed this one.
By the way, isn't using the NULL pattern acceptable when there are no additional lock variables involved (ie, unwind order doesn't matter)? Or should we always follow the pattern below regardless?
struct cca_host_dsc_pf0 *dsc_pf0 __free(vfree) = vcalloc(sizeof(*dsc_pf0), GFP_KERNEL);
-aneesh
Suzuki K Poulose
reviewerRe: [RFC PATCH v1 04/38] tsm: Support DMA Allocation from private memory
In reply to Jason GunthorpeQuoted 45 lines · Jason Gunthorpe
Quoted 33 lines · Aneesh Kumar K.V
Quoted 31 lines · Jason Gunthorpe
arch/arm64/mm/mem_encrypt.c int set_memory_decrypted(unsigned long addr, int numpages) · line 56
Quoted 9 lines · Aneesh Kumar K.V (Arm)
arch/arm64/mm/mem_encrypt.c +5
-0@@ -48,3 +49,12 @@ int set_memory_decrypted(unsigned long addr, int numpages)return crypt_ops->decrypt (addr, numpages); }EXPORT_SYMBOL_GPL (set_memory_decrypted); + +bool force_dma_unencrypted (struct device *dev) +{ +if (dev->tdi_enabled) +return false;Is this OK? I see code like this:
static inline
dma_addr_tphys_to_dma_direct(struct device *dev,
phys_addr_tphys){ if (force_dma_unencrypted(dev)) return phys_to_dma_unencrypted(dev, phys); return phys_to_dma(dev, phys);What are the ARM rules for generating dma addreses?
1) Device is T=0, memory is unencrypted, call
dma_addr_unencrypted()and do "top bit IBA set"2) Device is T=1, memory is encrypted, use the
phys_to_dma()normally3) Device it T=1, memory is uncrypted, use the
phys_to_dma()normally??? Seems odd, I would have guessed the DMA address sould be the same as case #1?Quoted 5 lines
Quoted 2 lines
Can you document this in a comment?
If a device is operating in secure mode (T=1), it is currently assumed that only access to private (encrypted) memory is supported.
No, this is no how the PCI specs were written as far as I understand. The XT bit thing is supposed to add more fine grained device side control over what memory the DMA can target. T alone does not do that.
Quoted 5 lines
Quoted 3 lines
It is unclear whether devices would need to perform DMA to shared (unencrypted) memory while operating in this mode, as TLPs with T=1 are generally expected to target private memory.
PCI SIG supports it, kernel should support it.
ACK. On Arm CCA, the device can access shared IPA, with T=1 transaction as long as the mapping is active in the Stage2 managed by RMM.
Rather than mapping the entire memory from the host, it would be ideal if the Coco vms have some sort of a callback to "make sure the DMA wouldn't fault for a device". e.g, it could be as simple as touching the page in Arm CCA (GFP_ZERO could do the trick, well one byte per Granule is good). or an ACCEPT a given page.
Is this a problem for AMDE SNP / Intel TDX ?
Suzuki
Quoted 2 lines
Jason
Re: [RFC PATCH v1 12/38] coco: host: arm64: CCA host platform device driver
In reply to Jason GunthorpeQuoted 26 lines · On Tue, 29 Jul 2025 20:22:43 -0300 Jason Gunthorpe <[email protected]> wrote:
Quoted 12 lines · Jonathan Cameron
Quoted 12 lines
drivers/virt/coco/arm-cca-host/arm-cca.c new file +11
-0+} +static void cca_tsm_remove (void *tsm_core) +{ +tsm_unregister (tsm_core); +} + +static int cca_tsm_probe (struct platform_device *pdev) +{ +struct tsm_core_dev *tsm_core; + + tsm_core =tsm_register (&pdev->dev,NULL , &cca_pci_ops);Quoted 8 lines
drivers/virt/coco/arm-cca-host/arm-cca.c line 188
Quoted 4 lines
drivers/virt/coco/arm-cca-host/arm-cca.c new file +4
-0+if (IS_ERR (tsm_core)) +return PTR_ERR (tsm_core); + +return devm_add_action_or_reset (&pdev->dev, cca_tsm_remove, tsm_core);So this makes two with the one in Dan's test code.
devm_tsm_register()seems to be a useful generic thing to add (implementation being exactly what you have here.Pelase no, this is insane, you have a probed driver with a probe/remove function pairing already. Why on earth would you use devm just to call a remove function :(
Quoted 7 lines
Just put
tsm_unregister()in the normal driver remove like it is supposed to be done and use the drvdata to pass thetsm_core_devpointer. It is easy and normal, look at fwctl for a very simple example.devm is useful to solve complex things, these trivial things should be done normally..
Sure, that would be fine for now. If we end up with a large complex flow that happens to have a tsm_register() in amongst various managed resources we can revisit. If they all end up looking like this then a manual call in remove is fine.
Quoted 2 lines
Jason
Re: [RFC PATCH v1 11/38] KVM: arm64: CCA: register host tsm platform device
In reply to Aneesh Kumar K.VQuoted 11 lines · On Wed, 30 Jul 2025 14:12:26 +0530 "Aneesh Kumar K.V"…
Quoted 9 lines · Jason Gunthorpe
Quoted 4 lines · Jonathan Cameron
Quoted 1 line
+static struct
platform_devicecca_host_dev= {Hmm. Greg is getting increasingly (and correctly in my view) grumpy with platform devices being registered with no underlying resources etc as glue layers. Maybe some of that will come later.
Is
faux_devicea better choice? I admit to not knowing entirely what it is for..
I'll go with a cautious yes to faux_device. This case of a glue device with no resources and no reason to be on a particular bus was definitely the intent but I'm not 100% sure without trying it that we don't run into any problems.
Not that many examples yet, but cpuidle-psci.c looks like a vaguely similar case to this one.
All it really does is move the location of the device and smash together the device registration with probe/remove. That means the device disappears if probe() fails, which is cleaner in many ways than leaving a pointless stub behind.
Maybe it isn't appropriate it if is actually useful to rmmod/modprobe the driver.
+CC Greg on basis I may have wrong end of the stick ;)
Quoted 22 lines
Quoted 7 lines
But alternatively, why do we need a dummy "hw" struct device at all? Typically a subsystem like TSM should be structured to create its own struct devices..
I would expect this to just call 'register tsm' ?
The goal is to have tsm class device to be parented by the platform device.
# ls -al total 0 drwxr-xr-x 2 root root 0 Jan 13 06:07 . drwxr-xr-x 23 root root 0 Jan 1 00:00 .. lrwxrwxrwx 1 root root 0 Jan 13 06:07 tsm0 -> ../../devices/platform/arm-rmi-dev/tsm/tsm0 # pwd /sys/class/tsm-aneesh
Re: [RFC PATCH v1 12/38] coco: host: arm64: CCA host platform device driver
In reply to Aneesh Kumar K.VQuoted 37 lines · On Wed, 30 Jul 2025 14:28:55 +0530 Aneesh Kumar K.V <[email protected]>…
Quoted 3 lines · Jonathan Cameron
On Mon, 28 Jul 2025 19:21:49 +0530
"Aneesh Kumar K.V (Arm)" <[email protected]> wrote:...
Quoted 20 lines
Quoted 5 lines
drivers/virt/coco/arm-cca-host/arm-cca.c new file +5
-0+ +#include "rmm-da.h" + +/* Number of streams that we can support at the hostbridge level */ +#define CCA_HB_PLATFORM_STREAMS 4drivers/virt/coco/arm-cca-host/arm-cca.c line 27
Quoted 10 lines
drivers/virt/coco/arm-cca-host/arm-cca.c new file +10
-0+ +/* Total number of stream id supported at root port level */ +#define MAX_STREAM_ID 256 + +DEFINE_FREE(vfree, void *, if (!IS_ERR_OR_NULL(_T)) vfree(_T)) +static struct pci_tsm *cca_tsm_pci_probe (struct pci_dev *pdev) +{ +int rc; +struct pci_host_bridge *hb; + struct cca_host_dsc_pf0 *dsc_pf0 __free(vfree) = NULL;Read the stuff in
cleanup.hand work out why this needs changing to be inline below and not use this NULL pattern here (unless you like grumpy Linus ;)Quoted 5 lines
Note that with the
err_out, even if you do that you'll still be breaking with the guidance doc (and actually causing undefined behavior :) Get rid of those gotos if you want to use__free()I’ve already fixed up similar cases by removing the goto based on
cleanup.hdocs in other functions.I must have missed this one.
Quoted 6 lines
By the way, isn't using the
NULLpattern acceptable when there are no additional lock variables involved (ie, unwind order doesn't matter)? Or should we always follow the pattern below regardless?struct cca_host_dsc_pf0 *dsc_pf0 __free(vfree) = vcalloc(sizeof(*dsc_pf0), GFP_KERNEL);
Always do this. It's not really about what happens today but more what we might break by failing to notice a future patch causes problems. Keeping the unwind ordering tightly couple with setup means we basically can't get it wrong (famous last words ;)
Jonathan
Quoted 2 lines
-aneesh
Re: [RFC PATCH v1 13/38] coco: host: arm64: Create a PDEV with rmm
In reply to Aneesh Kumar K.V (Arm)Quoted 3 lines · On Mon, 28 Jul 2025 19:21:50 +0530 "Aneesh Kumar K.V (Arm)"…
Create the realm physical device with RMM.
Signed-off-by Aneesh Kumar K.V (Arm) <[email protected]>
Hi Aneesh,
Various small things inline.
Jonathan
Quoted 30 lines
--- arch/arm64/include/asm/rmi_cmds.h | 31 +++++ arch/arm64/include/asm/rmi_smc.h | 72 ++++++++++- drivers/virt/coco/arm-cca-host/Makefile | 2 +- drivers/virt/coco/arm-cca-host/arm-cca.c | 10 +- drivers/virt/coco/arm-cca-host/rmm-da.c | 150 +++++++++++++++++++++++ drivers/virt/coco/arm-cca-host/rmm-da.h | 5 + 6 files changed, 267 insertions(+), 3 deletions(-) create mode 100644 drivers/virt/coco/arm-cca-host/rmm-da.carch/arm64/include/asm/rmi_cmds.h +12
-0@@ -505,4 +505,35 @@ static inline int rmi_rtt_unmap_unprotected(unsigned long rd,return res.a0; } +static inline unsigned long rmi_pdev_aux_count (unsigned long flags,u64 *aux_count) +{ +struct arm_smccc_res res; + +arm_smccc_1_1_invoke (SMC_RMI_PDEV_AUX_COUNT , flags, &res); + + *aux_count = res.a1; +return res.a0; +} + +static inline unsigned long rmi_pdev_create (unsigned long pdev_phys, +unsigned long pdev_params_phys)
arch/arm64/include/asm/rmi_cmds.h static inline int rmi_rtt_unmap_unprotected(unsigned long rd, · line 529
Quoted 10 lines
arch/arm64/include/asm/rmi_cmds.h +10
-0+{ +struct arm_smccc_res res; + +arm_smccc_1_1_invoke (SMC_RMI_PDEV_CREATE , + pdev_phys, pdev_params_phys, &res); + +return res.a0; +} + +static inline unsigned long rmi_pdev_get_state (unsigned long pdev_phys,unsigned long *state)
I'd use the enum for the state type and range check in here.
Quoted 15 lines
arch/arm64/include/asm/rmi_cmds.h +9
-0+{ +struct arm_smccc_res res; + +arm_smccc_1_1_invoke (SMC_RMI_PDEV_GET_STATE , pdev_phys, &res); + + *state = res.a1; +return res.a0; +} +#endif /* __ASM_RMI_CMDS_H */ arch/arm64/include/asm/rmi_smc.h +0
-0
arch/arm64/include/asm/rmi_smc.h line 29
Quoted 4 lines
arch/arm64/include/asm/rmi_smc.h +0
-1@@ -26,7 +26,7 @@#define SMC_RMI_DATA_CREATE SMC_RMI_CALL(0x0153)#define SMC_RMI_DATA_CREATE_UNKNOWN SMC_RMI_CALL(0x0154)#define SMC_RMI_DATA_DESTROY SMC_RMI_CALL(0x0155) -
Probably keep
arch/arm64/include/asm/rmi_smc.h line 29
Quoted 1 line
arch/arm64/include/asm/rmi_smc.h +1
-0+#define SMC_RMI_PDEV_AUX_COUNT SMC_RMI_CALL(0x0156)
and add one here as different groups.
Quoted 14 lines
arch/arm64/include/asm/rmi_smc.h +3
-0#define SMC_RMI_REALM_ACTIVATE SMC_RMI_CALL(0x0157)#define SMC_RMI_REALM_CREATE SMC_RMI_CALL(0x0158)#define SMC_RMI_REALM_DESTROY SMC_RMI_CALL(0x0159)@@ -47,6 +47,9 @@#define SMC_RMI_RTT_INIT_RIPAS SMC_RMI_CALL(0x0168)#define SMC_RMI_RTT_SET_RIPAS SMC_RMI_CALL(0x0169) +#define SMC_RMI_PDEV_CREATE SMC_RMI_CALL(0x0176) +#define SMC_RMI_PDEV_GET_STATE SMC_RMI_CALL(0x0178) +#define RMI_ABI_MAJOR_VERSION 1#define RMI_ABI_MINOR_VERSION 0
arch/arm64/include/asm/rmi_smc.h struct rec_run { · line 274
Quoted 4 lines
arch/arm64/include/asm/rmi_smc.h +1
-0@@ -268,4 +271,71 @@ struct rec_run {struct rec_exit exit; }; +enum rmi_pdev_state {
Type never used, but I think it should be. See above.
arch/arm64/include/asm/rmi_smc.h struct rec_run { · line 278
Quoted 4 lines
arch/arm64/include/asm/rmi_smc.h +4
-0+RMI_PDEV_NEW , +RMI_PDEV_NEEDS_KEY , +RMI_PDEV_HAS_KEY , + RMI_PDEV_READY,
Seems someone put another state in here, but I guess you'll update when the rest of the stack catches up.
RMI_PDEV_IDE_RESETTING,
Maybe throw a comment here for now.
Quoted 4 lines
arch/arm64/include/asm/rmi_smc.h +4
-0+RMI_PDEV_COMMUNICATING , +RMI_PDEV_STOPPING , +RMI_PDEV_STOPPED , + RMI_PDEV_ERROR,
arch/arm64/include/asm/rmi_smc.h struct rec_run { · line 292
Quoted 10 lines
arch/arm64/include/asm/rmi_smc.h +10
-0+}; + +#define MAX_PDEV_AUX_GRANULES 32 +#define MAX_IOCOH_ADDR_RANGE 16 +#define MAX_FCOH_ADDR_RANGE 4 + +#define RMI_PDEV_SPDM_TRUE BIT(0) +#define RMI_PDEV_IDE_TRUE BIT(1) +#define RMI_PDEV_FOCOH BIT(2) +#define RMI_PDEV_P2P_STREAM BIT(3)
I'd stick flags in the name (assuming this is
RmiPDevFlags? I'm not sure as the spec I could
find has different usages other after BIT(!))
arch/arm64/include/asm/rmi_smc.h struct rec_run { · line 299
Quoted 7 lines
arch/arm64/include/asm/rmi_smc.h +7
-0+ +#define RMI_HASH_SHA_256 0 +#define RMI_HASH_SHA_512 1 + +struct rmi_pdev_addr_range { +unsigned long base; +unsigned long top;
Whilst we only care about platforms where this is u64, maybe just make that explicit so we can trivially see this matches the spec?
arch/arm64/include/asm/rmi_smc.h struct rec_run { · line 302
Quoted 3 lines
arch/arm64/include/asm/rmi_smc.h +3
-0+}; + +struct rmi_pdev_params {
Reference? Looks like B4.4.25 RmiPdevParams type in alp15. Though as there are some fields missing I guess this chagned.
arch/arm64/include/asm/rmi_smc.h struct rec_run { · line 315
Quoted 13 lines
arch/arm64/include/asm/rmi_smc.h +13
-0+union { +struct { +u64 flags; +u64 pdev_id; +u64 segment_id; +u64 ecam_addr; +u64 root_id; +u64 cert_id; +u64 rid_base; +u64 rid_top; +u64 hash_algo; +u64 num_aux; +u64 ide_sid;
Called ncoh_ide_side in the alpha 15. Maybe match that unless it is changing name in future version.
Quoted 13 lines
arch/arm64/include/asm/rmi_smc.h +13
-0+u64 ncoh_num_addr_range; +u64 coh_num_addr_range; + }; +u8 padding1[0x100 ]; + }; + +union {/* 0x100 */ +u64 aux_granule[MAX_PDEV_AUX_GRANULES ]; +u8 padding2[0x100 ]; + }; + + union { /* 0x200 */ + struct {
arch/arm64/include/asm/rmi_smc.h struct rec_run { · line 338
Quoted 10 lines
arch/arm64/include/asm/rmi_smc.h +10
-0+struct rmi_pdev_addr_range ncoh_addr_range[MAX_IOCOH_ADDR_RANGE ]; + }; +u8 padding3[0x100 ]; + }; +union {/* 0x300 */ +struct { +struct rmi_pdev_addr_range coh_addr_range[MAX_FCOH_ADDR_RANGE ]; + }; +u8 padding4[0x100 ]; + };
Maybe pad out the rest? Mostly so we can see here that it is 4k.
drivers/virt/coco/arm-cca-host/arm-cca.c static int cca_tsm_connect(struct pci_dev *pdev) · line 127
Quoted 13 lines
arch/arm64/include/asm/rmi_smc.h +2
-0+}; +#endif /* __ASM_RMI_SMC_H */ drivers/virt/coco/arm-cca-host/arm-cca.c +0
-1@@ -124,7 +124,15 @@ static int cca_tsm_connect(struct pci_dev *pdev) rc =tsm_ide_stream_register (pdev, ide);if (rc) goto err_tsm; -
Tidy up. I'd leave it.
Quoted 18 lines
drivers/virt/coco/arm-cca-host/arm-cca.c +9
-0+/* +* Take a module reference so that we won't call unregister +* without rme_unasign_device +*/ +if (!try_module_get (THIS_MODULE )) { + rc = -ENXIO ; + goto err_tsm; + } +rme_asign_device (pdev);/* * Once ide is setup enable the stream at endpoint * Root port will be done by RMM drivers/virt/coco/arm-cca-host/rmm-da.c new file +0
-0
drivers/virt/coco/arm-cca-host/rmm-da.c line 41
Quoted 6 lines
drivers/virt/coco/arm-cca-host/rmm-da.c new file +4
-0@@ -0,0 +1,150 @@ + +static int init_pdev_params (struct pci_dev *pdev,struct rmi_pdev_params *params) +{ +void *aux;
Probably makes sense to reduce scope of aux to within the loop.
drivers/virt/coco/arm-cca-host/rmm-da.c line 45
Quoted 4 lines
drivers/virt/coco/arm-cca-host/rmm-da.c new file +4
-0+int rid, ret, i; +phys_addr_t aux_phys; +struct cca_host_dsc_pf0 *dsc_pf0; +struct pci_dev *rp =pcie_find_root_port (pdev);
Only used in one place for now anyway. Probably better to move it down there.
drivers/virt/coco/arm-cca-host/rmm-da.c line 48
Quoted 3 lines
drivers/virt/coco/arm-cca-host/rmm-da.c new file +3
-0+struct pci_config_window *cfg = pdev->bus->sysdata; + + dsc_pf0 =to_cca_dsc_pf0 (pdev);
I'd do that at declaration. Seems little advantage in doing it down here.
drivers/virt/coco/arm-cca-host/rmm-da.c line 58
Quoted 10 lines
drivers/virt/coco/arm-cca-host/rmm-da.c new file +10
-0+/* assign the ep device with RMM */ + rid =pci_dev_id (pdev); + params->pdev_id = rid; +/* slot number for certificate chain */ + params->cert_id =0 ; +/* io coherent spdm/ide and non p2p */ + params->flags =RMI_PDEV_SPDM_TRUE |RMI_PDEV_IDE_TRUE ; + params->ide_sid = dsc_pf0->sel_stream->stream_id; + params->hash_algo =RMI_HASH_SHA_256 ; +/* use the rid and MMIO resources of the epdev */
Spell out epdev or associate it with the pdev here more clearly.
drivers/virt/coco/arm-cca-host/rmm-da.c line 61
Quoted 3 lines
drivers/virt/coco/arm-cca-host/rmm-da.c new file +3
-0+ params->rid_top = params->rid_base = rid; + params->ecam_addr = cfg->res.start; + params->root_id =pci_dev_id (rp);
params->root_id = pci_dev_id(pcie_find_root_port(pdev));
to me acts as better documentation fo what is going on than using the local variable rp.
drivers/virt/coco/arm-cca-host/rmm-da.c line 71
Quoted 10 lines
drivers/virt/coco/arm-cca-host/rmm-da.c new file +10
-0+ + params->ncoh_num_addr_range =pci_res_to_pdev_addr (params->ncoh_addr_range, +ARRAY_SIZE (params->ncoh_addr_range), + pdev->resource, +DEVICE_COUNT_RESOURCE ); + +rmi_pdev_aux_count (params->flags, ¶ms->num_aux); +pr_debug ("%s using %ld pdev aux granules\n" , __func__, (unsigned long )params->num_aux); + dsc_pf0->num_aux = params->num_aux; +for (i =0 ; i < params->num_aux; i++) {
void *aux = (void *)__get_free_page(GFP_KERNEL);
Quoted 9 lines
drivers/virt/coco/arm-cca-host/rmm-da.c new file +9
-0+ aux = (void *)__get_free_page (GFP_KERNEL ); +if (!aux) { + ret = -ENOMEM ; + goto err_free_aux; + } + + aux_phys =virt_to_phys (aux); +if (rmi_granule_delegate (aux_phys)) { + ret = -ENXIO ;
drivers/virt/coco/arm-cca-host/rmm-da.c line 90
Quoted 10 lines
drivers/virt/coco/arm-cca-host/rmm-da.c new file +10
-0+free_page ((unsigned long )aux); + goto err_free_aux; + } + params->aux_granule[i] = aux_phys; + dsc_pf0->aux[i] = aux; + } +return 0 ; + +err_free_aux: +free_aux_pages (i, dsc_pf0->aux[i]);
I think you want
free_aux_pages(i, dsc_pf0->aux);
Assuming this is supposed to unwind the loop above.
drivers/virt/coco/arm-cca-host/rmm-da.c line 93
Quoted 3 lines
drivers/virt/coco/arm-cca-host/rmm-da.c new file +3
-0+return -ENOMEM ; +} +
Trivial: Single line probably fine here.
drivers/virt/coco/arm-cca-host/rmm-da.c line 104
Quoted 11 lines
drivers/virt/coco/arm-cca-host/rmm-da.c new file +11
-0+ +int rme_asign_device (struct pci_dev *pci_dev) +{ +int ret; +void *rmm_pdev; +unsigned long state; +phys_addr_t rmm_pdev_phys; +struct rmi_pdev_params *params; +struct cca_host_dsc_pf0 *dsc_pf0; + + dsc_pf0 =to_cca_dsc_pf0 (pci_dev);
Might as well save a line with
struct cca_host_dsc_pf0 *dsc_pf0 = to_cca_dsc_pf0(pci_dev);
drivers/virt/coco/arm-cca-host/rmm-da.c line 107
Quoted 3 lines
drivers/virt/coco/arm-cca-host/rmm-da.c new file +3
-0+ rmm_pdev = (void *)get_zeroed_page (GFP_KERNEL ); +if (!rmm_pdev) { + ret = -ENOMEM ;
return -ENOMEM;
Quoted 11 lines
drivers/virt/coco/arm-cca-host/rmm-da.c new file +11
-0+ goto err_out; + } + + rmm_pdev_phys =virt_to_phys (rmm_pdev); +if (rmi_granule_delegate (rmm_pdev_phys)) { + ret = -ENXIO ; + goto err_free_pdev; + } + + params = (struct rmi_pdev_params *)get_zeroed_page (GFP_KERNEL ); +if (!params) {
drivers/virt/coco/arm-cca-host/rmm-da.c line 128
Quoted 10 lines
drivers/virt/coco/arm-cca-host/rmm-da.c new file +10
-0+ ret = -ENOMEM ; + goto err_granule_undelegate; + } + + ret =init_pdev_params (pci_dev, params); +if (ret) + goto err_free_params; + + ret =rmi_pdev_create (rmm_pdev_phys,virt_to_phys (params)); +pr_info ("rmi_pdev_create(0x%llx, 0x%llx): %d\n" , rmm_pdev_phys,virt_to_phys (params), ret);
RFC, but even so I'd demote to debug and use dynamic debug stuff to enable them for your testing.
drivers/virt/coco/arm-cca-host/rmm-da.c line 134
Quoted 6 lines
drivers/virt/coco/arm-cca-host/rmm-da.c new file +6
-0+if (ret) + goto err_free_aux; + +rmi_pdev_get_state (rmm_pdev_phys, &state); +if (state !=RMI_PDEV_NEW ) + goto err_free_aux;
Nothing to unwind in rmi_pdev_create()? We've told the RMM about it then we blow away it's resources. Seems unwise! Maybe this is cleaned up elsewhere but if so a comment is probably good.
Quoted 4 lines
drivers/virt/coco/arm-cca-host/rmm-da.c new file +4
-0+ + dsc_pf0->rmm_pdev = rmm_pdev; +free_page ((unsigned long )params); +return 0 ;
drivers/virt/coco/arm-cca-host/rmm-da.c line 148
Quoted 10 lines
drivers/virt/coco/arm-cca-host/rmm-da.c new file +10
-0+ +err_free_aux: +free_aux_pages (dsc_pf0->num_aux, dsc_pf0->aux); +err_free_params: +free_page ((unsigned long )params); +err_granule_undelegate: +rmi_granule_undelegate (rmm_pdev_phys); +err_free_pdev: +free_page ((unsigned long )rmm_pdev); +err_out:
One of my favourite nitpicks. Why not just return if nothing to do? That tends to save a reviewer a bit of scrolling when checking error paths do correct unwinding.
Quoted 23 lines
drivers/virt/coco/arm-cca-host/rmm-da.c new file +2
-0+return ret; +}drivers/virt/coco/arm-cca-host/rmm-da.h +5
-0@@ -14,6 +14,10 @@struct cca_host_dsc_pf0 {struct pci_tsm_pf0 pci;struct pci_ide *sel_stream; + +void *rmm_pdev; +int num_aux; +void *aux[MAX_PDEV_AUX_GRANULES ]; };static inline struct cca_host_dsc_pf0 *to_cca_dsc_pf0 (struct pci_dev *pdev)@@ -26,4 +30,5 @@ static inline struct cca_host_dsc_pf0 *to_cca_dsc_pf0(struct pci_dev *pdev) return container_of(tsm, struct cca_host_dsc_pf0, pci.tsm); } +int rme_asign_device (struct pci_dev *pdev);#endif
Re: [RFC PATCH v1 11/38] KVM: arm64: CCA: register host tsm platform device
In reply to Jonathan CameronQuoted 28 lines · On Wed, 30 Jul 2025 11:38:27 +0100 Jonathan Cameron…
Quoted 11 lines · On Wed, 30 Jul 2025 14:12:26 +0530 "Aneesh Kumar K.V"…
Quoted 9 lines · Jason Gunthorpe
Quoted 4 lines · Jonathan Cameron
Quoted 1 line
+static struct
platform_devicecca_host_dev= {Hmm. Greg is getting increasingly (and correctly in my view) grumpy with platform devices being registered with no underlying resources etc as glue layers. Maybe some of that will come later.
Is
faux_devicea better choice? I admit to not knowing entirely what it is for..I'll go with a cautious yes to
faux_device. This case of a glue device with no resources and no reason to be on a particular bus was definitely the intent but I'm not 100% sure without trying it that we don't run into any problems.Not that many examples yet, but
cpuidle-psci.clooks like a vaguely similar case to this one.All it really does is move the location of the device and smash together the device registration with probe/remove. That means the device disappears if
probe()fails, which is cleaner in many ways than leaving a pointless stub behind.
Quoted 4 lines
Maybe it isn't appropriate it if is actually useful to rmmod/modprobe the driver.
+CC Greg on basis I may have wrong end of the stick ;)
This time with at least one less typo in Greg's email address.
Quoted 25 lines
Quoted 23 lines
Quoted 8 lines
But alternatively, why do we need a dummy "hw" struct device at all? Typically a subsystem like TSM should be structured to create its own struct devices..
I would expect this to just call 'register tsm' ?
The goal is to have tsm class device to be parented by the platform device.
# ls -al total 0 drwxr-xr-x 2 root root 0 Jan 13 06:07 . drwxr-xr-x 23 root root 0 Jan 1 00:00 .. lrwxrwxrwx 1 root root 0 Jan 13 06:07 tsm0 -> ../../devices/platform/arm-rmi-dev/tsm/tsm0 # pwd /sys/class/tsm-aneesh
Greg KH
maintainerRe: [RFC PATCH v1 11/38] KVM: arm64: CCA: register host tsm platform device
Quoted 31 lines · Jonathan Cameron
Quoted 28 lines · On Wed, 30 Jul 2025 11:38:27 +0100 Jonathan Cameron…
Quoted 11 lines · On Wed, 30 Jul 2025 14:12:26 +0530 "Aneesh Kumar K.V"…
Quoted 9 lines · Jason Gunthorpe
Quoted 4 lines · Jonathan Cameron
Quoted 1 line
+static struct
platform_devicecca_host_dev= {Hmm. Greg is getting increasingly (and correctly in my view) grumpy with platform devices being registered with no underlying resources etc as glue layers. Maybe some of that will come later.
Is
faux_devicea better choice? I admit to not knowing entirely what it is for..I'll go with a cautious yes to
faux_device. This case of a glue device with no resources and no reason to be on a particular bus was definitely the intent but I'm not 100% sure without trying it that we don't run into any problems.Not that many examples yet, but
cpuidle-psci.clooks like a vaguely similar case to this one.All it really does is move the location of the device and smash together the device registration with probe/remove. That means the device disappears if
probe()fails, which is cleaner in many ways than leaving a pointless stub behind.
Quoted 5 lines
Quoted 4 lines
Maybe it isn't appropriate it if is actually useful to rmmod/modprobe the driver.
+CC Greg on basis I may have wrong end of the stick ;)
This time with at least one less typo in Greg's email address.
Yes, use faux_device if you need/want a struct device to represent something in the tree and it does NOT have any real platform resources behind it. That's explicitly what it was designed for.
thanks,
greg k-h
Re: [RFC PATCH v1 14/38] coco: host: arm64: Device communication support
In reply to Aneesh Kumar K.V (Arm)Quoted 45 lines · On Mon, 28 Jul 2025 19:21:51 +0530 "Aneesh Kumar K.V (Arm)"…
Add helpers for device communication from RMM
Signed-off-by Aneesh Kumar K.V (Arm) <[email protected]>--- arch/arm64/include/asm/rmi_cmds.h | 11 ++ arch/arm64/include/asm/rmi_smc.h | 49 ++++++ drivers/virt/coco/arm-cca-host/arm-cca.c | 45 ++++++ drivers/virt/coco/arm-cca-host/rmm-da.c | 198 +++++++++++++++++++++++ drivers/virt/coco/arm-cca-host/rmm-da.h | 41 +++++ 5 files changed, 344 insertions(+)#endif /*
__ASM_RMI_CMDS_H*/arch/arm64/include/asm/rmi_smc.h +25
-0@@ -47,6 +47,7 @@ +#define RMI_DEV_COMM_EXIT_CACHE_REQ BIT(0) +#define RMI_DEV_COMM_EXIT_CACHE_RSP BIT(1) +#define RMI_DEV_COMM_EXIT_SEND BIT(2) +#define RMI_DEV_COMM_EXIT_WAIT BIT(3) +#define RMI_DEV_COMM_EXIT_MULTI BIT(4) + +#define RMI_DEV_COMM_NONE 0 +#define RMI_DEV_COMM_RESPONSE 1 +#define RMI_DEV_COMM_ERROR 2 + +#define RMI_PROTOCOL_SPDM 0 +#define RMI_PROTOCOL_SECURE_SPDM 1 + +#define RMI_DEV_VCA 0 +#define RMI_DEV_CERTIFICATE 1 +#define RMI_DEV_MEASUREMENTS 2 +#define RMI_DEV_INTERFACE_REPORT 3 + +struct rmi_dev_comm_enter { +u64 status; +u64 req_addr; +u64 resp_addr; +u64 resp_len; +}; +
arch/arm64/include/asm/rmi_smc.h struct rmi_pdev_params { · line 376
Quoted 10 lines
arch/arm64/include/asm/rmi_smc.h +10
-0+struct rmi_dev_comm_exit { +u64 flags; +u64 cache_req_offset; +u64 cache_req_len; +u64 cache_rsp_offset; +u64 cache_rsp_len; +u64 cache_obj_id; +u64 protocol; +u64 req_len; +u64 timeout;
In latest spec called rsp_timeout.
Not sure we care that much but if no strong reason otherwise, should aim to match the spec text. (Maybe this got renamed?)
Quoted 7 lines
arch/arm64/include/asm/rmi_smc.h +1
-0+};drivers/virt/coco/arm-cca-host/arm-cca.c +0
-0
drivers/virt/coco/arm-cca-host/arm-cca.c static void cca_tsm_pci_remove(struct pci_tsm *tsm) · line 92
Quoted 8 lines
drivers/virt/coco/arm-cca-host/arm-cca.c +5
-0@@ -85,6 +85,45 @@ static void cca_tsm_pci_remove(struct pci_tsm *tsm)vfree (dsc_pf0); } +static int init_dev_communication_buffers (struct cca_host_comm_data *comm_data) +{ +int ret = -ENOMEM ; + + comm_data->io_params = (struct rmi_dev_comm_data *)get_zeroed_page (GFP_KERNEL );
Hmm. There isn't a DEFINE_FREE() yet for free_page(). Maybe time to add one. If we did then we'd use local variables until all allocations succeed then assign with no_free_ptr()
Quoted 5 lines
drivers/virt/coco/arm-cca-host/arm-cca.c +5
-0+if (!comm_data->io_params) + goto err_out; + + comm_data->resp_buff = (void *)__get_free_page (GFP_KERNEL ); +if (!comm_data->resp_buff)
drivers/virt/coco/arm-cca-host/arm-cca.c static void cca_tsm_pci_remove(struct pci_tsm *tsm) · line 107
Quoted 10 lines
drivers/virt/coco/arm-cca-host/arm-cca.c +10
-0+ goto err_res_buff; + + comm_data->req_buff = (void *)__get_free_page (GFP_KERNEL ); +if (!comm_data->req_buff) + goto err_req_buff; + + + comm_data->io_params->enter.status =RMI_DEV_COMM_NONE ; + comm_data->io_params->enter.resp_addr =virt_to_phys (comm_data->resp_buff); + comm_data->io_params->enter.req_addr =virt_to_phys ((void *)comm_data->req_buff);
I think it's already a a void * and even if it were some other pointer type no cast would be necessary.
Quoted 26 lines
drivers/virt/coco/arm-cca-host/arm-cca.c +14
-0+ comm_data->io_params->enter.resp_len =0 ; + +return 0 ; + +err_req_buff: +free_page ((unsigned long )comm_data->resp_buff); +err_res_buff: +free_page ((unsigned long )comm_data->io_params); +err_out: +return ret; +} + +/* per root port unique with multiple restrictions. For now global */ static DECLARE_BITMAP(cca_stream_ids, MAX_STREAM_ID);@@ -124,6 +163,7 @@ static int cca_tsm_connect(struct pci_dev *pdev) rc =tsm_ide_stream_register (pdev, ide);if (rc) goto err_tsm; +init_dev_communication_buffers (&dsc_pf0->comm_data);/* * Take a module reference so that we won't call unregister * without rme_unasign_device
drivers/virt/coco/arm-cca-host/arm-cca.c static int cca_tsm_connect(struct pci_dev *pdev) · line 175
Quoted 3 lines
drivers/virt/coco/arm-cca-host/arm-cca.c +0
-0@@ -133,6 +173,11 @@ static int cca_tsm_connect(struct pci_dev *pdev) goto err_tsm; }rme_asign_device (pdev);
rme_assign_device() - I obviously missed this earlier!
drivers/virt/coco/arm-cca-host/arm-cca.c static int cca_tsm_connect(struct pci_dev *pdev) · line 177
Quoted 2 lines
drivers/virt/coco/arm-cca-host/arm-cca.c +2
-0+/* +* Schedule a work to fetch device certificate and setup IDE
Single line comment probably fine here. Though it perhaps doesn't add much over the function name.
Quoted 16 lines
drivers/virt/coco/arm-cca-host/arm-cca.c +3
-0+*/ +schedule_rme_ide_setup (pdev); +/* * Once ide is setup enable the stream at endpoint * Root port will be done by RMM drivers/virt/coco/arm-cca-host/rmm-da.c +2
-0@@ -148,3 +148,201 @@ int rme_asign_device(struct pci_dev *pci_dev) err_out:return ret; } + +static int doe_send_req_resp (struct pci_tsm *tsm)
drivers/virt/coco/arm-cca-host/rmm-da.c int rme_asign_device(struct pci_dev *pci_dev) · line 162
Quoted 10 lines
drivers/virt/coco/arm-cca-host/rmm-da.c +10
-0+{ +u8 protocol; +int ret, data_obj_type; +struct cca_host_comm_data *comm_data; +struct rmi_dev_comm_exit *io_exit; + + comm_data =to_cca_comm_data (tsm->pdev); + + io_exit = &comm_data->io_params->exit; + protocol = io_exit->protocol;
For all these I'd combine with the declarations.
Quoted 24 lines
drivers/virt/coco/arm-cca-host/rmm-da.c +24
-0+ +pr_debug ("doe_req size:%lld doe_io_type=%d\n" , io_exit->req_len, (int )protocol); + +if (protocol ==RMI_PROTOCOL_SPDM ) + data_obj_type =PCI_DOE_PROTO_CMA ; +else if (protocol ==RMI_PROTOCOL_SECURE_SPDM ) + data_obj_type =PCI_DOE_PROTO_SSESSION ; +else +return -EINVAL ; + + ret =pci_tsm_doe_transfer (tsm->dsm_dev, data_obj_type, + comm_data->req_buff, io_exit->req_len, + comm_data->resp_buff,PAGE_SIZE ); +pr_debug ("doe returned:%d\n" , ret); +return ret; +} + +/* Parallel update for cca_dsc contents FIXME!! */ +static int __do_dev_communicate (int type,struct pci_tsm *tsm) +{ +int ret; +bool is_multi; +u8 *cache_buf; +int *cache_offset;
drivers/virt/coco/arm-cca-host/rmm-da.c int rme_asign_device(struct pci_dev *pci_dev) · line 196
Quoted 10 lines
drivers/virt/coco/arm-cca-host/rmm-da.c +10
-0+int nbytes, cache_remaining; +struct cca_host_dsc_pf0 *dsc_pf0; +struct rmi_dev_comm_exit *io_exit; +struct rmi_dev_comm_enter *io_enter; +struct cca_host_comm_data *comm_data; + + + comm_data =to_cca_comm_data (tsm->pdev); + io_enter = &comm_data->io_params->enter; + io_exit = &comm_data->io_params->exit;
Might as well set these local variables as the declaration point above. None of them will be very long lines.
drivers/virt/coco/arm-cca-host/rmm-da.c int rme_asign_device(struct pci_dev *pci_dev) · line 206
Quoted 10 lines
drivers/virt/coco/arm-cca-host/rmm-da.c +10
-0+ + dsc_pf0 =to_cca_dsc_pf0 (tsm->dsm_dev); +redo_communicate: + is_multi = false; + +if (type ==PDEV_COMMUNICATE ) + ret =rmi_pdev_communicate (virt_to_phys (dsc_pf0->rmm_pdev), +virt_to_phys (comm_data->io_params)); +else + ret =RMI_ERROR_INPUT ;
I'd split this case out and return here farther than using the match below as it feels like the error message auto to be more specific. Something about type not matching.
Quoted 31 lines
drivers/virt/coco/arm-cca-host/rmm-da.c +31
-0+if (ret !=RMI_SUCCESS ) { +pr_err ("pdev communicate error\n" ); +return ret; + } + +/* caching request from RMM */ +if (io_exit->flags &RMI_DEV_COMM_EXIT_CACHE_RSP ) { +switch (io_exit->cache_obj_id) { +case RMI_DEV_VCA : + cache_buf = dsc_pf0->vca.buf; + cache_offset = &dsc_pf0->vca.size; + cache_remaining =sizeof (dsc_pf0->vca.buf) - *cache_offset; +break ; +case RMI_DEV_CERTIFICATE : + cache_buf = dsc_pf0->cert_chain.cache.buf; + cache_offset = &dsc_pf0->cert_chain.cache.size; + cache_remaining =sizeof (dsc_pf0->cert_chain.cache.buf) - *cache_offset; +break ; +default : +/* FIXME!! depending on the DevComms status, +* it might require to ABORT the communcation. +*/ +return -EINVAL ; + } + +if (io_exit->cache_rsp_len > cache_remaining) +return -EINVAL ; + +memcpy (cache_buf + *cache_offset, + (comm_data->resp_buff + io_exit->cache_rsp_offset), io_exit->cache_rsp_len); + *cache_offset += io_exit->cache_rsp_len;
drivers/virt/coco/arm-cca-host/rmm-da.c int rme_asign_device(struct pci_dev *pci_dev) · line 247
Quoted 10 lines
drivers/virt/coco/arm-cca-host/rmm-da.c +10
-0+ } + +/* +* wait for last packet request from RMM. +* We should not find this because our device communication in synchronous +*/ +if (io_exit->flags &RMI_DEV_COMM_EXIT_WAIT ) +return -ENXIO ; + + is_multi = !!(io_exit->flags &RMI_DEV_COMM_EXIT_MULTI );
!! doesn't add anything here that I can see over
is_multi = io_exit->flags & RMI_DEV_COMM_EXIT_MULTI;
Quoted 31 lines
drivers/virt/coco/arm-cca-host/rmm-da.c +31
-0+ +/* next packet to send */ +if (io_exit->flags &RMI_DEV_COMM_EXIT_SEND ) { + nbytes =doe_send_req_resp (tsm); +if (nbytes <0 ) { +/* report error back to RMM */ + io_enter->status =RMI_DEV_COMM_ERROR ; + }else { +/* send response back to RMM */ + io_enter->resp_len = nbytes; + io_enter->status =RMI_DEV_COMM_RESPONSE ; + } + }else { +/* no data transmitted => no data received */ + io_enter->resp_len =0 ; + } + +/* The call need to do multiple request/respnse */ +if (is_multi) + goto redo_communicate; + +return 0 ; +} + +static int do_dev_communicate (int type,struct pci_tsm *tsm,int target_state) +{ +int ret; +unsigned long state; +unsigned long error_state; +struct cca_host_dsc_pf0 *dsc_pf0; +struct rmi_dev_comm_enter *io_enter;
drivers/virt/coco/arm-cca-host/rmm-da.c int rme_asign_device(struct pci_dev *pci_dev) · line 288
Quoted 10 lines
drivers/virt/coco/arm-cca-host/rmm-da.c +10
-0+ + dsc_pf0 =to_cca_dsc_pf0 (tsm->dsm_dev); + io_enter = &dsc_pf0->comm_data.io_params->enter; + io_enter->resp_len =0 ; + io_enter->status =RMI_DEV_COMM_NONE ; + + state =-1 ; + do { + ret = __do_dev_communicate(type, tsm); + if (ret != 0) {
if (ret)
drivers/virt/coco/arm-cca-host/rmm-da.c int rme_asign_device(struct pci_dev *pci_dev) · line 290
Quoted 2 lines
drivers/virt/coco/arm-cca-host/rmm-da.c +2
-0+pr_err ("dev communication error\n" ); +break ;
I'd just return in error cases.
Quoted 17 lines
drivers/virt/coco/arm-cca-host/rmm-da.c +17
-0+ } + +if (type ==PDEV_COMMUNICATE ) { + ret =rmi_pdev_get_state (virt_to_phys (dsc_pf0->rmm_pdev), + &state); + error_state =RMI_PDEV_ERROR ; + } +if (ret !=0 ) { +pr_err ("Get dev state error\n" ); +break ; + } + }while (state != target_state && state != error_state); + +pr_info ("dev_io_complete: status: %d state:%ld\n" , ret, state); + +return state; +}
Re: [RFC PATCH v1 15/38] coco: host: arm64: Stop and destroy the physical device
In reply to Aneesh Kumar K.V (Arm)Quoted 1 line · On Mon, 28 Jul 2025 19:21:52 +0530 "Aneesh Kumar K.V (Arm)"…
Add support for stopping and destroying physical devices.
I think it's an odd mix to not do create and destroy in a single patch. Same with start and stop.
Leaves reviewers thinking perhaps you weren't cleaning up properly an any error paths are much less obvious.
Quoted 79 lines
Signed-off-by Aneesh Kumar K.V (Arm) <[email protected]>--- arch/arm64/include/asm/rmi_cmds.h | 18 ++++++++++++++++++ arch/arm64/include/asm/rmi_smc.h | 2 ++ drivers/virt/coco/arm-cca-host/arm-cca.c | 3 +++ drivers/virt/coco/arm-cca-host/rmm-da.c | 21 +++++++++++++++++++++ drivers/virt/coco/arm-cca-host/rmm-da.h | 1 + 5 files changed, 45 insertions(+)arch/arm64/include/asm/rmi_cmds.h +18
-0@@ -547,4 +547,22 @@ static inline unsigned long rmi_pdev_communicate(unsigned long pdev_phys, return res.a0; } +static inline unsigned long rmi_pdev_stop (unsigned long pdev_phys) +{ +struct arm_smccc_res res; + +arm_smccc_1_1_invoke (SMC_RMI_PDEV_STOP , pdev_phys, &res); + +return res.a0; +} + +static inline unsigned long rmi_pdev_destroy (unsigned long pdev_phys) +{ +struct arm_smccc_res res; + +arm_smccc_1_1_invoke (SMC_RMI_PDEV_DESTROY , pdev_phys, &res); + +return res.a0; +} +#endif /* __ASM_RMI_CMDS_H */ arch/arm64/include/asm/rmi_smc.h +2
-0@@ -49,7 +49,9 @@#define SMC_RMI_PDEV_COMMUNICATE SMC_RMI_CALL(0x0175)#define SMC_RMI_PDEV_CREATE SMC_RMI_CALL(0x0176) +#define SMC_RMI_PDEV_DESTROY SMC_RMI_CALL(0x0177)#define SMC_RMI_PDEV_GET_STATE SMC_RMI_CALL(0x0178) +#define SMC_RMI_PDEV_STOP SMC_RMI_CALL(0x017c)#define RMI_ABI_MAJOR_VERSION 1#define RMI_ABI_MINOR_VERSION 0drivers/virt/coco/arm-cca-host/arm-cca.c +3
-0@@ -210,12 +210,15 @@ static void cca_tsm_disconnect(struct pci_dev *pdev) ide = dsc_pf0->sel_stream; dsc_pf0->sel_stream =NULL ;pci_ide_stream_disable (pdev, ide); +rme_unassign_device (pdev); +module_put (THIS_MODULE );tsm_ide_stream_unregister (ide);pci_ide_stream_teardown (rp, ide);pci_ide_stream_teardown (pdev, ide);pci_ide_stream_unregister (ide);clear_bit (ide->stream_id, cca_stream_ids);pci_ide_stream_free (ide); +free_dev_communication_buffers (&dsc_pf0->comm_data); }static const struct pci_tsm_ops cca_pci_ops = {drivers/virt/coco/arm-cca-host/rmm-da.c +0
-0@@ -346,3 +346,24 @@ int schedule_rme_ide_setup(struct pci_dev *pdev)return 0 ; }
drivers/virt/coco/arm-cca-host/rmm-da.c int schedule_rme_ide_setup(struct pci_dev *pdev) · line 358
Quoted 10 lines
drivers/virt/coco/arm-cca-host/rmm-da.c +10
-0+ +void rme_unassign_device (struct pci_dev *pdev) +{ +unsigned long ret; +unsigned long state; +phys_addr_t rmm_pdev_phys; +struct cca_host_dsc_pf0 *dsc_pf0; + + dsc_pf0 =to_cca_dsc_pf0 (pdev); + rmm_pdev_phys =virt_to_phys (dsc_pf0->rmm_pdev);
As with previous patches, I'd set as many of these are seems reasonable at the variable declations.
drivers/virt/coco/arm-cca-host/rmm-da.c int schedule_rme_ide_setup(struct pci_dev *pdev) · line 364
Quoted 6 lines
drivers/virt/coco/arm-cca-host/rmm-da.c +6
-0+ ret =rmi_pdev_stop (rmm_pdev_phys); +if (WARN_ON (ret !=RMI_SUCCESS )) +return ; + + state =do_pdev_communicate (pdev->tsm,RMI_PDEV_STOPPED ); +/* ignore the error state and destroy the device */
WARN_ON is rather heavy if you want to ignore it.
Quoted 16 lines
drivers/virt/coco/arm-cca-host/rmm-da.c +5
-0+WARN_ON (state !=RMI_PDEV_STOPPED ); + ret =rmi_pdev_destroy (rmm_pdev_phys); +if (WARN_ON (ret !=RMI_SUCCESS )) +return ; +}drivers/virt/coco/arm-cca-host/rmm-da.h +1
-0@@ -71,5 +71,6 @@ static inline struct cca_host_comm_data *to_cca_comm_data(struct pci_dev *pdev) }int rme_asign_device (struct pci_dev *pdev); +void rme_unassign_device (struct pci_dev *pdev);int schedule_rme_ide_setup (struct pci_dev *pdev);#endif
Re: [RFC PATCH v1 19/38] coco: host: arm64: set_pubkey support
In reply to Aneesh Kumar K.V (Arm)Quoted 3 lines · On Mon, 28 Jul 2025 19:21:56 +0530 "Aneesh Kumar K.V (Arm)"…
Add changes to share the device's public key with the RMM.
Signed-off-by Aneesh Kumar K.V (Arm) <[email protected]>
A few minor comments inline.
Quoted 30 lines
drivers/virt/coco/arm-cca-host/rmm-da.c +15
-0@@ -6,6 +6,9 @@#include <linux/pci.h> #include <linux/pci-ecam.h> #include <asm/rmi_cmds.h> +#include <crypto/internal/rsa.h> +#include <keys/asymmetric-type.h> +#include <keys/x509-parser.h> #include "rmm-da.h" @@ -311,6 +314,136 @@ static int do_pdev_communicate(struct pci_tsm *tsm, int target_state)return do_dev_communicate (PDEV_COMMUNICATE , tsm, target_state); } +static int parse_certificate_chain (struct pci_tsm *tsm) +{ +struct cca_host_dsc_pf0 *dsc_pf0; +unsigned int chain_size; +unsigned int offset =0 ; +u8 *chain_data; +int ret =0 ; + + dsc_pf0 =to_cca_dsc_pf0 (tsm->pdev); + chain_size = dsc_pf0->cert_chain.cache.size; + chain_data = dsc_pf0->cert_chain.cache.buf; +
drivers/virt/coco/arm-cca-host/rmm-da.c static int do_pdev_communicate(struct pci_tsm *tsm, int target_state) · line 338
Quoted 10 lines
drivers/virt/coco/arm-cca-host/rmm-da.c +10
-0+while (offset < chain_size) { +unsigned int cert_len = +x509_get_certificate_length (chain_data + offset, + chain_size - offset); +struct x509_certificate *cert = +x509_cert_parse (chain_data + offset, cert_len); + +if (IS_ERR (cert)) { +pr_warn ("%s(): parsing of certificate chain not successful\n" , __func__); + ret =PTR_ERR (cert);
Direct return looks fine here. Maybe add a DEFINE_FREE(x509_cert,...) as then can use direct returns throughout.
Quoted 23 lines
drivers/virt/coco/arm-cca-host/rmm-da.c +23
-0+break ; + } + +if (offset + cert_len == chain_size) { + dsc_pf0->cert_chain.public_key =kzalloc (cert->pub->keylen,GFP_KERNEL ); +if (!dsc_pf0->cert_chain.public_key) { + ret = -ENOMEM ; +x509_free_certificate (cert); +break ; + } + +if (!strcmp ("ecdsa-nist-p256" , cert->pub->pkey_algo)) { + dsc_pf0->rmi_signature_algorithm =RMI_SIG_ECDSA_P256 ; + }else if (!strcmp ("ecdsa-nist-p384" , cert->pub->pkey_algo)) { + dsc_pf0->rmi_signature_algorithm =RMI_SIG_ECDSA_P384 ; + }else if (!strcmp ("rsa" , cert->pub->pkey_algo)) { + dsc_pf0->rmi_signature_algorithm =RMI_SIG_RSASSA_3072 ; + }else { + ret = -ENXIO ; +x509_free_certificate (cert); +break ; + } +memcpy (dsc_pf0->cert_chain.public_key, cert->pub->key, cert->pub->keylen);
drivers/virt/coco/arm-cca-host/rmm-da.c static int do_pdev_communicate(struct pci_tsm *tsm, int target_state) · line 371
Quoted 10 lines
drivers/virt/coco/arm-cca-host/rmm-da.c +10
-0+ dsc_pf0->cert_chain.public_key_size = cert->pub->keylen; + } + +x509_free_certificate (cert); + + offset += cert_len; + } + +if (ret ==0 ) + dsc_pf0->cert_chain.valid = true;
if (ret) return ret; dsc_pf0->cert_chain.valid = true; return 0;
would be my preference for style here but others may disagree.
drivers/virt/coco/arm-cca-host/rmm-da.c static int do_pdev_communicate(struct pci_tsm *tsm, int target_state) · line 379
Quoted 8 lines
drivers/virt/coco/arm-cca-host/rmm-da.c +8
-0+ +return ret; +} + +static int pdev_set_public_key (struct pci_tsm *tsm) +{ +struct rmi_public_key_params *key_shared; +unsigned long expected_key_len =0 ;
Don't set this. It's only used in places where it is explicitly set and if it is used anywhere else we want the compiler to tell us.
Quoted 30 lines
drivers/virt/coco/arm-cca-host/rmm-da.c +30
-0+struct cca_host_dsc_pf0 *dsc_pf0; +int ret; + + dsc_pf0 =to_cca_dsc_pf0 (tsm->pdev); +/* Check that all the necessary information was captured from communication */ +if (!dsc_pf0->cert_chain.valid) +return -EINVAL ; + + key_shared = (struct rmi_public_key_params *)get_zeroed_page (GFP_KERNEL ); +if (!key_shared) +return -ENOMEM ; + + key_shared->rmi_signature_algorithm = dsc_pf0->rmi_signature_algorithm; + +switch (key_shared->rmi_signature_algorithm) { +case RMI_SIG_ECDSA_P384 : + expected_key_len =97 ; + +if (dsc_pf0->cert_chain.public_key_size != expected_key_len) +return -EINVAL ; + key_shared->public_key_len = dsc_pf0->cert_chain.public_key_size; +memcpy (key_shared->public_key, + dsc_pf0->cert_chain.public_key, + dsc_pf0->cert_chain.public_key_size); + key_shared->metadata_len =0 ; +break ; +case RMI_SIG_ECDSA_P256 : + expected_key_len =65 ; + +if (dsc_pf0->cert_chain.public_key_size != expected_key_len)
drivers/virt/coco/arm-cca-host/rmm-da.c static int do_pdev_communicate(struct pci_tsm *tsm, int target_state) · line 419
Quoted 10 lines
drivers/virt/coco/arm-cca-host/rmm-da.c +10
-0+return -EINVAL ; + key_shared->public_key_len = dsc_pf0->cert_chain.public_key_size; +memcpy (key_shared->public_key, + dsc_pf0->cert_chain.public_key, + dsc_pf0->cert_chain.public_key_size); + key_shared->metadata_len =0 ; +break ; +case RMI_SIG_RSASSA_3072 : + expected_key_len =385 ; +struct rsa_key rsa_key = {0 };
Shouldn't define this inline. Maybe move up a line and add some {} to set the scope to this case statement.
Quoted 5 lines
drivers/virt/coco/arm-cca-host/rmm-da.c +5
-0+int ret_rsa_parse =rsa_parse_pub_key (&rsa_key, + dsc_pf0->cert_chain.public_key, + dsc_pf0->cert_chain.public_key_size); +/* This also checks the key_len */ +if (ret_rsa_parse)
drivers/virt/coco/arm-cca-host/rmm-da.c static int do_pdev_communicate(struct pci_tsm *tsm, int target_state) · line 434
Quoted 10 lines
drivers/virt/coco/arm-cca-host/rmm-da.c +10
-0+return ret_rsa_parse; +/* +* exponent is usally 65537 (size = 24bits) but in rare cases +* it size can be as large as the modulus +*/ +if (rsa_key.e_sz > expected_key_len) +return -EINVAL ; + key_shared->public_key_len = rsa_key.n_sz; + key_shared->metadata_len = rsa_key.e_sz; +memcpy (key_shared->public_key, (unsigned char *)rsa_key.n, rsa_key.n_sz);
Why is the cast needed?
Quoted 11 lines
drivers/virt/coco/arm-cca-host/rmm-da.c +11
-0+memcpy (key_shared->metadata, (unsigned char *)rsa_key.e, rsa_key.e_sz); +break ; +default : +return -EINVAL ; + } + + ret =rmi_pdev_set_pubkey (virt_to_phys (dsc_pf0->rmm_pdev), +virt_to_phys (key_shared)); +free_page ((unsigned long )key_shared); +return ret; +}
Re: [RFC PATCH v1 20/38] coco: host: arm64: Add support for creating a virtual device
In reply to Aneesh Kumar K.V (Arm)Quoted 3 lines · On Mon, 28 Jul 2025 19:21:57 +0530 "Aneesh Kumar K.V (Arm)"…
Changes to support the creation of virtual device objects with RMM.
Signed-off-by Aneesh Kumar K.V (Arm) <[email protected]>
Hi Aneesh,
Really trivial stuff in this one.
arch/arm64/include/asm/rmi_smc.h line 57
Quoted 10 lines
arch/arm64/include/asm/rmi_smc.h +2
-0@@ -53,6 +53,8 @@#define SMC_RMI_PDEV_GET_STATE SMC_RMI_CALL(0x0178)#define SMC_RMI_PDEV_SET_PUBKEY SMC_RMI_CALL(0x017b)#define SMC_RMI_PDEV_STOP SMC_RMI_CALL(0x017c) +#define SMC_RMI_VDEV_CREATE SMC_RMI_CALL(0x0187) +
One blankl line probably enough.
arch/arm64/include/asm/rmi_smc.h line 59
Quoted 2 lines
arch/arm64/include/asm/rmi_smc.h +0
-0#define RMI_ABI_MAJOR_VERSION 1
t/coco/arm-cca-host/arm-cca.c b/drivers/virt/coco/arm-cca-host/arm-cca.c
Quoted 13 lines
index
c65b81f0706f..2da513f45974100644drivers/virt/coco/arm-cca-host/arm-cca.c +2
-0@@ -10,6 +10,8 @@#include <linux/pci.h> #include <linux/tsm.h> #include <linux/vmalloc.h> +#include <linux/kvm_host.h> +#include <linux/pci.h> #include "rmm-da.h"
drivers/virt/coco/arm-cca-host/arm-cca.c static void cca_tsm_disconnect(struct pci_dev *pdev) · line 230
Quoted 8 lines
drivers/virt/coco/arm-cca-host/arm-cca.c +5
-0@@ -221,11 +223,39 @@ static void cca_tsm_disconnect(struct pci_dev *pdev)free_dev_communication_buffers (&dsc_pf0->comm_data); } +static struct pci_tdi *cca_tsm_bind (struct pci_dev *pdev,struct pci_dev *pf0_dev, +struct kvm *kvm,u64 tdi_id) +{ +void *rmm_vdev; + struct cca_host_tdi *host_tdi __free(kfree) = NULL;
Move declaration and registration of destructor down to where it's constructed. (Follow guidance in cleanup.h.
Quoted 18 lines
drivers/virt/coco/arm-cca-host/arm-cca.c +18
-0+struct realm *realm = &kvm->arch.realm; + +if (pdev->is_virtfn) +return ERR_PTR (-ENXIO ); + +if (!try_module_get (THIS_MODULE )) +return ERR_PTR (-ENXIO ); + + host_tdi =kmalloc (sizeof (struct cca_host_tdi ),GFP_KERNEL ); +if (!host_tdi) +return ERR_PTR (-ENOMEM ); + + rmm_vdev =rme_create_vdev (realm, pdev, pf0_dev, tdi_id); +if (!IS_ERR_OR_NULL (rmm_vdev)) { + host_tdi->rmm_vdev = rmm_vdev; +return &no_free_ptr (host_tdi)->tdi; + } +
drivers/virt/coco/arm-cca-host/arm-cca.c static void cca_tsm_disconnect(struct pci_dev *pdev) · line 258
Quoted 10 lines
drivers/virt/coco/arm-cca-host/arm-cca.c +5
-0+module_put (THIS_MODULE ); +return rmm_vdev; +} +static const struct pci_tsm_ops cca_pci_ops = { .probe = cca_tsm_pci_probe, .remove = cca_tsm_pci_remove, .connect = cca_tsm_connect, .disconnect = cca_tsm_disconnect, + .bind = cca_tsm_bind,
Odd spacing. Seems just bind is using a tab.
Quoted 1 line
drivers/virt/coco/arm-cca-host/arm-cca.c +0
-0};
Re: [RFC PATCH v1 21/38] coco: host: arm64: Add support for virtual device communication
In reply to Aneesh Kumar K.V (Arm)Quoted 3 lines · On Mon, 28 Jul 2025 19:21:58 +0530 "Aneesh Kumar K.V (Arm)"…
Add support for
vdev_communicatewith RMM.Signed-off-by Aneesh Kumar K.V (Arm) <[email protected]>
One minor comment.
drivers/virt/coco/arm-cca-host/rmm-da.c static int __do_dev_communicate(int type, struct pci_tsm *tsm)
Quoted 5 lines
drivers/virt/coco/arm-cca-host/rmm-da.c +0
-0@@ -207,8 +207,14 @@ static int __do_dev_communicate(int type, struct pci_tsm *tsm)
Can type parameter be an enum so we can see all cases are covered?
Quoted 12 lines
drivers/virt/coco/arm-cca-host/rmm-da.c +7
-2if (type ==PDEV_COMMUNICATE ) ret =rmi_pdev_communicate (virt_to_phys (dsc_pf0->rmm_pdev),virt_to_phys (comm_data->io_params)); -else - ret =RMI_ERROR_INPUT ; +else { + struct cca_host_tdi *host_tdi = container_of(tsm->tdi, struct cca_host_tdi, tdi); + + ret =rmi_vdev_communicate (virt_to_phys (dsc_pf0->rmm_pdev), +virt_to_phys (host_tdi->rmm_vdev), +virt_to_phys (comm_data->io_params)); + }
Re: [RFC PATCH v1 22/38] coco: host: arm64: Stop and destroy virtual device
In reply to Aneesh Kumar K.V (Arm)Quoted 3 lines · On Mon, 28 Jul 2025 19:21:59 +0530 "Aneesh Kumar K.V (Arm)"…
Add support for
vdev_stopandvdev_destroy.Signed-off-by Aneesh Kumar K.V (Arm) <[email protected]>
Really trivial comments.
Quoted 26 lines
--- arch/arm64/include/asm/rmi_cmds.h | 21 ++++++++ arch/arm64/include/asm/rmi_smc.h | 3 +- drivers/virt/coco/arm-cca-host/arm-cca.c | 10 ++++ drivers/virt/coco/arm-cca-host/rmm-da.c | 61 +++++++++++++++++++++++- drivers/virt/coco/arm-cca-host/rmm-da.h | 2 + 5 files changed, 95 insertions(+), 2 deletions(-)arch/arm64/include/asm/rmi_cmds.h +10
-0@@ -609,4 +609,25 @@ static inline unsigned long rmi_vdev_get_state(unsigned long vdev_phys, unsignedreturn res.a0; } +static inline unsigned long rmi_vdev_stop (unsigned long vdev_phys) +{ +struct arm_smccc_res res; + +arm_smccc_1_1_invoke (SMC_RMI_VDEV_STOP , vdev_phys, &res); + +return res.a0; +} + +static inline unsigned long rmi_vdev_destroy(unsigned long rd,
arch/arm64/include/asm/rmi_cmds.h static inline unsigned long rmi_vdev_get_state(unsigned long vdev_phys, unsigned · line 631
Quoted 10 lines
arch/arm64/include/asm/rmi_cmds.h +10
-0+unsigned long pdev_phys, + unsigned long vdev_phys) +{ + struct arm_smccc_res res; + +arm_smccc_1_1_invoke (SMC_RMI_VDEV_DESTROY , rd, pdev_phys, vdev_phys, &res); + +return res.a0; +} +
One is enough.
arch/arm64/include/asm/rmi_smc.h line 59
Quoted 13 lines
arch/arm64/include/asm/rmi_cmds.h +1
-0+#endif /* __ASM_RMI_CMDS_H */ arch/arm64/include/asm/rmi_smc.h +1
-1@@ -55,8 +55,9 @@#define SMC_RMI_PDEV_STOP SMC_RMI_CALL(0x017c)#define SMC_RMI_VDEV_COMMUNICATE SMC_RMI_CALL(0x0186)#define SMC_RMI_VDEV_CREATE SMC_RMI_CALL(0x0187) +#define SMC_RMI_VDEV_DESTROY SMC_RMI_CALL(0x0188)#define SMC_RMI_VDEV_GET_STATE SMC_RMI_CALL(0x0189) -
There have been a few of these. Check v2 carefully to make sure no more sneak in.
Quoted 27 lines
arch/arm64/include/asm/rmi_smc.h +1
-0+#define SMC_RMI_VDEV_STOP SMC_RMI_CALL(0x018A)#define RMI_ABI_MAJOR_VERSION 1#define RMI_ABI_MINOR_VERSION 0static void
cca_tsm_remove(void *tsm_core)drivers/virt/coco/arm-cca-host/rmm-da.c +16
-0+void rme_unbind_vdev (struct realm *realm,struct pci_dev *pdev,struct pci_dev *pf0_dev) +{ +int ret; +phys_addr_t rmm_pdev_phys; +phys_addr_t rmm_vdev_phys; +struct cca_host_dsc_pf0 *dsc_pf0; +struct cca_host_tdi *host_tdi; +phys_addr_t rd_phys =virt_to_phys (realm->rd); + + host_tdi = container_of(pdev->tsm->tdi, struct cca_host_tdi, tdi); + rmm_vdev_phys =virt_to_phys (host_tdi->rmm_vdev); + + dsc_pf0 =to_cca_dsc_pf0 (pf0_dev); + rmm_pdev_phys =virt_to_phys (dsc_pf0->rmm_pdev); +/* Request stopping the VDEV */ + ret =rmi_vdev_stop (rmm_vdev_phys);
drivers/virt/coco/arm-cca-host/rmm-da.c void *rme_create_vdev(struct realm *realm, struct pci_dev *pdev, · line 660
Quoted 10 lines
drivers/virt/coco/arm-cca-host/rmm-da.c +10
-0+if (ret) { +pr_err ("failed to stop vdev (%d)\n" , ret); +return ; + } + +schedule_vdev_unbind (pdev); + ret =rmi_vdev_destroy (rd_phys, rmm_pdev_phys, rmm_vdev_phys); +if (ret) { +pr_err ("failed to destroy vdev (%d)\n" , ret); +return ;
No point in returning here. Maybe fine to keep this if more code is coming after this in future patches.
Quoted 2 lines
drivers/virt/coco/arm-cca-host/rmm-da.c +2
-0+ } +}
Re: [RFC PATCH v1 23/38] coco: guest: arm64: Update arm CCA guest driver
In reply to Aneesh Kumar K.V (Arm)Quoted 11 lines · On Mon, 28 Jul 2025 19:22:00 +0530 "Aneesh Kumar K.V (Arm)"…
This patch includes renaming changes to simplify the registration of a TSM backend in the next patch. There are no functional changes in this update.
Signed-off-by Aneesh Kumar K.V (Arm) <[email protected]>index
0c9ea24a200c..547fc2c79f7d100644drivers/virt/coco/arm-cca-guest/arm-cca.c renamed from drivers/virt/coco/arm-cca-guest/arm-cca-guest.c +1
-0+static int cca_guest_probe (struct platform_device *pdev)
...
drivers/virt/coco/arm-cca-guest/arm-cca.c static int arm_cca_report_new(struct tsm_report *report, void *data) · line 208
Quoted 10 lines
drivers/virt/coco/arm-cca-guest/arm-cca.c renamed from drivers/virt/coco/arm-cca-guest/arm-cca-guest.c +2
-7+ ret =devm_add_action_or_reset (&pdev->dev, unregister_cca_tsm_report,NULL ); -/** -* arm_cca_guest_exit - unregister with the Trusted Security Module (TSM) -* interface. -*/ -static void __exit arm_cca_guest_exit(void) -{ -tsm_report_unregister (&arm_cca_tsm_ops); +return ret;
return devm_add_action_or_reset()
Mind you, Jason probably won't like this ;)
drivers/virt/coco/arm-cca-guest/arm-cca.c static int arm_cca_report_new(struct tsm_report *report, void *data) · line 227
Quoted 6 lines
}
MODULE_AUTHOR("Sami Mujawar <[email protected]>");drivers/virt/coco/arm-cca-guest/arm-cca.c renamed from drivers/virt/coco/arm-cca-guest/arm-cca-guest.c +1
-1-MODULE_DESCRIPTION ("Arm CCA Guest TSM Driver" ); +MODULE_DESCRIPTION ("Arm CCA Guest TSM driver" );
Is this D/d worth the noise?
Quoted 1 line
MODULE_LICENSE("GPL");
Re: [RFC PATCH v1 24/38] arm64: CCA: Register guest tsm callback
In reply to Aneesh Kumar K.V (Arm)Quoted 6 lines · On Mon, 28 Jul 2025 19:22:01 +0530 "Aneesh Kumar K.V (Arm)"…
Register the TSM callback if the DA feature is supported by RSI.
Additionally, adjust the build order so that the TSM class is created before the arm-cca-guest driver initialization.
Signed-off-by Aneesh Kumar K.V (Arm) <[email protected]>
See below.
Quoted 13 lines
arch/arm64/kernel/rsi.c +1
-0@@ -15,6 +15,7 @@#include <asm/rsi.h> static struct realm_config config; +static unsigned long rsi_feat_reg0;unsigned long prot_ns_shared;EXPORT_SYMBOL (prot_ns_shared);
arch/arm64/kernel/rsi.c EXPORT_SYMBOL(prot_ns_shared); · line 28
Quoted 6 lines
arch/arm64/kernel/rsi.c +3
-0@@ -22,6 +23,12 @@ EXPORT_SYMBOL(prot_ns_shared);DEFINE_STATIC_KEY_FALSE_RO (rsi_present);EXPORT_SYMBOL (rsi_present); +bool rsi_has_da_feature (void ) +{ +return !!u64_get_bits (rsi_feat_reg0,RSI_FEATURE_REGISTER_0_DA );
!! not needed.
Quoted 22 lines
arch/arm64/kernel/rsi.c +2
-0+} +EXPORT_SYMBOL_GPL (rsi_has_da_feature);drivers/virt/coco/arm-cca-guest/arm-cca.c +3
-1@@ -1,6 +1,6 @@static int cca_guest_probe (struct platform_device *pdev) {int ret;@@ -200,11 +256,22 @@ static int cca_guest_probe(struct platform_device *pdev)return -ENODEV ; ret =tsm_report_register (&arm_cca_tsm_report_ops,NULL ); -if (ret <0 ) +if (ret <0 ) {pr_err ("Error %d registering with TSM\n" , ret); + goto err_out; + }
drivers/virt/coco/arm-cca-guest/arm-cca.c static int cca_guest_probe(struct platform_device *pdev) · line 272
Quoted 10 lines
drivers/virt/coco/arm-cca-guest/arm-cca.c +8
-0ret =devm_add_action_or_reset (&pdev->dev, unregister_cca_tsm_report,NULL ); +if (ret <0 ) { +pr_err ("Error %d registering devm action\n" , ret); +unregister_cca_tsm_report (NULL ); + goto err_out; + } + +if (rsi_has_da_feature ()) + ret =cca_tsm_register (pdev);
Why do we not need to call unregister_cca_tsm_report() if this fails?
drivers/virt/coco/arm-cca-guest/arm-cca.c static int cca_guest_probe(struct platform_device *pdev) · line 274
Quoted 2 lines
drivers/virt/coco/arm-cca-guest/arm-cca.c +1
-0+err_out:
I'd just return above.
Quoted 11 lines
drivers/virt/coco/arm-cca-guest/arm-cca.c +0
-0return ret; }drivers/virt/coco/arm-cca-guest/rsi-da.h new file +2
-0@@ -0,0 +1,27 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/*
drivers/virt/coco/arm-cca-guest/rsi-da.h line 12
Quoted 10 lines
drivers/virt/coco/arm-cca-guest/rsi-da.h new file +10
-0+* Copyright (C) 2025 ARM Ltd. +*/ + +#ifndef RSI_DA_H_ +#define RSI_DA_H_ + +#include <linux/pci.h> +#include <linux/pci-tsm.h> +#include <asm/rsi_smc.h> +
One blank line probably enough.
Quoted 15 lines
drivers/virt/coco/arm-cca-guest/rsi-da.h new file +15
-0+ +struct cca_guest_dsc { +struct pci_tsm_pf0 pci; +}; + +static inline struct cca_guest_dsc *to_cca_guest_dsc (struct pci_dev *pdev) +{ +struct pci_tsm *tsm = pdev->tsm; + +if (!tsm) +return NULL ; + return container_of(tsm, struct cca_guest_dsc, pci.tsm); +} + +#endif
Re: [RFC PATCH v1 25/38] cca: guest: arm64: Realm device lock support
In reply to Aneesh Kumar K.V (Arm)Quoted 18 lines · On Mon, 28 Jul 2025 19:22:02 +0530 "Aneesh Kumar K.V (Arm)"…
Writing 1 to 'tsm/lock' will initiate the TDISP lock sequence.
Signed-off-by Aneesh Kumar K.V (Arm) <[email protected]>--- arch/arm64/include/asm/rsi_cmds.h | 32 +++++++++++++- arch/arm64/include/asm/rsi_smc.h | 5 +++ drivers/virt/coco/arm-cca-guest/Makefile | 2 +- drivers/virt/coco/arm-cca-guest/arm-cca.c | 18 ++++++++ drivers/virt/coco/arm-cca-guest/rsi-da.c | 52 +++++++++++++++++++++++ drivers/virt/coco/arm-cca-guest/rsi-da.h | 3 +- 6 files changed, 108 insertions(+), 4 deletions(-) create mode 100644 drivers/virt/coco/arm-cca-guest/rsi-da.carch/arm64/include/asm/rsi_cmds.h +0
-0
arch/arm64/include/asm/rsi_cmds.h static inline int rsi_features(unsigned long index, unsigned long *out) · line 176
Quoted 7 lines
arch/arm64/include/asm/rsi_cmds.h +2
-2@@ -172,8 +172,36 @@ static inline int rsi_features(unsigned long index, unsigned long *out)arm_smccc_1_1_invoke (SMC_RSI_FEATURES , index, &res); -if (out) - *out = res.a1; + *out = res.a1; +return res.a0;
Seems unrelated change.
Quoted 20 lines
arch/arm64/include/asm/rsi_cmds.h +2
-0+} +drivers/virt/coco/arm-cca-guest/rsi-da.c new file +10
-0@@ -0,0 +1,52 @@ +int rsi_device_lock (struct pci_dev *pdev) +{ +unsigned long ret; +struct cca_guest_dsc *dsm =to_cca_guest_dsc (pdev); +int vdev_id = (pci_domain_nr (pdev->bus) <<16 ) | +PCI_DEVID (pdev->bus->number, pdev->devfn); + + ret =rsi_rdev_get_instance_id (vdev_id, &dsm->instance_id); +if (ret !=RSI_SUCCESS ) { +pci_err (pdev,"failed to get the device instance id (%lu)\n" , ret);
drivers/virt/coco/arm-cca-guest/rsi-da.c line 51
Quoted 10 lines
drivers/virt/coco/arm-cca-guest/rsi-da.c new file +10
-0+return -EIO ; + } + + ret =rsi_rdev_lock (pdev, vdev_id, dsm->instance_id); +if (ret !=RSI_SUCCESS ) { +pci_err (pdev,"failed to lock the device (%lu)\n" , ret); +return -EIO ; + } + +return ret;
return 0?
You carefully overwrite other error codes. I assume RSI_SUCCESS == 0 but even better to just return 0 directly in the good path.
drivers/virt/coco/arm-cca-guest/rsi-da.h line 13
Quoted 10 lines
drivers/virt/coco/arm-cca-guest/rsi-da.c new file +1
-0+}drivers/virt/coco/arm-cca-guest/rsi-da.h +0
-1@@ -10,9 +10,9 @@#include <linux/pci-tsm.h> #include <asm/rsi_smc.h> -
Push back to earlier patch where I comment on this.
Quoted 12 lines
drivers/virt/coco/arm-cca-guest/rsi-da.h +2
-0struct cca_guest_dsc {struct pci_tsm_pf0 pci; +unsigned long instance_id; };static inline struct cca_guest_dsc *to_cca_guest_dsc (struct pci_dev *pdev)@@ -24,4 +24,5 @@ static inline struct cca_guest_dsc *to_cca_guest_dsc(struct pci_dev *pdev) return container_of(tsm, struct cca_guest_dsc, pci.tsm); } +int rsi_device_lock (struct pci_dev *pdev);#endif
Re: [RFC PATCH v1 26/38] KVM: arm64: Add exit handler related to device assignment
In reply to Aneesh Kumar K.V (Arm)Quoted 19 lines · On Mon, 28 Jul 2025 19:22:03 +0530 "Aneesh Kumar K.V (Arm)"…
Different RSI calls related to DA result in REC exits. Add a facility to register handlers for handling these REC exits.
Signed-off-by Aneesh Kumar K.V (Arm) <[email protected]>arch/arm64/include/asm/rmi_smc.h +0
-0arch/arm64/kvm/rme-exit.c +2
-0+static int rec_exit_dev_mem_map (struct kvm_vcpu *vcpu) +{
arch/arm64/kvm/rme-exit.c static int rec_exit_host_call(struct kvm_vcpu *vcpu) · line 181
Quoted 10 lines
arch/arm64/kvm/rme-exit.c +10
-0+int ret; +struct realm_rec *rec = &vcpu->arch.rec; + +if (realm_exit_dev_mem_map_handler) { + ret = (*realm_exit_dev_mem_map_handler)(rec); + }else { +kvm_pr_unimpl ("Unsupported exit reason: %u\n" , + rec->run->exit.exit_reason); + vcpu->run->exit_reason =KVM_EXIT_INTERNAL_ERROR ; + ret =0 ;
I guess maybe this gets more complex later, but right now
return 0;
And maybe return in the if branch as well.
Same for other similar code in this patch.
Quoted 20 lines
arch/arm64/kvm/rme-exit.c +10
-0+ } +return ret; +} +static void update_arch_timer_irq_lines (struct kvm_vcpu *vcpu) {struct realm_rec *rec = &vcpu->arch.rec;@@ -198,6 +252,12 @@ int handle_rec_exit(struct kvm_vcpu *vcpu, int rec_run_ret)return rec_exit_ripas_change (vcpu);case RMI_EXIT_HOST_CALL :return rec_exit_host_call (vcpu); +case RMI_EXIT_VDEV_REQUEST : +return rec_exit_vdev_request (vcpu); +case RMI_EXIT_VDEV_COMM : +return rec_exit_vdev_communication (vcpu); +case RMI_EXIT_DEV_MEM_MAP : +return rec_exit_dev_mem_map (vcpu); } kvm_pr_unimpl("Unsupported exit reason: %u\n",
Re: [RFC PATCH v1 30/38] coco: host: arm64: Add support for realm host interface (RHI)
In reply to Aneesh Kumar K.V (Arm)Quoted 4 lines · On Mon, 28 Jul 2025 19:22:07 +0530 "Aneesh Kumar K.V (Arm)"…
Device assignment-related RHI calls result in a REC exit, which is handled by the tsm
guest_requestcallback.Signed-off-by Aneesh Kumar K.V (Arm) <[email protected]>
Comments below.
Quoted 28 lines
--- arch/arm64/include/asm/rhi.h | 32 ++++++++++ drivers/virt/coco/arm-cca-host/arm-cca.c | 68 ++++++++++++++++++++ drivers/virt/coco/arm-cca-host/rmm-da.c | 81 ++++++++++++++++++++++++ drivers/virt/coco/arm-cca-host/rmm-da.h | 4 ++ include/linux/tsm.h | 3 + 5 files changed, 188 insertions(+) create mode 100644 arch/arm64/include/asm/rhi.hdrivers/virt/coco/arm-cca-host/arm-cca.c +14
-0+static int cca_tsm_guest_req (struct pci_dev *pdev,struct tsm_guest_req_info *info) +{ +int ret; + +switch (info->type) { +case ARM_CCA_DA_OBJECT_SIZE : + { +int object_size; +struct da_object_size_req req; + +if (sizeof (req) != info->req_len) +return -EINVAL ; + +if (copy_from_user (&req, info->req, info->req_len))
drivers/virt/coco/arm-cca-host/arm-cca.c static void cca_tsm_unbind(struct pci_tdi *tdi) · line 295
Quoted 10 lines
drivers/virt/coco/arm-cca-host/arm-cca.c +10
-0+return -EFAULT ; + + object_size =rme_get_da_object_size (pdev, req.object_type); +if (object_size >0 ) { +if (info->resp_len <sizeof (object_size)) +return -EINVAL ; +if (copy_to_user (info->resp, &object_size,sizeof (object_size))) +return -EFAULT ; + info->resp_len =sizeof (object_size); + ret =0 ;
return 0;
then else isn't needed.
drivers/virt/coco/arm-cca-host/arm-cca.c
Quoted 1 line
drivers/virt/coco/arm-cca-host/arm-cca.c +1
-0+ }else
#Style is {} on all legs if any need it.
drivers/virt/coco/arm-cca-host/arm-cca.c static void cca_tsm_unbind(struct pci_tdi *tdi) · line 298
Quoted 2 lines
drivers/virt/coco/arm-cca-host/arm-cca.c +2
-0+/* error */ + ret = object_size;
return object_size;
Quoted 9 lines
drivers/virt/coco/arm-cca-host/arm-cca.c +9
-0+break ; + } +case ARM_CCA_DA_OBJECT_READ : + { +int resp_len; +struct da_object_read_req req; + +if (sizeof (req) != info->req_len) +return -EINVAL ;
drivers/virt/coco/arm-cca-host/arm-cca.c static void cca_tsm_unbind(struct pci_tdi *tdi) · line 317
Quoted 10 lines
drivers/virt/coco/arm-cca-host/arm-cca.c +10
-0+ +if (copy_from_user (&req, info->req, info->req_len)) +return -EFAULT ; + + resp_len =rme_da_object_read (pdev, req.object_type, req.offset, + info->resp_len, + info->resp); +if (resp_len >0 ) { + info->resp_len = resp_len; + ret =0 ;
Similar to above.
drivers/virt/coco/arm-cca-host/arm-cca.c static void cca_tsm_unbind(struct pci_tdi *tdi) · line 327
Quoted 10 lines
drivers/virt/coco/arm-cca-host/arm-cca.c +10
-0+ }else +/* error */ + ret = resp_len; +break ; + } +default : + ret = -EINVAL ; +break ; + } +return ret;
As you've probably figured out, I love an early return. This sort of function shows why as it reduced indent etc in lots of places. Here you mix and match. Maybe it will make sense later in the series though!
Quoted 38 lines
drivers/virt/coco/arm-cca-host/arm-cca.c +1
-0+}drivers/virt/coco/arm-cca-host/rmm-da.c +20
-0@@ -10,7 +10,9 @@#include <keys/asymmetric-type.h> #include <keys/x509-parser.h> #include <linux/kvm_types.h> +#include <linux/kvm_host.h> #include <asm/kvm_rme.h> +#include <asm/kvm_emulate.h> #include "rmm-da.h" @@ -769,6 +771,85 @@ static int rme_exit_vdev_req_handler(struct realm_rec *rec)return 1 ; } +int rme_get_da_object_size (struct pci_dev *pdev,int type) +{ +int ret =0 ; +unsigned long len; +struct pci_tsm *tsm = pdev->tsm; +struct cca_host_dsc_pf0 *dsc_pf0; + +if (!tsm) +return -EINVAL ; + + dsc_pf0 =to_cca_dsc_pf0 (tsm->dsm_dev); + +/* Determine the buffer that should be used */ +if (type ==RHI_DA_OBJECT_INTERFACE_REPORT ) { + len = dsc_pf0->interface_report.size; + }else if (type ==RHI_DA_OBJECT_MEASUREMENT ) { + len = dsc_pf0->measurements.size; + }else if (type ==RHI_DA_OBJECT_CERTIFICATE ) {
drivers/virt/coco/arm-cca-host/rmm-da.c static int rme_exit_vdev_req_handler(struct realm_rec *rec) · line 801
Quoted 10 lines
drivers/virt/coco/arm-cca-host/rmm-da.c +10
-0+ len = dsc_pf0->cert_chain.cache.size; + }else if (type ==RHI_DA_OBJECT_VCA ) { + len = dsc_pf0->vca.size; + }else { + ret = -EINVAL ; + goto err_out; + } + +return len; +err_out:
Similar to below. This pattern just makes things more complex. If we need to introduce a label, do it in the patch where you add code to do something on error.
Quoted 38 lines
drivers/virt/coco/arm-cca-host/rmm-da.c +38
-0+return ret; +} + +int rme_da_object_read (struct pci_dev *pdev,int type,unsigned long offset, +unsigned long max_len,void __user *user_buf) +{ +void *buf; +int ret =0 ; +unsigned long len; +struct cca_host_dsc_pf0 *dsc_pf0; +struct pci_tsm *tsm = pdev->tsm; + +if (!tsm) +return -EINVAL ; + + dsc_pf0 =to_cca_dsc_pf0 (tsm->dsm_dev); + +/* Determine the buffer that should be used */ +if (type ==RHI_DA_OBJECT_INTERFACE_REPORT ) { + len = dsc_pf0->interface_report.size; + buf = dsc_pf0->interface_report.buf; + }else if (type ==RHI_DA_OBJECT_MEASUREMENT ) { + len = dsc_pf0->measurements.size; + buf = dsc_pf0->measurements.buf; + }else if (type ==RHI_DA_OBJECT_CERTIFICATE ) { + len = dsc_pf0->cert_chain.cache.size; + buf = dsc_pf0->cert_chain.cache.buf; + }else if (type ==RHI_DA_OBJECT_VCA ) { + len = dsc_pf0->vca.size; + buf = dsc_pf0->vca.buf; + }else { + ret = -EINVAL ; + goto err_out; + } + +/* Assume that the buffer is large enough for the whole report */ +if ((max_len - offset) < len) { +/* FIXME!! the error code */
drivers/virt/coco/arm-cca-host/rmm-da.c static int rme_exit_vdev_req_handler(struct realm_rec *rec) · line 849
Quoted 10 lines
drivers/virt/coco/arm-cca-host/rmm-da.c +10
-0+ ret = -ENOMEM ; + goto err_out; + } + +if (copy_to_user (user_buf + offset, buf, len)) { + ret = -EIO ; + goto err_out; + } + ret = len; +err_out:
Definitely makes sense to just return directly in the error paths above and just return len here
Quoted 39 lines
drivers/virt/coco/arm-cca-host/rmm-da.c +3
-0+return ret; +} +void rme_register_exit_handlers (void ) { realm_exit_vdev_comm_handler = rme_exit_vdev_comm_handler;drivers/virt/coco/arm-cca-host/rmm-da.h +4
-0@@ -10,6 +10,7 @@#include <linux/pci-ide.h> #include <linux/pci-tsm.h> #include <asm/rmi_smc.h> +#include <asm/rhi.h> #define MAX_CACHE_OBJ_SIZE 4096struct cache_object {@@ -101,4 +102,7 @@ void *rme_create_vdev(struct realm *realm, struct pci_dev *pdev,void rme_unbind_vdev (struct realm *realm,struct pci_dev *pdev,struct pci_dev *pf0_dev);void rme_register_exit_handlers (void ); +int rme_get_da_object_size (struct pci_dev *pdev,int type); +int rme_da_object_read (struct pci_dev *pdev,int type,unsigned long offset, +unsigned long max_len,void __user *user_buf);#endif include/linux/tsm.h +3
-0@@ -145,5 +145,8 @@ struct tsm_guest_req_info { size_t resp_len; }; +#define ARM_CCA_DA_OBJECT_SIZE 0x1 +#define ARM_CCA_DA_OBJECT_READ 0x2 +int tsm_guest_req (struct device *dev,struct tsm_guest_req_info *info);#endif /* __TSM_H */
Re: [RFC PATCH v1 31/38] coco: guest: arm64: Add support for fetching interface report and certificate chain from host
In reply to Aneesh Kumar K.V (Arm)Quoted 3 lines · On Mon, 28 Jul 2025 19:22:08 +0530 "Aneesh Kumar K.V (Arm)"…
Fetch interface report and certificate chain from the host using RHI calls.
Signed-off-by Aneesh Kumar K.V (Arm) <[email protected]>
Comments inline
Quoted 24 lines
drivers/virt/coco/arm-cca-guest/rsi-da.c +9
-0@@ -4,6 +4,7 @@*/ #include <linux/pci.h> +#include <linux/mem_encrypt.h> #include <asm/rsi_cmds.h> #include "rsi-da.h" @@ -50,6 +51,121 @@ rsi_rdev_get_interface_report(struct pci_dev *pdev, unsigned long vdev_id,return RSI_SUCCESS ; } +static long rhi_get_report (int vdev_id,int da_object_type,void **report,int *report_size) +{ +int ret, enc_ret =0 ; +int nr_pages; +int max_data_len; +void *data_buf_shared, *data_buf_private; +struct rsi_host_call *rhicall; +
drivers/virt/coco/arm-cca-guest/rsi-da.c rsi_rdev_get_interface_report(struct pci_dev *pdev, unsigned long vdev_id, · line 71
Quoted 10 lines
drivers/virt/coco/arm-cca-guest/rsi-da.c +10
-0+ rhicall =kmalloc (sizeof (struct rsi_host_call ),GFP_KERNEL ); +if (!rhicall) +return -ENOMEM ; + + rhicall->imm =0 ; + rhicall->gprs[0 ] =RHI_DA_FEATURES ; + + ret =rsi_host_call (virt_to_phys (rhicall)); +if (ret !=RSI_SUCCESS ) { + ret = -EIO ;
Extra space.
Quoted 34 lines
drivers/virt/coco/arm-cca-guest/rsi-da.c +34
-0+ goto err_out; + } + +if (rhicall->gprs[0 ] !=0x3 ) { + ret = -EIO ; + goto err_out; + } + + rhicall->imm =0 ; + rhicall->gprs[0 ] =RHI_DA_OBJECT_SIZE ; + rhicall->gprs[1 ] = vdev_id; + rhicall->gprs[2 ] = da_object_type; + + ret =rsi_host_call (virt_to_phys (rhicall)); +if (ret !=RSI_SUCCESS ) { + ret = -EIO ; + goto err_out; + } +if (rhicall->gprs[0 ] !=RHI_DA_SUCCESS ) { + ret = -EIO ; + goto err_out; + } + max_data_len = rhicall->gprs[1 ]; + *report_size = max_data_len; + +/* +* We need to share this memory with hypervisor. +* So it should be multiple of sharing unit. +*/ + max_data_len =ALIGN (max_data_len,PAGE_SIZE ); + nr_pages = max_data_len >>PAGE_SHIFT ; + +if (!max_data_len || nr_pages >MAX_ORDER_NR_PAGES ) { + ret = -ENOMEM ;
drivers/virt/coco/arm-cca-guest/rsi-da.c rsi_rdev_get_interface_report(struct pci_dev *pdev, unsigned long vdev_id, · line 115
Quoted 10 lines
drivers/virt/coco/arm-cca-guest/rsi-da.c +10
-0+ goto err_out; + } + +/* +* We need to share this memory with hypervisor. +* So it should be multiple of sharing unit. +*/ + data_buf_shared = (void *)__get_free_pages (GFP_KERNEL ,get_order (max_data_len)); +if (!data_buf_shared) { + ret = -ENOMEM ;
extra space. All of these seem to have one. Not seeing a reason for it though.
Quoted 37 lines
drivers/virt/coco/arm-cca-guest/rsi-da.c +37
-0+ goto err_out; + } + + data_buf_private =kmalloc (*report_size,GFP_KERNEL ); +if (!data_buf_private) { + ret = -ENOMEM ; + goto err_private_alloc; + } + + ret =set_memory_decrypted ((unsigned long )data_buf_shared, nr_pages); +if (ret) { + ret = -EIO ; + goto err_decrypt; + } + + rhicall->imm =0 ; + rhicall->gprs[0 ] =RHI_DA_OBJECT_READ ; + rhicall->gprs[1 ] = vdev_id; + rhicall->gprs[2 ] = da_object_type; + rhicall->gprs[3 ] =0 ;/* offset within the data buffer */ + rhicall->gprs[4 ] = max_data_len; + rhicall->gprs[5 ] =virt_to_phys (data_buf_shared); + ret =rsi_host_call (virt_to_phys (rhicall)); +if (ret !=RSI_SUCCESS || rhicall->gprs[0 ] !=RHI_DA_SUCCESS ) { + ret = -EIO ; + goto err_rhi_call; + } + +memcpy (data_buf_private, data_buf_shared, *report_size); + enc_ret =set_memory_encrypted ((unsigned long )data_buf_shared, nr_pages); +if (!enc_ret) +/* If we fail to mark it encrypted don't free it back */ +free_pages ((unsigned long )data_buf_shared,get_order (max_data_len)); + + *report = data_buf_private; +kfree (rhicall); +return 0 ;
drivers/virt/coco/arm-cca-guest/rsi-da.c rsi_rdev_get_interface_report(struct pci_dev *pdev, unsigned long vdev_id, · line 162
Quoted 10 lines
drivers/virt/coco/arm-cca-guest/rsi-da.c +10
-0+ +err_rhi_call: + enc_ret =set_memory_encrypted ((unsigned long )data_buf_shared, nr_pages); +err_decrypt: +kfree (data_buf_private); +err_private_alloc: +if (!enc_ret) +/* If we fail to mark it encrypted don't free it back */ +free_pages ((unsigned long )data_buf_shared,get_order (max_data_len)); +err_out:
I'd expect there to be nothing to do except return under an err_out label So rename it.
Quoted 20 lines
drivers/virt/coco/arm-cca-guest/rsi-da.c +13
-0+ *report =NULL ; + *report_size =0 ; +kfree (rhicall); +return ret; +} +int rsi_device_lock (struct pci_dev *pdev) {unsigned long ret;@@ -82,5 +198,20 @@ int rsi_device_lock(struct pci_dev *pdev)return -EOPNOTSUPP ; } +/* Now make a host call to copy the interface report to guest. */ + ret =rhi_get_report (vdev_id,RHI_DA_OBJECT_INTERFACE_REPORT , + &dsm->interface_report, &dsm->interface_report_size); +if (ret) { +pci_err (pdev,"failed to get interface report from the host (%lu)\n" , ret); +return -EIO ; + }
drivers/virt/coco/arm-cca-guest/rsi-da.c int rsi_device_lock(struct pci_dev *pdev) · line 217
Quoted 10 lines
drivers/virt/coco/arm-cca-guest/rsi-da.c +8
-0+ + ret =rhi_get_report (vdev_id,RHI_DA_OBJECT_CERTIFICATE , + &dsm->certificate, &dsm->certificate_size); +if (ret) { +pci_err (pdev,"failed to get device certificate from the host (%lu)\n" , ret); +return -EIO ; + } +return ret;
return 0;
Quoted 1 line
}
Re: [RFC PATCH v1 32/38] coco: guest: arm64: Add support for guest initiated TDI bind/unbind
In reply to Aneesh Kumar K.V (Arm)Quoted 6 lines · On Mon, 28 Jul 2025 19:22:09 +0530 "Aneesh Kumar K.V (Arm)"…
Add RHI for
VDEV_SET_TDI_STATENote: This is not part of RHI spec. This is a POC implementation and will be later switced to correct interface defined by RHI.
Signed-off-by Aneesh Kumar K.V (Arm) <[email protected]>
A few minor comments.
Maybe we need some discussion of how this is used.
Quoted 14 lines
arch/arm64/kernel/rhi.c new file +8
-0@@ -0,0 +1,35 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* +* Copyright (C) 2025 ARM Ltd. +*/ + +#include <asm/memory.h> +#include <asm/string.h> +#include <asm/rsi.h>
arch/arm64/kernel/rhi.c line 18
Quoted 10 lines
arch/arm64/kernel/rhi.c new file +10
-0+#include <asm/rhi.h> + +#include <linux/slab.h> + +long rhi_da_vdev_set_tdi_state (unsigned long guest_rid,unsigned long target_state) +{ +long ret; +struct rsi_host_call *rhi_call; + + rhi_call =kmalloc (sizeof (struct rsi_host_call ),GFP_KERNEL );
struct rsi_host_call *rhi_call __free(kfree) = kmalloc(sizeof(*rhi_call), GFP_KERNEL);
then direct returns on errors.
Quoted 22 lines
arch/arm64/kernel/rhi.c new file +17
-0+if (!rhi_call) +return -ENOMEM ; + + rhi_call->imm =0 ; + rhi_call->gprs[0 ] =RHI_DA_VDEV_SET_TDI_STATE ; + rhi_call->gprs[1 ] = guest_rid; + rhi_call->gprs[2 ] = target_state; + + ret =rsi_host_call (virt_to_phys (rhi_call)); +if (ret !=RSI_SUCCESS ) + ret = -EIO ; +else + ret = rhi_call->gprs[0 ]; + +kfree (rhi_call); +return ret; +}drivers/virt/coco/arm-cca-guest/arm-cca.c +0
-0
drivers/virt/coco/arm-cca-guest/arm-cca.c static void cca_tsm_pci_remove(struct pci_tsm *tsm) · line 225
Quoted 5 lines
drivers/virt/coco/arm-cca-guest/arm-cca.c +1
-1@@ -222,11 +222,20 @@ static void cca_tsm_pci_remove(struct pci_tsm *tsm)static int cca_tsm_lock (struct pci_dev *pdev) { -unsigned long ret; +long ret;
Push this earlier. It doesn't do any harm that I can see and will reduce churn
Quoted 27 lines
drivers/virt/coco/arm-cca-guest/arm-cca.c +10
-1+int vdev_id = (pci_domain_nr (pdev->bus) <<16 ) | +PCI_DEVID (pdev->bus->number, pdev->devfn); + ret =rhi_da_vdev_set_tdi_state (vdev_id,RHI_DA_TDI_CONFIG_LOCKED ); +if (ret) { +pci_err (pdev,"failed to TSM bind the device (%ld)\n" , ret); +return -EIO ; + } + +/* This will be done by above rhi in later spec */ ret =rsi_device_lock (pdev);if (ret) { -pci_err (pdev,"failed to lock the device (%lu)\n" , ret); +pci_err (pdev,"failed to lock the device (%ld)\n" , ret);return -EIO ; }return 0 ;static const struct
pci_tsm_opscca_pci_ops= {drivers/virt/coco/arm-cca-host/arm-cca.c +0
-0@@ -254,9 +254,13 @@ static struct pci_tdi *cca_tsm_bind(struct pci_dev *pdev, struct pci_dev *pf0_destatic void cca_tsm_unbind (struct pci_tdi *tdi) {
drivers/virt/coco/arm-cca-host/arm-cca.c static struct pci_tdi *cca_tsm_bind(struct pci_dev *pdev, struct pci_dev *pf0_de · line 259
Quoted 10 lines
drivers/virt/coco/arm-cca-host/arm-cca.c +6
-2struct realm *realm = &tdi->kvm->arch.realm; - +/* +* FIXME!! +* All the related DEV RIPAS regions should be unmapped by now. +* For now we handle them during stage2 teardown. There is no +* bound IPA address available here. Possibly dmabuf can help +*/ rme_unbind_vdev (realm, tdi->pdev, tdi->pdev->tsm->dsm_dev); -
Tidy up these whitespace changes. Just adds noise.
Quoted 3 lines
drivers/virt/coco/arm-cca-host/arm-cca.c +0
-0module_put (THIS_MODULE ); }
Re: [RFC PATCH v1 34/38] coco: guest: arm64: Validate mmio range found in the interface report
In reply to Aneesh Kumar K.V (Arm)Quoted 4 lines · On Mon, 28 Jul 2025 19:22:11 +0530 "Aneesh Kumar K.V (Arm)"…
This starts the sequence to transition the realm device to the TDISP RUN state by writing 1 to 'tsm/accept'.
Signed-off-by Aneesh Kumar K.V (Arm) <[email protected]>
Just some trivial stuff.
Quoted 15 lines
drivers/virt/coco/arm-cca-guest/rsi-da.c +6
-0@@ -215,3 +215,135 @@ int rsi_device_lock(struct pci_dev *pdev)return ret; } + +int rsi_update_interface_report (struct pci_dev *pdev,bool validate) +{ +int ret; +struct resource *r; +int msix_tbl_bar, msix_pba_bar;
drivers/virt/coco/arm-cca-guest/rsi-da.c int rsi_device_lock(struct pci_dev *pdev) · line 274
Quoted 10 lines
drivers/virt/coco/arm-cca-guest/rsi-da.c +10
-0+unsigned int range_id; +unsigned long mmio_start_phys; +unsigned long mmio_flags =0 ;/* non coherent, not limited order */ +struct pci_tdisp_mmio_range *mmio_range; +struct pci_tdisp_device_interface_report *interface_report; +struct cca_guest_dsc *dsm =to_cca_guest_dsc (pdev); +int vdev_id = (pci_domain_nr (pdev->bus) <<16 ) | +PCI_DEVID (pdev->bus->number, pdev->devfn); + +
Single line.
drivers/virt/coco/arm-cca-guest/rsi-da.c int rsi_device_lock(struct pci_dev *pdev) · line 278
Quoted 4 lines
drivers/virt/coco/arm-cca-guest/rsi-da.c +4
-0+ interface_report = (struct pci_tdisp_device_interface_report *)dsm->interface_report; + mmio_range = (struct pci_tdisp_mmio_range *)(interface_report +1 ); + +
Single line
drivers/virt/coco/arm-cca-guest/rsi-da.c int rsi_device_lock(struct pci_dev *pdev) · line 288
Quoted 10 lines
drivers/virt/coco/arm-cca-guest/rsi-da.c +10
-0+ msix_tbl_bar =get_msix_bar (pdev,PCI_MSIX_TABLE ); + msix_pba_bar =get_msix_bar (pdev,PCI_MSIX_PBA ); + +for (int i =0 ; i < interface_report->mmio_range_count; i++, mmio_range++) { + +/*FIXME!! units in 4K size*/ + range_id =FIELD_GET (TSM_INTF_REPORT_MMIO_RANGE_ID , mmio_range->range_attributes); + +/* no secure interrupts */ +if (msix_tbl_bar !=-1 && range_id == msix_tbl_bar) {
That first condition can get hiked out of the loop.
drivers/virt/coco/arm-cca-guest/rsi-da.c int rsi_device_lock(struct pci_dev *pdev) · line 293
Quoted 5 lines
drivers/virt/coco/arm-cca-guest/rsi-da.c +5
-0+pr_info ("Skipping misx table\n" ); +continue ; + } + +if (msix_pba_bar !=-1 && range_id == msix_pba_bar) {
Likewise.
drivers/virt/coco/arm-cca-guest/rsi-da.c int rsi_device_lock(struct pci_dev *pdev) · line 301
Quoted 8 lines
drivers/virt/coco/arm-cca-guest/rsi-da.c +8
-0+pr_info ("Skipping misx pba\n" ); +continue ; + } + + r =pci_resource_n (pdev, range_id); + +if (r->end == r->start || + ((r->end - r->start +1 ) & ~PAGE_MASK ) || !mmio_range->num_pages) {
resource_size()
Quoted 35 lines
drivers/virt/coco/arm-cca-guest/rsi-da.c +35
-0+pci_warn (pdev,"Skipping broken range [%d] #%d %d pages, %llx..%llx\n" , + i, range_id, mmio_range->num_pages, r->start, r->end); +continue ; + } + +if (FIELD_GET (TSM_INTF_REPORT_MMIO_IS_NON_TEE , mmio_range->range_attributes)) { +pci_info (pdev,"Skipping non-TEE range [%d] #%d %d pages, %llx..%llx\n" , + i, range_id, mmio_range->num_pages, r->start, r->end); +continue ; + } + +/* No secure interrupts, we should not find this set, ignore for now. */ +if (FIELD_GET (TSM_INTF_REPORT_MMIO_MSIX_TABLE , mmio_range->range_attributes) || +FIELD_GET (TSM_INTF_REPORT_MMIO_PBA , mmio_range->range_attributes)) { +pci_info (pdev,"Skipping MSIX (%ld/%ld) range [%d] #%d %d pages, %llx..%llx\n" , +FIELD_GET (TSM_INTF_REPORT_MMIO_MSIX_TABLE , mmio_range->range_attributes), +FIELD_GET (TSM_INTF_REPORT_MMIO_PBA , mmio_range->range_attributes), + i, range_id, mmio_range->num_pages, r->start, r->end); +continue ; + } + + mmio_start_phys = mmio_range->first_page; +if (validate) + ret =rsi_validate_dev_mapping (vdev_id, dsm->instance_id, + r->start, r->end +1 , + mmio_start_phys <<12 , mmio_flags); +else + ret =rsi_invalidate_dev_mapping (r->start, r->end +1 ); +if (ret) { +pci_err (pdev,"failed to set protection attributes for the address range\n" ); +return -EIO ; + } + } +return 0 ; +}
Quoted 3 lines · Aneesh Kumar K.V (Arm)
This patch series implements support for Device Assignment in the ARM CCA architecture. The code changes are based on Alp12 specification published here [1].
Robin and I were talking about CCA and DMA here:
https://lore.kernel.org/r/[email protected]
What do you think about pulling some of this out and trying to independently push a series getting the DMA API layers ready for device assignment?
I think there will be some discussion on these points, it would be good to get started.
Jason
Re: [RFC PATCH v1 37/38] coco: guest: arm64: Add support for fetching device measurements
In reply to Aneesh Kumar K.V (Arm)Quoted 3 lines · On Mon, 28 Jul 2025 19:22:14 +0530 "Aneesh Kumar K.V (Arm)"…
Fetch device measurements using
RSI_RDEV_GET_MEASUREMENTS.Signed-off-by Aneesh Kumar K.V (Arm) <[email protected]>
One completely trivial comment.
J
Quoted 7 lines
drivers/virt/coco/arm-cca-guest/rsi-da.c +0
-0@@ -213,6 +245,13 @@ int rsi_device_lock(struct pci_dev *pdev)return -EIO ;
drivers/virt/coco/arm-cca-guest/rsi-da.c int rsi_device_lock(struct pci_dev *pdev) · line 255
Quoted 10 lines
drivers/virt/coco/arm-cca-guest/rsi-da.c +7
-0} + ret =rhi_get_report (vdev_id,RHI_DA_OBJECT_MEASUREMENT , + &dsm->measurements, &dsm->measurements_size); +if (ret) { +pci_err (pdev,"failed to get device certificate from the host (%lu)\n" , ret); +return -EIO ; + } +return ret;
return 0; Always good to make it explicit when it can't take any other values. Looks like that belong sin an earlier patch though based on this snippet.
Quoted 1 line
drivers/virt/coco/arm-cca-guest/rsi-da.c +0
-0}
Re: [RFC PATCH v1 38/38] coco: guest: arm64: Add support for fetching device info
In reply to Aneesh Kumar K.V (Arm)Quoted 5 lines · On Mon, 28 Jul 2025 19:22:15 +0530 "Aneesh Kumar K.V (Arm)"…
RSI_RDEV_GET_INFOreturns different digest hash values, which can be compared with host cached values to ensure the host didn't tamper with the cached data.Signed-off-by Aneesh Kumar K.V (Arm) <[email protected]>
Hi Aneesh
A few comments on this one
Jonathan
Quoted 41 lines
drivers/virt/coco/arm-cca-guest/rsi-da.c +26
-0@@ -6,6 +6,7 @@#include <linux/pci.h> #include <linux/mem_encrypt.h> #include <asm/rsi_cmds.h> +#include <crypto/hash.h> #include "rsi-da.h" @@ -186,11 +187,102 @@ rsi_rdev_get_measurements(struct pci_dev *pdev, unsigned long vdev_id,return RSI_SUCCESS ; } +static int verify_digests (struct cca_guest_dsc *dsm) +{ +int i; +int ret; +u8 digest[SHA512_DIGEST_SIZE ]; +int sdesc_size; +size_t digest_size; +char *hash_alg_name; +struct shash_desc *shash; +struct crypto_shash *alg; +struct pci_dev *pdev = dsm->pci.tsm.pdev; +struct { +uint8_t *report; +size_t size; +uint8_t *digest; + } reports[] = { + { + dsm->interface_report, + dsm->interface_report_size, + dsm->dev_info.report_digest + }, + { + dsm->certificate, + dsm->certificate_size, + dsm->dev_info.cert_digest
drivers/virt/coco/arm-cca-guest/rsi-da.c rsi_rdev_get_measurements(struct pci_dev *pdev, unsigned long vdev_id, · line 224
Quoted 10 lines
drivers/virt/coco/arm-cca-guest/rsi-da.c +10
-0+ }, + { + dsm->measurements, + dsm->measurements_size, + dsm->dev_info.meas_digest + } + }; + + +if (dsm->dev_info.hash_algo ==RSI_HASH_SHA_256 ) {
This to me smells like a place that will need switch sooner or later, maybe just do it now.
drivers/virt/coco/arm-cca-guest/rsi-da.c rsi_rdev_get_measurements(struct pci_dev *pdev, unsigned long vdev_id, · line 232
Quoted 8 lines
drivers/virt/coco/arm-cca-guest/rsi-da.c +8
-0+ hash_alg_name ="sha256" ; + digest_size =SHA256_DIGEST_SIZE ; + }else if (dsm->dev_info.hash_algo ==RSI_HASH_SHA_512 ) { + hash_alg_name ="sha512" ; + digest_size =SHA512_DIGEST_SIZE ; + }else { +pci_err (pdev,"unknown realm hash algorithm!\n" ); + ret = -EINVAL ;
return -EINVAL;
drivers/virt/coco/arm-cca-guest/rsi-da.c rsi_rdev_get_measurements(struct pci_dev *pdev, unsigned long vdev_id, · line 236
Quoted 4 lines
drivers/virt/coco/arm-cca-guest/rsi-da.c +4
-0+ goto err_out; + } + + alg =crypto_alloc_shash (hash_alg_name,0 ,0 );
As below - I'd spin a DEFINE_FREE() for this to simplify error paths in here and remove the labels that had me confused briefly.
drivers/virt/coco/arm-cca-guest/rsi-da.c rsi_rdev_get_measurements(struct pci_dev *pdev, unsigned long vdev_id, · line 242
Quoted 6 lines
drivers/virt/coco/arm-cca-guest/rsi-da.c +6
-0+if (IS_ERR (alg)) { +pci_err (pdev,"cannot allocate %s\n" , hash_alg_name); +return PTR_ERR (alg); + } + + sdesc_size =sizeof (struct shash_desc ) +crypto_shash_descsize (alg);
Not common in crypto so perhaps just leave this as you have it.
sdesc_size = struct_size(struct shash_dec, __ctx, crypto_shash_desc_size(alg));
is more informative on what is going on here.
drivers/virt/coco/arm-cca-guest/rsi-da.c rsi_rdev_get_measurements(struct pci_dev *pdev, unsigned long vdev_id, · line 253
Quoted 11 lines
drivers/virt/coco/arm-cca-guest/rsi-da.c +11
-0+ shash =kzalloc (sdesc_size,GFP_KERNEL ); +if (!shash) { +pci_err (pdev,"cannot allocate sdesc\n" ); + ret = -ENOMEM ; + goto err_free_shash; + } + shash->tfm = alg; + +for (i =0 ; i <ARRAY_SIZE (reports); i++) { + ret =crypto_shash_digest (shash, reports[i].report, + reports[i].size, digest);
To me a bit marginal on whether this loop and the structures above are beneficial over straight line code.
Quoted 6 lines
drivers/virt/coco/arm-cca-guest/rsi-da.c +6
-0+if (ret) { +pci_err (pdev,"failed to compute digest, %d\n" , ret); + goto err_free_sdesc; + } + +if (memcmp (reports[i].digest, digest, digest_size)) {
drivers/virt/coco/arm-cca-guest/rsi-da.c rsi_rdev_get_measurements(struct pci_dev *pdev, unsigned long vdev_id, · line 269
Quoted 10 lines
drivers/virt/coco/arm-cca-guest/rsi-da.c +10
-0+pci_err (pdev,"invalid digest\n" ); + ret = -EINVAL ; + goto err_free_sdesc; + } + } + +kfree (shash); +crypto_free_shash (alg); + +pci_info (pdev,"Successfully verified the digests\n" );
debug.
drivers/virt/coco/arm-cca-guest/rsi-da.c rsi_rdev_get_measurements(struct pci_dev *pdev, unsigned long vdev_id, · line 272
Quoted 3 lines
drivers/virt/coco/arm-cca-guest/rsi-da.c +3
-0+return 0 ; + +err_free_sdesc:
I'd tweak these labels if you keep them. This isn't freeing the sdesc.
drivers/virt/coco/arm-cca-guest/rsi-da.c rsi_rdev_get_measurements(struct pci_dev *pdev, unsigned long vdev_id, · line 273
Quoted 1 line
drivers/virt/coco/arm-cca-guest/rsi-da.c +1
-0+kfree (shash);
Looks perfect for some __free() magic dust.
drivers/virt/coco/arm-cca-guest/rsi-da.c rsi_rdev_get_measurements(struct pci_dev *pdev, unsigned long vdev_id, · line 275
Quoted 2 lines
drivers/virt/coco/arm-cca-guest/rsi-da.c +2
-0+err_free_shash: +crypto_free_shash (alg);
DEFINE_FREE() needed for this but looks pretty uncontroversial.
drivers/virt/coco/arm-cca-guest/rsi-da.c rsi_rdev_get_measurements(struct pci_dev *pdev, unsigned long vdev_id, · line 276
Quoted 1 line
drivers/virt/coco/arm-cca-guest/rsi-da.c +1
-0+err_out:
As below. I'd not do this.
Quoted 13 lines
drivers/virt/coco/arm-cca-guest/rsi-da.c +4
-0+return ret; +} +int rsi_device_lock (struct pci_dev *pdev) {unsigned long ret;unsigned long tdisp_version;struct rsi_device_measurements_params *rsi_dev_meas; +struct rsi_device_info *dev_info;struct cca_guest_dsc *dsm =to_cca_guest_dsc (pdev);int vdev_id = (pci_domain_nr (pdev->bus) <<16 ) |PCI_DEVID (pdev->bus->number, pdev->devfn);
drivers/virt/coco/arm-cca-guest/rsi-da.c int rsi_device_lock(struct pci_dev *pdev) · line 348
Quoted 5 lines
drivers/virt/coco/arm-cca-guest/rsi-da.c +2
-0@@ -252,8 +344,44 @@ int rsi_device_lock(struct pci_dev *pdev)return -EIO ; } +/* RMM expects sizeof(dev_info) (512 bytes) aligned address */ + dev_info =kmalloc (sizeof (*dev_info),GFP_KERNEL );
Use a __free(kfree) here (and direct returns on errors) given it's freed in all paths and we don't care if it is freed before or after verifying the digests.
I'm being slow today, but what in that enforces the alignment? I guess it's that the structure happens to be big enough that it happens naturally?
I'd allocate max(512, sizeof(*dev_info)) to make it explicitly the case.
Quoted 10 lines
drivers/virt/coco/arm-cca-guest/rsi-da.c +10
-0+if (!dev_info) { + ret = -ENOMEM ; + goto err_out; + } + + ret =rsi_rdev_get_info (vdev_id, dsm->instance_id,virt_to_phys (dev_info)); +if (ret !=RSI_SUCCESS ) { +pci_err (pdev,"failed to get device digests (%lu)\n" , ret); + ret = -EIO ; +kfree (dev_info);
drivers/virt/coco/arm-cca-guest/rsi-da.c int rsi_device_lock(struct pci_dev *pdev) · line 368
Quoted 10 lines
drivers/virt/coco/arm-cca-guest/rsi-da.c +10
-0+ goto err_out; + } + + dsm->dev_info.attest_type = dev_info->attest_type; + dsm->dev_info.cert_id = dev_info->cert_id; + dsm->dev_info.hash_algo = dev_info->hash_algo; +memcpy (dsm->dev_info.cert_digest, dev_info->cert_digest,SHA512_DIGEST_SIZE ); +memcpy (dsm->dev_info.meas_digest, dev_info->meas_digest,SHA512_DIGEST_SIZE ); +memcpy (dsm->dev_info.report_digest, dev_info->report_digest,SHA512_DIGEST_SIZE ); +
Can't you memcpy the whole thing in one go?
drivers/virt/coco/arm-cca-guest/rsi-da.c int rsi_device_lock(struct pci_dev *pdev) · line 381
Quoted 13 lines
drivers/virt/coco/arm-cca-guest/rsi-da.c +13
-0+kfree (dev_info); +/* +* Verify that the digests of the provided reports match with the +* digests from RMM +*/ + ret =verify_digests (dsm); +if (ret) { +pci_err (pdev,"device digest validation failed (%ld)\n" , ret); +return ret; + } + +return 0 ; +err_out:
I'll always grumble about these. To me it always makes the code less readable. Some others disagree though ;(
drivers/virt/coco/arm-cca-guest/rsi-da.c int rsi_device_lock(struct pci_dev *pdev) · line 384
Quoted 3 lines
drivers/virt/coco/arm-cca-guest/rsi-da.c +1
-0return ret; } +
Looks like this should have been in an earlier patch.
Quoted 19 lines
drivers/virt/coco/arm-cca-guest/rsi-da.c +0
-0static inline unsigned long rsi_rdev_start (struct pci_dev *pdev,unsigned long vdev_id,unsigned long inst_id) {drivers/virt/coco/arm-cca-guest/rsi-da.h +1
-0@@ -10,6 +10,7 @@#include <linux/pci-tsm.h> #include <asm/rsi_smc.h> #include <asm/rhi.h> +#include <crypto/sha2.h> struct pci_tdisp_device_interface_report {u16 interface_info;@@ -33,6 +34,17 @@ struct pci_tdisp_mmio_range {#define TSM_INTF_REPORT_MMIO_RESERVED GENMASK(15, 4)#define TSM_INTF_REPORT_MMIO_RANGE_ID GENMASK(31, 16)
drivers/virt/coco/arm-cca-guest/rsi-da.h struct pci_tdisp_mmio_range { · line 46
Quoted 10 lines
drivers/virt/coco/arm-cca-guest/rsi-da.h +10
-0+struct dsm_device_info { +u64 flags; +u64 attest_type; +u64 cert_id; +u64 hash_algo; +u8 cert_digest[SHA512_DIGEST_SIZE ]; +u8 meas_digest[SHA512_DIGEST_SIZE ]; +u8 report_digest[SHA512_DIGEST_SIZE ]; +}; +
One probably enough.
Quoted 12 lines
drivers/virt/coco/arm-cca-guest/rsi-da.h +2
-0+struct cca_guest_dsc {struct pci_tsm_pf0 pci;unsigned long instance_id;@@ -42,6 +54,7 @@ struct cca_guest_dsc {int certificate_size;void *measurements;int measurements_size; +struct dsm_device_info dev_info; };static inline struct cca_guest_dsc *to_cca_guest_dsc (struct pci_dev *pdev)
Re: [RFC PATCH v1 35/38] coco: guest: arm64: Add Realm device start and stop support
In reply to Aneesh Kumar K.V (Arm)Quoted 3 lines · On Mon, 28 Jul 2025 19:22:12 +0530 "Aneesh Kumar K.V (Arm)"…
Writing 1 to 'tsm/acceept' will initiate the TDISP RUN sequence.
Signed-off-by Aneesh Kumar K.V (Arm) <[email protected]>
A few more trivial things on this first read through.
Jonathan
drivers/virt/coco/arm-cca-guest/rsi-da.c int rsi_device_start(struct pci_dev *pdev) · line 401
Quoted 11 lines
drivers/virt/coco/arm-cca-guest/rsi-da.c +6
-0+int rsi_device_stop (struct pci_dev *pdev) +{ +int ret; +struct cca_guest_dsc *dsm =to_cca_guest_dsc (pdev); +int vdev_id = (pci_domain_nr (pdev->bus) <<16 ) | +PCI_DEVID (pdev->bus->number, pdev->devfn);
Feels like this occurs so often we should add a helper. Can't be completely generic as pci_domain_nr can have more bits than I guess we are assuming here, but we can have something for use in the rsi code.
Quoted 13 lines
drivers/virt/coco/arm-cca-guest/rsi-da.c +6
-0+ + ret =rsi_rdev_stop (pdev, vdev_id, dsm->instance_id); +if (ret !=RSI_SUCCESS ) { +pci_err (pdev,"failed to stop the device (%u)\n" , ret); +return -EIO ; + }return 0 ; }drivers/virt/coco/arm-cca-guest/rsi-da.h +0
-0
drivers/virt/coco/arm-cca-guest/rsi-da.h static inline struct cca_guest_dsc *to_cca_guest_dsc(struct pci_dev *pdev) · line 57
Quoted 4 lines
drivers/virt/coco/arm-cca-guest/rsi-da.h +0
-1@@ -54,5 +54,5 @@ static inline struct cca_guest_dsc *to_cca_guest_dsc(struct pci_dev *pdev)int rsi_update_interface_report (struct pci_dev *pdev,bool validate);int rsi_device_lock (struct pci_dev *pdev);int rsi_device_start (struct pci_dev *pdev); -
Tidy this up.
Quoted 2 lines
drivers/virt/coco/arm-cca-guest/rsi-da.h +1
-0+int rsi_device_stop (struct pci_dev *pdev);#endif
Re: [RFC PATCH v1 34/38] coco: guest: arm64: Validate mmio range found in the interface report
In reply to Aneesh Kumar K.V (Arm)Quoted 6 lines · Aneesh Kumar K.V (Arm)
drivers/virt/coco/arm-cca-guest/rsi-da.c +6
-0+for (int i =0 ; i < interface_report->mmio_range_count; i++, mmio_range++) { + +/*FIXME!! units in 4K size*/ + range_id =FIELD_GET (TSM_INTF_REPORT_MMIO_RANGE_ID , mmio_range->range_attributes); + +/* no secure interrupts */
drivers/virt/coco/arm-cca-guest/rsi-da.c int rsi_device_lock(struct pci_dev *pdev) · line 297
Quoted 10 lines
drivers/virt/coco/arm-cca-guest/rsi-da.c +10
-0+if (msix_tbl_bar !=-1 && range_id == msix_tbl_bar) { +pr_info ("Skipping misx table\n" ); +continue ; + } + +if (msix_pba_bar !=-1 && range_id == msix_pba_bar) { +pr_info ("Skipping misx pba\n" ); +continue ; + } +
MSI-X and PBA can be placed to a BAR that has other registers as well. While the PCIe specification recommends BAR-level isolation for MSI-X structures, it is not mandated. It is enough to have sufficient isolation within the BAR. Therefore, skipping the MSI-X and PBA BARs altogether may leave registers unintentionally mapped via unprotected IPA when they should have been mapped via protected IPA.
Instead of skipping the whole BAR, would it make sense to determine where the MSI-X related regions reside, and skip validation only from these regions?
- R2
Re: [RFC PATCH v1 13/38] coco: host: arm64: Create a PDEV with rmm
In reply to Aneesh Kumar K.V (Arm)drivers/virt/coco/arm-cca-host/rmm-da.c line 21
Quoted 10 lines · Aneesh Kumar K.V (Arm)
drivers/virt/coco/arm-cca-host/rmm-da.c new file +10
-0+static int pci_res_to_pdev_addr (struct rmi_pdev_addr_range *pdev_addr, +unsigned int naddr,struct resource *res, +unsigned int nres) +{ +int i, j; + +for (i =0 , j =0 ; i < naddr && j < nres; j++) { +if (res[j].flags &IORESOURCE_MEM ) { + pdev_addr[i].base = res[j].start; + pdev_addr[i].top = res[j].end;
I think there is an off-by-one bug in res[j].end. As per the RMM specification the base address is inclusive and the top address is exclusive. Both res[j].start and res[j].end are inclusive, and hence res[j].end seems wrong.
drivers/virt/coco/arm-cca-host/rmm-da.c line 59
Quoted 2 lines
drivers/virt/coco/arm-cca-host/rmm-da.c new file +2
-0+/* use the rid and MMIO resources of the epdev */ + params->rid_top = params->rid_base = rid;
Similar issue here. As per the specification the rid_base is inclusive and the rid_top exclusive.
- R2
Re: [RFC PATCH v1 11/38] KVM: arm64: CCA: register host tsm platform device
In reply to Jonathan CameronQuoted 23 lines · Jonathan Cameron
Quoted 11 lines · On Wed, 30 Jul 2025 14:12:26 +0530 "Aneesh Kumar K.V"…
Quoted 9 lines · Jason Gunthorpe
Quoted 4 lines · Jonathan Cameron
Quoted 1 line
+static struct
platform_devicecca_host_dev= {Hmm. Greg is getting increasingly (and correctly in my view) grumpy with platform devices being registered with no underlying resources etc as glue layers. Maybe some of that will come later.
Is
faux_devicea better choice? I admit to not knowing entirely what it is for..I'll go with a cautious yes to
faux_device. This case of a glue device with no resources and no reason to be on a particular bus was definitely the intent but I'm not 100% sure without trying it that we don't run into any problems.Not that many examples yet, but
cpuidle-psci.clooks like a vaguely similar case to this one.
Quoted 7 lines
All it really does is move the location of the device and smash together the device registration with probe/remove. That means the device disappears if
probe()fails, which is cleaner in many ways than leaving a pointless stub behind.Maybe it isn't appropriate it if is actually useful to rmmod/modprobe the driver.
Yeah, exactly. Can a TSM driver even be modular? If it has to be built in then there is no reason to do this:
Quoted 2 lines
Quoted 2 lines
The goal is to have tsm class device to be parented by the platform device.
IMHO the only real point of that is to trigger module autoloading.
Otherwise the tsm core should accept NULL as the parent pointer during registration, it probably already does..
Jason
Re: [RFC PATCH v1 04/38] tsm: Support DMA Allocation from private memory
In reply to Suzuki K PouloseQuoted 8 lines · Suzuki K Poulose
Quoted 5 lines
Quoted 3 lines
It is unclear whether devices would need to perform DMA to shared (unencrypted) memory while operating in this mode, as TLPs with T=1 are generally expected to target private memory.
PCI SIG supports it, kernel should support it.
ACK. On Arm CCA, the device can access shared IPA, with T=1 transaction as long as the mapping is active in the Stage2 managed by RMM.
Right, I expect that the T=0 SMMU S2 translation is a perfect subset of the T=1 S2 rmm translation. At most pages that are not available to T=0 should be removed when making the subset.
I'm not sure what the plan is here on ARM though, do you expect to pre-load the entire T=0 SMMU S2 with the shared IPA aliases and rely on the GPT for protection or will the hypervisor dynamically change the T=0 SMMU S2 after each shared/private change? Same question for the RMM S2?
The first option sounds fairly appealing, IMHO
Quoted 3 lines
Rather than mapping the entire memory from the host, it would be ideal if the Coco vms have some sort of a callback to "make sure the DMA wouldn't fault for a device".
Isn't that a different topic? For right now we expect that all pages are pinned and loaded into both S2s. Upon any private/shared conversion the pages should be reloaded into the appropriate S2s if required. The VM never needs to tell the hypervisor that it wants to do DMA.
There are all sorts of options here to relax this but exploring them it an entirely different project that CCA, IMHO.
Jason
Re: [RFC PATCH v1 06/38] iommufd: Add and option to request for bar mapping with IORESOURCE_EXCLUSIVE
In reply to Xu YilunQuoted 14 lines · Xu Yilun
Quoted 13 lines · Jason Gunthorpe
Quoted 12 lines · Aneesh Kumar K.V
Quoted 10 lines · Jason Gunthorpe
Quoted 1 line · Aneesh Kumar K.V (Arm)
Signed-off-by Aneesh Kumar K.V (Arm) <[email protected]>Why would we need this?
I can sort of understand why Intel would need it due to their issues with MCE, but ARM shouldn't care either way, should it?
But also why is it an iommufd option? That doesn't seem right..
Quoted 8 lines
Quoted 6 lines
Quoted 3 lines
Quoted 1 line
Jason
This is based on our previous discussion https://lore.kernel.org/all/[email protected]
I suggested a global option, this is a per-device option, and that especially seems wrong for iommufd. If it is per-device that is vfio,
I think this should be per-device.
IMHO there is no use case for that, it should arguably be global to the whole kernel.
Quoted 2 lines
The original purpose of this pci_region_request_*() is to prevent further mmap/read/write against a
vfio_cdevFD which would be used
No way, the VFIO internal mmap should be controled by VFIO not by request region. If you want to block that it should be blocked by iommufd telling VFIO that the device is bound which revokes the mmaps/dmabufs/etc and prevents opening new ones.
The only thing request region should do is prevent /sys/../resource, /dev/mem users and so on, which is why it can and should be global. Arguably VFIO should always block those things but historically hasn't..
There should only be one request region call in VFIO, it should ideally happen when the VFIO driver probes the device.
Jason
Re: [RFC PATCH v1 12/38] coco: host: arm64: CCA host platform device driver
In reply to Jonathan CameronQuoted 7 lines · Jonathan Cameron
Quoted 2 lines
devm is useful to solve complex things, these trivial things should be done normally..
Sure, that would be fine for now. If we end up with a large complex flow that happens to have a
tsm_register()in amongst various managed resources we can revisit. If they all end up looking like this then a manual call in remove is fine.
IMHO just don't use devm, it is so easy to use devm wrong and get out of order clean up. It works well for extremely simple case where 100% of cleanup is in devm (but then it is questionable if the overhead is worthwehile), and it is necessary for extremely hard cases where writing a manual unwind is too hard. But the middle ground it tends to just make ordering bugs and not provide alot of value, IMHO.
Jason
Re: [RFC PATCH v1 14/38] coco: host: arm64: Device communication support
In reply to Jonathan CameronQuoted 9 lines · Jonathan Cameron
drivers/virt/coco/arm-cca-host/arm-cca.c static void cca_tsm_pci_remove(struct pci_tsm *tsm) · line 92
Quoted 5 lines
drivers/virt/coco/arm-cca-host/arm-cca.c +5
-0+static int init_dev_communication_buffers (struct cca_host_comm_data *comm_data) +{ +int ret = -ENOMEM ; + + comm_data->io_params = (struct rmi_dev_comm_data *)get_zeroed_page (GFP_KERNEL );Hmm. There isn't a
DEFINE_FREE()yet forfree_page(). Maybe time to add one. If we did then we'd use local variables until all allocations succeed then assign withno_free_ptr()
Maybe think carefully if you really need a "page".
What would prevent just using kzalloc(PAGE_SIZE)? Under the covers it is almost the same thing.
Jason
Re: [RFC PATCH v1 23/38] coco: guest: arm64: Update arm CCA guest driver
In reply to Jonathan CameronQuoted 8 lines · Jonathan Cameron
drivers/virt/coco/arm-cca-guest/arm-cca.c static int arm_cca_report_new(struct tsm_report *report, void *data) · line 208
Quoted 4 lines
drivers/virt/coco/arm-cca-guest/arm-cca.c renamed from drivers/virt/coco/arm-cca-guest/arm-cca-guest.c +1
-3-static void __exit arm_cca_guest_exit(void) -{ -tsm_report_unregister (&arm_cca_tsm_ops); +return ret;return
devm_add_action_or_reset()Mind you, Jason probably won't like this ;)
devm in a module __exit function? How ?
Jason
Re: [RFC PATCH v1 11/38] KVM: arm64: CCA: register host tsm platform device
In reply to Jason GunthorpeQuoted 35 lines · On Thu, 31 Jul 2025 09:11:33 -0300 Jason Gunthorpe <[email protected]> wrote:
Quoted 23 lines · Jonathan Cameron
Quoted 11 lines · On Wed, 30 Jul 2025 14:12:26 +0530 "Aneesh Kumar K.V"…
Quoted 9 lines · Jason Gunthorpe
Quoted 4 lines · Jonathan Cameron
Quoted 1 line
+static struct
platform_devicecca_host_dev= {Hmm. Greg is getting increasingly (and correctly in my view) grumpy with platform devices being registered with no underlying resources etc as glue layers. Maybe some of that will come later.
Is
faux_devicea better choice? I admit to not knowing entirely what it is for..I'll go with a cautious yes to
faux_device. This case of a glue device with no resources and no reason to be on a particular bus was definitely the intent but I'm not 100% sure without trying it that we don't run into any problems.Not that many examples yet, but
cpuidle-psci.clooks like a vaguely similar case to this one.Quoted 7 lines
All it really does is move the location of the device and smash together the device registration with probe/remove. That means the device disappears if
probe()fails, which is cleaner in many ways than leaving a pointless stub behind.Maybe it isn't appropriate it if is actually useful to rmmod/modprobe the driver.
Yeah, exactly. Can a TSM driver even be modular? If it has to be built in then there is no reason to do this:
Quoted 7 lines
Quoted 2 lines
Quoted 2 lines
The goal is to have tsm class device to be parented by the platform device.
IMHO the only real point of that is to trigger module autoloading.
Otherwise the tsm core should accept NULL as the parent pointer during registration, it probably already does..
If you mean create a class device with no parent, that's also something we are slowly trying to fix. Reminds me that fixing up more perf devices is still on my todo list.
Should be a child of something, so maybe that is a good reason for a faux_device here if there is nothing else to use.
Jonathan
Quoted 2 lines
Jason
Suzuki K Poulose
reviewerRe: [RFC PATCH v1 04/38] tsm: Support DMA Allocation from private memory
In reply to Jason GunthorpeQuoted 5 lines · Jason Gunthorpe
Quoted 4 lines · Suzuki K Poulose
Quoted 4 lines
Quoted 3 lines
It is unclear whether devices would need to perform DMA to shared (unencrypted) memory while operating in this mode, as TLPs with T=1 are generally expected to target private memory.
Quoted 8 lines
Quoted 4 lines
Quoted 1 line
PCI SIG supports it, kernel should support it.
ACK. On Arm CCA, the device can access shared IPA, with T=1 transaction as long as the mapping is active in the Stage2 managed by RMM.
Right, I expect that the T=0 SMMU S2 translation is a perfect subset of the T=1 S2 rmm translation. At most pages that are not available to T=0 should be removed when making the subset.
Yes, this is what the VMM is supposed to do today, see [0] & [1].
Quoted 5 lines
I'm not sure what the plan is here on ARM though, do you expect to pre-load the entire T=0 SMMU S2 with the shared IPA aliases and rely on the GPT for protection or will the hypervisor dynamically change the T=0 SMMU S2 after each shared/private change? Same question for
Yes, share/private transitions do go all the way back to VMM and it is supposed to make the necessary changes to the SMMU S2 (as in [1]).
Quoted 2 lines
the RMM S2?
As for the RMM S2, the current plan is to re-use the CPU S2 managed by RMM.
Quoted 8 lines
The first option sounds fairly appealing, IMHO
Quoted 3 lines
Rather than mapping the entire memory from the host, it would be ideal if the Coco vms have some sort of a callback to "make sure the DMA wouldn't fault for a device".
Isn't that a different topic? For right now we expect that all pages are pinned and loaded into both S2s. Upon any private/shared
Actually it is. But might solve the problem for confidential VMs, where the S2 mapping is kind of pinned.
Population of S2 is a bit tricky for CVMs, as there are restrictions due to :
1) Pre-boot measurements
2) Restrictions on modifying the S2 (at least on CCA).
Thus, "the preload S2 and pin" must be done, after the "Initial images are loaded". And that becomes tricky from the Hypervisor (thinking of this, the VMM may be able to do this properly, as long as it remembers which areas where loaded).
Filling in the S2, with already populated S2 is complicated for CCA (costly, but not impossible). But the easier way is for the Realm to fault in the pages before they are used for DMA (and S2 mappings can be pinned by the hyp as default). Hence that suggestion.
Suzuki
[0] https://gitlab.arm.com/linux-arm/kvmtool-cca/-/commit/7c34972ddc
[1] https://gitlab.arm.com/linux-arm/kvmtool-cca/-/commit/ab4e654c4
Quoted 9 lines
conversion the pages should be reloaded into the appropriate S2s if required. The VM never needs to tell the hypervisor that it wants to do DMA.
There are all sorts of options here to relax this but exploring them it an entirely different project that CCA, IMHO.
Jason
Re: [RFC PATCH v1 23/38] coco: guest: arm64: Update arm CCA guest driver
In reply to Jason GunthorpeQuoted 11 lines · On Thu, 31 Jul 2025 09:29:48 -0300 Jason Gunthorpe <[email protected]> wrote:
Quoted 8 lines · Jonathan Cameron
drivers/virt/coco/arm-cca-guest/arm-cca.c static int arm_cca_report_new(struct tsm_report *report, void *data) · line 208
Quoted 4 lines
drivers/virt/coco/arm-cca-guest/arm-cca.c renamed from drivers/virt/coco/arm-cca-guest/arm-cca-guest.c +1
-3-static void __exit arm_cca_guest_exit(void) -{ -tsm_report_unregister (&arm_cca_tsm_ops); +return ret;return
devm_add_action_or_reset()Mind you, Jason probably won't like this ;)
devm in a module
__exitfunction? How ?
More coffee time...
Quoted 2 lines
Jason
Re: [RFC PATCH v1 04/38] tsm: Support DMA Allocation from private memory
In reply to Suzuki K PouloseQuoted 8 lines · Suzuki K Poulose
Quoted 7 lines · Jason Gunthorpe
Quoted 6 lines · Suzuki K Poulose
Quoted 5 lines
Quoted 3 lines
It is unclear whether devices would need to perform DMA to shared (unencrypted) memory while operating in this mode, as TLPs with T=1 are generally expected to target private memory.
PCI SIG supports it, kernel should support it.
Quoted 8 lines
Quoted 6 lines
Quoted 2 lines
ACK. On Arm CCA, the device can access shared IPA, with T=1 transaction as long as the mapping is active in the Stage2 managed by RMM.
Right, I expect that the T=0 SMMU S2 translation is a perfect subset of the T=1 S2 rmm translation. At most pages that are not available to T=0 should be removed when making the subset.
Yes, this is what the VMM is supposed to do today, see [0] & [1].
Okay great!
Quoted 7 lines
Quoted 4 lines
I'm not sure what the plan is here on ARM though, do you expect to pre-load the entire T=0 SMMU S2 with the shared IPA aliases and rely on the GPT for protection or will the hypervisor dynamically change the T=0 SMMU S2 after each shared/private change? Same question for
Yes, share/private transitions do go all the way back to VMM and it is supposed to make the necessary changes to the SMMU S2 (as in [1]).
Okay, it works, but also why?
From a hypervisor perspective when using VFIO I'd like the guestmemfd to fix all the physical memory immediately, so the entire physical map is fixed and known. Backed by 1G huge pages most likely.
Is there a reason not to just dump that into the T=0 SMMU using 1G huge pages and never touch it again? The GPT provides protection?
Sure sounds appealing..
Quoted 2 lines
As for the RMM S2, the current plan is to re-use the CPU S2 managed by RMM.
Yes, but my question is if the CPU will be prepopulated
Quoted 2 lines
Actually it is. But might solve the problem for confidential VMs, where the S2 mapping is kind of pinned.
Not kind of pinned, it is pinned in the hypervisor..
Quoted 4 lines
Population of S2 is a bit tricky for CVMs, as there are restrictions due to :
1) Pre-boot measurements
2) Restrictions on modifying the S2 (at least on CCA).
I haven't dug into any of this, but I'd challenge you to try to make it run fast if the guestmemfd has a full fixed address map in 1G pages and could just dump them into the RMM efficiently once during boot.
Perhaps there are ways to optimize the measurements for huge amounts of zero'd memory.
Quoted 4 lines
Filling in the S2, with already populated S2 is complicated for CCA (costly, but not impossible). But the easier way is for the Realm to fault in the pages before they are used for DMA (and S2 mappings can be pinned by the hyp as default). Hence that suggestion.
I guess, but it's weird, kinda slow, and the RMM can never unfault them..
How will you reconstruct the 1G huge pages in the S2 if you are only populating on faults? Can you really fault the entire 1G page? If so why can't it be prepopulated?
Jason
Re: [RFC PATCH v1 11/38] KVM: arm64: CCA: register host tsm platform device
In reply to Jonathan CameronQuoted 3 lines · Jonathan Cameron
If you mean create a class device with no parent, that's also something we are slowly trying to fix. Reminds me that fixing up more perf devices is still on my todo list.
IIRC if you create a class device with no parent it gets placed on the virtual bus...
Do you mean we should not do that?
Quoted 2 lines
Should be a child of something, so maybe that is a good reason for a
faux_devicehere if there is nothing else to use.
Don't see such a big difference to have it be the child of a faux device on the faux bus than to just be directly on the virtual bus?
Jason
Re: [RFC PATCH v1 34/38] coco: guest: arm64: Validate mmio range found in the interface report
In reply to Arto MerilainenQuoted 15 lines · Arto Merilainen
Quoted 13 lines · Aneesh Kumar K.V (Arm)
drivers/virt/coco/arm-cca-guest/rsi-da.c +13
-0+for (int i =0 ; i < interface_report->mmio_range_count; i++, mmio_range++) { + +/*FIXME!! units in 4K size*/ + range_id =FIELD_GET (TSM_INTF_REPORT_MMIO_RANGE_ID , mmio_range->range_attributes); + +/* no secure interrupts */ +if (msix_tbl_bar !=-1 && range_id == msix_tbl_bar) { +pr_info ("Skipping misx table\n" ); +continue ; + } + +if (msix_pba_bar !=-1 && range_id == msix_pba_bar) { +pr_info ("Skipping misx pba\n" );
Quoted 8 lines
drivers/virt/coco/arm-cca-guest/rsi-da.c int rsi_device_lock(struct pci_dev *pdev) · line 292
Quoted 3 lines
drivers/virt/coco/arm-cca-guest/rsi-da.c +3
-0+continue ; + } +MSI-X and PBA can be placed to a BAR that has other registers as well. While the PCIe specification recommends BAR-level isolation for MSI-X structures, it is not mandated.
Right, there are not enough BARs in most devices to give MSI its own BAR.
Quoted 4 lines
It is enough to have sufficient isolation within the
BAR. Therefore, skipping the MSI-X and PBA BARs altogether may leave registers unintentionally mapped via unprotected IPA when they should have been mapped via protected IPA.
Right, this sounds bad.
Quoted 3 lines
Instead of skipping the whole BAR, would it make sense to determine where the MSI-X related regions reside, and skip validation only from these regions?
IMHO this is a mess. The virtualization must end up putting a shared page(s) covering the MSI space in the middle of the MMIO region.
I think this should be done by fragmenting the layout in the IPA where the private MMIO is within the protected IPA space with an unmapped hole covering the MSIX registers. The acceptance process should validate this.
The MSIX registers would then be located in the shared IPA space.
A normal driver mmaping it's BAR will then crash if it tries to access the MSIX registers. This is good, we want to catch these non-secure configurations and block them.
The MSI code will have to know to compute the shared IPA alias and use that.
Jason
Quoted 9 lines · Aneesh Kumar K.V (Arm)
This patch series implements support for Device Assignment in the ARM CCA architecture. The code changes are based on Alp12 specification published here [1].
The code builds on the TSM framework patches posted at [2]. We add extension to that framework so that TSM is now used in both the host and the guest.
A DA workflow can be summarized as below:
Quoted 8 lines
Host:
step 1.
echo ${DEVICE} >/sys/bus/pci/devices/${DEVICE}/driver/unbindecho vfio-pci >/sys/bus/pci/devices/${DEVICE}/driver_overrideecho ${DEVICE} >/sys/bus/pci/drivers_probestep 2.
echo 1 >/sys/bus/pci/devices/$DEVICE/tsm/connect
Just for my own understanding... presumably there is no ordering constraint for ARM CCA between step1 and step2, right? I.e. The connect state is independent of the bind state.
In the v4 PCI/TSM scheme the connect command is now:
echo $tsm_dev > /sys/bus/pci/devices/$DEVICE/tsm/connect
Quoted 1 line
Now in the guest we follow the below steps
I assume a signifcant amount of kvmtool magic happens here to get the TDI into a "bind capable" state, can you share that command?
I had been assuming that everyone was prototyping with QEMU. Not a problem per se, but the memory management for shared device assignment / bounce buffering has had a quite of bit of work on the QEMU side, so just curious about the difference in approach here. Like, does kvmtool support operating the device in shared mode with bounce buffering and page conversion (shared <=> private) support? In any event, happy to see mutiple simultaneous consumers of this new kernel infrastructure.
Quoted 5 lines
step 1:
echo ${DEVICE} >/sys/bus/pci/devices/${DEVICE}/driver/unbindstep 2: Move the device to TDISP LOCK state
echo 1 >/sys/bus/pci/devices/${DEVICE}/tsm/lock
Ok, so my stance has recently picked up some nuance here. As Jason mentions here:
http://lore.kernel.org/[email protected]
"However it works, it should be done before the driver is probed and remain stable for the duration of the driver attachment. From the iommu side the correct iommu domain, on the correct IOMMU instance to handle the expected traffic should be setup as the DMA API's iommu domain."
I agree with that up until the point where the implication is userspace control of the UNLOCKED->LOCKED transition. That transition requires enabling bus-mastering (BME), configuring the device into an expected state, and *then* locking the device. That means userspace is blindly hoping that the device is in a state where it will remain quiet on the bus between BME and LOCKED, and that the previous unbind left the device in a state where it is prepared to be locked again.
The BME concern may be overblown given major PCI drivers blindly set BME without validating the device is in a quiesced state, but the "device is prepped for locking" problem seems harder.
2 potential ways to solve this, but open to other ideas:
- Userspace only picks the iommu domain context for the device not the lock state. Something like:
private > /sys/bus/pci/devices/${DEVICE}/tsm/domain ...where the default is "shared" and from that point the device can not issue DMA until a driver attaches. Driver controls UNLOCKED->LOCKED->RUN.
- Userspace is not involved in this transition and the dma mapping API is updated to allow a driver to switch the iommu domain at runtime, but only if the device has no outstanding mappings and the transition can only happen from ->probe() context. Driver controls joining secure-world-DMA and UNLOCKED->LOCKED->RUN.
Clearly the first option is less work in the kernel, but in both options the driver is in control of when BME is set relative to being ready for the LOCKED transition.
Quoted 2 lines
step 3: Moves the device to TDISP RUN state echo 1 >
/sys/bus/pci/devices/${DEVICE}/tsm/accept
This has the same concern from me about userspace being in control of BME. It feels like a departure from typical expectations. At least in the case of a driver setting BME the driver's probe routine is going to get the device in order shortly and otherwise have error handlers at the ready to effect any needed recovery.
Userspace just leaves the device enabled indefinitely and hopes.
Now, the nice thing about the scheme as proposed in this set is that userspace has all the time in the world between "lock" and "accept" to talk to a verifier.
With the driver in control there would need to be something like a usermodehelper to notify userspace that the device is in the locked state and to go ahead and run the attestation while the driver waits*.
* or driver could decide to not wait, especially useful for debug and development
Quoted 2 lines
step 4: Load the driver again.
echo ${DEVICE} >/sys/bus/pci/drivers_probe
TIL drivers_probe
Maybe want to recommend:
echo ${DEVICE} > /sys/bus/pci/drivers/${DRIVER}/bind
...to users just in case there are multiple drivers loaded for the device for the "shared" vs "private" case?
Quoted 2 lines
I'm currently working against TSM v3, as TSM v4 lacks the necessary callbacks—bind, unbind, and guest_req—required for guest interactions.
For staging purposes I wanted to put the "connect" flow to bed before moving on to the guest side.
Quoted 16 lines
The implementation also makes use of RHI interfaces that fall outside the current RHI specification [5]. Once the spec is finalized, the code will be aligned accordingly.
For now, I’ve retained
validate_mmioandvdev_reqexit handling within KVM. This will transition to a guest_req-based mechanism once the specification is updated.At that point, all device assignment (DA)-specific VM exits will exit directly to the VMM, and will use the
guest_reqioctl to handle exit reasons. As part of this change, the handlersrealm_exit_vdev_req_handler,realm_exit_vdev_comm_handler, andrealm_exit_dev_mem_map_handlerwill be removed.Full patchset for the kernel and kvmtool can be found at [3] and [4]
Quoted 7 lines
[1] https://developer.arm.com/-/cdn-downloads/permalink/Architectures/Armv9/DEN0137_1.1-alp12.zip
[2] https://lore.kernel.org/all/[email protected]
[3] https://git.gitlab.arm.com/linux-arm/linux-cca.git cca/tdisp-upstream-post-v1
[4] https://git.gitlab.arm.com/linux-arm/kvmtool-cca.git cca/tdisp-upstream-post-v1
[5] https://developer.arm.com/documentation/den0148/latest/
Thanks for this and the help reviewing PCI/TSM so far! I want to get this into tsm.git#staging so we can start to make hard claims ("look at the shared tree!") of hardware vendor consensus.
Greg KH
maintainerRe: [RFC PATCH v1 11/38] KVM: arm64: CCA: register host tsm platform device
In reply to Jason GunthorpeQuoted 9 lines · Jason Gunthorpe
Quoted 3 lines · Jonathan Cameron
If you mean create a class device with no parent, that's also something we are slowly trying to fix. Reminds me that fixing up more perf devices is still on my todo list.
IIRC if you create a class device with no parent it gets placed on the virtual bus...
Quoted 7 lines
Do you mean we should not do that?
Quoted 2 lines
Should be a child of something, so maybe that is a good reason for a
faux_devicehere if there is nothing else to use.Don't see such a big difference to have it be the child of a faux device on the faux bus than to just be directly on the virtual bus?
Either is fine, but just never use a platform device for a non-platform-resource-backed device please.
thanks,
greg k-h
Suzuki K Poulose
reviewerRe: [RFC PATCH v1 04/38] tsm: Support DMA Allocation from private memory
In reply to Jason GunthorpeQuoted 28 lines · Jason Gunthorpe
Quoted 8 lines · Suzuki K Poulose
Quoted 7 lines · Jason Gunthorpe
Quoted 6 lines · Suzuki K Poulose
Quoted 5 lines
Quoted 3 lines
It is unclear whether devices would need to perform DMA to shared (unencrypted) memory while operating in this mode, as TLPs with T=1 are generally expected to target private memory.
PCI SIG supports it, kernel should support it.
Quoted 8 lines
Quoted 6 lines
Quoted 2 lines
ACK. On Arm CCA, the device can access shared IPA, with T=1 transaction as long as the mapping is active in the Stage2 managed by RMM.
Right, I expect that the T=0 SMMU S2 translation is a perfect subset of the T=1 S2 rmm translation. At most pages that are not available to T=0 should be removed when making the subset.
Yes, this is what the VMM is supposed to do today, see [0] & [1].
Okay great!
Quoted 7 lines
Quoted 4 lines
I'm not sure what the plan is here on ARM though, do you expect to pre-load the entire T=0 SMMU S2 with the shared IPA aliases and rely on the GPT for protection or will the hypervisor dynamically change the T=0 SMMU S2 after each shared/private change? Same question for
Yes, share/private transitions do go all the way back to VMM and it is supposed to make the necessary changes to the SMMU S2 (as in [1]).
Quoted 8 lines
Okay, it works, but also why?
From a hypervisor perspective when using VFIO I'd like the guestmemfd
to fix all the physical memory immediately, so the entire physical map is fixed and known. Backed by 1G huge pages most likely.Is there a reason not to just dump that into the T=0 SMMU using 1G huge pages and never touch it again? The GPT provides protection?
That is possible, once we get guest_memfd mmap support merged upstream. GPT does provide protection. The only caveat is, does the guest_memfd support this at all ? i.e., shared->private transitions with a shared mapping in place (Though this is in SMMU only, not the Host CPU pagetables)
Quoted 18 lines
Sure sounds appealing..
Quoted 2 lines
As for the RMM S2, the current plan is to re-use the CPU S2 managed by RMM.
Yes, but my question is if the CPU will be prepopulated
Quoted 2 lines
Actually it is. But might solve the problem for confidential VMs, where the S2 mapping is kind of pinned.
Not kind of pinned, it is pinned in the hypervisor..
Quoted 4 lines
Population of S2 is a bit tricky for CVMs, as there are restrictions due to :
1) Pre-boot measurements
2) Restrictions on modifying the S2 (at least on CCA).
Quoted 6 lines
I haven't dug into any of this, but I'd challenge you to try to make it run fast if the guestmemfd has a full fixed address map in 1G pages and could just dump them into the RMM efficiently once during boot.
Perhaps there are ways to optimize the measurements for huge amounts of zero'd memory.
There is. We (VMM) can choose not to "measure" the zero'd pages.
Quoted 10 lines
Quoted 4 lines
Filling in the S2, with already populated S2 is complicated for CCA (costly, but not impossible). But the easier way is for the Realm to fault in the pages before they are used for DMA (and S2 mappings can be pinned by the hyp as default). Hence that suggestion.
I guess, but it's weird, kinda slow, and the RMM can never unfault them..
How will you reconstruct the 1G huge pages in the S2 if you are only populating on faults? Can you really fault the entire 1G page? If so why can't it be prepopulated?
It is tricky to prepopulate the 1G page, as parts of the pages may be "populated" with contents. We can recreate the 1G block mapping by "FOLD" ing the leaf level tables, all the way upto 1G, after the mappings are created. We have to do that anyway for CCA.
I think we can go ahead with VMM pre-populating the entire DRAM and keeping it pinned for DA. Rather than doing this from the vfio kernel, it could be done by the VMM as it has better knowledge of the populated contents and map the rest as "unmeasured" 0s.
Suzuki
Re: [RFC PATCH v1 04/38] tsm: Support DMA Allocation from private memory
Quoted 8 lines · Suzuki K Poulose
Quoted 2 lines
Is there a reason not to just dump that into the T=0 SMMU using 1G huge pages and never touch it again? The GPT provides protection?
That is possible, once we get
guest_memfdmmap support merged upstream. GPT does provide protection. The only caveat is, does theguest_memfdsupport this at all ? i.e.,shared->privatetransitions with a shared mapping in place (Though this is in SMMU only, not the Host CPU pagetables)
I don't know, we haven't got to the guestmemfd/IOMMU integration yet, which is why I ask the questions.
I think AMD and ARM would both be interested in guestmemfd <-> iommu working this way, at least.
Quoted 4 lines
I think we can go ahead with VMM pre-populating the entire DRAM and keeping it pinned for DA. Rather than doing this from the vfio kernel, it could be done by the VMM as it has better knowledge of the populated contents and map the rest as "unmeasured" 0s.
Yes, if done it should be done by the VMM and run through guestmemfd/kvm however that is agreed to.
Jason
Quoted 10 lines · On Thu, Jul 31, 2025 at 07:07:17PM -0700, [email protected] wrote:
Quoted 8 lines · Aneesh Kumar K.V (Arm)
Host:
step 1.
echo ${DEVICE} >/sys/bus/pci/devices/${DEVICE}/driver/unbindecho vfio-pci >/sys/bus/pci/devices/${DEVICE}/driver_overrideecho ${DEVICE} >/sys/bus/pci/drivers_probestep 2.
echo 1 >/sys/bus/pci/devices/$DEVICE/tsm/connect
Quoted 7 lines
Just for my own understanding... presumably there is no ordering constraint for ARM CCA between step1 and step2, right? I.e. The connect state is independent of the bind state.
In the v4 PCI/TSM scheme the connect command is now:
echo $tsm_dev >
/sys/bus/pci/devices/$DEVICE/tsm/connect
What does this do on the host? It seems to somehow prep it for VM assignment? Seems pretty strange this is here in sysfs and not part of creating the vPCI function in the VM through VFIO and iommufd?
Frankly, I'm nervous about making any uAPI whatsoever for the hypervisor side at this point. I don't think we have enough of the solution even in draft format. I'd really like your first merged TSM series to only have uAPI for the guest side where things are hopefully closer to complete..
Quoted 9 lines
Quoted 5 lines
step 1:
echo ${DEVICE} >/sys/bus/pci/devices/${DEVICE}/driver/unbindstep 2: Move the device to TDISP LOCK state
echo 1 >/sys/bus/pci/devices/${DEVICE}/tsm/lockOk, so my stance has recently picked up some nuance here. As Jason mentions here:
Quoted 7 lines
http://lore.kernel.org/[email protected]
"However it works, it should be done before the driver is probed and remain stable for the duration of the driver attachment. From the iommu side the correct iommu domain, on the correct IOMMU instance to handle the expected traffic should be setup as the DMA API's iommu domain."
I think it is not just the dma api, but also the MMIO registers may move location (form shared to protected IPA space for example). Meaning any attached driver is completely wrecked.
Quoted 3 lines
I agree with that up until the point where the implication is userspace control of the
UNLOCKED->LOCKEDtransition. That transition requires enabling bus-mastering (BME),
Why? That's sad. BME should be controlled by the VM driver not the TSM, and it should be set only when a VM driver is probed to the RUN state device?
Quoted 4 lines
and *then* locking the device. That means userspace is blindly hoping that the device is in a state where it will remain quiet on the bus between BME and LOCKED, and that the previous unbind left the device in a state where it is prepared to be locked again.
Yes, but we broadly assume this already in Linux. Drivers assume their devices are quiet when they are bound the first time, we expect on unbinding a driver quiets the device before removing.
So broadly I think you can assume that a device with no driver is quiet regardless of BME.
Quoted 10 lines
2 potential ways to solve this, but open to other ideas:
- Userspace only picks the iommu domain context for the device not the lock state. Something like:
private > /sys/bus/pci/devices/${DEVICE}/tsm/domain...where the default is "shared" and from that point the device can not issue DMA until a driver attaches. Driver controls
UNLOCKED->LOCKED->RUN.
What? Gross, no way can we let userspace control such intimate details of the kernel. The kernel must auto set based on what T=x mode the device driver binds into.
Quoted 5 lines
- Userspace is not involved in this transition and the dma mapping API is updated to allow a driver to switch the iommu domain at runtime, but only if the device has no outstanding mappings and the transition can only happen from ->probe() context. Driver controls joining secure-world-DMA and
UNLOCKED->LOCKED->RUN.
I don't see why it is so complicated. The driver is unbound before it reaches T=1 so we expect the device to be quiet (bigger problems if not). When the PCI core reaches T=1 it tells the DMA API to reconfigure things for the unbound struct device. Then we bind a driver as normal.
Driver controls nothing. All existing T=0 drivers "just work" with no source changes in T=1 mode. DMA API magically hides the bounce buffering. Surely this should be the baseline target functionality from a Linux perspective?
So we should not have "driver controls" statements at all. Userspace prepares the PCI device, driver probes onto a T=1 environment and just works.
Quoted 5 lines
Quoted 2 lines
step 3: Moves the device to TDISP RUN state echo 1 >
/sys/bus/pci/devices/${DEVICE}/tsm/acceptThis has the same concern from me about userspace being in control of BME. It feels like a departure from typical expectations.
It is, it is architecturally broken for BME to be controlled by the TSM. BME is controlled by the guest OS driver only.
IMHO if this is a real worry (and I don't think it is) then the right answer is for physical BME to be set on during locking, but VIRTUAL BME is left off. Virtual BME is created by the hypervisor/tsm by telling the IOMMU to block DMA.
The Guest OS should not participate in this broken design, the hypervisor can set pBME automatically when the lock request comes in, and the quality of vBME emulation is left up to the implementation, but the implementation must provide at least a NOP vBME once locked.
Quoted 3 lines
Now, the nice thing about the scheme as proposed in this set is that userspace has all the time in the world between "lock" and "accept" to talk to a verifier.
Seems right to me. There should be NO trusted kernel driver bound until the verifier accepts the attestation. Anything else allows un accepted devices to attack the kernel drivers. Few kernel drivers today distrust their HW interfaces as hostile actors and security defend against them. Therefore we should be very reluctant to bind drivers to anything..
Arguably a CC secure kernel should have an allow list of audited secure drivers that can autoprobe and all other drivers must be approved by userspace in some way, either through T=1 and attestation or some customer-aware risk assumption.
From that principal the kernel should NOT auto probe drivers to T=0 devices that can be made T=1. Userspace should handle attaching HW to such devices, and userspace can sequence whatever is required, including the attestation and verifying.
Otherwise, if you say, have a TDISP capable mlx5 device and boot up the cVM in a comporomised host the host can probably completely hack your cVM by exploiting the mlx5 drivers's total trust in the HW interface while running in T=0 mode.
You must attest it and switch to T=1 before binding any driver if you care about mitigating this risk.
Quoted 3 lines
With the driver in control there would need to be something like a usermodehelper to notify userspace that the device is in the locked state and to go ahead and run the attestation while the driver waits*.
It doesn't make sense to require modification to all existing drivers in Linux! The starting point must have the core code do this sequence for every driver. Once that is working we can talk about if other flows are needed.
Quoted 11 lines
Quoted 2 lines
step 4: Load the driver again.
echo ${DEVICE} >/sys/bus/pci/drivers_probeTIL
drivers_probeMaybe want to recommend:
echo ${DEVICE} >
/sys/bus/pci/drivers/${DRIVER}/bind...to users just in case there are multiple drivers loaded for the device for the "shared" vs "private" case?
Generic userspace will have a hard time to know what the driver names are..
The driver_probe option looks good to me as the default.
I'm not sure how generic code can handle "multiple drivers".. Most devices will be able to work just fine with T=0 mode with bounce buffers so we should generally not encourage people to make completely different drivers for T=0/T=1 mode.
I think what is needed is some way for userspace to trigger the "locking configuration" you mentioned, that may need a special driver, but ONLY if the userspace is sequencing the device to T=1 mode. Not sure how to make that generic, but I think so long as userspace is explicitly controlling driver binding we can punt on that solution to the userspace project :)
The real nastyness is RAS - what do you do when the device falls out of RUN, the kernel driver should pretty much explode. But lots of people would like the kernel driver to stay alive and somehow we FLR, re-attest and "resume" the kernel driver without allowing any T=0 risks. For instance you can keep your netdev and just see a lot of lost packets while the driver thrashes.
But I think we can start with the idea that such RAS failures have to reload the driver too and work on improvements. Realistically few drivers have the sort of RAS features to consume this anyhow and maybe we introduce some "enhanced" driver mode to opt-into down the road.
Jason
Quoted 15 lines · Jason Gunthorpe
Quoted 14 lines · On Thu, Jul 31, 2025 at 07:07:17PM -0700, [email protected] wrote:
Quoted 8 lines · Aneesh Kumar K.V (Arm)
Host:
step 1.
echo ${DEVICE} >/sys/bus/pci/devices/${DEVICE}/driver/unbindecho vfio-pci >/sys/bus/pci/devices/${DEVICE}/driver_overrideecho ${DEVICE} >/sys/bus/pci/drivers_probestep 2.
echo 1 >/sys/bus/pci/devices/$DEVICE/tsm/connectJust for my own understanding... presumably there is no ordering constraint for ARM CCA between step1 and step2, right? I.e. The connect state is independent of the bind state.
Quoted 7 lines
Quoted 3 lines
In the v4 PCI/TSM scheme the connect command is now:
echo $tsm_dev >
/sys/bus/pci/devices/$DEVICE/tsm/connectWhat does this do on the host? It seems to somehow prep it for VM assignment? Seems pretty strange this is here in sysfs and not part of creating the vPCI function in the VM through VFIO and iommufd?
vPCI is out of the picture at this phase.
On the host this establishes an SPDM session and sets up link encryption (IDE) with the physical device. Leave VMs out of the picture, this capability in isolation is a useful property. It addresses the similar threat model that Intel Total Memory Encryption (TME) or AMD Secure Memory Encryption (SME) go after, i.e. interposer on a physical link capturing data in flight.
With that established then one can go futher to do the full TDISP dance.
Quoted 5 lines
Frankly, I'm nervous about making any uAPI whatsoever for the hypervisor side at this point. I don't think we have enough of the solution even in draft format. I'd really like your first merged TSM series to only have uAPI for the guest side where things are hopefully closer to complete.
Aligned. I am not comfortable merging any of this until we have that end to end reliably stable for a kernel cycle or 2. The proposal is soak all the vendor solutions together in tsm.git#staging.
Now, if the guest side graduates out of that staging before the host side, I am ok with that.
Quoted 17 lines
Quoted 9 lines
Quoted 5 lines
step 1:
echo ${DEVICE} >/sys/bus/pci/devices/${DEVICE}/driver/unbindstep 2: Move the device to TDISP LOCK state
echo 1 >/sys/bus/pci/devices/${DEVICE}/tsm/lockOk, so my stance has recently picked up some nuance here. As Jason mentions here:
Quoted 7 lines
http://lore.kernel.org/[email protected]
"However it works, it should be done before the driver is probed and remain stable for the duration of the driver attachment. From the iommu side the correct iommu domain, on the correct IOMMU instance to handle the expected traffic should be setup as the DMA API's iommu domain."
Quoted 3 lines
I think it is not just the dma api, but also the MMIO registers may move location (form shared to protected IPA space for example). Meaning any attached driver is completely wrecked.
True.
Quoted 7 lines
Quoted 3 lines
I agree with that up until the point where the implication is userspace control of the
UNLOCKED->LOCKEDtransition. That transition requires enabling bus-mastering (BME),Why? That's sad. BME should be controlled by the VM driver not the TSM, and it should be set only when a VM driver is probed to the RUN state device?
To me it is an unfortunate PCI specification wrinkle that writing to the command register drops the device from RUN to ERROR. So you can LOCK without setting BME, but then no DMA.
Quoted 19 lines
Quoted 4 lines
and *then* locking the device. That means userspace is blindly hoping that the device is in a state where it will remain quiet on the bus between BME and LOCKED, and that the previous unbind left the device in a state where it is prepared to be locked again.
Yes, but we broadly assume this already in Linux. Drivers assume their devices are quiet when they are bound the first time, we expect on unbinding a driver quiets the device before removing.
So broadly I think you can assume that a device with no driver is quiet regardless of BME.
Quoted 7 lines
2 potential ways to solve this, but open to other ideas:
- Userspace only picks the iommu domain context for the device not the lock state. Something like:
private > /sys/bus/pci/devices/${DEVICE}/tsm/domain
Quoted 7 lines
Quoted 3 lines
...where the default is "shared" and from that point the device can not issue DMA until a driver attaches. Driver controls
UNLOCKED->LOCKED->RUN.What? Gross, no way can we let userspace control such intimate details of the kernel. The kernel must auto set based on what T=x mode the device driver binds into.
Flummoxed. Any way this gets sliced, userspace is asking for "private world attach" because it alone knows that this device is acceptable, and devices need to arrive in "shared world attach" mode.
Quoted 12 lines
Quoted 5 lines
- Userspace is not involved in this transition and the dma mapping API is updated to allow a driver to switch the iommu domain at runtime, but only if the device has no outstanding mappings and the transition can only happen from ->probe() context. Driver controls joining secure-world-DMA and
UNLOCKED->LOCKED->RUN.I don't see why it is so complicated. The driver is unbound before it reaches T=1 so we expect the device to be quiet (bigger problems if not). When the PCI core reaches T=1 it tells the DMA API to reconfigure things for the unbound struct device. Then we bind a driver as normal.
Quoted 4 lines
Driver controls nothing. All existing T=0 drivers "just work" with no source changes in T=1 mode. DMA API magically hides the bounce buffering. Surely this should be the baseline target functionality from a Linux perspective?
I started this project with "all existing T=0 drivers 'just work'" as a goal and a virtue. I have been begrudgingly pulled away from it from the slow drip of complexity it appears to push into the PCI core.
Now, I suspect the number of devices that are willing to spend gates and firmware on TDISP capabilities in the near term is small. The "just works" case is saved for either an L1 VMM to hide all this from an L2 guest, or a simplified TDISP specification that actually allows an OS PCI core to handle these details in a standard way.
Quoted 3 lines
So we should not have "driver controls" statements at all. Userspace prepares the PCI device, driver probes onto a T=1 environment and just works.
The concern is neither userspace nor the PCI core have everything it needs to get the device to T=1. PCI core knows that the device is T=1 capable, but does not know how to preconfigure the device-specific lock state, needs to wait for attestation. Userpace knows how to attest/verify the device but really has no business running the device outside of binding a driver, and can not rely on the PCI core to have prepped the device's device-specific lock state.
Userspace might be able to bind a new driver that leaves the device in a lockable state on unbind, but that is not "just works" that is, "introduce a new concept of skinny TDISP setup drivers that leave devices in LOCKED state on driver unbind, so that userspace can do the work to verify the device and move it to RUN before loading the main driver that expects the device arrives already running. Also, that main driver needs to be careful not to trigger typically benign actions like touch the command register to trip the device into ERROR state, or any device-specific actions that trip ERROR state but would otherwise be benign outside of TDISP."
If locking the device was just a toggle it would be possible. As far as I can see it is a "prep+toggle" where "prep" needs a driver.
Quoted 8 lines
Quoted 5 lines
Quoted 2 lines
step 3: Moves the device to TDISP RUN state echo 1 >
/sys/bus/pci/devices/${DEVICE}/tsm/acceptThis has the same concern from me about userspace being in control of BME. It feels like a departure from typical expectations.
It is, it is architecturally broken for BME to be controlled by the TSM. BME is controlled by the guest OS driver only.
Agree. That "accept" attribute does not belong with TSM. That is where Aneesh has it in this RFC. "Accept" as an action is the combination of device entered the LOCKED state in a configuration the verifier is willing to accept and the mechanics of triggering the LOCKED->RUN transition.
Quoted 9 lines
IMHO if this is a real worry (and I don't think it is) then the right answer is for physical BME to be set on during locking, but VIRTUAL BME is left off. Virtual BME is created by the hypervisor/tsm by telling the IOMMU to block DMA.
The Guest OS should not participate in this broken design, the hypervisor can set pBME automatically when the lock request comes in, and the quality of vBME emulation is left up to the implementation, but the implementation must provide at least a NOP vBME once locked.
I can let go of the "BME without driver" worry, but that does nothing to solve the "device specific configuration required before lock" problem.
Quoted 11 lines
Quoted 3 lines
Now, the nice thing about the scheme as proposed in this set is that userspace has all the time in the world between "lock" and "accept" to talk to a verifier.
Seems right to me. There should be NO trusted kernel driver bound until the verifier accepts the attestation. Anything else allows un accepted devices to attack the kernel drivers. Few kernel drivers today distrust their HW interfaces as hostile actors and security defend against them. Therefore we should be very reluctant to bind drivers to anything.
Quoted 4 lines
Arguably a CC secure kernel should have an allow list of audited secure drivers that can autoprobe and all other drivers must be approved by userspace in some way, either through T=1 and attestation or some customer-aware risk assumption.
Yes, today, where nothing is T=1 capable for an L1 guest*, the onus is 100% on the distribution, not the kernel. I.e. trim kernel config and set modprobe policy to prevent unwanted drivers.
* For L2 there are proposals like this, where if you already trust your paravisor also pre-trust all the devices it tells you to trust.
[1]: http://lore.kernel.org/[email protected]
Quoted 4 lines
From that principal the kernel should NOT auto probe drivers to T=0 devices that can be made T=1. Userspace should handle attaching HW to such devices, and userspace can sequence whatever is required, including the attestation and verifying.
Agree, for PCI it would be simple to set a no-auto-probe policy for T=1 capable devices.
Quoted 7 lines
Otherwise, if you say, have a TDISP capable mlx5 device and boot up the cVM in a comporomised host the host can probably completely hack your cVM by exploiting the mlx5 drivers's total trust in the HW interface while running in T=0 mode.
You must attest it and switch to T=1 before binding any driver if you care about mitigating this risk.
Yes, userspace must have a chance to say "no" before a driver attempts to launch DMA to private memory after secrets have been deployed to the TVM.
Quoted 6 lines
Quoted 3 lines
With the driver in control there would need to be something like a usermodehelper to notify userspace that the device is in the locked state and to go ahead and run the attestation while the driver waits*.
It doesn't make sense to require modification to all existing drivers in Linux!
I do not want to burden the PCI core with TDISP compatibility hacks and workarounds if it turns out only a small handful of devices ever deploy a first generation TDISP Device Security Manager (DSM). L1 aiding L2, or TDISP simplicity improvements to allow the PCI core to handle this in a non-broken way, are what I expect if secure device assignment takes off.
Quoted 3 lines
The starting point must have the core code do this sequence for every driver. Once that is working we can talk about if other flows are needed.
Do you agree that "device-specific-prep+lock" is the problem to solve?
Quoted 29 lines
Quoted 11 lines
Quoted 2 lines
step 4: Load the driver again.
echo ${DEVICE} >/sys/bus/pci/drivers_probeTIL
drivers_probeMaybe want to recommend:
echo ${DEVICE} >
/sys/bus/pci/drivers/${DRIVER}/bind...to users just in case there are multiple drivers loaded for the device for the "shared" vs "private" case?
Generic userspace will have a hard time to know what the driver names are..
The
driver_probeoption looks good to me as the default.I'm not sure how generic code can handle "multiple drivers".. Most devices will be able to work just fine with T=0 mode with bounce buffers so we should generally not encourage people to make completely different drivers for T=0/T=1 mode.
I think what is needed is some way for userspace to trigger the "locking configuration" you mentioned, that may need a special driver, but ONLY if the userspace is sequencing the device to T=1 mode. Not sure how to make that generic, but I think so long as userspace is explicitly controlling driver binding we can punt on that solution to the userspace project :)
Quoted 6 lines
The real nastyness is RAS - what do you do when the device falls out of RUN, the kernel driver should pretty much explode. But lots of people would like the kernel driver to stay alive and somehow we FLR, re-attest and "resume" the kernel driver without allowing any T=0 risks. For instance you can keep your netdev and just see a lot of lost packets while the driver thrashes.
Ideally the RUN->ERROR->UNLOCKED->LOCKED->RUN recovery can fit into the existing 'struct pci_error_handlers' regime in some farther out future.
It was a "fun" discovery to see that virtual AER injection does not exist in QEMU (at least last time I checked) and assigned devices that throw physical AER events just kill the VM.
Quoted 4 lines
But I think we can start with the idea that such RAS failures have to reload the driver too and work on improvements. Realistically few drivers have the sort of RAS features to consume this anyhow and maybe we introduce some "enhanced" driver mode to opt-into down the road.
Hmm, having trouble not reading that back supporting my argument above:
Realistically few devices support TDISP lets require enhanced drivers to opt-into TDISP for the time being.
Re: [RFC PATCH v1 11/38] KVM: arm64: CCA: register host tsm platform device
In reply to Jason GunthorpeQuoted 29 lines · Jason Gunthorpe
Quoted 28 lines · Jonathan Cameron
Quoted 11 lines · On Wed, 30 Jul 2025 14:12:26 +0530 "Aneesh Kumar K.V"…
Quoted 9 lines · Jason Gunthorpe
Quoted 4 lines · Jonathan Cameron
Quoted 1 line
+static struct
platform_devicecca_host_dev= {Hmm. Greg is getting increasingly (and correctly in my view) grumpy with platform devices being registered with no underlying resources etc as glue layers. Maybe some of that will come later.
Is
faux_devicea better choice? I admit to not knowing entirely what it is for..I'll go with a cautious yes to
faux_device. This case of a glue device with no resources and no reason to be on a particular bus was definitely the intent but I'm not 100% sure without trying it that we don't run into any problems.Not that many examples yet, but
cpuidle-psci.clooks like a vaguely similar case to this one.All it really does is move the location of the device and smash together the device registration with probe/remove. That means the device disappears if
probe()fails, which is cleaner in many ways than leaving a pointless stub behind.
Quoted 5 lines
Quoted 2 lines
Maybe it isn't appropriate it if is actually useful to rmmod/modprobe the driver.
Yeah, exactly. Can a TSM driver even be modular? If it has to be built in then there is no reason to do this:
For example, CRYPTO_DEV_CCP_DD, the AMD PCI device driver that will call tsm_register(), is already modular.
Quoted 4 lines
Quoted 2 lines
Quoted 2 lines
The goal is to have tsm class device to be parented by the platform device.
IMHO the only real point of that is to trigger module autoloading.
Right. For TDX, and I expect CCA as well, the arch code that knows that PCI/TSM functionality is available and can register a device, may be running too early to attach a driver to that device.
I.e. I would like to just use faux_device, but without the ability to do EPROBE_DEFER, for example to await the plaform IOMMU driver. It needs to move to its own bus so the attach event can be handled at a better time.
Quoted 2 lines
Otherwise the tsm core should accept NULL as the parent pointer during registration, it probably already does..
Yes, NULL @parent "just works" with tsm_register().
However, I expect all tsm_register() callers to be from modular drivers.
Aneesh Kumar K.V
maintainerRe: [RFC PATCH v1 04/38] tsm: Support DMA Allocation from private memory
In reply to Jason GunthorpeQuoted 45 lines · Jason Gunthorpe
Quoted 33 lines · Aneesh Kumar K.V
Quoted 31 lines · Jason Gunthorpe
arch/arm64/mm/mem_encrypt.c int set_memory_decrypted(unsigned long addr, int numpages) · line 56
Quoted 9 lines · Aneesh Kumar K.V (Arm)
arch/arm64/mm/mem_encrypt.c +5
-0@@ -48,3 +49,12 @@ int set_memory_decrypted(unsigned long addr, int numpages)return crypt_ops->decrypt (addr, numpages); }EXPORT_SYMBOL_GPL (set_memory_decrypted); + +bool force_dma_unencrypted (struct device *dev) +{ +if (dev->tdi_enabled) +return false;Is this OK? I see code like this:
static inline
dma_addr_tphys_to_dma_direct(struct device *dev,
phys_addr_tphys){ if (force_dma_unencrypted(dev)) return phys_to_dma_unencrypted(dev, phys); return phys_to_dma(dev, phys);What are the ARM rules for generating dma addreses?
1) Device is T=0, memory is unencrypted, call
dma_addr_unencrypted()and do "top bit IBA set"2) Device is T=1, memory is encrypted, use the
phys_to_dma()normally3) Device it T=1, memory is uncrypted, use the
phys_to_dma()normally??? Seems odd, I would have guessed the DMA address sould be the same as case #1?Quoted 5 lines
Quoted 2 lines
Can you document this in a comment?
If a device is operating in secure mode (T=1), it is currently assumed that only access to private (encrypted) memory is supported.
No, this is no how the PCI specs were written as far as I understand. The XT bit thing is supposed to add more fine grained device side control over what memory the DMA can target. T alone does not do that.
Quoted 6 lines
Quoted 3 lines
It is unclear whether devices would need to perform DMA to shared (unencrypted) memory while operating in this mode, as TLPs with T=1 are generally expected to target private memory.
PCI SIG supports it, kernel should support it.
Would we also need a separate DMA allocation API for allocating addresses intended to be shared with the non-secure hypervisor?
Are there any existing drivers in the kernel that already require such shared allocations, which I could use as a reference?
-aneesh
Aneesh Kumar K.V
maintainerRe: [RFC PATCH v1 08/38] iommufd/tsm: Add tsm_op iommufd ioctls
In reply to Jonathan CameronQuoted 139 lines · Jonathan Cameron
Quoted 3 lines · On Mon, 28 Jul 2025 19:21:45 +0530 "Aneesh Kumar K.V (Arm)"…
Add operations bind and unbind used to bind a TDI to the secure guest.
Signed-off-by Aneesh Kumar K.V (Arm) <[email protected]>Hi Aneesh,
I'm mostly reading this to get head around it rather than fully review at this point.
A few things inline though that I noticed whilst doing so.
Jonathan
Quoted 26 lines
--- drivers/iommu/iommufd/iommufd_private.h | 1 + drivers/iommu/iommufd/main.c | 3 ++ drivers/iommu/iommufd/viommu.c | 50 +++++++++++++++++++++++++ drivers/vfio/pci/vfio_pci_core.c | 10 +++++ include/uapi/linux/iommufd.h | 18 +++++++++ 5 files changed, 82 insertions(+)drivers/iommu/iommufd/iommufd_private.h +1
-0@@ -697,6 +697,7 @@ void iommufd_vdevice_destroy(struct iommufd_object *obj);void iommufd_vdevice_abort (struct iommufd_object *obj);int iommufd_hw_queue_alloc_ioctl (struct iommufd_ucmd *ucmd);void iommufd_hw_queue_destroy (struct iommufd_object *obj); +int iommufd_vdevice_tsm_op_ioctl (struct iommufd_ucmd *ucmd);static inline struct iommufd_vdevice *iommufd_get_vdevice (struct iommufd_ctx *ictx,u32 id)drivers/iommu/iommufd/viommu.c +0
-0drivers/iommu/iommufd/viommu.c void iommufd_vdevice_abort(struct iommufd_object *obj) · line 165
Quoted 4 lines
drivers/iommu/iommufd/viommu.c +1
-0@@ -162,6 +162,9 @@ void iommufd_vdevice_abort(struct iommufd_object *obj)lockdep_assert_held (&idev->igroup->lock); +#ifdef CONFIG_TSM Can we use stubs for some of this stuff so we don't need ifdefs in as many places.
drivers/iommu/iommufd/viommu.c int iommufd_hw_queue_alloc_ioctl(struct iommufd_ucmd *ucmd) · line 479
Quoted 12 lines
drivers/iommu/iommufd/viommu.c +5
-0+tsm_unbind (idev->dev); +#endif if (vdev->destroy) vdev->destroy (vdev);/* xa_cmpxchg is okay to fail if alloc failed xa_cmpxchg previously */ @@ -471,3 +474,50 @@ int iommufd_hw_queue_alloc_ioctl(struct iommufd_ucmd *ucmd)iommufd_put_object (ucmd->ictx, &viommu->obj);return rc; } + +#ifdef CONFIG_TSM +int iommufd_vdevice_tsm_op_ioctl (struct iommufd_ucmd *ucmd)Might want to split this out to a separate c file and use stubs in a header to keep the code clean here.
Quoted 9 lines
drivers/iommu/iommufd/viommu.c +9
-0+{ +struct iommu_vdevice_tsm_op *cmd = ucmd->cmd; +struct iommufd_vdevice *vdev; +struct kvm *kvm; +int rc = -ENODEV ; + +if (cmd->flags) +return -EOPNOTSUPP ; +drivers/iommu/iommufd/viommu.c int iommufd_hw_queue_alloc_ioctl(struct iommufd_ucmd *ucmd) · line 498
Quoted 10 lines
drivers/iommu/iommufd/viommu.c +10
-0+ vdev =container_of (iommufd_get_object (ucmd->ictx, cmd->vdevice_id, +IOMMUFD_OBJ_VDEVICE ), + struct iommufd_vdevice, obj); +if (IS_ERR (vdev)) +return PTR_ERR (vdev); + + kvm = vdev->viommu->kvm_filp->private_data; +if (kvm) { +/* +* tsm layer will make take care of parallel calls to tsm_bind/unbind Wrap comment to say under 80 chars. Or if file goes higher, use a single line comment.
tsm layer will take care ...
(stray 'make')
drivers/iommu/iommufd/viommu.c int iommufd_hw_queue_alloc_ioctl(struct iommufd_ucmd *ucmd) · line 506
Quoted 8 lines
drivers/iommu/iommufd/viommu.c +8
-0+*/ +if (cmd->op ==IOMMU_VDEICE_TSM_BIND ) + rc =tsm_bind (vdev->idev->dev, kvm, vdev->virt_id); +else if (cmd->op ==IOMMU_VDEICE_TSM_UNBIND ) + rc =tsm_unbind (vdev->idev->dev); + +if (rc) { + rc = -ENODEV ;If we want to eat an error code coming from elsewhere, maybe a comment on why?
drivers/iommu/iommufd/viommu.c int iommufd_hw_queue_alloc_ioctl(struct iommufd_ucmd *ucmd) · line 510
Quoted 4 lines
drivers/iommu/iommufd/viommu.c +4
-0+ goto out_put_vdev; + } + }else { + goto out_put_vdev;If this always skips the next line, does that imply that line should have been under if (kvm)? Maybe this makes more sense in later patches - if so ignore this comment.
Quoted 18 lines
drivers/iommu/iommufd/viommu.c +13
-0+ } + rc =iommufd_ucmd_respond (ucmd,sizeof (*cmd)); + +out_put_vdev: +iommufd_put_object (ucmd->ictx, &vdev->obj); +return rc; +} +#else /* !CONFIG_TSM */ +int iommufd_vdevice_tsm_op_ioctl (struct iommufd_ucmd *ucmd) +{ +return -ENODEV ; +} +#endif drivers/vfio/pci/vfio_pci_core.c +0
-0drivers/vfio/pci/vfio_pci_core.c void vfio_pci_core_close_device(struct vfio_device *core_vdev) · line 698
Quoted 5 lines
drivers/vfio/pci/vfio_pci_core.c +2
-0@@ -694,6 +694,16 @@ void vfio_pci_core_close_device(struct vfio_device *core_vdev)#if IS_ENABLED (CONFIG_EEH )eeh_dev_release (vdev->pdev);#endif + +#if 0 If you really need to do this, add a comment on why if 0
Quoted 10 lines
drivers/vfio/pci/vfio_pci_core.c void vfio_pci_core_close_device(struct vfio_device *core_vdev) · line 708
Quoted 10 lines
drivers/vfio/pci/vfio_pci_core.c +8
-0+/* +* destroy vdevice which involves tsm unbind before we disable pci disable +* A MSE/BME clear will transition the device to error state. +*/ +if (core_vdev->iommufd_device) +iommufd_device_tombstone_vdevice (core_vdev->iommufd_device); +#endif +vfio_pci_core_disable (vdev);
This is something I’d like to get feedback on. According to the TSM specification, we’re required to unlock before clearing MSE/BME via calling vfio_pci_core_disable(vdev).
However, in the current iommufd branch, we seem to call vdevice_destroy a bit too late in the sequence to meet this requirement.
Quoted 27 lines
Quoted 9 lines
drivers/vfio/pci/vfio_pci_core.c +0
-0mutex_lock (&vdev->igate);include/uapi/linux/iommufd.h +0
-0@@ -57,6 +57,7 @@ enum {/** include/uapi/linux/iommufd.h enum iommu_veventq_flag { · line 1133
Quoted 6 lines
include/uapi/linux/iommufd.h +3
-0@@ -1127,6 +1128,23 @@ enum iommu_veventq_flag { IOMMU_VEVENTQ_FLAG_LOST_EVENTS = (1U << 0), }; +/** +* struct iommu_vdevice_tsm_OP - ioctl(IOMMU_VDEVICE_TSM_OP) +* @size: sizeof(struct iommu_vdevice_tsm_OP)
_opI guess?Quoted 9 lines
include/uapi/linux/iommufd.h +9
-0+* @op: Either TSM_BIND or TSM_UNBIMD +* @flags: Must be 0 +* @vdevice_id: Object handle for the vDevice. Returned from IOMMU_VDEVICE_ALLOC +*/ +struct iommu_vdevice_tsm_op { +__u32 size; +__u32 op; +__u32 flags; +__u32 vdevice_id;
Quoted 8 lines
include/uapi/linux/iommufd.h enum iommu_veventq_flag { · line 1150
Quoted 8 lines
include/uapi/linux/iommufd.h +5
-0+}; +#define IOMMU_VDEVICE_TSM_OP _IO(IOMMUFD_TYPE, IOMMUFD_CMD_VDEVICE_TSM_OP) +#define IOMMU_VDEICE_TSM_BIND 0x1 +#define IOMMU_VDEICE_TSM_UNBIND 0x2 +/** * struct iommufd_vevent_header - Virtual Event Header for a vEVENTQ Status * @flags: Combination of enum iommu_veventq_flag
Thanks for the review comments. I'll update the patch with the suggested changes.
-aneesh
Aneesh Kumar K.V
maintainerRe: [RFC PATCH v1 13/38] coco: host: arm64: Create a PDEV with rmm
In reply to Jonathan CameronQuoted 144 lines · Jonathan Cameron
Quoted 3 lines · On Mon, 28 Jul 2025 19:21:50 +0530 "Aneesh Kumar K.V (Arm)"…
Create the realm physical device with RMM.
Signed-off-by Aneesh Kumar K.V (Arm) <[email protected]>Hi Aneesh,
Various small things inline.
Jonathan
Quoted 30 lines
--- arch/arm64/include/asm/rmi_cmds.h | 31 +++++ arch/arm64/include/asm/rmi_smc.h | 72 ++++++++++- drivers/virt/coco/arm-cca-host/Makefile | 2 +- drivers/virt/coco/arm-cca-host/arm-cca.c | 10 +- drivers/virt/coco/arm-cca-host/rmm-da.c | 150 +++++++++++++++++++++++ drivers/virt/coco/arm-cca-host/rmm-da.h | 5 + 6 files changed, 267 insertions(+), 3 deletions(-) create mode 100644 drivers/virt/coco/arm-cca-host/rmm-da.carch/arm64/include/asm/rmi_cmds.h +12
-0@@ -505,4 +505,35 @@ static inline int rmi_rtt_unmap_unprotected(unsigned long rd,return res.a0; } +static inline unsigned long rmi_pdev_aux_count (unsigned long flags,u64 *aux_count) +{ +struct arm_smccc_res res; + +arm_smccc_1_1_invoke (SMC_RMI_PDEV_AUX_COUNT , flags, &res); + + *aux_count = res.a1; +return res.a0; +} + +static inline unsigned long rmi_pdev_create (unsigned long pdev_phys, +unsigned long pdev_params_phys)arch/arm64/include/asm/rmi_cmds.h static inline int rmi_rtt_unmap_unprotected(unsigned long rd, · line 529
Quoted 10 lines
arch/arm64/include/asm/rmi_cmds.h +10
-0+{ +struct arm_smccc_res res; + +arm_smccc_1_1_invoke (SMC_RMI_PDEV_CREATE , + pdev_phys, pdev_params_phys, &res); + +return res.a0; +} + +static inline unsigned long rmi_pdev_get_state (unsigned long pdev_phys,unsigned long *state)I'd use the enum for the state type and range check in here.
Quoted 15 lines
arch/arm64/include/asm/rmi_cmds.h +9
-0+{ +struct arm_smccc_res res; + +arm_smccc_1_1_invoke (SMC_RMI_PDEV_GET_STATE , pdev_phys, &res); + + *state = res.a1; +return res.a0; +} +#endif /* __ASM_RMI_CMDS_H */ arch/arm64/include/asm/rmi_smc.h +0
-0arch/arm64/include/asm/rmi_smc.h line 29
Quoted 4 lines
arch/arm64/include/asm/rmi_smc.h +0
-1@@ -26,7 +26,7 @@#define SMC_RMI_DATA_CREATE SMC_RMI_CALL(0x0153)#define SMC_RMI_DATA_CREATE_UNKNOWN SMC_RMI_CALL(0x0154)#define SMC_RMI_DATA_DESTROY SMC_RMI_CALL(0x0155) -Probably keep
arch/arm64/include/asm/rmi_smc.h line 29
Quoted 1 line
arch/arm64/include/asm/rmi_smc.h +1
-0+#define SMC_RMI_PDEV_AUX_COUNT SMC_RMI_CALL(0x0156)and add one here as different groups.
Quoted 14 lines
arch/arm64/include/asm/rmi_smc.h +3
-0#define SMC_RMI_REALM_ACTIVATE SMC_RMI_CALL(0x0157)#define SMC_RMI_REALM_CREATE SMC_RMI_CALL(0x0158)#define SMC_RMI_REALM_DESTROY SMC_RMI_CALL(0x0159)@@ -47,6 +47,9 @@#define SMC_RMI_RTT_INIT_RIPAS SMC_RMI_CALL(0x0168)#define SMC_RMI_RTT_SET_RIPAS SMC_RMI_CALL(0x0169) +#define SMC_RMI_PDEV_CREATE SMC_RMI_CALL(0x0176) +#define SMC_RMI_PDEV_GET_STATE SMC_RMI_CALL(0x0178) +#define RMI_ABI_MAJOR_VERSION 1#define RMI_ABI_MINOR_VERSION 0arch/arm64/include/asm/rmi_smc.h struct rec_run { · line 274
Quoted 4 lines
arch/arm64/include/asm/rmi_smc.h +1
-0@@ -268,4 +271,71 @@ struct rec_run {struct rec_exit exit; }; +enum rmi_pdev_state {Type never used, but I think it should be. See above.
arch/arm64/include/asm/rmi_smc.h struct rec_run { · line 278
Quoted 4 lines
arch/arm64/include/asm/rmi_smc.h +4
-0+RMI_PDEV_NEW , +RMI_PDEV_NEEDS_KEY , +RMI_PDEV_HAS_KEY , + RMI_PDEV_READY,Seems someone put another state in here, but I guess you'll update when the rest of the stack catches up.
RMI_PDEV_IDE_RESETTING,Maybe throw a comment here for now.
Quoted 4 lines
arch/arm64/include/asm/rmi_smc.h +4
-0+RMI_PDEV_COMMUNICATING , +RMI_PDEV_STOPPING , +RMI_PDEV_STOPPED , + RMI_PDEV_ERROR,arch/arm64/include/asm/rmi_smc.h struct rec_run { · line 292
Quoted 10 lines
arch/arm64/include/asm/rmi_smc.h +10
-0+}; + +#define MAX_PDEV_AUX_GRANULES 32 +#define MAX_IOCOH_ADDR_RANGE 16 +#define MAX_FCOH_ADDR_RANGE 4 + +#define RMI_PDEV_SPDM_TRUE BIT(0) +#define RMI_PDEV_IDE_TRUE BIT(1) +#define RMI_PDEV_FOCOH BIT(2) +#define RMI_PDEV_P2P_STREAM BIT(3)I'd stick flags in the name (assuming this is
RmiPDevFlags? I'm not sure as the spec I could
find has different usages other after BIT(!))arch/arm64/include/asm/rmi_smc.h struct rec_run { · line 299
Quoted 7 lines
arch/arm64/include/asm/rmi_smc.h +7
-0+ +#define RMI_HASH_SHA_256 0 +#define RMI_HASH_SHA_512 1 + +struct rmi_pdev_addr_range { +unsigned long base; +unsigned long top;Whilst we only care about platforms where this is u64, maybe just make that explicit so we can trivially see this matches the spec?
Quoted 8 lines
arch/arm64/include/asm/rmi_smc.h struct rec_run { · line 302
Quoted 3 lines
arch/arm64/include/asm/rmi_smc.h +3
-0+}; + +struct rmi_pdev_params {Reference? Looks like B4.4.25 RmiPdevParams type in alp15. Though as there are some fields missing I guess this chagned.
This patch series is currently based on the ALP12 specification, available here:
https://developer.arm.com/-/cdn-downloads/permalink/Architectures/Armv9/DEN0137_1.1-alp12.zip
The next revision of the series will be updated to align with the ALP15 specification.
Quoted 35 lines
Quoted 4 lines
arch/arm64/include/asm/rmi_smc.h +3
-0+ union { + struct { + u64 flags;arch/arm64/include/asm/rmi_smc.h struct rec_run { · line 315
Quoted 10 lines
arch/arm64/include/asm/rmi_smc.h +10
-0+u64 pdev_id; +u64 segment_id; +u64 ecam_addr; +u64 root_id; +u64 cert_id; +u64 rid_base; +u64 rid_top; +u64 hash_algo; +u64 num_aux; +u64 ide_sid;Called
ncoh_ide_sidein the alpha 15. Maybe match that unless it is changing name in future version.Quoted 17 lines
arch/arm64/include/asm/rmi_smc.h +17
-0+u64 ncoh_num_addr_range; +u64 coh_num_addr_range; + }; +u8 padding1[0x100 ]; + }; + +union {/* 0x100 */ +u64 aux_granule[MAX_PDEV_AUX_GRANULES ]; +u8 padding2[0x100 ]; + }; + +union {/* 0x200 */ +struct { +struct rmi_pdev_addr_range ncoh_addr_range[MAX_IOCOH_ADDR_RANGE ]; + }; +u8 padding3[0x100 ]; + };
Quoted 9 lines
arch/arm64/include/asm/rmi_smc.h struct rec_run { · line 338
Quoted 6 lines
arch/arm64/include/asm/rmi_smc.h +6
-0+union {/* 0x300 */ +struct { +struct rmi_pdev_addr_range coh_addr_range[MAX_FCOH_ADDR_RANGE ]; + }; +u8 padding4[0x100 ]; + };Maybe pad out the rest? Mostly so we can see here that it is 4k.
ok
Quoted 167 lines
Quoted 10 lines
arch/arm64/include/asm/rmi_smc.h +2
-0+}; +#endif /* __ASM_RMI_SMC_H */ drivers/virt/coco/arm-cca-host/arm-cca.c +0
-0drivers/virt/coco/arm-cca-host/arm-cca.c static int cca_tsm_connect(struct pci_dev *pdev) · line 127
Quoted 4 lines
drivers/virt/coco/arm-cca-host/arm-cca.c +0
-1@@ -124,7 +124,15 @@ static int cca_tsm_connect(struct pci_dev *pdev) rc =tsm_ide_stream_register (pdev, ide);if (rc) goto err_tsm; -Tidy up. I'd leave it.
Quoted 18 lines
drivers/virt/coco/arm-cca-host/arm-cca.c +9
-0+/* +* Take a module reference so that we won't call unregister +* without rme_unasign_device +*/ +if (!try_module_get (THIS_MODULE )) { + rc = -ENXIO ; + goto err_tsm; + } +rme_asign_device (pdev);/* * Once ide is setup enable the stream at endpoint * Root port will be done by RMM drivers/virt/coco/arm-cca-host/rmm-da.c new file +0
-0drivers/virt/coco/arm-cca-host/rmm-da.c line 41
Quoted 6 lines
drivers/virt/coco/arm-cca-host/rmm-da.c new file +4
-0@@ -0,0 +1,150 @@ + +static int init_pdev_params (struct pci_dev *pdev,struct rmi_pdev_params *params) +{ +void *aux;Probably makes sense to reduce scope of aux to within the loop.
drivers/virt/coco/arm-cca-host/rmm-da.c line 45
Quoted 4 lines
drivers/virt/coco/arm-cca-host/rmm-da.c new file +4
-0+int rid, ret, i; +phys_addr_t aux_phys; +struct cca_host_dsc_pf0 *dsc_pf0; +struct pci_dev *rp =pcie_find_root_port (pdev);Only used in one place for now anyway. Probably better to move it down there.
drivers/virt/coco/arm-cca-host/rmm-da.c line 48
Quoted 3 lines
drivers/virt/coco/arm-cca-host/rmm-da.c new file +3
-0+struct pci_config_window *cfg = pdev->bus->sysdata; + + dsc_pf0 =to_cca_dsc_pf0 (pdev);I'd do that at declaration. Seems little advantage in doing it down here.
drivers/virt/coco/arm-cca-host/rmm-da.c line 58
Quoted 10 lines
drivers/virt/coco/arm-cca-host/rmm-da.c new file +10
-0+/* assign the ep device with RMM */ + rid =pci_dev_id (pdev); + params->pdev_id = rid; +/* slot number for certificate chain */ + params->cert_id =0 ; +/* io coherent spdm/ide and non p2p */ + params->flags =RMI_PDEV_SPDM_TRUE |RMI_PDEV_IDE_TRUE ; + params->ide_sid = dsc_pf0->sel_stream->stream_id; + params->hash_algo =RMI_HASH_SHA_256 ; +/* use the rid and MMIO resources of the epdev */ Spell out epdev or associate it with the pdev here more clearly.
drivers/virt/coco/arm-cca-host/rmm-da.c line 61
Quoted 3 lines
drivers/virt/coco/arm-cca-host/rmm-da.c new file +3
-0+ params->rid_top = params->rid_base = rid; + params->ecam_addr = cfg->res.start; + params->root_id =pci_dev_id (rp);params->root_id = pci_dev_id(pcie_find_root_port(pdev));to me acts as better documentation fo what is going on than using the local variable rp.
drivers/virt/coco/arm-cca-host/rmm-da.c line 71
Quoted 10 lines
drivers/virt/coco/arm-cca-host/rmm-da.c new file +10
-0+ + params->ncoh_num_addr_range =pci_res_to_pdev_addr (params->ncoh_addr_range, +ARRAY_SIZE (params->ncoh_addr_range), + pdev->resource, +DEVICE_COUNT_RESOURCE ); + +rmi_pdev_aux_count (params->flags, ¶ms->num_aux); +pr_debug ("%s using %ld pdev aux granules\n" , __func__, (unsigned long )params->num_aux); + dsc_pf0->num_aux = params->num_aux; +for (i =0 ; i < params->num_aux; i++) {void *aux = (void *)__get_free_page(GFP_KERNEL);Quoted 9 lines
drivers/virt/coco/arm-cca-host/rmm-da.c new file +9
-0+ aux = (void *)__get_free_page (GFP_KERNEL ); +if (!aux) { + ret = -ENOMEM ; + goto err_free_aux; + } + + aux_phys =virt_to_phys (aux); +if (rmi_granule_delegate (aux_phys)) { + ret = -ENXIO ;drivers/virt/coco/arm-cca-host/rmm-da.c line 90
Quoted 10 lines
drivers/virt/coco/arm-cca-host/rmm-da.c new file +10
-0+free_page ((unsigned long )aux); + goto err_free_aux; + } + params->aux_granule[i] = aux_phys; + dsc_pf0->aux[i] = aux; + } +return 0 ; + +err_free_aux: +free_aux_pages (i, dsc_pf0->aux[i]);I think you want
free_aux_pages(i, dsc_pf0->aux);Assuming this is supposed to unwind the loop above.
drivers/virt/coco/arm-cca-host/rmm-da.c line 93
Quoted 3 lines
drivers/virt/coco/arm-cca-host/rmm-da.c new file +3
-0+return -ENOMEM ; +} +Trivial: Single line probably fine here.
drivers/virt/coco/arm-cca-host/rmm-da.c line 104
Quoted 11 lines
drivers/virt/coco/arm-cca-host/rmm-da.c new file +11
-0+ +int rme_asign_device (struct pci_dev *pci_dev) +{ +int ret; +void *rmm_pdev; +unsigned long state; +phys_addr_t rmm_pdev_phys; +struct rmi_pdev_params *params; +struct cca_host_dsc_pf0 *dsc_pf0; + + dsc_pf0 =to_cca_dsc_pf0 (pci_dev);Might as well save a line with
struct cca_host_dsc_pf0 *dsc_pf0 = to_cca_dsc_pf0(pci_dev);drivers/virt/coco/arm-cca-host/rmm-da.c line 107
Quoted 3 lines
drivers/virt/coco/arm-cca-host/rmm-da.c new file +3
-0+ rmm_pdev = (void *)get_zeroed_page (GFP_KERNEL ); +if (!rmm_pdev) { + ret = -ENOMEM ;return
-ENOMEM;Quoted 11 lines
drivers/virt/coco/arm-cca-host/rmm-da.c new file +11
-0+ goto err_out; + } + + rmm_pdev_phys =virt_to_phys (rmm_pdev); +if (rmi_granule_delegate (rmm_pdev_phys)) { + ret = -ENXIO ; + goto err_free_pdev; + } + + params = (struct rmi_pdev_params *)get_zeroed_page (GFP_KERNEL ); +if (!params) {drivers/virt/coco/arm-cca-host/rmm-da.c line 128
Quoted 10 lines
drivers/virt/coco/arm-cca-host/rmm-da.c new file +10
-0+ ret = -ENOMEM ; + goto err_granule_undelegate; + } + + ret =init_pdev_params (pci_dev, params); +if (ret) + goto err_free_params; + + ret =rmi_pdev_create (rmm_pdev_phys,virt_to_phys (params)); +pr_info ("rmi_pdev_create(0x%llx, 0x%llx): %d\n" , rmm_pdev_phys,virt_to_phys (params), ret);RFC, but even so I'd demote to debug and use dynamic debug stuff to enable them for your testing.
Quoted 12 lines
drivers/virt/coco/arm-cca-host/rmm-da.c line 134
Quoted 6 lines
drivers/virt/coco/arm-cca-host/rmm-da.c new file +6
-0+if (ret) + goto err_free_aux; + +rmi_pdev_get_state (rmm_pdev_phys, &state); +if (state !=RMI_PDEV_NEW ) + goto err_free_aux;Nothing to unwind in
rmi_pdev_create()? We've told the RMM about it then we blow away it's resources. Seems unwise! Maybe this is cleaned up elsewhere but if so a comment is probably good.
pdev_create is expected to set the device state to RMI_PDEV_NEW, so the subsequent check is redundant if rmi_pdev_create returns success. I will drop that.
Quoted 36 lines
Quoted 5 lines
drivers/virt/coco/arm-cca-host/rmm-da.c new file +4
-0+ + dsc_pf0->rmm_pdev = rmm_pdev; +free_page ((unsigned long )params); +return 0 ;drivers/virt/coco/arm-cca-host/rmm-da.c line 148
Quoted 10 lines
drivers/virt/coco/arm-cca-host/rmm-da.c new file +10
-0+ +err_free_aux: +free_aux_pages (dsc_pf0->num_aux, dsc_pf0->aux); +err_free_params: +free_page ((unsigned long )params); +err_granule_undelegate: +rmi_granule_undelegate (rmm_pdev_phys); +err_free_pdev: +free_page ((unsigned long )rmm_pdev); +err_out:One of my favourite nitpicks. Why not just return if nothing to do? That tends to save a reviewer a bit of scrolling when checking error paths do correct unwinding.
Quoted 16 lines
drivers/virt/coco/arm-cca-host/rmm-da.c new file +2
-0+return ret; +}drivers/virt/coco/arm-cca-host/rmm-da.h +4
-0@@ -14,6 +14,10 @@struct cca_host_dsc_pf0 {struct pci_tsm_pf0 pci;struct pci_ide *sel_stream; + +void *rmm_pdev; +int num_aux; +void *aux[MAX_PDEV_AUX_GRANULES ]; };
Quoted 7 lines
drivers/virt/coco/arm-cca-host/rmm-da.h static inline struct cca_host_dsc_pf0 *to_cca_dsc_pf0(struct pci_dev *pdev) · line 34
Quoted 7 lines
drivers/virt/coco/arm-cca-host/rmm-da.h +1
-0static inline struct cca_host_dsc_pf0 *to_cca_dsc_pf0 (struct pci_dev *pdev)@@ -26,4 +30,5 @@ static inline struct cca_host_dsc_pf0 *to_cca_dsc_pf0(struct pci_dev *pdev) return container_of(tsm, struct cca_host_dsc_pf0, pci.tsm); } +int rme_asign_device (struct pci_dev *pdev);#endif
Thanks for the review comments. I'll update the patch with the suggested changes.
-aneesh
Aneesh Kumar K.V
maintainerRe: [RFC PATCH v1 13/38] coco: host: arm64: Create a PDEV with rmm
In reply to Arto MerilainenQuoted 18 lines · Arto Merilainen
drivers/virt/coco/arm-cca-host/rmm-da.c line 21
Quoted 10 lines · Aneesh Kumar K.V (Arm)
drivers/virt/coco/arm-cca-host/rmm-da.c new file +10
-0+static int pci_res_to_pdev_addr (struct rmi_pdev_addr_range *pdev_addr, +unsigned int naddr,struct resource *res, +unsigned int nres) +{ +int i, j; + +for (i =0 , j =0 ; i < naddr && j < nres; j++) { +if (res[j].flags &IORESOURCE_MEM ) { + pdev_addr[i].base = res[j].start; + pdev_addr[i].top = res[j].end;I think there is an off-by-one bug in res[j].end. As per the RMM specification the base address is inclusive and the top address is exclusive. Both res[j].start and res[j].end are inclusive, and hence res[j].end seems wrong.
Quoted 6 lines
drivers/virt/coco/arm-cca-host/rmm-da.c line 59
Quoted 2 lines
drivers/virt/coco/arm-cca-host/rmm-da.c new file +2
-0+/* use the rid and MMIO resources of the epdev */ + params->rid_top = params->rid_base = rid;Similar issue here. As per the specification the
rid_baseis inclusive and therid_topexclusive.
Thanks for the review comments. I'll update the patch with the suggested changes.
-aneesh
Re: [RFC PATCH v1 04/38] tsm: Support DMA Allocation from private memory
In reply to Aneesh Kumar K.VQuoted 51 lines · Aneesh Kumar K.V
Quoted 49 lines · Jason Gunthorpe
Quoted 33 lines · Aneesh Kumar K.V
Quoted 31 lines · Jason Gunthorpe
arch/arm64/mm/mem_encrypt.c int set_memory_decrypted(unsigned long addr, int numpages) · line 56
Quoted 9 lines · Aneesh Kumar K.V (Arm)
arch/arm64/mm/mem_encrypt.c +5
-0@@ -48,3 +49,12 @@ int set_memory_decrypted(unsigned long addr, int numpages)return crypt_ops->decrypt (addr, numpages); }EXPORT_SYMBOL_GPL (set_memory_decrypted); + +bool force_dma_unencrypted (struct device *dev) +{ +if (dev->tdi_enabled) +return false;Is this OK? I see code like this:
static inline
dma_addr_tphys_to_dma_direct(struct device *dev,
phys_addr_tphys){ if (force_dma_unencrypted(dev)) return phys_to_dma_unencrypted(dev, phys); return phys_to_dma(dev, phys);What are the ARM rules for generating dma addreses?
1) Device is T=0, memory is unencrypted, call
dma_addr_unencrypted()and do "top bit IBA set"2) Device is T=1, memory is encrypted, use the
phys_to_dma()normally3) Device it T=1, memory is uncrypted, use the
phys_to_dma()normally??? Seems odd, I would have guessed the DMA address sould be the same as case #1?Quoted 5 lines
Quoted 2 lines
Can you document this in a comment?
If a device is operating in secure mode (T=1), it is currently assumed that only access to private (encrypted) memory is supported.
No, this is no how the PCI specs were written as far as I understand. The XT bit thing is supposed to add more fine grained device side control over what memory the DMA can target. T alone does not do that.
Quoted 3 lines
It is unclear whether devices would need to perform DMA to shared (unencrypted) memory while operating in this mode, as TLPs with T=1 are generally expected to target private memory.
Quoted 8 lines
Quoted 2 lines
PCI SIG supports it, kernel should support it.
Would we also need a separate DMA allocation API for allocating addresses intended to be shared with the non-secure hypervisor?
Are there any existing drivers in the kernel that already require such shared allocations, which I could use as a reference?
The most likely case in the near term is PCI P2P to shared MMIO.
I don't know any way to allocate shared memory in a driver??
At the bare minimum this patch should be documenting the correct architecture and outlining any gaps in the current implementation.
I also don't really understand what the above code is even doing.. Isn't the design on ARM that the IPA always encodes the shared/private in the top bit?
How do we get a shared page that does not already have a phys_addr_t in the shared IPA? Shouldn't the kernel have switched to the shared IPA alias when it returned the swiotlb buffer? eg why do we need to do:
#define dma_addr_unencrypted(x) ((x) | PROT_NS_SHARED)
At all?
Suzuki ?
Jason
Quoted 6 lines · On Fri, Aug 01, 2025 at 02:19:54PM -0700, [email protected] wrote:
On the host this establishes an SPDM session and sets up link encryption (IDE) with the physical device. Leave VMs out of the picture, this capability in isolation is a useful property. It addresses the similar threat model that Intel Total Memory Encryption (TME) or AMD Secure Memory Encryption (SME) go after, i.e. interposer on a physical link capturing data in flight.
Okay, maybe connect is not an intuitive name for opening IDE sessions..
Quoted 3 lines
I started this project with "all existing T=0 drivers 'just work'" as a goal and a virtue. I have been begrudgingly pulled away from it from the slow drip of complexity it appears to push into the PCI core.
Do you have some examples? I don't really see what complexity there is if the solution it simply not auto bind any drivers to TDISP capable devices and userspace is responsible to manually bind a driver once it has reached T=1.
This seems like the minimum possible simplicitly for the kernel as simply everything is managed by userspace, and there is really no special kernel behavior beyond switching the DMA API of an unbound driver on the T=0/1 change.
Quoted 2 lines
The concern is neither userspace nor the PCI core have everything it needs to get the device to T=1.
Disagree, I think userspace can have everything. It may need some per-device userspace support in difficult cases, but userspace can deal with it..
Quoted 2 lines
PCI core knows that the device is T=1 capable, but does not know how to preconfigure the device-specific lock state,
Userspace can do this. Can we define exactly what is needed to do this "pre-configure the device specific lock state"? At the very worst, for the most poorly designed device, userspace would have to bind a T=0 driver and then unbind it.
Again, I am trying to make something simple for the kernel that gets us to a working solution before we jump ahead to far more complex in the kernel models, like aware drivers that can toggle themselves between T=0/1.
Quoted 2 lines
Userspace might be able to bind a new driver that leaves the device in a lockable state on unbind, but that is not "just works" that is,
I wouldn't have the kernel leave the device in the locked state. That should always be userspace. The special driver may do whatever special setup is needed, then unbind and leave a normal unlocked device "prepped" for userspace locking without doing a FLR or something. Realistically I expect this to be a very rare requirement, I think this coming up just reflects the HW immaturity of some early TDISP devices.
Sensible mature devices should have no need of a pre-locking step. I think we should design toward that goal as the stable future and only try to enable a hacky work around for the problematic early devices. I certainly am not keen on seeing significant permanent kernel complexity to support this device design defect.
Quoted 5 lines
driver that expects the device arrives already running. Also, that main driver needs to be careful not to trigger typically benign actions like touch the command register to trip the device into ERROR state, or any device-specific actions that trip ERROR state but would otherwise be benign outside of TDISP."
As I said below, I disagree with this. You can't touch the *physical* command register but the cVM can certainly touch the *virtualized* command register. It up to the VMM To ensure this doesn't cause the device to fall out of RUN as part of virtualization.
I'd also say that the VMM should be responsible to set pBME=1 even if vBME=0? Shouldn't it? That simplifies even more things for the guest.
Quoted 7 lines
Quoted 4 lines
From that principal the kernel should NOT auto probe drivers to T=0 devices that can be made T=1. Userspace should handle attaching HW to such devices, and userspace can sequence whatever is required, including the attestation and verifying.
Agree, for PCI it would be simple to set a no-auto-probe policy for T=1 capable devices.
So then it is just a question of what does a userspace component need to do.
Quoted 5 lines
I do not want to burden the PCI core with TDISP compatibility hacks and workarounds if it turns out only a small handful of devices ever deploy a first generation TDISP Device Security Manager (DSM). L1 aiding L2, or TDISP simplicity improvements to allow the PCI core to handle this in a non-broken way, are what I expect if secure device assignment takes off.
Same feeling about pre-configuration :)
Quoted 5 lines
Quoted 3 lines
The starting point must have the core code do this sequence for every driver. Once that is working we can talk about if other flows are needed.
Do you agree that "device-specific-prep+lock" is the problem to solve?
Not "the" problem, but an design issue we need to accommodate but not endorse.
Quoted 9 lines
Quoted 4 lines
But I think we can start with the idea that such RAS failures have to reload the driver too and work on improvements. Realistically few drivers have the sort of RAS features to consume this anyhow and maybe we introduce some "enhanced" driver mode to opt-into down the road.
Hmm, having trouble not reading that back supporting my argument above:
Realistically few devices support TDISP lets require enhanced drivers to opt-into TDISP for the time being.
I would be comfortable if hitless RAS recovery for TDISP devices requires some kernel opt-in. But also I'm not sure how this should work from a security perspective. Should userspace also have to re-attest before allowing back to RUN? Clearly this is complicated.
Also, I would be comfortable to support this only for devices that do not require pre-configuration.
Jason
Quoted 11 lines · Jason Gunthorpe
Quoted 6 lines · On Fri, Aug 01, 2025 at 02:19:54PM -0700, [email protected] wrote:
On the host this establishes an SPDM session and sets up link encryption (IDE) with the physical device. Leave VMs out of the picture, this capability in isolation is a useful property. It addresses the similar threat model that Intel Total Memory Encryption (TME) or AMD Secure Memory Encryption (SME) go after, i.e. interposer on a physical link capturing data in flight.
Okay, maybe connect is not an intuitive name for opening IDE sessions..
Part of the rationale for a generic name is the TSM is free to assert that the link is secure without IDE. Think integrated devices where there is no expectation the link can be observed.
The host and guest side TSM operations are split into link/transport security and device/state security (private MMIO/DMA) concerns respectively. So maybe "secure_link" would be a better name for this host-side-only operation.
Quoted 8 lines
Quoted 3 lines
I started this project with "all existing T=0 drivers 'just work'" as a goal and a virtue. I have been begrudgingly pulled away from it from the slow drip of complexity it appears to push into the PCI core.
Do you have some examples? I don't really see what complexity there is if the solution it simply not auto bind any drivers to TDISP capable devices and userspace is responsible to manually bind a driver once it has reached T=1.
The example I have front of mind (confirmed by 2 vendors) is deferring the loading of guest-side device/state security capable firmware to the guest driver when the full device is assigned. In that scenario default device power-on firmware is capable of link/transport security, enough to get the device assigned. Guest needs to get the device/state security firmware loaded before TDISP state transitions are possible.
I do think RAS recovery needs it too, but like you say below that should come with conditions.
Quoted 11 lines
This seems like the minimum possible simplicitly for the kernel as simply everything is managed by userspace, and there is really no special kernel behavior beyond switching the DMA API of an unbound driver on the T=0/1 change.
Quoted 2 lines
The concern is neither userspace nor the PCI core have everything it needs to get the device to T=1.
Disagree, I think userspace can have everything. It may need some per-device userspace support in difficult cases, but userspace can deal with it..
I do think userspace can / must deal with it. Let me come back with actual patches and a sample test case. I see a potential path to support the above "prep" scenario without the mess of TDISP setup drivers, or the ugly complexity of driver toggles or a usermodehelper.
Quoted 8 lines
Quoted 2 lines
PCI core knows that the device is T=1 capable, but does not know how to preconfigure the device-specific lock state,
Userspace can do this. Can we define exactly what is needed to do this "pre-configure the device specific lock state"? At the very worst, for the most poorly designed device, userspace would have to bind a T=0 driver and then unbind it.
Quoted 4 lines
Again, I am trying to make something simple for the kernel that gets us to a working solution before we jump ahead to far more complex in the kernel models, like aware drivers that can toggle themselves between T=0/1.
Agree. When I talked about wishing for the simple TDISP case that is userspace can always "just lock" and "driver bind" without needing to worry about "prep", i.e any "prep" is always implied by "lock". That should be the baseline.
Quoted 11 lines
Quoted 2 lines
Userspace might be able to bind a new driver that leaves the device in a lockable state on unbind, but that is not "just works" that is,
I wouldn't have the kernel leave the device in the locked state. That should always be userspace. The special driver may do whatever special setup is needed, then unbind and leave a normal unlocked device "prepped" for userspace locking without doing a FLR or something. Realistically I expect this to be a very rare requirement, I think this coming up just reflects the HW immaturity of some early TDISP devices.
Quoted 5 lines
Sensible mature devices should have no need of a pre-locking step. I think we should design toward that goal as the stable future and only try to enable a hacky work around for the problematic early devices. I certainly am not keen on seeing significant permanent kernel complexity to support this device design defect.
Yeah, that is the nightmare I had last night. I completed the thought exercise about driver toggle and said, "whoops, nope, Jason is right, we can't design for that without leaving a permanent mess to cleanup". The end goal needs to look like straight line typical driver probe path for TDISP capable devices.
Quoted 6 lines
Quoted 5 lines
driver that expects the device arrives already running. Also, that main driver needs to be careful not to trigger typically benign actions like touch the command register to trip the device into ERROR state, or any device-specific actions that trip ERROR state but would otherwise be benign outside of TDISP."
Quoted 7 lines
As I said below, I disagree with this. You can't touch the *physical* command register but the cVM can certainly touch the *virtualized* command register. It up to the VMM To ensure this doesn't cause the device to fall out of RUN as part of virtualization.
I'd also say that the VMM should be responsible to set pBME=1 even if vBME=0? Shouldn't it? That simplifies even more things for the guest.
True. Although, now I am going back on my PCI core burden concern to wonder if *it* should handle a vBME on behalf of the driver if only because it may want to force the device out of the RUN state on driver unbind to meet typical pci_disable_device() expectations.
Alexey had this, I thought it was burdensome, now coming around.
Quoted 19 lines
Quoted 7 lines
Quoted 4 lines
From that principal the kernel should NOT auto probe drivers to T=0 devices that can be made T=1. Userspace should handle attaching HW to such devices, and userspace can sequence whatever is required, including the attestation and verifying.
Agree, for PCI it would be simple to set a no-auto-probe policy for T=1 capable devices.
So then it is just a question of what does a userspace component need to do.
Quoted 5 lines
I do not want to burden the PCI core with TDISP compatibility hacks and workarounds if it turns out only a small handful of devices ever deploy a first generation TDISP Device Security Manager (DSM). L1 aiding L2, or TDISP simplicity improvements to allow the PCI core to handle this in a non-broken way, are what I expect if secure device assignment takes off.
Same feeling about pre-configuration :)
Quoted 8 lines
Quoted 5 lines
Quoted 3 lines
The starting point must have the core code do this sequence for every driver. Once that is working we can talk about if other flows are needed.
Do you agree that "device-specific-prep+lock" is the problem to solve?
Not "the" problem, but an design issue we need to accommodate but not endorse.
I hear you, let me walk back from the cliff with patches.
Quoted 11 lines
Quoted 10 lines
Quoted 5 lines
But I think we can start with the idea that such RAS failures have to reload the driver too and work on improvements. Realistically few drivers have the sort of RAS features to consume this anyhow and maybe we introduce some "enhanced" driver mode to opt-into down the road.
Hmm, having trouble not reading that back supporting my argument above:
Realistically few devices support TDISP lets require enhanced drivers to opt-into TDISP for the time being.
Quoted 7 lines
I would be comfortable if hitless RAS recovery for TDISP devices requires some kernel opt-in. But also I'm not sure how this should work from a security perspective. Should userspace also have to re-attest before allowing back to RUN? Clearly this is complicated.
Also, I would be comfortable to support this only for devices that do not require pre-configuration.
That seems reasonable. You want hitless RAS? Give us hitless init.
Quoted 11 lines · On Sat, Aug 02, 2025 at 04:50:50PM -0700, [email protected] wrote:
Quoted 4 lines
Do you have some examples? I don't really see what complexity there is if the solution it simply not auto bind any drivers to TDISP capable devices and userspace is responsible to manually bind a driver once it has reached T=1.
The example I have front of mind (confirmed by 2 vendors) is deferring the loading of guest-side device/state security capable firmware to the guest driver when the full device is assigned. In that scenario default device power-on firmware is capable of link/transport security, enough to get the device assigned. Guest needs to get the device/state security firmware loaded before TDISP state transitions are possible.
Yeah, those are the only cases I know of too, and IMHO, they are just early devices. Clearly the clean answer is to put enough boot FW on the device's flash to get to T=1 mode, then have the trusted OS driver load the operating firmware from the trusted OS filesystem though the trusted bootloader T=1 device.
You effectively attest the bootloader, and then if you trust the bootloader you know that when the device gets to T=1 it can be trusted to properly run the FW the trusted driver provides.
Think about this more broadly, does the prep FW load idea make sense for something like SRIOV? No, it really doesn't. The hypervisor loaded FW that is running the PF should definately be strong enough to get to T=1 on the VM/VF side as well.
The non-SRIOV cases are quite often whole machine assignment scenarios. But I'm sensing alot of that space is moving toward bare metal machines instead of VMs.
I wonder if you can use all the CC machinery to attest and secure a bare metal host?
Quoted 2 lines
I do think RAS recovery needs it too, but like you say below that should come with conditions.
Especially RAS becomes simple because it basically follows the normal flows that existed prior to TDISP, with the exception of needing some attestation step.
I don't know alot about CC attestation, but maybe we can have userspace provide the kernel with the accepted measurement and then for RAS the kernel can FLR, remeasure and if the measurement is exactly the same go back into T=1 automatically as part of the PCI core FLR logic.
Quoted 4 lines
I do think userspace can / must deal with it. Let me come back with actual patches and a sample test case. I see a potential path to support the above "prep" scenario without the mess of TDISP setup drivers, or the ugly complexity of driver toggles or a usermodehelper.
I don't see how, something nasty has to be done in the kernel to allow an attached driver to switch between T=1 and T=0 "views" of the device and lockstep those changes with userspace. This is not so simple and it really basically exactly the same as driver binding.
I don't think we should be afraid of T=0 prep drivers in these early days.
Something more complex could come later if it is really warranted and people really insist on continuing this unclean device design strategy.
Quoted 5 lines
Yeah, that is the nightmare I had last night. I completed the thought exercise about driver toggle and said, "whoops, nope, Jason is right, we can't design for that without leaving a permanent mess to cleanup". The end goal needs to look like straight line typical driver probe path for TDISP capable devices.
Yeah, maybe it is worthwhile to someday try to figure out an alternative - keep in mind that critically this requires someone to also come with an intree driver that will use all these new APIs and capabilities!!!
So lets get walking first and then someone can come with some proposal, complete with a driver implementing it, and it can be judged. This project is already so big, and I'm pretty sure if you start to also need entirely new operating modes for drivers the basics will just get bogged down in that discussion, and very likely killed anyhow due to a lack of user.
Even if we decide that is prefered it is better to separate it and discuss it after the basics are merged. At least where I sit getting basic guest support is a big priority so I strongly want to strip it down to minimal as possible to make consistent progress steps.
Quoted 4 lines
True. Although, now I am going back on my PCI core burden concern to wonder if *it* should handle a vBME on behalf of the driver if only because it may want to force the device out of the RUN state on driver unbind to meet typical
pci_disable_device()expectations.
Hiding some vBME in the PCI core might make sense if we can't get the VMM owners to agree to do it on the hypervisor side. It works better on the VMM side because there is always an IOMMU and the VMM can emulate BME by blocking DMA with the IOMMU.
But I would not allow/expect kernel device drivers to have anything to do with the TDISP states. Getting into RUN is fully sequenced by userspace, getting out of run should also be sequenced only by userspace.
Removing a driver does not change the trust state of the PCI device, so it shouldn't drop out of RUN. If userspace wishes to FLR the device after userspace asked to unbind it can, there are already sysfs controls for this IIRC.
Basically, all this says that Linux drivers that want to be used with T=1 should be well behaved, fully quite all their DMA on remove, and have no *functional* need for BME to do anyhting. We pretty much already expect this of drivers today, so I don't see an issue with strongly requiring it for T=1.
Keep in mind the flip side, almost no drivers are structured properly to forcibly quiet any DMA before pci_enable_device(). Some HW, like mlx5, can't do this at all without either using DMA to send a reset command or through FLR.
Quoted 9 lines
Quoted 7 lines
I would be comfortable if hitless RAS recovery for TDISP devices requires some kernel opt-in. But also I'm not sure how this should work from a security perspective. Should userspace also have to re-attest before allowing back to RUN? Clearly this is complicated.
Also, I would be comfortable to support this only for devices that do not require pre-configuration.
That seems reasonable. You want hitless RAS? Give us hitless init.
Yeah.. Realistically there are few drivers that can even do this today, mlx5 for example has such code (and it is hard!).
There is alot of investment required in the driver's core subsystem to make this work. netdev and RDMA can support a 'rebirth' sort of flow where the driver can disconnect the SW APIs, FLR the device, then reconnect in some way. However, for example, I recently had a discussion with DRM guys about RAS and they are not even doing the basic locking/etc to be able to do this. :\
Jason
Alexey Kardashevskiy
maintainerRe: [RFC PATCH v1 10/38] iommufd/vdevice: Add TSM map ioctl
In reply to Jason GunthorpeQuoted 7 lines · Jason Gunthorpe
Quoted 5 lines · Aneesh Kumar K.V (Arm)
With passthrough devices, we need to make sure private memory is allocated and assigned to the secure guest before we can issue the DMA. For ARM RMM, we only need to map and the secure SMMU management is internal to RMM. For shared IPA, vfio/iommufd DMA MAP/UNMAP interface does the equivalent
Quoted 6 lines
I'm not really sure what this is about? It is about getting KVM to pin all the memory and commit it to the RMM so it can be used for DMA?
But it looks really strange to have an iommufd ioctl that just calls a KVM function. Feeling this should be a KVM function, or a guestmfd behavior??
I ended up exporting the guestmemfd's kvm_gmem_get_folio() for gfn->pfn and its fd a bit differently in iommufd - "no extra referencing":
https://github.com/AMDESE/linux-kvm/commit/f1ebd358327f026f413f8d3d64d46decfd6ab7f6
It is a new iommufd->kvm dependency though.
Quoted 4 lines
I was kind of thinking it would be nice to have a guestmemfd mode that was "pinned", meaning the memory is allocated and remains almost always mapped into the TSM's page tables automatically. VFIO using guests would set things this way.
Yeah while doing the above, I was wondering if I want to pass the fd type when DMA-mapping from an fd or "detect" it as I do in the above commit or have some iommufd_fdmap_ops in this fd saying "(no) pinning needed" (or make this a flag of IOMMU_IOAS_MAP_FILE).
The "detection" is (mapping_inaccessible(mapping) && mapping_unevictable(mapping)), works for now.
btw in the AMD case, here it does not matter as much if it is private or shared, I map everything and let RMP and the VM deal with the permissions. Thanks,
Quoted 2 lines
Jason
Signature · 1 line
Alexey
Aneesh Kumar K.V
maintainerRe: [RFC PATCH v1 14/38] coco: host: arm64: Device communication support
In reply to Jonathan CameronQuoted 202 lines · Jonathan Cameron
Quoted 45 lines · On Mon, 28 Jul 2025 19:21:51 +0530 "Aneesh Kumar K.V (Arm)"…
Add helpers for device communication from RMM
Signed-off-by Aneesh Kumar K.V (Arm) <[email protected]>--- arch/arm64/include/asm/rmi_cmds.h | 11 ++ arch/arm64/include/asm/rmi_smc.h | 49 ++++++ drivers/virt/coco/arm-cca-host/arm-cca.c | 45 ++++++ drivers/virt/coco/arm-cca-host/rmm-da.c | 198 +++++++++++++++++++++++ drivers/virt/coco/arm-cca-host/rmm-da.h | 41 +++++ 5 files changed, 344 insertions(+)#endif /*
__ASM_RMI_CMDS_H*/arch/arm64/include/asm/rmi_smc.h +25
-0@@ -47,6 +47,7 @@ +#define RMI_DEV_COMM_EXIT_CACHE_REQ BIT(0) +#define RMI_DEV_COMM_EXIT_CACHE_RSP BIT(1) +#define RMI_DEV_COMM_EXIT_SEND BIT(2) +#define RMI_DEV_COMM_EXIT_WAIT BIT(3) +#define RMI_DEV_COMM_EXIT_MULTI BIT(4) + +#define RMI_DEV_COMM_NONE 0 +#define RMI_DEV_COMM_RESPONSE 1 +#define RMI_DEV_COMM_ERROR 2 + +#define RMI_PROTOCOL_SPDM 0 +#define RMI_PROTOCOL_SECURE_SPDM 1 + +#define RMI_DEV_VCA 0 +#define RMI_DEV_CERTIFICATE 1 +#define RMI_DEV_MEASUREMENTS 2 +#define RMI_DEV_INTERFACE_REPORT 3 + +struct rmi_dev_comm_enter { +u64 status; +u64 req_addr; +u64 resp_addr; +u64 resp_len; +}; +arch/arm64/include/asm/rmi_smc.h struct rmi_pdev_params { · line 376
Quoted 10 lines
arch/arm64/include/asm/rmi_smc.h +10
-0+struct rmi_dev_comm_exit { +u64 flags; +u64 cache_req_offset; +u64 cache_req_len; +u64 cache_rsp_offset; +u64 cache_rsp_len; +u64 cache_obj_id; +u64 protocol; +u64 req_len; +u64 timeout;In latest spec called
rsp_timeout.
Not sure we care that much but if no strong reason otherwise, should aim to match the spec text. (Maybe this got renamed?)Quoted 7 lines
arch/arm64/include/asm/rmi_smc.h +1
-0+};drivers/virt/coco/arm-cca-host/arm-cca.c +0
-0drivers/virt/coco/arm-cca-host/arm-cca.c static void cca_tsm_pci_remove(struct pci_tsm *tsm) · line 92
Quoted 8 lines
drivers/virt/coco/arm-cca-host/arm-cca.c +5
-0@@ -85,6 +85,45 @@ static void cca_tsm_pci_remove(struct pci_tsm *tsm)vfree (dsc_pf0); } +static int init_dev_communication_buffers (struct cca_host_comm_data *comm_data) +{ +int ret = -ENOMEM ; + + comm_data->io_params = (struct rmi_dev_comm_data *)get_zeroed_page (GFP_KERNEL );Hmm. There isn't a
DEFINE_FREE()yet forfree_page(). Maybe time to add one. If we did then we'd use local variables until all allocations succeed then assign withno_free_ptr()Quoted 5 lines
drivers/virt/coco/arm-cca-host/arm-cca.c +5
-0+if (!comm_data->io_params) + goto err_out; + + comm_data->resp_buff = (void *)__get_free_page (GFP_KERNEL ); +if (!comm_data->resp_buff)drivers/virt/coco/arm-cca-host/arm-cca.c static void cca_tsm_pci_remove(struct pci_tsm *tsm) · line 107
Quoted 10 lines
drivers/virt/coco/arm-cca-host/arm-cca.c +10
-0+ goto err_res_buff; + + comm_data->req_buff = (void *)__get_free_page (GFP_KERNEL ); +if (!comm_data->req_buff) + goto err_req_buff; + + + comm_data->io_params->enter.status =RMI_DEV_COMM_NONE ; + comm_data->io_params->enter.resp_addr =virt_to_phys (comm_data->resp_buff); + comm_data->io_params->enter.req_addr =virt_to_phys ((void *)comm_data->req_buff);I think it's already a a void * and even if it were some other pointer type no cast would be necessary.
Quoted 26 lines
drivers/virt/coco/arm-cca-host/arm-cca.c +14
-0+ comm_data->io_params->enter.resp_len =0 ; + +return 0 ; + +err_req_buff: +free_page ((unsigned long )comm_data->resp_buff); +err_res_buff: +free_page ((unsigned long )comm_data->io_params); +err_out: +return ret; +} + +/* per root port unique with multiple restrictions. For now global */ static DECLARE_BITMAP(cca_stream_ids, MAX_STREAM_ID);@@ -124,6 +163,7 @@ static int cca_tsm_connect(struct pci_dev *pdev) rc =tsm_ide_stream_register (pdev, ide);if (rc) goto err_tsm; +init_dev_communication_buffers (&dsc_pf0->comm_data);/* * Take a module reference so that we won't call unregister * without rme_unasign_device drivers/virt/coco/arm-cca-host/arm-cca.c static int cca_tsm_connect(struct pci_dev *pdev) · line 175
Quoted 3 lines
drivers/virt/coco/arm-cca-host/arm-cca.c +0
-0@@ -133,6 +173,11 @@ static int cca_tsm_connect(struct pci_dev *pdev) goto err_tsm; }rme_asign_device (pdev);
rme_assign_device()- I obviously missed this earlier!drivers/virt/coco/arm-cca-host/arm-cca.c static int cca_tsm_connect(struct pci_dev *pdev) · line 177
Quoted 2 lines
drivers/virt/coco/arm-cca-host/arm-cca.c +2
-0+/* +* Schedule a work to fetch device certificate and setup IDE Single line comment probably fine here. Though it perhaps doesn't add much over the function name.
Quoted 16 lines
drivers/virt/coco/arm-cca-host/arm-cca.c +3
-0+*/ +schedule_rme_ide_setup (pdev); +/* * Once ide is setup enable the stream at endpoint * Root port will be done by RMM drivers/virt/coco/arm-cca-host/rmm-da.c +2
-0@@ -148,3 +148,201 @@ int rme_asign_device(struct pci_dev *pci_dev) err_out:return ret; } + +static int doe_send_req_resp (struct pci_tsm *tsm)drivers/virt/coco/arm-cca-host/rmm-da.c int rme_asign_device(struct pci_dev *pci_dev) · line 162
Quoted 10 lines
drivers/virt/coco/arm-cca-host/rmm-da.c +10
-0+{ +u8 protocol; +int ret, data_obj_type; +struct cca_host_comm_data *comm_data; +struct rmi_dev_comm_exit *io_exit; + + comm_data =to_cca_comm_data (tsm->pdev); + + io_exit = &comm_data->io_params->exit; + protocol = io_exit->protocol;For all these I'd combine with the declarations.
Quoted 24 lines
drivers/virt/coco/arm-cca-host/rmm-da.c +24
-0+ +pr_debug ("doe_req size:%lld doe_io_type=%d\n" , io_exit->req_len, (int )protocol); + +if (protocol ==RMI_PROTOCOL_SPDM ) + data_obj_type =PCI_DOE_PROTO_CMA ; +else if (protocol ==RMI_PROTOCOL_SECURE_SPDM ) + data_obj_type =PCI_DOE_PROTO_SSESSION ; +else +return -EINVAL ; + + ret =pci_tsm_doe_transfer (tsm->dsm_dev, data_obj_type, + comm_data->req_buff, io_exit->req_len, + comm_data->resp_buff,PAGE_SIZE ); +pr_debug ("doe returned:%d\n" , ret); +return ret; +} + +/* Parallel update for cca_dsc contents FIXME!! */ +static int __do_dev_communicate (int type,struct pci_tsm *tsm) +{ +int ret; +bool is_multi; +u8 *cache_buf; +int *cache_offset;drivers/virt/coco/arm-cca-host/rmm-da.c int rme_asign_device(struct pci_dev *pci_dev) · line 196
Quoted 10 lines
drivers/virt/coco/arm-cca-host/rmm-da.c +10
-0+int nbytes, cache_remaining; +struct cca_host_dsc_pf0 *dsc_pf0; +struct rmi_dev_comm_exit *io_exit; +struct rmi_dev_comm_enter *io_enter; +struct cca_host_comm_data *comm_data; + + + comm_data =to_cca_comm_data (tsm->pdev); + io_enter = &comm_data->io_params->enter; + io_exit = &comm_data->io_params->exit;Might as well set these local variables as the declaration point above. None of them will be very long lines.
Quoted 15 lines
drivers/virt/coco/arm-cca-host/rmm-da.c int rme_asign_device(struct pci_dev *pci_dev) · line 206
Quoted 10 lines
drivers/virt/coco/arm-cca-host/rmm-da.c +10
-0+ + dsc_pf0 =to_cca_dsc_pf0 (tsm->dsm_dev); +redo_communicate: + is_multi = false; + +if (type ==PDEV_COMMUNICATE ) + ret =rmi_pdev_communicate (virt_to_phys (dsc_pf0->rmm_pdev), +virt_to_phys (comm_data->io_params)); +else + ret =RMI_ERROR_INPUT ;I'd split this case out and return here farther than using the match below as it feels like the error message auto to be more specific. Something about type not matching.
This get updated in the follow up patch as below.
if (type == PDEV_COMMUNICATE)
ret = rmi_pdev_communicate(virt_to_phys(dsc_pf0->rmm_pdev),
virt_to_phys(comm_data->io_params));
else {
struct cca_host_tdi *host_tdi = container_of(tsm->tdi, struct cca_host_tdi, tdi);
ret = rmi_vdev_communicate(virt_to_phys(dsc_pf0->rmm_pdev),
virt_to_phys(host_tdi->rmm_vdev),
virt_to_phys(comm_data->io_params));
}Quoted 105 lines
Quoted 32 lines
drivers/virt/coco/arm-cca-host/rmm-da.c +31
-0+if (ret !=RMI_SUCCESS ) { +pr_err ("pdev communicate error\n" ); +return ret; + } + +/* caching request from RMM */ +if (io_exit->flags &RMI_DEV_COMM_EXIT_CACHE_RSP ) { +switch (io_exit->cache_obj_id) { +case RMI_DEV_VCA : + cache_buf = dsc_pf0->vca.buf; + cache_offset = &dsc_pf0->vca.size; + cache_remaining =sizeof (dsc_pf0->vca.buf) - *cache_offset; +break ; +case RMI_DEV_CERTIFICATE : + cache_buf = dsc_pf0->cert_chain.cache.buf; + cache_offset = &dsc_pf0->cert_chain.cache.size; + cache_remaining =sizeof (dsc_pf0->cert_chain.cache.buf) - *cache_offset; +break ; +default : +/* FIXME!! depending on the DevComms status, +* it might require to ABORT the communcation. +*/ +return -EINVAL ; + } + +if (io_exit->cache_rsp_len > cache_remaining) +return -EINVAL ; + +memcpy (cache_buf + *cache_offset, + (comm_data->resp_buff + io_exit->cache_rsp_offset), io_exit->cache_rsp_len); + *cache_offset += io_exit->cache_rsp_len;drivers/virt/coco/arm-cca-host/rmm-da.c int rme_asign_device(struct pci_dev *pci_dev) · line 247
Quoted 10 lines
drivers/virt/coco/arm-cca-host/rmm-da.c +10
-0+ } + +/* +* wait for last packet request from RMM. +* We should not find this because our device communication in synchronous +*/ +if (io_exit->flags &RMI_DEV_COMM_EXIT_WAIT ) +return -ENXIO ; + + is_multi = !!(io_exit->flags &RMI_DEV_COMM_EXIT_MULTI );!! doesn't add anything here that I can see over
is_multi = io_exit->flags & RMI_DEV_COMM_EXIT_MULTI;Quoted 31 lines
drivers/virt/coco/arm-cca-host/rmm-da.c +31
-0+ +/* next packet to send */ +if (io_exit->flags &RMI_DEV_COMM_EXIT_SEND ) { + nbytes =doe_send_req_resp (tsm); +if (nbytes <0 ) { +/* report error back to RMM */ + io_enter->status =RMI_DEV_COMM_ERROR ; + }else { +/* send response back to RMM */ + io_enter->resp_len = nbytes; + io_enter->status =RMI_DEV_COMM_RESPONSE ; + } + }else { +/* no data transmitted => no data received */ + io_enter->resp_len =0 ; + } + +/* The call need to do multiple request/respnse */ +if (is_multi) + goto redo_communicate; + +return 0 ; +} + +static int do_dev_communicate (int type,struct pci_tsm *tsm,int target_state) +{ +int ret; +unsigned long state; +unsigned long error_state; +struct cca_host_dsc_pf0 *dsc_pf0; +struct rmi_dev_comm_enter *io_enter;drivers/virt/coco/arm-cca-host/rmm-da.c int rme_asign_device(struct pci_dev *pci_dev) · line 288
Quoted 10 lines
drivers/virt/coco/arm-cca-host/rmm-da.c +10
-0+ + dsc_pf0 =to_cca_dsc_pf0 (tsm->dsm_dev); + io_enter = &dsc_pf0->comm_data.io_params->enter; + io_enter->resp_len =0 ; + io_enter->status =RMI_DEV_COMM_NONE ; + + state =-1 ; + do { + ret = __do_dev_communicate(type, tsm); + if (ret != 0) {if (ret)
drivers/virt/coco/arm-cca-host/rmm-da.c int rme_asign_device(struct pci_dev *pci_dev) · line 290
Quoted 2 lines
drivers/virt/coco/arm-cca-host/rmm-da.c +2
-0+pr_err ("dev communication error\n" ); +break ;I'd just return in error cases.
Quoted 9 lines
drivers/virt/coco/arm-cca-host/rmm-da.c +9
-0+ } + +if (type ==PDEV_COMMUNICATE ) { + ret =rmi_pdev_get_state (virt_to_phys (dsc_pf0->rmm_pdev), + &state); + error_state =RMI_PDEV_ERROR ; + } +if (ret !=0 ) { +pr_err ("Get dev state error\n" );
Quoted 8 lines
drivers/virt/coco/arm-cca-host/rmm-da.c int rme_asign_device(struct pci_dev *pci_dev) · line 307
Quoted 8 lines
drivers/virt/coco/arm-cca-host/rmm-da.c +8
-0+break ; + } + }while (state != target_state && state != error_state); + +pr_info ("dev_io_complete: status: %d state:%ld\n" , ret, state); + +return state; +}
Thanks for the review comments. I'll update the patch with the suggested changes.
-aneesh
Aneesh Kumar K.V
maintainerRe: [RFC PATCH v1 15/38] coco: host: arm64: Stop and destroy the physical device
In reply to Jonathan CameronQuoted 10 lines · Jonathan Cameron
Quoted 1 line · On Mon, 28 Jul 2025 19:21:52 +0530 "Aneesh Kumar K.V (Arm)"…
Add support for stopping and destroying physical devices.
I think it's an odd mix to not do create and destroy in a single patch. Same with start and stop.
Leaves reviewers thinking perhaps you weren't cleaning up properly an any error paths are much less obvious.
I’ll fold this into the patch that adds the pdev_create functionality.
-aneesh
Aneesh Kumar K.V
maintainerRe: [RFC PATCH v1 19/38] coco: host: arm64: set_pubkey support
In reply to Jonathan CameronQuoted 170 lines · Jonathan Cameron
Quoted 3 lines · On Mon, 28 Jul 2025 19:21:56 +0530 "Aneesh Kumar K.V (Arm)"…
Add changes to share the device's public key with the RMM.
Signed-off-by Aneesh Kumar K.V (Arm) <[email protected]>A few minor comments inline.
Quoted 30 lines
drivers/virt/coco/arm-cca-host/rmm-da.c +15
-0@@ -6,6 +6,9 @@#include <linux/pci.h> #include <linux/pci-ecam.h> #include <asm/rmi_cmds.h> +#include <crypto/internal/rsa.h> +#include <keys/asymmetric-type.h> +#include <keys/x509-parser.h> #include "rmm-da.h" @@ -311,6 +314,136 @@ static int do_pdev_communicate(struct pci_tsm *tsm, int target_state)return do_dev_communicate (PDEV_COMMUNICATE , tsm, target_state); } +static int parse_certificate_chain (struct pci_tsm *tsm) +{ +struct cca_host_dsc_pf0 *dsc_pf0; +unsigned int chain_size; +unsigned int offset =0 ; +u8 *chain_data; +int ret =0 ; + + dsc_pf0 =to_cca_dsc_pf0 (tsm->pdev); + chain_size = dsc_pf0->cert_chain.cache.size; + chain_data = dsc_pf0->cert_chain.cache.buf; +drivers/virt/coco/arm-cca-host/rmm-da.c static int do_pdev_communicate(struct pci_tsm *tsm, int target_state) · line 338
Quoted 10 lines
drivers/virt/coco/arm-cca-host/rmm-da.c +10
-0+while (offset < chain_size) { +unsigned int cert_len = +x509_get_certificate_length (chain_data + offset, + chain_size - offset); +struct x509_certificate *cert = +x509_cert_parse (chain_data + offset, cert_len); + +if (IS_ERR (cert)) { +pr_warn ("%s(): parsing of certificate chain not successful\n" , __func__); + ret =PTR_ERR (cert);Direct return looks fine here. Maybe add a
DEFINE_FREE(x509_cert,...)as then can use direct returns throughout.Quoted 23 lines
drivers/virt/coco/arm-cca-host/rmm-da.c +23
-0+break ; + } + +if (offset + cert_len == chain_size) { + dsc_pf0->cert_chain.public_key =kzalloc (cert->pub->keylen,GFP_KERNEL ); +if (!dsc_pf0->cert_chain.public_key) { + ret = -ENOMEM ; +x509_free_certificate (cert); +break ; + } + +if (!strcmp ("ecdsa-nist-p256" , cert->pub->pkey_algo)) { + dsc_pf0->rmi_signature_algorithm =RMI_SIG_ECDSA_P256 ; + }else if (!strcmp ("ecdsa-nist-p384" , cert->pub->pkey_algo)) { + dsc_pf0->rmi_signature_algorithm =RMI_SIG_ECDSA_P384 ; + }else if (!strcmp ("rsa" , cert->pub->pkey_algo)) { + dsc_pf0->rmi_signature_algorithm =RMI_SIG_RSASSA_3072 ; + }else { + ret = -ENXIO ; +x509_free_certificate (cert); +break ; + } +memcpy (dsc_pf0->cert_chain.public_key, cert->pub->key, cert->pub->keylen);drivers/virt/coco/arm-cca-host/rmm-da.c static int do_pdev_communicate(struct pci_tsm *tsm, int target_state) · line 371
Quoted 10 lines
drivers/virt/coco/arm-cca-host/rmm-da.c +10
-0+ dsc_pf0->cert_chain.public_key_size = cert->pub->keylen; + } + +x509_free_certificate (cert); + + offset += cert_len; + } + +if (ret ==0 ) + dsc_pf0->cert_chain.valid = true;if (ret) return ret; dsc_pf0->cert_chain.valid = true; return 0;would be my preference for style here but others may disagree.
drivers/virt/coco/arm-cca-host/rmm-da.c static int do_pdev_communicate(struct pci_tsm *tsm, int target_state) · line 379
Quoted 8 lines
drivers/virt/coco/arm-cca-host/rmm-da.c +8
-0+ +return ret; +} + +static int pdev_set_public_key (struct pci_tsm *tsm) +{ +struct rmi_public_key_params *key_shared; +unsigned long expected_key_len =0 ;Don't set this. It's only used in places where it is explicitly set and if it is used anywhere else we want the compiler to tell us.
Quoted 30 lines
drivers/virt/coco/arm-cca-host/rmm-da.c +30
-0+struct cca_host_dsc_pf0 *dsc_pf0; +int ret; + + dsc_pf0 =to_cca_dsc_pf0 (tsm->pdev); +/* Check that all the necessary information was captured from communication */ +if (!dsc_pf0->cert_chain.valid) +return -EINVAL ; + + key_shared = (struct rmi_public_key_params *)get_zeroed_page (GFP_KERNEL ); +if (!key_shared) +return -ENOMEM ; + + key_shared->rmi_signature_algorithm = dsc_pf0->rmi_signature_algorithm; + +switch (key_shared->rmi_signature_algorithm) { +case RMI_SIG_ECDSA_P384 : + expected_key_len =97 ; + +if (dsc_pf0->cert_chain.public_key_size != expected_key_len) +return -EINVAL ; + key_shared->public_key_len = dsc_pf0->cert_chain.public_key_size; +memcpy (key_shared->public_key, + dsc_pf0->cert_chain.public_key, + dsc_pf0->cert_chain.public_key_size); + key_shared->metadata_len =0 ; +break ; +case RMI_SIG_ECDSA_P256 : + expected_key_len =65 ; + +if (dsc_pf0->cert_chain.public_key_size != expected_key_len)drivers/virt/coco/arm-cca-host/rmm-da.c static int do_pdev_communicate(struct pci_tsm *tsm, int target_state) · line 419
Quoted 10 lines
drivers/virt/coco/arm-cca-host/rmm-da.c +10
-0+return -EINVAL ; + key_shared->public_key_len = dsc_pf0->cert_chain.public_key_size; +memcpy (key_shared->public_key, + dsc_pf0->cert_chain.public_key, + dsc_pf0->cert_chain.public_key_size); + key_shared->metadata_len =0 ; +break ; +case RMI_SIG_RSASSA_3072 : + expected_key_len =385 ; +struct rsa_key rsa_key = {0 };Shouldn't define this inline. Maybe move up a line and add some {} to set the scope to this case statement.
Quoted 5 lines
drivers/virt/coco/arm-cca-host/rmm-da.c +5
-0+int ret_rsa_parse =rsa_parse_pub_key (&rsa_key, + dsc_pf0->cert_chain.public_key, + dsc_pf0->cert_chain.public_key_size); +/* This also checks the key_len */ +if (ret_rsa_parse)drivers/virt/coco/arm-cca-host/rmm-da.c static int do_pdev_communicate(struct pci_tsm *tsm, int target_state) · line 434
Quoted 10 lines
drivers/virt/coco/arm-cca-host/rmm-da.c +10
-0+return ret_rsa_parse; +/* +* exponent is usally 65537 (size = 24bits) but in rare cases +* it size can be as large as the modulus +*/ +if (rsa_key.e_sz > expected_key_len) +return -EINVAL ; + key_shared->public_key_len = rsa_key.n_sz; + key_shared->metadata_len = rsa_key.e_sz; +memcpy (key_shared->public_key, (unsigned char *)rsa_key.n, rsa_key.n_sz);Why is the cast needed?
Quoted 11 lines
drivers/virt/coco/arm-cca-host/rmm-da.c static int do_pdev_communicate(struct pci_tsm *tsm, int target_state) · line 445
Quoted 11 lines
drivers/virt/coco/arm-cca-host/rmm-da.c +11
-0+memcpy (key_shared->metadata, (unsigned char *)rsa_key.e, rsa_key.e_sz); +break ; +default : +return -EINVAL ; + } + + ret =rmi_pdev_set_pubkey (virt_to_phys (dsc_pf0->rmm_pdev), +virt_to_phys (key_shared)); +free_page ((unsigned long )key_shared); +return ret; +}
Thanks for the review comments. I'll update the patch with the suggested changes.
-aneesh
Aneesh Kumar K.V
maintainerRe: [RFC PATCH v1 21/38] coco: host: arm64: Add support for virtual device communication
In reply to Jonathan CameronQuoted 9 lines · Jonathan Cameron
Quoted 3 lines · On Mon, 28 Jul 2025 19:21:58 +0530 "Aneesh Kumar K.V (Arm)"…
Add support for
vdev_communicatewith RMM.Signed-off-by Aneesh Kumar K.V (Arm) <[email protected]>One minor comment.
Quoted 8 lines
drivers/virt/coco/arm-cca-host/rmm-da.c static int __do_dev_communicate(int type, struct pci_tsm *tsm)
Quoted 5 lines
drivers/virt/coco/arm-cca-host/rmm-da.c +0
-0@@ -207,8 +207,14 @@ static int __do_dev_communicate(int type, struct pci_tsm *tsm)Can type parameter be an enum so we can see all cases are covered?
Sure, will update the patch
Quoted 13 lines
drivers/virt/coco/arm-cca-host/rmm-da.c static int __do_dev_communicate(int type, struct pci_tsm *tsm) · line 216
Quoted 13 lines
drivers/virt/coco/arm-cca-host/rmm-da.c +7
-2if (type ==PDEV_COMMUNICATE ) ret =rmi_pdev_communicate (virt_to_phys (dsc_pf0->rmm_pdev),virt_to_phys (comm_data->io_params)); -else - ret =RMI_ERROR_INPUT ; +else { + struct cca_host_tdi *host_tdi = container_of(tsm->tdi, struct cca_host_tdi, tdi); + + ret =rmi_vdev_communicate (virt_to_phys (dsc_pf0->rmm_pdev), +virt_to_phys (host_tdi->rmm_vdev), +virt_to_phys (comm_data->io_params)); + }
-aneesh
Aneesh Kumar K.V
maintainerRe: [RFC PATCH v1 24/38] arm64: CCA: Register guest tsm callback
In reply to Jonathan CameronQuoted 56 lines · Jonathan Cameron
Quoted 6 lines · On Mon, 28 Jul 2025 19:22:01 +0530 "Aneesh Kumar K.V (Arm)"…
Register the TSM callback if the DA feature is supported by RSI.
Additionally, adjust the build order so that the TSM class is created before the arm-cca-guest driver initialization.
Signed-off-by Aneesh Kumar K.V (Arm) <[email protected]>See below.
Quoted 13 lines
arch/arm64/kernel/rsi.c +1
-0@@ -15,6 +15,7 @@#include <asm/rsi.h> static struct realm_config config; +static unsigned long rsi_feat_reg0;unsigned long prot_ns_shared;EXPORT_SYMBOL (prot_ns_shared);arch/arm64/kernel/rsi.c EXPORT_SYMBOL(prot_ns_shared); · line 28
Quoted 6 lines
arch/arm64/kernel/rsi.c +3
-0@@ -22,6 +23,12 @@ EXPORT_SYMBOL(prot_ns_shared);DEFINE_STATIC_KEY_FALSE_RO (rsi_present);EXPORT_SYMBOL (rsi_present); +bool rsi_has_da_feature (void ) +{ +return !!u64_get_bits (rsi_feat_reg0,RSI_FEATURE_REGISTER_0_DA );!! not needed.
Quoted 23 lines
arch/arm64/kernel/rsi.c +2
-0+} +EXPORT_SYMBOL_GPL (rsi_has_da_feature);drivers/virt/coco/arm-cca-guest/arm-cca.c +3
-1@@ -1,6 +1,6 @@static int cca_guest_probe (struct platform_device *pdev) {int ret;@@ -200,11 +256,22 @@ static int cca_guest_probe(struct platform_device *pdev)return -ENODEV ; ret =tsm_report_register (&arm_cca_tsm_report_ops,NULL ); -if (ret <0 ) +if (ret <0 ) {pr_err ("Error %d registering with TSM\n" , ret); + goto err_out; + }
Quoted 12 lines
drivers/virt/coco/arm-cca-guest/arm-cca.c static int cca_guest_probe(struct platform_device *pdev) · line 272
Quoted 9 lines
drivers/virt/coco/arm-cca-guest/arm-cca.c +8
-0ret =devm_add_action_or_reset (&pdev->dev, unregister_cca_tsm_report,NULL ); +if (ret <0 ) { +pr_err ("Error %d registering devm action\n" , ret); +unregister_cca_tsm_report (NULL ); + goto err_out; + } + +if (rsi_has_da_feature ()) + ret =cca_tsm_register (pdev);Why do we not need to call
unregister_cca_tsm_report()if this fails?
tsm_report and DA are independent functionalities. I’ll update the guest probe to return success so that tsm_report remains available even if the DA tsm registration fails.
-aneesh
Aneesh Kumar K.V
maintainerRe: [RFC PATCH v1 34/38] coco: guest: arm64: Validate mmio range found in the interface report
In reply to Arto MerilainenQuoted 27 lines · Arto Merilainen
Quoted 6 lines · Aneesh Kumar K.V (Arm)
drivers/virt/coco/arm-cca-guest/rsi-da.c +6
-0+for (int i =0 ; i < interface_report->mmio_range_count; i++, mmio_range++) { + +/*FIXME!! units in 4K size*/ + range_id =FIELD_GET (TSM_INTF_REPORT_MMIO_RANGE_ID , mmio_range->range_attributes); + +/* no secure interrupts */ drivers/virt/coco/arm-cca-guest/rsi-da.c int rsi_device_lock(struct pci_dev *pdev) · line 297
Quoted 10 lines
drivers/virt/coco/arm-cca-guest/rsi-da.c +10
-0+if (msix_tbl_bar !=-1 && range_id == msix_tbl_bar) { +pr_info ("Skipping misx table\n" ); +continue ; + } + +if (msix_pba_bar !=-1 && range_id == msix_pba_bar) { +pr_info ("Skipping misx pba\n" ); +continue ; + } +MSI-X and PBA can be placed to a BAR that has other registers as well. While the PCIe specification recommends BAR-level isolation for MSI-X structures, it is not mandated. It is enough to have sufficient isolation within the BAR. Therefore, skipping the MSI-X and PBA BARs altogether may leave registers unintentionally mapped via unprotected IPA when they should have been mapped via protected IPA.
Quoted 4 lines
Instead of skipping the whole BAR, would it make sense to determine where the MSI-X related regions reside, and skip validation only from these regions?
Yes, that was added because at one point the FVP model was including the MSI-X table and PBA regions in the interface report.
If I understand correctly, we shouldn't expect to see those regions in the report unless secure interrupts are supported. The BAR-based skipping was added as a workaround to handle the FVP issue.
I believe we can drop that workaround now—if those regions are incorrectly present, the below validation logic should catch and reject them appropriately. Does that sound reasonable?
/* No secure interrupts, we should not find this set, ignore for now. */
if (FIELD_GET(TSM_INTF_REPORT_MMIO_MSIX_TABLE, mmio_range->range_attributes) ||
FIELD_GET(TSM_INTF_REPORT_MMIO_PBA, mmio_range->range_attributes)) {
pci_info(pdev, "Skipping MSIX (%ld/%ld) range [%d] #%d %d pages, %llx..%llx\n",
FIELD_GET(TSM_INTF_REPORT_MMIO_MSIX_TABLE, mmio_range->range_attributes),
FIELD_GET(TSM_INTF_REPORT_MMIO_PBA, mmio_range->range_attributes),
i, range_id, mmio_range->num_pages, r->start, r->end);
continue;
}-aneesh
Aneesh Kumar K.V
maintainerRe: [RFC PATCH v1 38/38] coco: guest: arm64: Add support for fetching device info
In reply to Jonathan CameronQuoted 3 lines · Jonathan Cameron
On Mon, 28 Jul 2025 19:22:15 +0530
"Aneesh Kumar K.V (Arm)" <[email protected]> wrote:
...
Quoted 22 lines
Quoted 13 lines
drivers/virt/coco/arm-cca-guest/rsi-da.c +4
-0+return ret; +} +int rsi_device_lock (struct pci_dev *pdev) {unsigned long ret;unsigned long tdisp_version;struct rsi_device_measurements_params *rsi_dev_meas; +struct rsi_device_info *dev_info;struct cca_guest_dsc *dsm =to_cca_guest_dsc (pdev);int vdev_id = (pci_domain_nr (pdev->bus) <<16 ) |PCI_DEVID (pdev->bus->number, pdev->devfn);drivers/virt/coco/arm-cca-guest/rsi-da.c int rsi_device_lock(struct pci_dev *pdev) · line 348
Quoted 5 lines
drivers/virt/coco/arm-cca-guest/rsi-da.c +2
-0@@ -252,8 +344,44 @@ int rsi_device_lock(struct pci_dev *pdev)return -EIO ; } +/* RMM expects sizeof(dev_info) (512 bytes) aligned address */ + dev_info =kmalloc (sizeof (*dev_info),GFP_KERNEL );Use a
__free(kfree)here (and direct returns on errors) given it's freed in all paths and we don't care if it is freed before or after verifying the digests.
Quoted 5 lines
I'm being slow today, but what in that enforces the alignment? I guess it's that the structure happens to be big enough that it happens naturally?
I'd allocate
max(512, sizeof(*dev_info))to make it explicitly the case.
yes, struct rsi_device_info is 512 bytes.
Quoted 15 lines
Quoted 15 lines
drivers/virt/coco/arm-cca-guest/rsi-da.c +14
-0+if (!dev_info) { + ret = -ENOMEM ; + goto err_out; + } + + ret =rsi_rdev_get_info (vdev_id, dsm->instance_id,virt_to_phys (dev_info)); +if (ret !=RSI_SUCCESS ) { +pci_err (pdev,"failed to get device digests (%lu)\n" , ret); + ret = -EIO ; +kfree (dev_info); + goto err_out; + } + + dsm->dev_info.attest_type = dev_info->attest_type;
Quoted 9 lines
drivers/virt/coco/arm-cca-guest/rsi-da.c int rsi_device_lock(struct pci_dev *pdev) · line 368
Quoted 6 lines
drivers/virt/coco/arm-cca-guest/rsi-da.c +6
-0+ dsm->dev_info.cert_id = dev_info->cert_id; + dsm->dev_info.hash_algo = dev_info->hash_algo; +memcpy (dsm->dev_info.cert_digest, dev_info->cert_digest,SHA512_DIGEST_SIZE ); +memcpy (dsm->dev_info.meas_digest, dev_info->meas_digest,SHA512_DIGEST_SIZE ); +memcpy (dsm->dev_info.report_digest, dev_info->report_digest,SHA512_DIGEST_SIZE ); +Can't you memcpy the whole thing in one go?
yes. But won't that be confusing? Is there a difference? Also struct dsm_device_info is not same as struct rsi_device_info. We don't need to keep all that padding in dsm_device_info.
Quoted 65 lines
Quoted 4 lines
drivers/virt/coco/arm-cca-guest/rsi-da.c +3
-0+kfree (dev_info); +/* +* Verify that the digests of the provided reports match with the drivers/virt/coco/arm-cca-guest/rsi-da.c int rsi_device_lock(struct pci_dev *pdev) · line 381
Quoted 10 lines
drivers/virt/coco/arm-cca-guest/rsi-da.c +10
-0+* digests from RMM +*/ + ret =verify_digests (dsm); +if (ret) { +pci_err (pdev,"device digest validation failed (%ld)\n" , ret); +return ret; + } + +return 0 ; +err_out:I'll always grumble about these. To me it always makes the code less readable. Some others disagree though ;(
drivers/virt/coco/arm-cca-guest/rsi-da.c int rsi_device_lock(struct pci_dev *pdev) · line 384
Quoted 3 lines
drivers/virt/coco/arm-cca-guest/rsi-da.c +1
-0return ret; } +Looks like this should have been in an earlier patch.
Quoted 19 lines
drivers/virt/coco/arm-cca-guest/rsi-da.c +0
-0static inline unsigned long rsi_rdev_start (struct pci_dev *pdev,unsigned long vdev_id,unsigned long inst_id) {drivers/virt/coco/arm-cca-guest/rsi-da.h +1
-0@@ -10,6 +10,7 @@#include <linux/pci-tsm.h> #include <asm/rsi_smc.h> #include <asm/rhi.h> +#include <crypto/sha2.h> struct pci_tdisp_device_interface_report {u16 interface_info;@@ -33,6 +34,17 @@ struct pci_tdisp_mmio_range {#define TSM_INTF_REPORT_MMIO_RESERVED GENMASK(15, 4)#define TSM_INTF_REPORT_MMIO_RANGE_ID GENMASK(31, 16)drivers/virt/coco/arm-cca-guest/rsi-da.h struct pci_tdisp_mmio_range { · line 46
Quoted 10 lines
drivers/virt/coco/arm-cca-guest/rsi-da.h +10
-0+struct dsm_device_info { +u64 flags; +u64 attest_type; +u64 cert_id; +u64 hash_algo; +u8 cert_digest[SHA512_DIGEST_SIZE ]; +u8 meas_digest[SHA512_DIGEST_SIZE ]; +u8 report_digest[SHA512_DIGEST_SIZE ]; +}; +One probably enough.
Quoted 11 lines
drivers/virt/coco/arm-cca-guest/rsi-da.h +2
-0+struct cca_guest_dsc {struct pci_tsm_pf0 pci;unsigned long instance_id;@@ -42,6 +54,7 @@ struct cca_guest_dsc {int certificate_size;void *measurements;int measurements_size; +struct dsm_device_info dev_info; };
Quoted 1 line
drivers/virt/coco/arm-cca-guest/rsi-da.h struct cca_guest_dsc { · line 60
Quoted 1 line
drivers/virt/coco/arm-cca-guest/rsi-da.h +0
-0static inline struct cca_guest_dsc *to_cca_guest_dsc (struct pci_dev *pdev)
-aneesh
Aneesh Kumar K.V
maintainerRe: [RFC PATCH v1 04/38] tsm: Support DMA Allocation from private memory
In reply to Jason GunthorpeQuoted 72 lines · Jason Gunthorpe
Quoted 51 lines · Aneesh Kumar K.V
Quoted 49 lines · Jason Gunthorpe
Quoted 33 lines · Aneesh Kumar K.V
Quoted 31 lines · Jason Gunthorpe
arch/arm64/mm/mem_encrypt.c int set_memory_decrypted(unsigned long addr, int numpages) · line 56
Quoted 9 lines · Aneesh Kumar K.V (Arm)
arch/arm64/mm/mem_encrypt.c +5
-0@@ -48,3 +49,12 @@ int set_memory_decrypted(unsigned long addr, int numpages)return crypt_ops->decrypt (addr, numpages); }EXPORT_SYMBOL_GPL (set_memory_decrypted); + +bool force_dma_unencrypted (struct device *dev) +{ +if (dev->tdi_enabled) +return false;Is this OK? I see code like this:
static inline
dma_addr_tphys_to_dma_direct(struct device *dev,
phys_addr_tphys){ if (force_dma_unencrypted(dev)) return phys_to_dma_unencrypted(dev, phys); return phys_to_dma(dev, phys);What are the ARM rules for generating dma addreses?
1) Device is T=0, memory is unencrypted, call
dma_addr_unencrypted()and do "top bit IBA set"2) Device is T=1, memory is encrypted, use the
phys_to_dma()normally3) Device it T=1, memory is uncrypted, use the
phys_to_dma()normally??? Seems odd, I would have guessed the DMA address sould be the same as case #1?Quoted 5 lines
Quoted 2 lines
Can you document this in a comment?
If a device is operating in secure mode (T=1), it is currently assumed that only access to private (encrypted) memory is supported.
No, this is no how the PCI specs were written as far as I understand. The XT bit thing is supposed to add more fine grained device side control over what memory the DMA can target. T alone does not do that.
Quoted 3 lines
It is unclear whether devices would need to perform DMA to shared (unencrypted) memory while operating in this mode, as TLPs with T=1 are generally expected to target private memory.
Quoted 8 lines
Quoted 2 lines
PCI SIG supports it, kernel should support it.
Would we also need a separate DMA allocation API for allocating addresses intended to be shared with the non-secure hypervisor?
Are there any existing drivers in the kernel that already require such shared allocations, which I could use as a reference?
The most likely case in the near term is PCI P2P to shared MMIO.
I don't know any way to allocate shared memory in a driver??
At the bare minimum this patch should be documenting the correct architecture and outlining any gaps in the current implementation.
I also don't really understand what the above code is even doing.. Isn't the design on ARM that the IPA always encodes the shared/private in the top bit?
Quoted 6 lines
How do we get a shared page that does not already have a
phys_addr_tin the shared IPA? Shouldn't the kernel have switched to the shared IPA alias when it returned the swiotlb buffer? eg why do we need to do:#define
dma_addr_unencrypted(x)((x) |PROT_NS_SHARED)
swiotlb virt addr is updated in the direct map page table such that we have the correct attribute set. For ex: swiotlb_update_mem_attributes uses set_memory_decrypted() to mark the memory as shared.
set_memory_decrypted((unsigned long)mem->vaddr, bytes >> PAGE_SHIFT);
However, when mapping swiotlb regions to obtain a dma_addr_t, we still need to explicitly convert the physical address:
swiotlb_map()
swiotlb_addr = swiotlb_tbl_map_single(dev, paddr, size, 0, dir, attrs);
...
/* Ensure that the address returned is DMA'ble */
dma_addr = phys_to_dma_unencrypted(dev, swiotlb_addr);Note that we don’t update the phys_addr_t to set the top bit. For reference:
tlb_addr = slot_addr(pool->start, index) + offset;
-aneesh
Re: [RFC PATCH v1 34/38] coco: guest: arm64: Validate mmio range found in the interface report
In reply to Aneesh Kumar K.VQuoted 29 lines · Aneesh Kumar K.V
Quoted 27 lines · Arto Merilainen
Quoted 6 lines · Aneesh Kumar K.V (Arm)
drivers/virt/coco/arm-cca-guest/rsi-da.c +6
-0+for (int i =0 ; i < interface_report->mmio_range_count; i++, mmio_range++) { + +/*FIXME!! units in 4K size*/ + range_id =FIELD_GET (TSM_INTF_REPORT_MMIO_RANGE_ID , mmio_range->range_attributes); + +/* no secure interrupts */ drivers/virt/coco/arm-cca-guest/rsi-da.c int rsi_device_lock(struct pci_dev *pdev) · line 297
Quoted 10 lines
drivers/virt/coco/arm-cca-guest/rsi-da.c +10
-0+if (msix_tbl_bar !=-1 && range_id == msix_tbl_bar) { +pr_info ("Skipping misx table\n" ); +continue ; + } + +if (msix_pba_bar !=-1 && range_id == msix_pba_bar) { +pr_info ("Skipping misx pba\n" ); +continue ; + } +MSI-X and PBA can be placed to a BAR that has other registers as well. While the PCIe specification recommends BAR-level isolation for MSI-X structures, it is not mandated. It is enough to have sufficient isolation within the BAR. Therefore, skipping the MSI-X and PBA BARs altogether may leave registers unintentionally mapped via unprotected IPA when they should have been mapped via protected IPA.
Quoted 7 lines
Quoted 4 lines
Instead of skipping the whole BAR, would it make sense to determine where the MSI-X related regions reside, and skip validation only from these regions?
Yes, that was added because at one point the FVP model was including the MSI-X table and PBA regions in the interface report.
The issue I am raising is a different one. The MSI-X table and PBA may reside in the middle of a BAR range, and the BAR range may contain registers which should be accessible only via protected IPA. In this case I would expect the pages before and after the MSI-X table (and PBA) to be validated while the pages related to MSI-X structures would be left unprotected.
Given that the MSI-X table or PBA regions shouldn't be present in the interface report, the code needs to first find out where the MSI-X structures reside, and use this information to determine which "sub-ranges" should be skipped over during validation.
Quoted 4 lines
If I understand correctly, we shouldn't expect to see those regions in the report unless secure interrupts are supported. The BAR-based skipping was added as a workaround to handle the FVP issue.
Correct. Assuming that RMM doesn't lock the MSI-X table, I'd expect these regions to be omitted in the interface report.
Quoted 4 lines
I believe we can drop that workaround now—if those regions are incorrectly present, the below validation logic should catch and reject them appropriately. Does that sound reasonable?
Quoted 11 lines
/* No secure interrupts, we should not find this set, ignore for now. */
if (FIELD_GET(TSM_INTF_REPORT_MMIO_MSIX_TABLE, mmio_range->range_attributes) ||
FIELD_GET(TSM_INTF_REPORT_MMIO_PBA, mmio_range->range_attributes)) {
pci_info(pdev, "Skipping MSIX (%ld/%ld) range [%d] #%d %d pages, %llx..%llx\n",
FIELD_GET(TSM_INTF_REPORT_MMIO_MSIX_TABLE, mmio_range->range_attributes),
FIELD_GET(TSM_INTF_REPORT_MMIO_PBA, mmio_range->range_attributes),
i, range_id, mmio_range->num_pages, r->start, r->end);
continue;
}First, I think this is skipping over the whole range => if there are registers outside of the MSI-X structures (but within the same BAR), they won't be validated.
Second, if the MSI-X regions are present in the interface report, wouldn't it - in the common case - mean that the device expects the ranges to be accessed with T=1? If this happens unexpectedly, it sounds that MSI-X wouldn't be usable. I wonder if the code should simply return an error instead if informing the user via pci_info()...
- R2
Aneesh Kumar K.V
maintainerRe: [RFC PATCH v1 10/38] iommufd/vdevice: Add TSM map ioctl
In reply to Alexey KardashevskiyQuoted 16 lines · Alexey Kardashevskiy
Quoted 7 lines · Jason Gunthorpe
Quoted 5 lines · Aneesh Kumar K.V (Arm)
With passthrough devices, we need to make sure private memory is allocated and assigned to the secure guest before we can issue the DMA. For ARM RMM, we only need to map and the secure SMMU management is internal to RMM. For shared IPA, vfio/iommufd DMA MAP/UNMAP interface does the equivalent
Quoted 6 lines
I'm not really sure what this is about? It is about getting KVM to pin all the memory and commit it to the RMM so it can be used for DMA?
But it looks really strange to have an iommufd ioctl that just calls a KVM function. Feeling this should be a KVM function, or a guestmfd behavior??
Quoted 5 lines
I ended up exporting the guestmemfd's
kvm_gmem_get_folio()forgfn->pfnand its fd a bit differently in iommufd - "no extra referencing":
https://github.com/AMDESE/linux-kvm/commit/f1ebd358327f026f413f8d3d64d46decfd6ab7f6It is a new
iommufd->kvmdependency though.
Was the motivation for that design choice the fact that in case of AMD VFIO/IOMMUFD manages both private memory allocation and updates to the IOMMU page tables?
On the ARM side, the requirement is to ensure that pages are present in the stage-2 page table, which is managed by the firmware (RMM). Because of this, we need an interface that VFIO/IOMMUFD can use to trigger stage-2 mappings within KVM.
Alternatively, we could introduce a dedicated KVM ioctl for this purpose, avoiding the need to rely on IOMMUFD.
For reference, TDX uses a similar ioctl—KVM_TDX_INIT_MEM_REGION—to initialize guest memory. However, that interface isn’t well-suited for dynamic updates to stage-2 mappings during shared-to-private or private-to-shared transitions.
Quoted 6 lines
Quoted 5 lines
I was kind of thinking it would be nice to have a guestmemfd mode that was "pinned", meaning the memory is allocated and remains almost always mapped into the TSM's page tables automatically. VFIO using guests would set things this way.
Quoted 7 lines
Yeah while doing the above, I was wondering if I want to pass the fd type when DMA-mapping from an fd or "detect" it as I do in the above commit or have some
iommufd_fdmap_opsin this fd saying "(no) pinning needed" (or make this a flag ofIOMMU_IOAS_MAP_FILE).The "detection" is (
mapping_inaccessible(mapping)&&mapping_unevictable(mapping)), works for now.btw in the AMD case, here it does not matter as much if it is private or shared, I map everything and let RMP and the VM deal with the permissions. Thanks,
-aneesh
Re: [RFC PATCH v1 38/38] coco: guest: arm64: Add support for fetching device info
In reply to Aneesh Kumar K.VQuoted 26 lines
Quoted 16 lines
Quoted 16 lines
drivers/virt/coco/arm-cca-guest/rsi-da.c +14
-0+if (!dev_info) { + ret = -ENOMEM ; + goto err_out; + } + + ret =rsi_rdev_get_info (vdev_id, dsm->instance_id,virt_to_phys (dev_info)); +if (ret !=RSI_SUCCESS ) { +pci_err (pdev,"failed to get device digests (%lu)\n" , ret); + ret = -EIO ; +kfree (dev_info); + goto err_out; + } + + dsm->dev_info.attest_type = dev_info->attest_type;Quoted 9 lines
drivers/virt/coco/arm-cca-guest/rsi-da.c int rsi_device_lock(struct pci_dev *pdev) · line 368
Quoted 6 lines
drivers/virt/coco/arm-cca-guest/rsi-da.c +6
-0+ dsm->dev_info.cert_id = dev_info->cert_id; + dsm->dev_info.hash_algo = dev_info->hash_algo; +memcpy (dsm->dev_info.cert_digest, dev_info->cert_digest,SHA512_DIGEST_SIZE ); +memcpy (dsm->dev_info.meas_digest, dev_info->meas_digest,SHA512_DIGEST_SIZE ); +memcpy (dsm->dev_info.report_digest, dev_info->report_digest,SHA512_DIGEST_SIZE ); +Can't you memcpy the whole thing in one go?
Quoted 3 lines
yes. But won't that be confusing? Is there a difference? Also struct
dsm_device_infois not same as structrsi_device_info. We don't need to keep all that padding indsm_device_info.
Ah. I misread and thought they were the same structure. No problem copying only relevant fields then!
Jonathan
Re: [RFC PATCH v1 03/38] tsm: Move dsm_dev from pci_tdi to pci_tsm
In reply to Aneesh Kumar K.V (Arm)On Mon, Jul 28, 2025 at 07:21:40PM +0530, Aneesh Kumar K.V (Arm) wrote:
Subject line should include "PCI" prefix.
Needs a commit log, even if it repeats what's in the subject. Would also be good to know *why* this is desirable.
Quoted 11 lines
Signed-off-by Aneesh Kumar K.V (Arm) <[email protected]>--- drivers/pci/tsm.c | 72 ++++++++++++++++++++++++----------------- include/linux/pci-tsm.h | 4 +-- 2 files changed, 45 insertions(+), 31 deletions(-)drivers/pci/tsm.c +0
-0
drivers/pci/tsm.c static enum pci_tsm_type pci_tsm_type(struct pci_dev *pdev) · line 418
Quoted 4 lines
drivers/pci/tsm.c +1
-0@@ -415,15 +415,55 @@ static enum pci_tsm_type pci_tsm_type(struct pci_dev *pdev)return PCI_TSM_INVALID ; } +/* lookup the Device Security Manager (DSM) pf0 for @pdev */
s/lookup/Look up/ (it's a verb here)
drivers/pci/tsm.c static enum pci_tsm_type pci_tsm_type(struct pci_dev *pdev) · line 422
Quoted 4 lines
drivers/pci/tsm.c +4
-0+static struct pci_dev *dsm_dev_get (struct pci_dev *pdev) +{ +struct pci_dev *uport_pf0; +
Remove this blank line ...
drivers/pci/tsm.c static enum pci_tsm_type pci_tsm_type(struct pci_dev *pdev) · line 423
Quoted 1 line
drivers/pci/tsm.c +1
-0+ struct pci_dev *pf0 __free(pci_dev_put) = pf0_dev_get(pdev);
and add one here.
Quoted 4 lines
drivers/pci/tsm.c +4
-0+if (!pf0) +return NULL ; + +if (pf0 == pdev)
drivers/pci/tsm.c static enum pci_tsm_type pci_tsm_type(struct pci_dev *pdev) · line 437
Quoted 10 lines
drivers/pci/tsm.c +10
-0+return no_free_ptr (pf0); + +/* Check that @pf0 was not initialized as PCI_TSM_DOWNSTREAM */ +if (pf0->tsm && pf0->tsm->type ==PCI_TSM_PF0 ) +return no_free_ptr (pf0); + +/* +* For cases where a switch may be hosting TDISP services on +* behalf of downstream devices, check the first usptream port +* relative to this endpoint.
s/usptream/upstream/
Quoted 14 lines
drivers/pci/tsm.c +10
-0+*/ +if (!pdev->dev.parent || !pdev->dev.parent->parent) +return NULL ; + + uport_pf0 =to_pci_dev (pdev->dev.parent->parent); +if (!uport_pf0->tsm) +return NULL ; +return pci_dev_get (uport_pf0); +} +/** * pci_tsm_initialize() - base 'struct pci_tsm' initialization * @pdev: The PCI device * @tsm: context to initialize
drivers/pci/tsm.c static enum pci_tsm_type pci_tsm_type(struct pci_dev *pdev) · line 460
Quoted 10 lines
drivers/pci/tsm.c +5
-1*/ -void pci_tsm_initialize (struct pci_dev *pdev,struct pci_tsm *tsm) +int pci_tsm_initialize (struct pci_dev *pdev,struct pci_tsm *tsm) { + struct pci_dev *dsm_dev __free(pci_dev_put) = dsm_dev_get(pdev); +if (!dsm_dev) +return -EINVAL ; + tsm->type =pci_tsm_type (pdev); tsm->pdev = pdev;
Add blank line before comment.
drivers/pci/tsm.c static enum pci_tsm_type pci_tsm_type(struct pci_dev *pdev) · line 463
Quoted 3 lines
drivers/pci/tsm.c +3
-0+/* +* No reference needed because when we destroy +* dsm_dev all the tdis get destroyed before that.
"tdi" looks like an initialism, which would normally be capitalized.
Quoted 36 lines
drivers/pci/tsm.c +5
-17+*/ + tsm->dsm_dev = dsm_dev; +return 0 ; }EXPORT_SYMBOL_GPL (pci_tsm_initialize);@@ -447,7 +487,8 @@ int pci_tsm_pf0_initialize(struct pci_dev *pdev, struct pci_tsm_pf0 *tsm) } tsm->state =PCI_TSM_INIT ; -pci_tsm_initialize (pdev, &tsm->tsm); +if (pci_tsm_initialize (pdev, &tsm->tsm)) +return -ENODEV ;return 0 ; }@@ -612,32 +653,6 @@ int pci_tsm_doe_transfer(struct pci_dev *pdev, enum pci_doe_proto type, }EXPORT_SYMBOL_GPL (pci_tsm_doe_transfer); -/* lookup the Device Security Manager (DSM) pf0 for @pdev */ -static struct pci_dev *dsm_dev_get (struct pci_dev *pdev) -{ -struct pci_dev *uport_pf0; - - struct pci_dev *pf0 __free(pci_dev_put) = pf0_dev_get(pdev); -if (!pf0) -return NULL ; - -/* Check that @pf0 was not initialized as PCI_TSM_DOWNSTREAM */ -if (pf0->tsm && pf0->tsm->type ==PCI_TSM_PF0 ) -return no_free_ptr (pf0); - -/* -* For cases where a switch may be hosting TDISP services on -* behalf of downstream devices, check the first usptream port
drivers/pci/tsm.c int pci_tsm_doe_transfer(struct pci_dev *pdev, enum pci_doe_proto type, · line 640
Quoted 10 lines
drivers/pci/tsm.c +0
-10-* relative to this endpoint. -*/ -if (!pdev->dev.parent || !pdev->dev.parent->parent) -return NULL ; - - uport_pf0 =to_pci_dev (pdev->dev.parent->parent); -if (!uport_pf0->tsm) -return NULL ; -return pci_dev_get (uport_pf0); -}
This code move looks like it could be a separate patch that only moves (and fixes the typos I mentioned).
Then a second patch could do what the subject claims (moving dsm_dev from pci_tdi to pci_tsm) so it's not buried in the simple move.
Quoted 42 lines
drivers/pci/tsm.c +0
-1/* Only implement non-interruptible lock for now */ static struct mutex *tdi_ops_lock (struct pci_dev *pf0_dev)@@ -695,7 +710,6 @@ int pci_tsm_bind(struct pci_dev *pdev, struct kvm *kvm, u64 tdi_id)return -ENXIO ; tdi->pdev = pdev; - tdi->dsm_dev = dsm_dev; tdi->kvm = kvm; pdev->tsm->tdi = tdi;include/linux/pci-tsm.h +2
-2@@ -38,7 +38,6 @@ enum pci_tsm_type {*/ struct pci_tdi {struct pci_dev *pdev; -struct pci_dev *dsm_dev;struct kvm *kvm; };@@ -56,6 +55,7 @@ struct pci_tdi {*/ struct pci_tsm {struct pci_dev *pdev; +struct pci_dev *dsm_dev;enum pci_tsm_type type;struct pci_tdi *tdi; };@@ -173,7 +173,7 @@ void pci_tsm_core_unregister(const struct pci_tsm_ops *ops);int pci_tsm_doe_transfer (struct pci_dev *pdev,enum pci_doe_proto type,const void *req,size_t req_sz,void *resp,size_t resp_sz); -void pci_tsm_initialize (struct pci_dev *pdev,struct pci_tsm *tsm); +int pci_tsm_initialize (struct pci_dev *pdev,struct pci_tsm *tsm);int pci_tsm_pf0_initialize (struct pci_dev *pdev,struct pci_tsm_pf0 *tsm);int pci_tsm_bind (struct pci_dev *pdev,struct kvm *kvm,u64 tdi_id);int pci_tsm_unbind (struct pci_dev *pdev);Signature · 1 line
2.43.0
Re: [RFC PATCH v1 04/38] tsm: Support DMA Allocation from private memory
In reply to Aneesh Kumar K.V (Arm)drivers/pci/tsm.c static void __pci_tsm_init(struct pci_dev *pdev) · line 564
Quoted 10 lines · Aneesh Kumar K.V (Arm)
Currently, we enforce the use of bounce buffers to ensure that memory accessed by non-secure devices is explicitly shared with the host [1]. However, for secure devices, this approach must be avoided.
drivers/pci/tsm.c +2
-0@@ -557,6 +560,9 @@ static void __pci_tsm_init(struct pci_dev *pdev)default :break ; } + +/* FIXME!! should this be default true and switch to false for TEE capable device */
Fix whatever this is, or make it a real sentence and wrap to fit in 80 columns.
Quoted 2 lines
(quoted diff) +1
-0+ pdev->dev.tdi_enabled = false; }
Re: [RFC PATCH v1 05/38] tsm: Don't overload connect
In reply to Aneesh Kumar K.V (Arm)Only touches drivers/pci, so needs a "PCI" prefix of some flavor.
Quoted 5 lines · Aneesh Kumar K.V (Arm)
We need separate handling in the guest while destroying the device. Hence switch to new callback lock and unlock. Hide the connect sysfs file in guest.
guest sysfs will now looks like
s/guest/Guest/
s/will now looks/will now look/
Review
Quoted 6 lines
ls -al /sys/bus/pci/devices/0000:02:00.0/tsm/
total 0drwxr-xr-x 2 root root 0 Jan 1 00:00 . drwxr-xr-x 7 root root 0 Jan 1 00:00 ..(quoted diff) +0
-2-rw-r--r-- 1 root root 4096 Jan 1 00:00 accept -rw-r--r-- 1 root root 4096 Jan 1 00:00 lock
Indent quoted material a couple spaces.
I don't know about TSM, so this commit log doesn't quite tell me why we need this.
It appears this also adds a sysfs file that could be documented somehow.
Quoted 42 lines
Signed-off-by Aneesh Kumar K.V (Arm) <[email protected]>--- drivers/pci/tsm.c | 136 ++++++++++++++++++++++++++++++++++------ include/linux/pci-tsm.h | 3 + 2 files changed, 121 insertions(+), 18 deletions(-)drivers/pci/tsm.c +1
-16@@ -125,18 +125,6 @@ static int pci_tsm_disconnect(struct pci_dev *pdev)return 0 ; } -/* -* TDISP locked state temporarily makes the device inaccessible, do not -* surprise live attached drivers -*/ -static int __driver_idle_connect (struct pci_dev *pdev) -{ -guard (device)(&pdev->dev); -if (pdev->dev.driver) -return -EBUSY ; -return tsm_ops->connect (pdev); -} -/* * When the registered ops support accept it indicates that this is a * TVM-side (guest) TSM operations structure. In this mode ->connect() @@ -162,10 +150,7 @@ static int pci_tsm_connect(struct pci_dev *pdev)if (tsm->state >=PCI_TSM_CONNECT )return 0 ; -if (tvm_mode ()) - rc =__driver_idle_connect (pdev); -else - rc = tsm_ops->connect (pdev); + rc = tsm_ops->connect (pdev);if (rc)return rc; tsm->state =PCI_TSM_CONNECT ;
drivers/pci/tsm.c static ssize_t accept_show(struct device *dev, struct device_attribute *attr, · line 290
Quoted 7 lines
drivers/pci/tsm.c +4
-0@@ -299,6 +284,99 @@ static ssize_t accept_show(struct device *dev, struct device_attribute *attr, }static DEVICE_ATTR_RW (accept); +static int pci_tsm_unlock (struct pci_dev *pdev) +{ +struct pci_tsm_pf0 *tsm =to_pci_tsm_pf0 (pdev->tsm); +
Drop this blank line ...
drivers/pci/tsm.c static ssize_t accept_show(struct device *dev, struct device_attribute *attr, · line 291
Quoted 1 line
drivers/pci/tsm.c +1
-0+ struct mutex *lock __free(tsm_ops_unlock) = tsm_ops_lock(tsm);
add one here. I'm seeing a new pattern; maybe the trend is to put declarations of __free() things outside the usual local variable list?
Quoted 176 lines
drivers/pci/tsm.c +112
-2+if (!lock) +return -EINTR ; + +if (tsm->state <PCI_TSM_INIT ) +return -ENXIO ; +if (tsm->state <PCI_TSM_LOCK ) +return 0 ; + + tsm_ops->unlock (pdev); + tsm->state =PCI_TSM_INIT ; + pdev->dev.tdi_enabled = false; + +return 0 ; +} + +/* +* TDISP locked state temporarily makes the device inaccessible, do not +* surprise live attached drivers +*/ +static int __driver_idle_lock (struct pci_dev *pdev) +{ +guard (device)(&pdev->dev); +if (pdev->dev.driver) +return -EBUSY ; +return tsm_ops->lock (pdev); +} + +static int pci_tsm_lock (struct pci_dev *pdev) +{ +struct pci_tsm_pf0 *tsm =to_pci_tsm_pf0 (pdev->tsm); +int rc; + + struct mutex *lock __free(tsm_ops_unlock) = tsm_ops_lock(tsm); +if (!lock) +return -EINTR ; + +if (tsm->state <PCI_TSM_INIT ) +return -ENXIO ; + + rc =__driver_idle_lock (pdev); +if (rc) +return rc; + tsm->state =PCI_TSM_CONNECT ; +return 0 ; +} + +static ssize_t lock_store (struct device *dev,struct device_attribute *attr, +const char *buf,size_t len) +{ +int rc; +bool connect; +struct pci_dev *pdev =to_pci_dev (dev); + + rc =kstrtobool (buf, &connect); +if (rc) +return rc; + + struct rw_semaphore *lock __free(tsm_read_unlock) = tsm_read_lock(); +if (!lock) +return -EINTR ; + +if (connect) + rc =pci_tsm_lock (pdev); +else + rc =pci_tsm_unlock (pdev); +if (rc) +return rc; +return len; +} + +static ssize_t lock_show (struct device *dev,struct device_attribute *attr, +char *buf) +{ +struct pci_dev *pdev =to_pci_dev (dev); +struct pci_tsm_pf0 *tsm; + + struct rw_semaphore *lock __free(tsm_read_unlock) = tsm_read_lock(); +if (!lock) +return -EINTR ; + +if (!pdev->tsm) +return -ENXIO ; + + tsm =to_pci_tsm_pf0 (pdev->tsm); +return sysfs_emit (buf,"%d\n" , tsm->state >=PCI_TSM_LOCK ); +} +static DEVICE_ATTR_RW (lock); +static umode_t pci_tsm_pf0_attr_visible (struct kobject *kobj,struct attribute *a,int n) {@@ -306,6 +384,11 @@ static umode_t pci_tsm_pf0_attr_visible(struct kobject *kobj,/* Host context, filter out guest only attributes */ if (a == &dev_attr_accept.attr)return 0 ; +if (a == &dev_attr_lock.attr) +return 0 ; + }else { +if (a == &dev_attr_connect.attr) +return 0 ; }return a->mode;@@ -325,6 +408,7 @@ DEFINE_SYSFS_GROUP_VISIBLE(pci_tsm_pf0);static struct attribute *pci_tsm_pf0_attrs[] = { &dev_attr_connect.attr, &dev_attr_accept.attr, + &dev_attr_lock.attr,NULL };@@ -537,7 +621,8 @@ static void pci_tsm_pf0_init(struct pci_dev *pdev)return ; pdev->tsm =no_free_ptr (pci_tsm); -sysfs_update_group (&pdev->dev.kobj, &pci_tsm_auth_attr_group); +if (!tvm_mode ()) +sysfs_update_group (&pdev->dev.kobj, &pci_tsm_auth_attr_group);sysfs_update_group (&pdev->dev.kobj, &pci_tsm_pf0_attr_group);if (pci_tsm_owner_attr_group)sysfs_merge_group (&pdev->dev.kobj, pci_tsm_owner_attr_group);@@ -602,6 +687,19 @@ static void pci_tsm_pf0_destroy(struct pci_dev *pdev)__pci_tsm_pf0_destroy (tsm); } +static void pci_tsm_guest_destroy (struct pci_dev *pdev) +{ +struct pci_tsm_pf0 *tsm =to_pci_tsm_pf0 (pdev->tsm); + +if (tsm->state >PCI_TSM_INIT ) +pci_tsm_unlock (pdev); + pdev->tsm =NULL ; +if (pci_tsm_owner_attr_group) +sysfs_unmerge_group (&pdev->dev.kobj, pci_tsm_owner_attr_group); +sysfs_update_group (&pdev->dev.kobj, &pci_tsm_pf0_attr_group); +__pci_tsm_pf0_destroy (tsm); +} +static void __pci_tsm_destroy (struct pci_dev *pdev) {struct pci_tsm *pci_tsm = pdev->tsm;@@ -611,7 +709,9 @@ static void __pci_tsm_destroy(struct pci_dev *pdev)lockdep_assert_held_write (&pci_tsm_rwsem); -if (is_pci_tsm_pf0 (pdev)) { +if (tvm_mode ()) { +pci_tsm_guest_destroy (pdev); + }else if (is_pci_tsm_pf0 (pdev)) {pci_tsm_pf0_destroy (pdev); }else {__pci_tsm_unbind (pdev);include/linux/pci-tsm.h +3
-0@@ -11,6 +11,7 @@ enum pci_tsm_state {PCI_TSM_ERR =-1 ,PCI_TSM_INIT ,PCI_TSM_CONNECT , +PCI_TSM_LOCK , PCI_TSM_ACCEPT, };@@ -153,6 +154,8 @@ struct pci_tsm_ops {void (*remove)(struct pci_tsm *tsm);int (*connect)(struct pci_dev *pdev);void (*disconnect)(struct pci_dev *pdev); +int (*lock)(struct pci_dev *pdev); +void (*unlock)(struct pci_dev *pdev);struct pci_tdi *(*bind)(struct pci_dev *pdev,struct pci_dev *pf0_dev,struct kvm *kvm,u64 tdi_id);void (*unbind)(struct pci_tdi *tdi);Signature · 1 line
2.43.0